Develop Reference

Inserting batches of elements to a database

So i try to input all the keys from a HashMap to a database. My first approach was to insert all the keys one by one to my database. Note that the HashMap size is some million keys long so this process took a lot of time.
I did some research and stumbled upon the preparedStatement interface. So i came up with this piece of code to create a Batch of 10000 elements and then input them all together to the database.
final int batchSize = 10000;
int count = 0;
Connection dbConnection = null;
try {
dbConnection = getDBConnection();
String SQL = "INSERT INTO masterdict (window) " +
"VALUES(?)";
PreparedStatement ps = (PreparedStatement) dbConnection.prepareStatement(SQL);
for (String k : masterDict.keySet()) {
ps.setString(1,k);
ps.addBatch();
if(++count % batchSize == 0) {
System.out.println(count);
ps.executeBatch();
}
}
ps.executeBatch();
ps.close();
dbConnection.close();
for some reason though this approach takes exactly the same time to complete as the first one. Can anyone explain to me why is this the case?

After reading through the comments i ended up with this new version of the code that works just fine.
final int batchSize = 10000;
int count = 0;
Connection dbConnection = null;
try {
dbConnection = getDBConnection();
String SQL = "INSERT INTO masterdict (window) " +
"VALUES(?)";
PreparedStatement ps = (PreparedStatement) dbConnection.prepareStatement(SQL);
dbConnection.setAutoCommit(false);
for (String k : masterDict.keySet()) {
ps.setString(1,k);
ps.addBatch();
if(++count % batchSize == 0) {
System.out.println(count);
ps.executeBatch();
dbConnection.commit();
}
}
ps.executeBatch();
dbConnection.commit();
ps.close();
} catch (SQLException e) {
if (dbConnection != null) {
dbConnection.rollback();
}
System.out.println(e.getMessage());
} finally {
dbConnection.close();
}

Java JDBC display first 500 records at a time, commit, than display the next 500 records and etc

So I want to be able to display 500 records at a time, commit and print that it has been displayed records 1 to 500 records have been committed. And than do the next 500 records and commit again until reached the maximum records which is over 20k records. I managed to get the first 500 records but I am stuck how can I commit them and in commit them and continue to get the next 500 records and so on.
public static void selectRecordsIcore() throws SQLException {
Connection dbConnection = null;
PreparedStatement preparedStatement = null;
Statement statement = null;
String selectTableSQL = "SELECT profile_id, ingress_flag, egress_flag, ce_ingress_flag, ce_egress_flag from COS_PROFILE"
+ " WHERE profile_id >= ? AND profile_id <= ?;";
try {
dbConnection = getInformixConnection(); //connects to ICORE database
System.out.println("I am in the try");
//Gets the max profile_id record
statement = dbConnection.createStatement();
ResultSet r = statement.executeQuery("SELECT max(profile_id) AS rowcount FROM COS_PROFILE");
r.next();
int maxCount = r.getInt("rowcount");
System.out.println("COS_PROFILE table before update has " + maxCount + " row(s).");
preparedStatement = dbConnection.prepareStatement(selectTableSQL);
preparedStatement.setInt(1, 1);
preparedStatement.setInt(2, maxCount);
// execute select SQL statement
rs = preparedStatement.executeQuery();
updateRecordIntoBids();
} catch (SQLException e) {
System.out.println(e.getMessage());
} finally {
if (rs != null) {
rs.close();
}
if (statement != null) {
statement.close();
}
if (preparedStatement != null) {
preparedStatement.close();
}
if (dbConnection != null) {
dbConnection.close();
System.out.println("Database ICORE Connection is closed");
}
}
}
private static void updateRecordIntoBids() throws SQLException {
System.out.println("I am inside update method");
Connection dbConnection = null;
PreparedStatement preparedStatement = null;
dbConnection = getOracleConnection(); //connects to BIDS database
String updateTableSQL =
"UPDATE traffic_profile_temp SET pe_ingress_flag = ?, "
+ " pe_egress_flag = ?,"
+ " ce_ingress_flag = ?,"
+ " ce_egress_flag = ? "
+ " WHERE traffic_profile_id = ? ";
preparedStatement = dbConnection.prepareStatement(updateTableSQL);
try {
int rowCount = 0;
while (rs.next() && rowCount < 500) {
// System.out.println("inside the while loop");
String ingressflag = rs.getString("ingress_flag"); //BIDS column is pe_ingress_flag
String egressflag = rs.getString("egress_flag"); //BIDS column is pe_egress_flag
String ceingressflag = rs.getString("ce_ingress_flag"); //BIDS column is ce_ingress_flag
String ceegressflag = rs.getString("ce_egress_flag"); //BIDS column is ce_egress_flag
int profileid = rs.getInt("profile_id"); //BIDS column is traffic_profile_id
preparedStatement.setString(1, ingressflag);
preparedStatement.setString(2, egressflag);
preparedStatement.setString(3, ceingressflag);
preparedStatement.setString(4, ceegressflag);
preparedStatement.setInt(5, profileid);
// System.out.println(updateTableSQL);
System.out.println("Record " +profileid +" is updated to traffic_profile_temp table!");
// execute update SQL stetement
preparedStatement.addBatch();
rowCount++;
System.out.println(rowCount);
}
preparedStatement.executeBatch();
} catch (SQLException e) {
System.out.println(e.getMessage());
} finally {
if (preparedStatement != null) {
preparedStatement.close();
}
if (dbConnection != null) {
dbConnection.close();
System.out.println("Database BIDS Connection is closed");
}
}
}

update this part
while (rs.next() && rowCount < 500) {
with
while (rs.next()) {
and
// execute update SQL stetement
preparedStatement.addBatch();
rowCount++;
System.out.println(rowCount);
with
// execute update SQL stetement
preparedStatement.addBatch();
rowCount++;
if(rowCount % 500 == 0){
preparedStatement.executeBatch();
}
System.out.println(rowCount);
This check if the rowCount can be divided by 500, execute the batch. Don't forget to execute the batch after all statements finish to execute the remaining batches which couldn't divided by 500 . for more details regarding batches

jdbc sqlite returns no rows

public ArrayList<Booking> getAllBookings(){
ArrayList<Booking> list = new ArrayList<Booking>();
int rowCount = 0;
try {
stmt = connection.createStatement();
String sql = "SELECT * FROM Bookings";
ResultSet rows = stmt.executeQuery(sql);
rows.last();
System.out.println("Row Count "+rows.getRow());
rows.beforeFirst();
connection.commit();
while (rows.next()){
Booking b = new Booking();
otherStuff();
list.add(b);
rowCount++;
}
stmt.close();
rows.close();
} catch (Exception e) {
}
System.out.println("There were " +rowCount + " records.");
return list;
}
Why do I not get any rows returned? I am connecting using this:
public static Connection dbConnect(){
try {
Class.forName("org.sqlite.JDBC");
Connection connection = DriverManager.getConnection("jdbc:sqlite:D:/test/test_db.sqlite3");
System.out.println("Connection Succesful");
return connection;
} catch (Exception e) {
System.out.println(e);
return null;
} }
When running SQLite Studio my query returns the correct results, but here my output on console is:
Connection Succesful
(Row Count + rows.getRow() is not printing here for some reason)
There were 0 records.

How can I retry the statements that were not executed after a batch execution fails?

I'm need to update a table with data from a CSV. All data is validated before the update takes place: a validation method (witch is not presented bellow) checks if some assumptions are true and "flags" the object as valid or invalid. I've already test it a lot and it's working exactly as I want.
Even so, I would like to guarantee that all Statements will be executed even if there's a fail on a batch, something that I was not able to think about. If this happens, I want the batch in witch this fail statement is to be skipped and that the next one is executed.
public void updateTable(List<PersonBean> personList) {
Connection connection = null;
PreparedStatement ps = null;
String updateDBPersonSQL = "UPDATE Person set merge_parent_id = ? WHERE id = ?";
try {
logger.info("DATA UPDATING STARTED");
input = new FileInputStream("resources/propertiesFiles/applications.properties");
properties.load(input);
final int batchSize = Integer.parseInt(properties.getProperty("batchSize"));
connection = DBConnection.getConnection();
connection.setAutoCommit(false);
int validObj = 0;
ps = connection.prepareStatement(updateDBPersonSQL);
for (int i = 0; i < personList.size(); i++) {
PersonBean person = personList.get(i);
if (person.getValidationStatus().equals("valid")) {
ps.setInt(1, person.getMerge_parent_id());
ps.setInt(2, person.getId());
ps.addBatch();
validObj++;
if (validObj % batchSize == 0 && validObj != 0) {
ps.executeBatch();
connection.commit();
logger.info((batchSize) + " rows updated");
}
}
}
int [] batchCount = ps.executeBatch();
connection.commit();
logger.info(batchCount.length + " rows updated");
writeValidationStatusToCSV(personList);
} catch (BatchUpdateException e) {
int [] updateCount = e.getUpdateCounts();
for (int i = 0; i < updateCount.length; i++) {
if (updateCount[i] >= 0) {
logger.info(updateCount.length + " objects updated.");
} else if (updateCount[i] == Statement.EXECUTE_FAILED) {
?????
}
}
logger.error(updateCount.length);
logger.error("BatchUpdateException: " + e);
logger.error("getNextException: " + e.getNextException());
try {
connection.rollback();
} catch (SQLException e1) {
logger.error("Rollback error: " + e1, e1);
}
} finally {
if (ps!= null) {
try {
ps.close();
} catch (SQLException e) {
logger.info(e);
}
}
}
logger.info("DATA UPDATING FINISHED");
}
I saw a lot of material about how to handle the exception, but none explained or pointed me to the direction of how to retry the next Statements, it means, how to execute the next batch.
How do I manage to do this?
EDIT: I'm using Postgresql

I manage to retry the next batches by surrounding the batch execution with try and catch statements. This way I'm able to catch the BatchUpdateException and call a continue statement.
try {
ps.executeBatch();
connection.commit();
/*Some more code*/
} catch (BatchUpdateException e) {
connection.rollback();
/*Some more code*/
continue;
}
I also used some control logic to "flag" the statements and batches that were already executed and logged them, making it easier to troubleshoot if some statement fails.
Here is the full code:
public void updateTable(List<PersonBean> personList) throws Exception {
logger.info("TABLE UPDATE STARTED");
List <PersonBean> personListValidated = createValidStmtList(personList);
Connection connection = null;
PreparedStatement ps = null;
String updatePersonSQL = "UPDATE Person SET merge_parent_id = ? WHERE id = ?";
input = new FileInputStream("resources/propertiesFiles/applications.properties");
properties.load(input);
final int batchSize = Integer.parseInt(properties.getProperty("batchSize"));
/*A list was used to "flag" the batches that were already executed. BatchStatus objs have only two parameters, number (incremented as the batches are being executed) and status (success or fail).*/
List <BatchStatus> batchStatusList = new ArrayList<BatchStatus>();
/*This variables will be used to help flag the batches and statements that were already executed.*/
int batchCount = 0;
int stmtAddedToBatchCount = 0;
try {
connection = DBConnection.getConnection();
connection.setAutoCommit(false);
ps = connection.prepareStatement(updatePersonSQL);
/*personListValidated contains the objects that will be updated in the table. Instead of doing the validation on the update method, I decomposed
* this part in other 2 methods, making it easier to control of the statements added to the batch.
*/
for (int i = 0; i < personListValidated.size(); i++) {
PersonBean personValid = personListValidated.get(i);
ps.setInt(1, personValid.getMerge_parent_id());
ps.setInt(2, personValid.getId());
ps.addBatch();
personValid.setToBatch("true");
stmtAddedToBatchCount++;
logger.info("Row added to batch (count: " + stmtAddedToBatchCount + ")");
if (stmtAddedToBatchCount % batchSize == 0) {
batchCount++;
try {
ps.executeBatch();
connection.commit();
for (int j = stmtAddedToBatchCount - batchSize; j < stmtAddedToBatchCount; j++){
personValid = personListValidated.get(j);
personValid.setValidationStatus("success");
}
BatchStatus batchStatusObj = new BatchStatus(batchCount, "sucess");
batchStatusList.add(batchStatusObj);
logger.info(batchStatusList.get(batchCount - 1));
} catch (BatchUpdateException e) {
connection.rollback();
for (int j = stmtAddedToBatchCount - batchSize; j < stmtAddedToBatchCount; j++){
personValid = personListValidated.get(j);
personValid.setValidationStatus("fail");
}
BatchStatus batchStatusObj = new BatchStatus(batchCount, "fail");
batchStatusList.add(batchStatusObj);
logger.info(batchStatusList.get(batchCount - 1));
logger.error("Bacth execution fail: " + e, e);
continue;
}
}
}
} catch (SQLException e) {
logger.error(e, e);
}
int[] lastBatchCount = null;
/*Try and catch to handle the statements executed on the last batch*/
try {
lastBatchCount = ps.executeBatch();
connection.commit();
for (int j = batchStatusList.size() * batchSize; j < stmtAddedToBatchCount; j++){
PersonBean personValid = personListValidated.get(j);
personValid.setValidationStatus("success");
}
logger.info(lastBatchCount.length + " rows inserted on the last batch");
logger.info("Last batch excuted");
} catch (BatchUpdateException e) {
connection.rollback();
for (int j = batchStatusList.size() * batchSize; j < stmtAddedToBatchCount; j++){
PersonBean personValid = personListValidated.get(j);
personValid.setValidationStatus("fail");
}
logger.error("Last batch fail to execute: " + e, e);
}
writeValidationStatusToCSV(personList);
logger.info("TABLE UPDATE FINISHED");
}

Inserting information from one mysql table to another

I am writing a program that will take in a student ID and verify if that ID exists in a mysql table. If it does exist, I would like to take the entire row that it exists in and copy that row to another table. Currently the program will just copy all rows in a table to the other. Any help appreciated. I have inserted a snippet of code below.
try {
String compareText = IDField.getText().trim();
if(compareText.length() > 0){
Class.forName("com.mysql.jdbc.Driver");
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/simlab","root","password");
System.out.println("Connected to database");
Statement stmt1 = conn.createStatement();
ResultSet rs1 = stmt1.executeQuery("select * from students where LUID='"+IDField.getText()+"' ");
boolean isPresent = rs1.next();
if (isPresent)
{
Class.forName("com.mysql.jdbc.Driver");
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/simlab","root","password");
System.out.println("Connected to database");
int rows = stmt1.executeUpdate("INSERT INTO skills(ID_Student,LUID_Student)SELECT ID, LUID FROM students");
if (rows == 0)
{
System.out.println("Don't add any row!");
}
else
{
System.out.println(rows + " row(s)affected.");
conn.close();
}
//System.out.println("Already exists!!");
}

You could all do that in a single SQL statement:
INSERT INTO <Dest-Table>
(SELECT * FROM <Src-Table> WHERE ID=?);
It will only copy rows that exist.

I suspect it's due to this line:
int rows = stmt1.executeUpdate("INSERT INTO skills(ID_Student,LUID_Student)SELECT ID, LUID FROM students");
As, if that line is parsed, the SELECT statement has no WHERE clause, and will therefore get every row, and therefore insert everything.

With Prepared statements
String sql = "INSERT INTO abc"
+ "(SELECT id1,id2 FROM pqr)";
ps1 = con.prepareStatement(sql);
int rs = ps1.executeUpdate();
if (rs > 0) {
update = true;
} else {
update = false;
}
} catch (Exception ex) {
ex.printStackTrace();
} finally {
try {
if (ps1 != null) {
ps1.close();
ps1 = null;
}
if (con != null) {
con.close();
con = null;
}
} catch (Exception e) {
}
}
return update;