Develop Reference

How do I update thousands of records into MySQL DB in milliseconds

I want to update about 10K records into MySQL DB in less than a second. I have written below code which takes about 6-8 seconds to update a list of records into DB.
public void updateResultList(List<?> list) {
String user = "root";
String pass = "root";
String jdbcUrl = "jdbc:mysql://12.1.1.1/db_1?useSSL=false";
String driver = "com.mysql.jdbc.Driver";
PreparedStatement pstm = null;
try {
Class.forName(driver);
Connection myConn = DriverManager.getConnection(jdbcUrl, user, pass);
myConn.setAutoCommit(false);
for(int i=0; i<list.size(); i++) {
Object[] row = (Object[]) list.get(i);
int candidateID = Integer.valueOf(String.valueOf(row[0]));
String result = String.valueOf(row[14]);
int score = Integer.valueOf(String.valueOf(row[19]));
String uploadState = (String) row[20];
String sql = "UPDATE personal_info SET result = ?, score = ?, uploadState = ? "
+ " WHERE CandidateID = ?";
pstm = (PreparedStatement) myConn.prepareStatement(sql);
pstm.setString(1, result);
pstm.setInt(2, score);
pstm.setString(3, uploadState);
pstm.setInt(4, candidateID);
pstm.addBatch();
pstm.executeBatch();
}
myConn.commit();
myConn.setAutoCommit(true);
pstm.close();
myConn.close();
}
catch (Exception exc) {
exc.printStackTrace();
try {
throw new ServletException(exc);
} catch (ServletException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
Please let me know your inputs to optimize this code for performance improvement.

First, you need to init prepareStatement only once,you need to init it before the for loop
Second,you should avoid excute pstm.executeBatch(); for every loop it will cost much more resource,you need to execute it for a specified amount,such as 100,500 or more,also do not execute it outside the for loop for only once,due to it will cost more memory resource
Class.forName(driver);
Connection myConn = DriverManager.getConnection(jdbcUrl, user, pass);
myConn.setAutoCommit(false);
String sql = "UPDATE personal_info SET result = ?, score = ?, uploadState = ? "
+ " WHERE CandidateID = ?";
pstm = (PreparedStatement) myConn.prepareStatement(sql);
for(int i=0; i<list.size(); i++) {
Object[] row = (Object[]) list.get(i);
int candidateID = Integer.valueOf(String.valueOf(row[0]));
String result = String.valueOf(row[14]);
int score = Integer.valueOf(String.valueOf(row[19]));
String uploadState = (String) row[20];
pstm.setString(1, result);
pstm.setInt(2, score);
pstm.setString(3, uploadState);
pstm.setInt(4, candidateID);
pstm.addBatch();
if(i%500==0){//execute when it meet a specified amount
pstm.executeBatch();
}
}
pstm.executeBatch();
myConn.commit();
myConn.setAutoCommit(true);

Rather than batching the individual UPDATEs, you could batch INSERTs into a temporary table with rewriteBatchedStatements=true and then use a single UPDATE statement to update the main table. On my machine with a local MySQL instance, the following code takes about 2.5 seconds ...
long t0 = System.nanoTime();
conn.setAutoCommit(false);
String sql = null;
sql = "UPDATE personal_info SET result=?, score=?, uploadState=? WHERE CandidateID=?";
PreparedStatement ps = conn.prepareStatement(sql);
String tag = "X";
for (int i = 1; i <= 10000; i++) {
ps.setString(1, String.format("result_%s_%d", tag, i));
ps.setInt(2, 200000 + i);
ps.setString(3, String.format("state_%s_%d", tag, i));
ps.setInt(4, i);
ps.addBatch();
}
ps.executeBatch();
conn.commit();
System.out.printf("%d ms%n", (System.nanoTime() - t0) / 1000000);
... while this version takes about 1.3 seconds:
long t0 = System.nanoTime();
conn.setAutoCommit(false);
String sql = null;
Statement st = conn.createStatement();
st.execute("CREATE TEMPORARY TABLE tmp (CandidateID INT, result VARCHAR(255), score INT, uploadState VARCHAR(255))");
sql = "INSERT INTO tmp (result, score, uploadState, CandidateID) VALUES (?,?,?,?)";
PreparedStatement ps = conn.prepareStatement(sql);
String tag = "Y";
for (int i = 1; i <= 10000; i++) {
ps.setString(1, String.format("result_%s_%d", tag, i));
ps.setInt(2, 400000 + i);
ps.setString(3, String.format("state_%s_%d", tag, i));
ps.setInt(4, i);
ps.addBatch();
}
ps.executeBatch();
sql =
"UPDATE personal_info pi INNER JOIN tmp ON tmp.CandidateID=pi.CandidateID "
+ "SET pi.result=tmp.result, pi.score=tmp.score, pi.uploadState=tmp.uploadState";
st.execute(sql);
conn.commit();
System.out.printf("%d ms%n", (System.nanoTime() - t0) / 1000000);

your pstm.executeBatch() should be after forloop
refer How to insert List into database

looping through resultset to search another value on different table

I want to search for some specific results in Results table, using an id from a different table to calculate cgpa based on the students id selected from studentInfo table, below is what I tried, the issue is that it just computes for the first Id or rather student
String lev = levelCombo.getSelectedItem().toString();
int leve = Integer.parseInt(lev);
int le = leve / 100;
try {
String sql
= "select idNumber from studentInfo where level ='" + lev + "' ";
pst = conn.prepareStatement(sql);
rs = pst.executeQuery();
while (rs.next()) {
String idNumb = rs.getString("idNumber");
cgpa(idNumb, le);
System.out.print("" + idNumb);
}
} catch (Exception e) {
}
}
public void cgpa(String idnumber, int level) {
float units = 0;
float ugps = 0;
float cgp = 0;
for (int i = 1; i <= level; i++) {
try {
String sql = "select sum(UNIT) ,sum(UGP) ,round(sum(UGP) / NULLIF(sum(UNIT), 0), 2) from Results where ID_NUMBER = ? and level ='" + i + "' ";
pst = conn.prepareStatement(sql);
pst.setString(1, idnumber);
rs = pst.executeQuery();
while (rs.next()) {
Float un = rs.getFloat("sum(UNIT)");
Float gp = rs.getFloat("sum(UGP)");
// Float sum=rs.getFloat("round(sum(UGP)/NULLIF(sum(UNIT), 0),2)");
units += un;
ugps += gp;
cgp = ugps / units;
String sql1 = "update academicstatus set cgpa ='" + cgp + "' ,product='" + ugps
+ "',unit='" + units + "' where idnumber = ? ";
updateAcademicstatus(cgp, ugps, units, idnumber);
pst = conn.prepareStatement(sql1);
pst.setString(1, idnumber);
pst.executeUpdate();
}
} catch (Exception ex) {
ex.printStackTrace();
} finally {
try {
rs.close();
pst.close();
} catch (Exception e) {
}
}
}
}

Are You trying in to single query
select sum(UNIT),sum(UGP),round(sum(UGP)/ ULLIF(sum(UNIT),0),2)idNumber from studentInfo sinfo
join Results re on re.ID_NUMBER = sinfo.idNumber where level='"+lev+"'"

How to make one mySQL's table column invisible

I am running a query on ID column but I don't want it to be visible in my frame/pane. How can I achieve this? Shall I make another table, is there a function in sql/mysql which allows to hide columns? I tried to google it but havent found anything yet.
Here is the code:
public void tableChanged(TableModelEvent e) {
int row = e.getFirstRow();
int col = e.getColumn();
model = (MyTableModel) e.getSource();
String stulpPav = model.getColumnName(col);
Object data = model.getValueAt(row, col);
Object studId = model.getValueAt(row, 0);
System.out.println("tableChanded works");
try {
new ImportData(stulpPav, data, studId);
} catch (ClassNotFoundException e1) {
e1.printStackTrace();
} catch (SQLException e1) {
e1.printStackTrace();
}
}
public class ImportData {
Connection connection = TableWithBottomLine.getConnection();
public ImportData(String a, Object b, Object c)
throws ClassNotFoundException, SQLException {
Statement stmt = null;
try {
String stulpPav = a;
String duom = b.toString();
String studId = c.toString();
System.out.println(duom);
connection.setAutoCommit(false);
stmt = connection.createStatement();
stmt.addBatch("update finance.fin set " + stulpPav + " = " + duom
+ " where ID = " + studId + ";");
stmt.executeBatch();
connection.commit();
} catch (BatchUpdateException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
} finally {
if (stmt != null)
stmt.close();
connection.setAutoCommit(true);
System.out.println("Data was imported to database");
}
}
}
public class MyTableModel extends AbstractTableModel{
int rowCount;
Object data [][];
String columnNames [];
public MyTableModel() throws SQLException{
String query ="SELECT ID, tbl_Date as Date, Flat, Mobile, Food, Alcohol, Transport, Outdoor, Pauls_stuff, Income, Stuff FROM finance.fin";
ResultSet rs ;
Connection connection = TableWithBottomLine.getConnection();
Statement stmt = null;
stmt = connection.createStatement();
rs = stmt.executeQuery(query);
rs.last();
rowCount = rs.getRow();
data = new Object[rowCount][11];
rs = stmt.executeQuery(query);
for (int iEil = 0; iEil < rowCount; iEil++){
rs.next();
data[iEil][0] = rs.getInt("ID");
data[iEil][1] = rs.getDate("Date");
data[iEil][2] = rs.getFloat("Flat");
data[iEil][3] = rs.getFloat("Mobile");
data[iEil][4] = rs.getFloat("Food");
data[iEil][5] = rs.getFloat("Alcohol");
data[iEil][6] = rs.getFloat("Transport");
data[iEil][7] = rs.getFloat("Outdoor");
data[iEil][8] = rs.getFloat("Pauls_stuff");
data[iEil][9] = rs.getFloat("Income");
data[iEil][10] = rs.getFloat("Stuff");
}
String[] columnName = {"ID", "Date","Flat","Mobile"
,"Food","Alcohol","Transport", "Outdoor", "Pauls_stuff", "Income", "Stuff"};
columnNames = columnName;
}

This has solved my problem:
table.removeColumn(table.getColumnModel().getColumn(0));
I placed this in my class contructor. This lets remove the column from the view of the table but column 'ID' is still contained in the TableModel. I found that many people looking for an option to exclude specific column (like autoincrement) from SELECT statement in sql / mysql but the language itself doesn't have that feature. So I hope this solution will help others as well.

Don't put ID in the select part of the query
String query ="SELECT tbl_Date as Date, Flat, Mobile, Food, Alcohol, Transport,
Outdoor, Pauls_stuff, Income, Stuff FROM finance.fin";

populate SWT table with database content

I am using PhpMyAdmin to save my data in database. I have a SWT table to populate with database content.
here is my code..
public static void fetchDatafromDB(String StartIndex, String FinalIndex) {
try {
Class.forName(GlobalVariables.SQL_driver).newInstance();
Connection conn = DriverManager.getConnection(GlobalVariables.DB_url + GlobalVariables.DB_name, GlobalVariables.DB_Username, GlobalVariables.DB_password);
Statement st = conn.createStatement();
String query = "SELECT `From`, `To`, `IDno`, `TimeStamp` FROM `callsheet` WHERE TimeStamp BETWEEN '" + StartIndex + "' AND '" + FinalIndex + "'";
ResultSet rs = st.executeQuery(query);
java.sql.ResultSetMetaData rsmd = rs.getMetaData();
int columnsNumber = rsmd.getColumnCount();
while (rs.next()) {
for (int i = 1; i <= columnsNumber; i++) {
// System.out.print(rs.getString(i));
item.setText(i, rs.getString(i));
}
// System.out.println();
}
} catch (Exception P) {
P.printStackTrace();
}
}
it worked.
Now I am getting some problem with tabling the DB content in my swt table. What my program does, is that, it sets the selected (defined by limit in program above) content of DB in one row (one by one manner) but I want the next row of DB table to be tabled in next row of SWT table. Could you suggest something about this? ! Screenshot of my swtTable

It should look something like this:
public static void fetchDatafromDB(String startIndex, String finalIndex) {
try {
Class.forName(GlobalVariables.SQL_driver).newInstance();
Connection conn = DriverManager.getConnection(GlobalVariables.DB_url + GlobalVariables.DB_name, GlobalVariables.DB_Username, GlobalVariables.DB_password);
Statement st = conn.createStatement();
String query = "SELECT `FROM`, `To`, `IDno`, `TimeStamp` FROM `callsheet` WHERE TimeStamp BETWEEN '" + startIndex + "' AND '" + finalIndex + "'";
ResultSet rs = st.executeQuery(query);
java.sql.ResultSetMetaData rsmd = rs.getMetaData();
int columnsNumber = rsmd.getColumnCount();
TableItem item;
while (rs.next()) {
// Create a new TableItem for each entry in the result set (each row)
item = new TableItem(table, SWT.NONE);
for (int i = 1; i <= columnsNumber; i++) {
// Populate the item (mind the index!!)
item.setText(i - 1, rs.getString(i));
}
}
} catch (Exception e) {
e.printStackTrace();
}
}

Inserting records into a MySQL table using Java

I created a database with one table in MySQL:
CREATE DATABASE iac_enrollment_system;
USE iac_enrollment_system;
CREATE TABLE course(
course_code CHAR(7),
course_desc VARCHAR(255) NOT NULL,
course_chair VARCHAR(255),
PRIMARY KEY(course_code)
);
I tried to insert a record using Java:
// STEP 1: Import required packages
import java.sql.*;
import java.util.*;
public class SQLInsert {
// JDBC driver name and database URL
static final String JDBC_DRIVER = "com.mysql.jdbc.Driver";
static final String DB_URL = "jdbc:mysql://localhost:3306/iac_enrollment_system";
// Database credentials
static final String USER = "root";
static final String PASS = "1234";
public static void main(String[] args) {
Connection conn = null;
Statement stmt = null;
Scanner scn = new Scanner(System.in);
String course_code = null, course_desc = null, course_chair = null;
try {
// STEP 2: Register JDBC driver
Class.forName("com.mysql.jdbc.Driver");
// STEP 3: Open a connection
System.out.print("\nConnecting to database...");
conn = DriverManager.getConnection(DB_URL, USER, PASS);
System.out.println(" SUCCESS!\n");
// STEP 4: Ask for user input
System.out.print("Enter course code: ");
course_code = scn.nextLine();
System.out.print("Enter course description: ");
course_desc = scn.nextLine();
System.out.print("Enter course chair: ");
course_chair = scn.nextLine();
// STEP 5: Excute query
System.out.print("\nInserting records into table...");
stmt = conn.createStatement();
String sql = "INSERT INTO course " +
"VALUES (course_code, course_desc, course_chair)";
stmt.executeUpdate(sql);
System.out.println(" SUCCESS!\n");
} catch(SQLException se) {
se.printStackTrace();
} catch(Exception e) {
e.printStackTrace();
} finally {
try {
if(stmt != null)
conn.close();
} catch(SQLException se) {
}
try {
if(conn != null)
conn.close();
} catch(SQLException se) {
se.printStackTrace();
}
}
System.out.println("Thank you for your patronage!");
}
}
The output appears to return successfully:
But when I select from MySQL, the inserted record is blank:
Why is it inserting a blank record?

no that cannot work(not with real data):
String sql = "INSERT INTO course " +
"VALUES (course_code, course_desc, course_chair)";
stmt.executeUpdate(sql);
change it to:
String sql = "INSERT INTO course (course_code, course_desc, course_chair)" +
"VALUES (?, ?, ?)";
Create a PreparedStatment with that sql and insert the values with index:
PreparedStatement preparedStatement = conn.prepareStatement(sql);
preparedStatement.setString(1, "Test");
preparedStatement.setString(2, "Test2");
preparedStatement.setString(3, "Test3");
preparedStatement.executeUpdate();

this can also be done like this if you don't want to use prepared statements.
String sql = "INSERT INTO course(course_code,course_desc,course_chair)"+"VALUES('"+course_code+"','"+course_desc+"','"+course_chair+"');"
Why it didnt insert value is because you were not providing values, but you were providing names of variables that you have used.

This should work for any table, instead of hard-coding the columns.
//Source details
String sourceUrl = "jdbc:oracle:thin:#//server:1521/db";
String sourceUserName = "src";
String sourcePassword = "***";
// Destination details
String destinationUserName = "dest";
String destinationPassword = "***";
String destinationUrl = "jdbc:mysql://server:3306/db";
Connection srcConnection = getSourceConnection(sourceUrl, sourceUserName, sourcePassword);
Connection destConnection = getDestinationConnection(destinationUrl, destinationUserName, destinationPassword);
PreparedStatement sourceStatement = srcConnection.prepareStatement("SELECT * FROM src_table ");
ResultSet rs = sourceStatement.executeQuery();
rs.setFetchSize(1000); // not needed
ResultSetMetaData meta = rs.getMetaData();
List<String> columns = new ArrayList<>();
for (int i = 1; i <= meta.getColumnCount(); i++)
columns.add(meta.getColumnName(i));
try (PreparedStatement destStatement = destConnection.prepareStatement(
"INSERT INTO dest_table ("
+ columns.stream().collect(Collectors.joining(", "))
+ ") VALUES ("
+ columns.stream().map(c -> "?").collect(Collectors.joining(", "))
+ ")"
)
)
{
int count = 0;
while (rs.next()) {
for (int i = 1; i <= meta.getColumnCount(); i++) {
destStatement.setObject(i, rs.getObject(i));
}
destStatement.addBatch();
count++;
}
destStatement.executeBatch(); // you will see all the rows in dest once this statement is executed
System.out.println("done " + count);
}

There is a mistake in your insert statement chage it to below and try :
String sql = "insert into table_name values ('" + Col1 +"','" + Col2 + "','" + Col3 + "')";