Develop Reference

Migrating a Big Derby embebed database to HSQLDB throw java.lang.OutOfMemoryError: Java heap space

I'm trying to migrate a big db from derby to HSQLDB in a Spring Boot service, like 1.5M regs in few tables of 10 columns. I'm checking with VisualVM; byte and char consume a lot of memory. But the biggest delta in time are in the derby classes.
Sometimes the error is thrown here, but other times thrown in other controllers. I don't want to touch all files to add my catchOutofMemory to restart.
Following is a version of my code, the block comment shows the resume of the process:
run(){//thread inside static function.
while(keepMigrating){
keepMigrating=Migrate();
}
}
private static boolean Migrate(JdbcTemplate derby,JdbcTemplate hsql){
int regs = 100000;
PreparedStatement statement = null;
ResultSet rs = null;
PreparedStatement statementHSQL = null;
try {
for (String table : tables) {//tables contains all tables to migrate
//check how many registers left asd asign to cant, if cant is 0 the empty is true.
PreparedStatement statementUpd[];
while (!empty) {
if (trys <= 0) throw new Exception("redo");
//check how many registers left asd asign to cant, if cant is 0 the empty is true and out of bucle and ready to check next table
/*
*Next process resume as:
*fetch data from derby that hasnt been migrated limited by cant
*create a batch to insert in hsql
*create a update for derby
*create a delete in case someting goes wrong
*excecute insert and update, if someting in batch fail delete the entry in migrate table
*reduce regs to get out of migrate method at some ponint.
*/
statement = derby.getDataSource().getConnection().prepareStatement(
MessageFormat.format(select_all_migrate_false_and_fetch_cant,table));
statementUpd = new PreparedStatement[cant];
ArrayList<String> deleteIds = new ArrayList<>();
StringBuilder columnNames = new StringBuilder();
StringBuilder updateSQL = new StringBuilder();
StringBuilder bindVariables = new StringBuilder();
try {
ResultSetMetaData meta = rs.getMetaData();
for (int i = 1; i <= meta.getColumnCount(); i++) {
if (!meta.getColumnName(i).equals("MIGRATED")) {
if (i > 1) {
columnNames.append(", ");
bindVariables.append(", ");
}
columnNames.append(meta.getColumnName(i));
bindVariables.append('?');
}
}
String sql = "INSERT INTO " + table.substring(4) + " ("
+ columnNames
+ ") VALUES ("
+ bindVariables
+ ")";
statementHSQL = hsql.getDataSource().getConnection().prepareStatement(sql);
HashMap<String, Object> data = new HashMap<>();
int row = 0;
int lastId = 0;
String columnName;
while (rs.next()) {
for (int i = 1; i <= meta.getColumnCount(); i++) {
columnName = meta.getColumnName(i);
Object o = rs.getObject(i);
statementHSQL.setObject(i, o);
if (columnName.equals(mainColumn))
deleteIds.add(String.valueOf(o));
if (!(meta.getColumnType(i) == 2004)) data.put(columnName, o);
if (columnName.equals(mainColumn)) id = rs.getObject(i);
}
int c = 1;
String update = MessageFormat.format("INSERT INTO {0}M ({1}M, MIGRATED) VALUES(?, TRUE)",
table.substring(4), mainColumn).replace("\"M", "M\"");//migrated state is saved in other table
lastId = Integer.valueOf(String.valueOf(id));
statementUpd[row] = derby.getDataSource().getConnection().prepareStatement(update);
statementUpd[row].setObject(1, rs.getObject(mainColumn));
updateSQL = new StringBuilder();
statementHSQL.addBatch();
row += 1;
}
/*
* Build delete query in case of inserted values in HSQLDB but not updated in DERBY
*/
StringBuilder builder = new StringBuilder();
builder.append("(");
int count = 1;
for (String s : deleteIds) {
if (count > 1) builder.append(", ");
builder.append("?");
count++;
}
builder.append(")");
String str = builder.toString();
String queryDelete = "DELETE FROM " + table.substring(4) + " WHERE " + mainColumn + " IN " + str;
PreparedStatement statementHSQLDel = hsql.getDataSource().getConnection().prepareStatement
(queryDelete);
int c = 1;
for (String s : deleteIds) {
statementHSQLDel.setObject(c, s);
c++;
}
boolean deletes = statementHSQLDel.execute();
statementHSQLDel.close();
try {
DatabaseUtils.close(statementHSQLDel);
} catch (Exception e) {
catchOutOfMemory(e);
}
int[] result = statementHSQL.executeBatch();
StringBuilder resultS = new StringBuilder();
int stCounter = 0;
int stCounterInsert = 0;
int stCounterUpdate = 0;
String notarydebug;
for (int i : result) {
int upd = 0;
try {
if (i == 1) upd = statementUpd[stCounter].executeUpdate();
} catch (Exception e) {
catchOutOfMemory(e);
}
stCounterInsert += i;
stCounterUpdate += upd;
resultS.append(",").append(String.valueOf(i)).append("-").append(String.valueOf(upd));
stCounter += 1;
}
statementHSQL.clearBatch();
try {
DatabaseUtils.close(statementHSQL);
} catch (Exception e) {
catchOutOfMemory(e);
}
} catch (SQLException se) {
catchOutOfMemory(se);//otherstuff
} catch (Exception e) {
catchOutOfMemory(e);
}
try {
DatabaseUtils.close(rs);
DatabaseUtils.close(statement);
} catch (Exception e) {
catchOutOfMemory(e);
}
regs=regs-cant;
}
}
}catch (Exception e) {
if (e.getMessage().equals("redo")) return true;//end the loop of regs maximun and get out of method.
}
return false;//end migration succesfully
}
private static int catchOutOfMemory(Throwable e) {
if (e == null) return 0;
if (e instanceof OutOfMemoryError) {
Application.restartBat();
return 1;
} else {
return catchOutOfMemory(e.getCause());
}
}
edit:
So I change as sugested inthe comment to accept a commit, something like this:
Connection hsqlCon;
PrepareStatement hsqlStm;
hsqlCon = JdbcHSQLDB.getDataSource().getConnection();
hsqlStm = hsqlCon.prepareStatement(sql);
hsqlStm.addBatch();
hsqlStm.execute();
hsqlStm.close();
hsqlCon.close();
but i got the same heap memory consumpsion:

The type of table in HSQLDB is not clear from the supplied code. You must use this statement once for each table, to make sure the table data is stored in the finename.data file:
SET TABLE tableName TYPE CACHED
The reported sequence of batch INSERT is not correct. Use this sequence:
Connection hsqlCon;
PrepareStatement hsqlStm;
hsqlCon = JdbcHSQLDB.getDataSource().getConnection();
hsqlStm = hsqlCon.prepareStatement(sql);
{ // repeat this block until all is finished
{ // repeat for 1000 rows
hsqlStm.addBatch();
}
hsqlStm.executeBatch(); // after every 1000 rows
}
hsqlStm.close();
hsqlCon.close();

How to deal with multiple queries in a java servlet

I'm trying to send an increase count variable of a picture (which is increased by just increasing +1 everytime a new session hits a picture). I'm getting the following error message however, i'm checking for an empty result set. My thought process is that I can try to select the picturesNo that has been called and if it can't find that pictureNo we simply insert the first count to the table, and if it can find it, we then update this.
Error message:
"SQLException: Illegal operation on empty result set."
Code to increase the count for the session
HttpSession session = request.getSession() ;
Integer counter = (Integer) session.getAttribute("counter");
String accCount = (String) session.getAttribute("attributeKey") ;
String url = "http://localhost:8080/techfore";
String encodedURL = url + ";jsessionid=" + request.getSession().getId();
if (accCount == null || encodedURL == null) { // New session?
response.sendRedirect("/techfore/WelcomePage");
}
else{
if(counter == 0) {
counter = new Integer(counter.intValue() + 1);
session.setAttribute("counter", counter);
}
}
Utilities.initalCount(out, pictureName, counter);
Code to run the queries
public static void initalCount(PrintWriter out, String pictureName, int accessCount) {
Connection con = null;
try { // Connect to the database
con = openConnection(out);
}
catch (Exception e) { // Failed to open the connection
out.println("<P>" + e.getMessage());
}
try {
Statement stmt = con.createStatement();
String query0;
ResultSet rs1;
query0="SELECT PictureNo FROM Statistics WHERE PictureNo = (SELECT PictureNo FROM Pictures WHERE ShortName = '"+pictureName+"')";
rs1 = stmt.executeQuery(query0);
if(rs1.next()){
//yes exist
String description = rs1.getString("Description");
int pictureNo = rs1.getInt("PictureNo");
IncreaseCount(out, pictureNo, accessCount);
}
else {
//if rs is null insert
int pictureNo = rs1.getInt("PictureNo");
AddCount(out, pictureNo, accessCount);
}
stmt.close() ;
}
catch(SQLException ex) {
out.println("<P>SQLException: " + ex.getMessage()) ;
}
}
public static void AddCount(PrintWriter out, int pictureNo, int accessCount) {
Connection con = null;
try { // Connect to the database
con = openConnection(out);
}
catch (Exception e) { // Failed to open the connection
out.println("<P>" + e.getMessage());
return;
}
try {
Statement stmt = con.createStatement();
String query;
ResultSet rs1;
query="INSERT INTO Statistics VALUES "+pictureNo+","+accessCount+" ";
stmt.executeUpdate(query);
stmt.close() ;
}
catch(SQLException ex) {
out.println("<P>SQLException: " + ex.getMessage()) ;
}
}
public static void IncreaseCount(PrintWriter out, int pictureNo, int accessCount) {
Connection con = null;
try { // Connect to the database
con = openConnection(out);
}
catch (Exception e) { // Failed to open the connection
out.println("<P>" + e.getMessage());
return;
}
try {
Statement stmt = con.createStatement();
String query;
ResultSet rs1;
query="UPDATE Statistics SET AccessCount = "+accessCount+" + 1 WHERE PictureNo = "+pictureNo+"";
stmt.executeUpdate(query);
stmt.close() ;
}
catch(SQLException ex) {
out.println("<P>SQLException: " + ex.getMessage()) ;
}
}
New insert
query="INSERT INTO Statistics VALUES (SELECT PictureNo FROM Pictures WHERE FileName = '"+pictureName+"'),"+accessCount+" ";

Unable to convert the CLOB data into String

I am trying to convert java.sql.Clob data into String by using SubString method (This method giving good performance compared with other). The clob data having near or morethan to 32MB. AS my observation substring method able to to return upto 33554342 bytes only.
if clob data is crossing 33554342 bytes then this it's throwing below sql exception
ORA-24817: Unable to allocate the given chunk for current lob operation
EDIT
CODE:
public static void main(String[] args) throws SQLException {
Main main = new Main();
Connection con = main.getConnection();
if (con == null) {
return;
}
PreparedStatement pstmt = null;
ResultSet rs = null;
String sql = "SELECT Table_ID,CLOB_FILE FROM TableName WHERE SOMECONDITION ";
String table_Id = null;
String directClobInStr = null;
CLOB clobObj = null;
String clobStr = null;
Object obj= null;
try {
pstmt = con.prepareStatement(sql);
rs = pstmt.executeQuery();
while (rs.next()) {
table_Id = rs.getString( "Table_ID" ) ;
directClobInStr = rs.getString( "clob_FILE" ) ;
obj = rs.getObject( "CLOB_FILE");
clobObj = (CLOB) obj;
System.out.println("Table id " + table_Id);
System.out.println("directClobInStr " + directClobInStr);
clobStr = clobObj.getSubString(1L, (int)clobObj.length() );//33554342
System.out.println("clobDataStr = " + clobStr);
}
}
catch (SQLException e) {
e.printStackTrace();
return;
}
catch (Exception e) {
e.printStackTrace();
return;
}
finally {
try {
rs.close();
pstmt.close();
con.close();
}
catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
NOTE:- here obj = rs.getObject( "CLOB_FILE"); working but I am not expecting this. because I am getting ResultSet object from somewhere as Object. I have to convert and get the data from CLOB
Any Idea how to achieve this?

Instead:
clobStr = clobObj.getSubString(1L, (int)clobObj.length() );
Try something like:
int toread = (int) clobObj.length();
int read = 0;
final int block_size = 8*1024*1024;
StringBuilder str = new StringBuilder(toread);
while (toread > 0) {
int current_block = Math.min(toread, block_size);
str.append(clobObj.getSubString(read+1, current_block));
read += current_block;
toread -= current_block;
}
clobStr = str.toString();
It extracts substrings using a loop (8MB per iteration).
But remember that, as far as I known, Java Strings are limited to 2 GB (this is the reason why read is declared as int instead of long) and Oracle CLOBs are limited to 128 TB.

jdbc sqlite returns no rows

public ArrayList<Booking> getAllBookings(){
ArrayList<Booking> list = new ArrayList<Booking>();
int rowCount = 0;
try {
stmt = connection.createStatement();
String sql = "SELECT * FROM Bookings";
ResultSet rows = stmt.executeQuery(sql);
rows.last();
System.out.println("Row Count "+rows.getRow());
rows.beforeFirst();
connection.commit();
while (rows.next()){
Booking b = new Booking();
otherStuff();
list.add(b);
rowCount++;
}
stmt.close();
rows.close();
} catch (Exception e) {
}
System.out.println("There were " +rowCount + " records.");
return list;
}
Why do I not get any rows returned? I am connecting using this:
public static Connection dbConnect(){
try {
Class.forName("org.sqlite.JDBC");
Connection connection = DriverManager.getConnection("jdbc:sqlite:D:/test/test_db.sqlite3");
System.out.println("Connection Succesful");
return connection;
} catch (Exception e) {
System.out.println(e);
return null;
} }
When running SQLite Studio my query returns the correct results, but here my output on console is:
Connection Succesful
(Row Count + rows.getRow() is not printing here for some reason)
There were 0 records.

How to make one mySQL's table column invisible

I am running a query on ID column but I don't want it to be visible in my frame/pane. How can I achieve this? Shall I make another table, is there a function in sql/mysql which allows to hide columns? I tried to google it but havent found anything yet.
Here is the code:
public void tableChanged(TableModelEvent e) {
int row = e.getFirstRow();
int col = e.getColumn();
model = (MyTableModel) e.getSource();
String stulpPav = model.getColumnName(col);
Object data = model.getValueAt(row, col);
Object studId = model.getValueAt(row, 0);
System.out.println("tableChanded works");
try {
new ImportData(stulpPav, data, studId);
} catch (ClassNotFoundException e1) {
e1.printStackTrace();
} catch (SQLException e1) {
e1.printStackTrace();
}
}
public class ImportData {
Connection connection = TableWithBottomLine.getConnection();
public ImportData(String a, Object b, Object c)
throws ClassNotFoundException, SQLException {
Statement stmt = null;
try {
String stulpPav = a;
String duom = b.toString();
String studId = c.toString();
System.out.println(duom);
connection.setAutoCommit(false);
stmt = connection.createStatement();
stmt.addBatch("update finance.fin set " + stulpPav + " = " + duom
+ " where ID = " + studId + ";");
stmt.executeBatch();
connection.commit();
} catch (BatchUpdateException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
} finally {
if (stmt != null)
stmt.close();
connection.setAutoCommit(true);
System.out.println("Data was imported to database");
}
}
}
public class MyTableModel extends AbstractTableModel{
int rowCount;
Object data [][];
String columnNames [];
public MyTableModel() throws SQLException{
String query ="SELECT ID, tbl_Date as Date, Flat, Mobile, Food, Alcohol, Transport, Outdoor, Pauls_stuff, Income, Stuff FROM finance.fin";
ResultSet rs ;
Connection connection = TableWithBottomLine.getConnection();
Statement stmt = null;
stmt = connection.createStatement();
rs = stmt.executeQuery(query);
rs.last();
rowCount = rs.getRow();
data = new Object[rowCount][11];
rs = stmt.executeQuery(query);
for (int iEil = 0; iEil < rowCount; iEil++){
rs.next();
data[iEil][0] = rs.getInt("ID");
data[iEil][1] = rs.getDate("Date");
data[iEil][2] = rs.getFloat("Flat");
data[iEil][3] = rs.getFloat("Mobile");
data[iEil][4] = rs.getFloat("Food");
data[iEil][5] = rs.getFloat("Alcohol");
data[iEil][6] = rs.getFloat("Transport");
data[iEil][7] = rs.getFloat("Outdoor");
data[iEil][8] = rs.getFloat("Pauls_stuff");
data[iEil][9] = rs.getFloat("Income");
data[iEil][10] = rs.getFloat("Stuff");
}
String[] columnName = {"ID", "Date","Flat","Mobile"
,"Food","Alcohol","Transport", "Outdoor", "Pauls_stuff", "Income", "Stuff"};
columnNames = columnName;
}

This has solved my problem:
table.removeColumn(table.getColumnModel().getColumn(0));
I placed this in my class contructor. This lets remove the column from the view of the table but column 'ID' is still contained in the TableModel. I found that many people looking for an option to exclude specific column (like autoincrement) from SELECT statement in sql / mysql but the language itself doesn't have that feature. So I hope this solution will help others as well.

Don't put ID in the select part of the query
String query ="SELECT tbl_Date as Date, Flat, Mobile, Food, Alcohol, Transport,
Outdoor, Pauls_stuff, Income, Stuff FROM finance.fin";