Develop Reference

Migrating a Big Derby embebed database to HSQLDB throw java.lang.OutOfMemoryError: Java heap space

I'm trying to migrate a big db from derby to HSQLDB in a Spring Boot service, like 1.5M regs in few tables of 10 columns. I'm checking with VisualVM; byte and char consume a lot of memory. But the biggest delta in time are in the derby classes.
Sometimes the error is thrown here, but other times thrown in other controllers. I don't want to touch all files to add my catchOutofMemory to restart.
Following is a version of my code, the block comment shows the resume of the process:
run(){//thread inside static function.
while(keepMigrating){
keepMigrating=Migrate();
}
}
private static boolean Migrate(JdbcTemplate derby,JdbcTemplate hsql){
int regs = 100000;
PreparedStatement statement = null;
ResultSet rs = null;
PreparedStatement statementHSQL = null;
try {
for (String table : tables) {//tables contains all tables to migrate
//check how many registers left asd asign to cant, if cant is 0 the empty is true.
PreparedStatement statementUpd[];
while (!empty) {
if (trys <= 0) throw new Exception("redo");
//check how many registers left asd asign to cant, if cant is 0 the empty is true and out of bucle and ready to check next table
/*
*Next process resume as:
*fetch data from derby that hasnt been migrated limited by cant
*create a batch to insert in hsql
*create a update for derby
*create a delete in case someting goes wrong
*excecute insert and update, if someting in batch fail delete the entry in migrate table
*reduce regs to get out of migrate method at some ponint.
*/
statement = derby.getDataSource().getConnection().prepareStatement(
MessageFormat.format(select_all_migrate_false_and_fetch_cant,table));
statementUpd = new PreparedStatement[cant];
ArrayList<String> deleteIds = new ArrayList<>();
StringBuilder columnNames = new StringBuilder();
StringBuilder updateSQL = new StringBuilder();
StringBuilder bindVariables = new StringBuilder();
try {
ResultSetMetaData meta = rs.getMetaData();
for (int i = 1; i <= meta.getColumnCount(); i++) {
if (!meta.getColumnName(i).equals("MIGRATED")) {
if (i > 1) {
columnNames.append(", ");
bindVariables.append(", ");
}
columnNames.append(meta.getColumnName(i));
bindVariables.append('?');
}
}
String sql = "INSERT INTO " + table.substring(4) + " ("
+ columnNames
+ ") VALUES ("
+ bindVariables
+ ")";
statementHSQL = hsql.getDataSource().getConnection().prepareStatement(sql);
HashMap<String, Object> data = new HashMap<>();
int row = 0;
int lastId = 0;
String columnName;
while (rs.next()) {
for (int i = 1; i <= meta.getColumnCount(); i++) {
columnName = meta.getColumnName(i);
Object o = rs.getObject(i);
statementHSQL.setObject(i, o);
if (columnName.equals(mainColumn))
deleteIds.add(String.valueOf(o));
if (!(meta.getColumnType(i) == 2004)) data.put(columnName, o);
if (columnName.equals(mainColumn)) id = rs.getObject(i);
}
int c = 1;
String update = MessageFormat.format("INSERT INTO {0}M ({1}M, MIGRATED) VALUES(?, TRUE)",
table.substring(4), mainColumn).replace("\"M", "M\"");//migrated state is saved in other table
lastId = Integer.valueOf(String.valueOf(id));
statementUpd[row] = derby.getDataSource().getConnection().prepareStatement(update);
statementUpd[row].setObject(1, rs.getObject(mainColumn));
updateSQL = new StringBuilder();
statementHSQL.addBatch();
row += 1;
}
/*
* Build delete query in case of inserted values in HSQLDB but not updated in DERBY
*/
StringBuilder builder = new StringBuilder();
builder.append("(");
int count = 1;
for (String s : deleteIds) {
if (count > 1) builder.append(", ");
builder.append("?");
count++;
}
builder.append(")");
String str = builder.toString();
String queryDelete = "DELETE FROM " + table.substring(4) + " WHERE " + mainColumn + " IN " + str;
PreparedStatement statementHSQLDel = hsql.getDataSource().getConnection().prepareStatement
(queryDelete);
int c = 1;
for (String s : deleteIds) {
statementHSQLDel.setObject(c, s);
c++;
}
boolean deletes = statementHSQLDel.execute();
statementHSQLDel.close();
try {
DatabaseUtils.close(statementHSQLDel);
} catch (Exception e) {
catchOutOfMemory(e);
}
int[] result = statementHSQL.executeBatch();
StringBuilder resultS = new StringBuilder();
int stCounter = 0;
int stCounterInsert = 0;
int stCounterUpdate = 0;
String notarydebug;
for (int i : result) {
int upd = 0;
try {
if (i == 1) upd = statementUpd[stCounter].executeUpdate();
} catch (Exception e) {
catchOutOfMemory(e);
}
stCounterInsert += i;
stCounterUpdate += upd;
resultS.append(",").append(String.valueOf(i)).append("-").append(String.valueOf(upd));
stCounter += 1;
}
statementHSQL.clearBatch();
try {
DatabaseUtils.close(statementHSQL);
} catch (Exception e) {
catchOutOfMemory(e);
}
} catch (SQLException se) {
catchOutOfMemory(se);//otherstuff
} catch (Exception e) {
catchOutOfMemory(e);
}
try {
DatabaseUtils.close(rs);
DatabaseUtils.close(statement);
} catch (Exception e) {
catchOutOfMemory(e);
}
regs=regs-cant;
}
}
}catch (Exception e) {
if (e.getMessage().equals("redo")) return true;//end the loop of regs maximun and get out of method.
}
return false;//end migration succesfully
}
private static int catchOutOfMemory(Throwable e) {
if (e == null) return 0;
if (e instanceof OutOfMemoryError) {
Application.restartBat();
return 1;
} else {
return catchOutOfMemory(e.getCause());
}
}
edit:
So I change as sugested inthe comment to accept a commit, something like this:
Connection hsqlCon;
PrepareStatement hsqlStm;
hsqlCon = JdbcHSQLDB.getDataSource().getConnection();
hsqlStm = hsqlCon.prepareStatement(sql);
hsqlStm.addBatch();
hsqlStm.execute();
hsqlStm.close();
hsqlCon.close();
but i got the same heap memory consumpsion:

The type of table in HSQLDB is not clear from the supplied code. You must use this statement once for each table, to make sure the table data is stored in the finename.data file:
SET TABLE tableName TYPE CACHED
The reported sequence of batch INSERT is not correct. Use this sequence:
Connection hsqlCon;
PrepareStatement hsqlStm;
hsqlCon = JdbcHSQLDB.getDataSource().getConnection();
hsqlStm = hsqlCon.prepareStatement(sql);
{ // repeat this block until all is finished
{ // repeat for 1000 rows
hsqlStm.addBatch();
}
hsqlStm.executeBatch(); // after every 1000 rows
}
hsqlStm.close();
hsqlCon.close();

Java - executebatch is not working when inserting records to db

I am trying to insert records into my table in MYSQL after extracting the words from a file and stored them in a hashset.
I tried using executeBatch() to insert into my db after getting 500 records but when the execution finished, I checked my table and there's no record inserted at all.
Note: When I use ExecuteUpdate() then the records will show in my table. But not ExecuteBatch() since I want to insert by batch, not one by one.
May I know what did I do wrong?
Code:
public void readDataBase(String path,String word) throws Exception {
try {
// Result set get the result of the SQL query
int i=0;
// This will load the MySQL driver, each DB has its own driver
Class.forName("com.mysql.jdbc.Driver");
// Setup the connection with the DB
connect = DriverManager
.getConnection("jdbc:mysql://126.32.3.20/fulltext_ltat?"
+ "user=root&password=root");
// Statements allow to issue SQL queries to the database
// statement = connect.createStatement();
System.out.print("Connected");
// Result set get the result of the SQL query
preparedStatement = connect
.prepareStatement("insert IGNORE into fulltext_ltat.indextable values (default,?, ?) ");
preparedStatement.setString( 1, path);
preparedStatement.setString(2, word);
preparedStatement.addBatch();
i++;
// preparedStatement.executeUpdate();
if(i%500==0){
preparedStatement.executeBatch();
}
preparedStatement.close();
// writeResultSet(resultSet);
} catch (Exception e) {
throw e;
} finally {
close();
}
}
This is my loop to call that method(words is just an array that contains the words which is going to be inserted to the table):
for(int i = 1 ; i <= words.length - 1 ; i++ ) {
connection.readDataBase(path, words[i].toString());
}
My main method:
public static void main(String[] args) throws Exception {
StopWatch stopwatch = new StopWatch();
stopwatch.start();
File folder = new File("D:\\PDF1");
File[] listOfFiles = folder.listFiles();
for (File file : listOfFiles) {
if (file.isFile()) {
HashSet<String> uniqueWords = new HashSet<>();
String path = "D:\\PDF1\\" + file.getName();
try (PDDocument document = PDDocument.load(new File(path))) {
if (!document.isEncrypted()) {
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
String lines[] = pdfFileInText.split("\\r?\\n");
for (String line : lines) {
String[] words = line.split(" ");
for (String word : words) {
uniqueWords.add(word)
;
}
}
// System.out.println(uniqueWords);
}
} catch (IOException e) {
System.err.println("Exception while trying to read pdf document - " + e);
}
Object[] words = uniqueWords.toArray();
MysqlAccessIndex connection = new MysqlAccessIndex();
for(int i = 1 ; i <= words.length - 1 ; i++ ) {
connection.readDataBase(path, words[i].toString());
}
System.out.println("Completed");
}
}

Your pattern for doing batch updates is off. You should be opening the connection and preparing the statement only once. Then, iterate multiple times, binding parameters, and add that statement to the batch.
// define a collection of paths and words somewhere
List<String> paths = new ArrayList<>();
List<String> words = new ArrayList<>();
try {
// presumably you only want to insert so many records
int LIMIT = 10000;
Class.forName("com.mysql.jdbc.Driver");
connect = DriverManager
.getConnection("jdbc:mysql://126.32.3.20/fulltext_ltat?"
+ "user=root&password=root");
String sql = "INSERT IGNORE INTO fulltext_ltat.indextable VALUES (default, ?, ?);";
preparedStatement = connect.prepareStatement(sql);
for (int i=0; i < LIMIT; ++i) {
preparedStatement.setString(1, paths.get(i));
preparedStatement.setString(2, word.get(i));
preparedStatement.addBatch();
if (i % 500 == 0) {
preparedStatement.executeBatch();
}
}
// execute remaining batches
preparedStatement.executeBatch();
}
catch (SQLException e) {
e.printStackTrace();
}
finally {
try {
preparedStatement.close();
connect.close();
}
catch (SQLException e) {
e.printStackTrace();
}
}
One key change I made here is to add logic for when you should stop doing inserts. Currently, your code looks to have an infinite loop, which means it would run forever. This is probably not what you were intending to do.

where is your loop. try this
connect = DriverManager
.getConnection("jdbc:mysql://126.32.3.20/fulltext_ltat?"
+ "user=root&password=root&rewriteBatchedStatements=true");

Java - Improving performance for building up index table

I am working on a full text indexing using inverted file method where it extracts all the word in a document, and inserts each word one by one into my table in MYSQL.
So far, my program works perfectly fine but I am stuck in thinking how it could be optimize further to improve the time it takes to insert into db. I am aware inverted file has a disadvantage of slow time for building up the index table.
Here is my code:
public class IndexTest {
public static void main(String[] args) throws Exception {
StopWatch stopwatch = new StopWatch();
stopwatch.start();
File folder = new File("D:\\PDF1");
File[] listOfFiles = folder.listFiles();
for (File file : listOfFiles) {
if (file.isFile()) {
HashSet<String> uniqueWords = new HashSet<>();
String path = "D:\\PDF1\\" + file.getName();
try (PDDocument document = PDDocument.load(new File(path))) {
if (!document.isEncrypted()) {
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
String lines[] = pdfFileInText.split("\\r?\\n");
for (String line : lines) {
String[] words = line.split(" ");
for (String word : words) {
uniqueWords.add(word);
}
}
// System.out.println(uniqueWords);
}
} catch (IOException e) {
System.err.println("Exception while trying to read pdf document - " + e);
}
Object[] words = uniqueWords.toArray();
String unique = uniqueWords.toString();
// System.out.println(words[1].toString());
for(int i = 1 ; i <= words.length - 1 ; i++ ) {
MysqlAccessIndex connection = new MysqlAccessIndex();
connection.readDataBase(path, words[i].toString());
}
System.out.println("Completed");
}
}
stopwatch.stop();
long timeTaken = stopwatch.getTime();
System.out.println(timeTaken);
MYSQL connection:
public class MysqlAccessIndex {
public Connection connect = null;
public Statement statement = null;
public PreparedStatement preparedStatement = null;
public ResultSet resultSet = null;
public MysqlAccessIndex() throws Exception {
Class.forName("com.mysql.jdbc.Driver");
connect = DriverManager
.getConnection("jdbc:mysql://126.32.3.178/fulltext_ltat?"
+ "user=root&password=root123");
// statement = connect.createStatement();
System.out.print("Connected");
}
public void readDataBase(String path,String word) throws Exception {
try {
preparedStatement = connect
.prepareStatement("insert IGNORE into fulltext_ltat.test_text values (?, ?) ");
preparedStatement.setString(1, path);
preparedStatement.setString(2, word);
preparedStatement.executeUpdate();
} catch (Exception e) {
throw e;
} finally {
close();
}
}
Is it possible if I could use some sort of multi threading to say insert three words in three rows at the same time to speed up the insert process or some sort?
I would appreciate any suggestion.

I think solution to your problem - is to use bulk insert.
You could try to do something like this:
public void readDataBase(String path, HashSet<String> uniqueWords) throws Exception {
PreparedStatement preparedStatement;
try {
String compiledQuery = "insert IGNORE into fulltext_ltat.test_text values (?, ?) ";
preparedStatement = connect.prepareStatement(compiledQuery);
for(String word : uniqueWords) {
preparedStatement.setString(1, path);
preparedStatement.setString(2, word);
preparedStatement.addBatch();
}
long start = System.currentTimeMillis();
int[] inserted = preparedStatement.executeBatch();
} catch (Exception e) {
throw e;
} finally {
close();
}
}
Modify your readDataBase method to have HashSet<String> uniqueWords in params.
After that you should add preparedStatement.addBatch() call after each item to insert and execute preparedStatement.executeBatch() instead of preparedStatement.executeUpdate() in the end.
I hope it would help.

Unable to convert the CLOB data into String

I am trying to convert java.sql.Clob data into String by using SubString method (This method giving good performance compared with other). The clob data having near or morethan to 32MB. AS my observation substring method able to to return upto 33554342 bytes only.
if clob data is crossing 33554342 bytes then this it's throwing below sql exception
ORA-24817: Unable to allocate the given chunk for current lob operation
EDIT
CODE:
public static void main(String[] args) throws SQLException {
Main main = new Main();
Connection con = main.getConnection();
if (con == null) {
return;
}
PreparedStatement pstmt = null;
ResultSet rs = null;
String sql = "SELECT Table_ID,CLOB_FILE FROM TableName WHERE SOMECONDITION ";
String table_Id = null;
String directClobInStr = null;
CLOB clobObj = null;
String clobStr = null;
Object obj= null;
try {
pstmt = con.prepareStatement(sql);
rs = pstmt.executeQuery();
while (rs.next()) {
table_Id = rs.getString( "Table_ID" ) ;
directClobInStr = rs.getString( "clob_FILE" ) ;
obj = rs.getObject( "CLOB_FILE");
clobObj = (CLOB) obj;
System.out.println("Table id " + table_Id);
System.out.println("directClobInStr " + directClobInStr);
clobStr = clobObj.getSubString(1L, (int)clobObj.length() );//33554342
System.out.println("clobDataStr = " + clobStr);
}
}
catch (SQLException e) {
e.printStackTrace();
return;
}
catch (Exception e) {
e.printStackTrace();
return;
}
finally {
try {
rs.close();
pstmt.close();
con.close();
}
catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
NOTE:- here obj = rs.getObject( "CLOB_FILE"); working but I am not expecting this. because I am getting ResultSet object from somewhere as Object. I have to convert and get the data from CLOB
Any Idea how to achieve this?

Instead:
clobStr = clobObj.getSubString(1L, (int)clobObj.length() );
Try something like:
int toread = (int) clobObj.length();
int read = 0;
final int block_size = 8*1024*1024;
StringBuilder str = new StringBuilder(toread);
while (toread > 0) {
int current_block = Math.min(toread, block_size);
str.append(clobObj.getSubString(read+1, current_block));
read += current_block;
toread -= current_block;
}
clobStr = str.toString();
It extracts substrings using a loop (8MB per iteration).
But remember that, as far as I known, Java Strings are limited to 2 GB (this is the reason why read is declared as int instead of long) and Oracle CLOBs are limited to 128 TB.

com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException UNKNOWN COLUMN

I am currently trying to scan and parse the file that is not in sql format. I am trying to input all the data into the SQL table but for some reason every time i run the program, i get the error saying unknown column 'what' in 'field list.' So the neither of the data goes through. 'what' is one of the names that is on the text. The table currently has 11 columns. I know I am parsing or scanning it wrong but I cannot figure out where. Here is my code:
public class parseTable {
public parseTable (String name) throws FileNotFoundException
{
File file = new File(name);
parse(file);
}
private void parse(File file) throws FileNotFoundException
{
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try{
Class.forName("com.mysql.jdbc.Driver").newInstance();
String connectionUrl = "jdbc:mysql://localhost:3306/";
String connectionUser = "";
String connectionPassword = "";
conn = DriverManager.getConnection(connectionUrl, connectionUser, connectionPassword);
stmt = conn.createStatement();
Scanner scan = new Scanner(file);
String[] rowInfo = new String[11];
int count = 0;
while(scan.hasNextLine()){
//String data = scan.nextLine();
Scanner lineScan = new Scanner(scan.nextLine());
while(lineScan.hasNext()){
String words = lineScan.next();
if(count < 11){
rowInfo[count] = words;
count++;
}
else if(count == 11 && words.equals("States")){
rowInfo[count - 1] = rowInfo[count - 1] + " " + words;
}
else{
String query = "";
for(int i = 0; i < rowInfo.length; i++)
{
if(query.equals(""))
{
query = rowInfo[i];
}
else if(i == 9){
query = query + "," + rowInfo[i];
}
else if(rowInfo[i].equals(null)){
query = query + ", " + "NULL";
}
else
query = query + ", " + "'" + rowInfo[i] + "'";
}
stmt.executeUpdate("INSERT INTO dup VALUES(" + query + ")");
count = 0;
rowInfo = new String[11];
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try { if (rs != null) rs.close(); } catch (SQLException e) { e.printStackTrace(); }
try { if (stmt != null) stmt.close(); } catch (SQLException e) { e.printStackTrace(); }
try { if (conn != null) conn.close(); } catch (SQLException e) { e.printStackTrace(); }
}
}
}
And this is the data I'm trying to input:
1 hello cheese 1111 what#yahoo.com user adm street zip what USA
2 Alex cheese 1111 what#yahoo.com user adm street zip what USA
So this is my new code now, using PrepareStatement. However I still get an error and I looked online for the solution on where I'm making a mistake, but I cant seem to figure out where.
String query = "INSERT INTO mil_table (UserName, NameFirst, NameLast, supportID, EmailAddress, Password,
IDQ, AddressCity, AddressState, AddressZip, AddressCountry) VALUES(?,?,?,?,?,?,?,?,?,?,?)";
pstmt = conn.prepareStatement(query);
Scanner scan = new Scanner(file);
String[] rowInfo = new String[11];
int count = 0;
while(scan.hasNextLine()){
//String data = scan.nextLine();
Scanner lineScan = new Scanner(scan.nextLine());
while(lineScan.hasNext()){
String words = lineScan.next();
if(count < 11){
rowInfo[count] = words;
count++;
}
else if(count == 11 && words.equals("States")){
rowInfo[count - 1] = rowInfo[count - 1] + " " + words;
}
else{
for(int i = 0; i <rowInfo.length; i++)
{
pstmt.setString(i + 1, rowInfo[i]);
}
//stmt.executeUpdate("INSERT INTO mil_table VALUES(" + query + ")");
//System.out.println("#" + query + "#");
pstmt.executeUpdate();
count = 0;
rowInfo = new String[11];
}
}

As you are using MySQL, you will need to enclose the text inputs with quotes. Try enclosing the String values that you are inserting in quotes and then execute your code.