Develop Reference

Bulk insert from a csv to a table in an Oracle DB using Java

I am attempting to insert a table in an Oracle DB using java. I am reading a csv file line by line using OpenCSV. The csv is about 50000 rows and 9 columns. Here is some of my code:
/* Create Connection objects */
Class.forName ("oracle.jdbc.OracleDriver");
Connection conn = DriverManager.getConnection("jdbc:oracle:thin:#HoSt", "UsErNaMe", "PaSsWoRd");
PreparedStatement sql_statement = null;
String jdbc_insert_sql = "INSERT INTO METATADA_AUTOSYS"
+ "(MACH_NAME,JOB_NAME,SCRIPT_COMMAND,APPLICATION_NAME,JOB_ID,STATUS,CREATE_DATE,LAST_START_DT,LAST_END_DT) VALUES"
+ "(?,?,?,?,?,?,?,?,?)";
sql_statement = conn.prepareStatement(jdbc_insert_sql);
/* Read CSV file in OpenCSV */
String inputCSVFile = "C:/Users/conwacx/Desktop/meta_auto_v3/Autosys_Metadata.csv";
CSVReader reader = new CSVReader(new FileReader(inputCSVFile));
String [] nextLine;
int lnNum = 0;
int batchSize = 5000;
//loop file , add records to batch
try{
while ((nextLine = reader.readNext()) != null) {
lnNum++;
/* Bind CSV file input to table columns */
sql_statement.setString(1, nextLine[0]);
sql_statement.setString(2,nextLine[1]);
sql_statement.setString(3,nextLine[2]);
sql_statement.setString(4,nextLine[3]);
sql_statement.setString(5,nextLine[4]); //setInt(Integer.parseInt(nextLine[4].trim());
sql_statement.setString(6,nextLine[5]);
sql_statement.setObject(7, nextLine[5]);
sql_statement.setString(8,nextLine[7]);
sql_statement.setString(9,nextLine[8]);
sql_statement.addBatch();
// Add the record to batch
if (++batchSize % 5000 == 0){
sql_statement.executeBatch();
}
}
sql_statement.executeBatch();
}
catch(SQLException e){
e.printStackTrace();
}
//Perform a bulk batch insert
int[] totalRecords = new int[7];
try {
totalRecords = sql_statement.executeBatch();
} catch(BatchUpdateException e) {
//handle exception for failed records here
totalRecords = e.getUpdateCounts();
} catch(SQLException ex){
ex.printStackTrace();
}
System.out.println ("Total records inserted in bulk from CSV file " + totalRecords.length);
/* Close prepared statement */
sql_statement.close();
/* COMMIT transaction */
conn.commit();
/* Close connection */
conn.close();
}
I am not receiving an error while running this. It is printing:
Total records inserted in bulk from CSV file 0
The table is not being updated with the new values in Oracle. Any suggestions?

Execute sql_statement.executeBatch() only once if your batch size is reached.
executeBatch() is returning an array with the results (how many rows are affected).
So you have to add each element of the array to compute the total count.
The condition for execute the batch is also wrong.
I cant proof, but i would change your example like this (only changed section):
public void insertData() throws ClassNotFoundException, SQLException, IOException {
/* Create Connection objects */
Class.forName("oracle.jdbc.OracleDriver");
String jdbc_insert_sql = "INSERT INTO METATADA_AUTOSYS"
+ "(MACH_NAME,JOB_NAME,SCRIPT_COMMAND,APPLICATION_NAME,JOB_ID,STATUS,CREATE_DATE,LAST_START_DT,LAST_END_DT) VALUES"
+ "(?,?,?,?,?,?,?,?,?)";
int totalRecords = 0;
final int batchSize = 5000;
/* Read CSV file in OpenCSV */
String inputCSVFile = "C:/Users/conwacx/Desktop/meta_auto_v3/Autosys_Metadata.csv";
try (CSVReader reader = new CSVReader(new FileReader(inputCSVFile))) {
try (Connection conn = DriverManager.getConnection("jdbc:oracle:thin:#HoSt", "UsErNaMe", "PaSsWoRd")) {
try (PreparedStatement sql_statement = conn.prepareStatement(jdbc_insert_sql);) {
String[] nextLine;
int lnNum = 0;
// loop file , add records to batch
try {
while ((nextLine = reader.readNext()) != null) {
lnNum++;
/* Bind CSV file input to table columns */
sql_statement.setString(1, nextLine[0]);
sql_statement.setString(2, nextLine[1]);
sql_statement.setString(3, nextLine[2]);
sql_statement.setString(4, nextLine[3]);
sql_statement.setString(5, nextLine[4]);
sql_statement.setString(6, nextLine[5]);
sql_statement.setObject(7, nextLine[5]);
sql_statement.setString(8, nextLine[7]);
sql_statement.setString(9, nextLine[8]);
sql_statement.addBatch();
// Add the record to batch
if (lnNum >= batchSize) {
// Perform a bulk batch insert
totalRecords += doExecute(sql_statement);
lnNum = 0;
}
}
// insert the last rows
if ( lnNum >= 0 ) {
totalRecords += doExecute(sql_statement);
}
} catch (SQLException e) {
e.printStackTrace();
}
}
System.out.println("Total records inserted in bulk from CSV file " + totalRecords);
/* COMMIT transaction */
conn.commit();
}
}
}
private int doExecute(PreparedStatement sql_statement) {
int totalRecords = 0;
int[] results = null;
try {
results = sql_statement.executeBatch();
for (int i = 0; i < results.length; i++) {
totalRecords += results[i];
}
} catch (BatchUpdateException e) {
// handle exception for failed records here
results = e.getUpdateCounts();
for (int i = 0; i < results.length; i++) {
totalRecords += results[i];
}
} catch (SQLException ex) {
ex.printStackTrace();
}
return totalRecords;
}

Java CSVReader ignore commas in double quotes

I have a CSV file that I am having trouble parsing. I am using the opencsv library. Here is what my data looks like and what I am trying to achieve.
RPT_PE,CLASS,RPT_MKT,PROV_CTRCT,CENTER_NM,GK_TY,MBR_NM,MBR_PID
"20150801","NULL","33612","00083249P PCP602","JOE SMITH ARNP","NULL","FRANK, LUCAS E","50004655200"
The issue I am having is the member name ("FRANK, LUCAS E") is being split into two columns and the member name should be one. Again I'm using opencsv and a comma as the separator. Is there any way I can ignore the commas inside the double-quotes?
public void loadCSV(String csvFile, String tableName,
boolean truncateBeforeLoad) throws Exception {
CSVReader csvReader = null;
if (null == this.connection) {
throw new Exception("Not a valid connection.");
}
try {
csvReader = new CSVReader(new FileReader(csvFile), this.seprator);
} catch (Exception e) {
e.printStackTrace();
throw new Exception("Error occured while executing file. "
+ e.getMessage());
}
String[] headerRow = csvReader.readNext();
if (null == headerRow) {
throw new FileNotFoundException(
"No columns defined in given CSV file."
+ "Please check the CSV file format.");
}
String questionmarks = StringUtils.repeat("?,", headerRow.length);
questionmarks = (String) questionmarks.subSequence(0, questionmarks
.length() - 1);
String query = SQL_INSERT.replaceFirst(TABLE_REGEX, tableName);
System.out.println("Base Query: " + query);
String headerRowMod = Arrays.toString(headerRow).replaceAll(", ]", "]");
String[] strArray = headerRowMod.split(",");
query = query
.replaceFirst(KEYS_REGEX, StringUtils.join(strArray, ","));
System.out.println("Add Headers: " + query);
query = query.replaceFirst(VALUES_REGEX, questionmarks);
System.out.println("Add questionmarks: " + query);
String[] nextLine;
Connection con = null;
PreparedStatement ps = null;
try {
con = this.connection;
con.setAutoCommit(false);
ps = con.prepareStatement(query);
if (truncateBeforeLoad) {
//delete data from table before loading csv
con.createStatement().execute("DELETE FROM " + tableName);
}
final int batchSize = 1000;
int count = 0;
Date date = null;
while ((nextLine = csvReader.readNext()) != null) {
System.out.println("Next Line: " + Arrays.toString(nextLine));
if (null != nextLine) {
int index = 1;
for (String string : nextLine) {
date = DateUtil.convertToDate(string);
if (null != date) {
ps.setDate(index++, new java.sql.Date(date
.getTime()));
} else {
ps.setString(index++, string);
}
}
ps.addBatch();
}
if (++count % batchSize == 0) {
ps.executeBatch();
}
}
ps.executeBatch(); // insert remaining records
con.commit();
} catch (SQLException | IOException e) {
con.rollback();
e.printStackTrace();
throw new Exception(
"Error occured while loading data from file to database."
+ e.getMessage());
} finally {
if (null != ps) {
ps.close();
}
if (null != con) {
con.close();
}
csvReader.close();
}
}
public char getSeprator() {
return seprator;
}
public void setSeprator(char seprator) {
this.seprator = seprator;
}
public char getQuoteChar() {
return quoteChar;
}
public void setQuoteChar(char quoteChar) {
this.quoteChar = quoteChar;
}
}

Did you try the the following?
CSVReader reader = new CSVReader(new FileReader("yourfile.csv"), ',');
I wrote a following program and it works for me, I got the following result:
[20150801] [NULL] [33612] [00083249P PCP602] [JOE SMITH ARNP] [NULL]
[FRANK, LUCAS E] [50004655200]
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import au.com.bytecode.opencsv.CSVReader;
public class CVSTest {
/**
* #param args
*/
public static void main(String[] args) {
CSVReader reader = null;
try {
reader = new CSVReader(new FileReader(
"C:/Work/Dev/Projects/Pure_Test/Test/src/cvs"), ',');
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
String[] nextLine;
try {
while ((nextLine = reader.readNext()) != null) {
// nextLine[] is an array of values from the line
System.out.println("[" + nextLine[0] + "] [" + nextLine[1]
+ "] [" + nextLine[2] + "] [" + nextLine[3] + "] ["
+ nextLine[4] + "] [" + nextLine[5] + "] ["
+ nextLine[6] + "] [" + nextLine[7] + "]");
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

According to the documentation, you can supply custom separator and quote characters in the constructor, which should deal with it:
CSVReader(Reader reader, char separator, char quotechar)
Construct your reader with , as separator and " as quotechar.

It is simple to load your CSV as an SQL table into HSQLDB, then select rows from the table to insert into another database. HSQLDB handles commas inside quotes. You need to define your text source as "quoted". See this:
http://hsqldb.org/doc/2.0/guide/texttables-chapt.html

Your case should be handled out of the box with no special configuration required.
If you can't make it work, then just switch to uniVocity-parsers to do this for you - it's twice as fast in comparison to OpenCSV, requires much less code and is packed with features.
CsvParserSettings settings = new CsvParserSettings(); // you have many configuration options here - check the tutorial.
CsvParser parser = new CsvParser(settings);
List<String[]> allRows = parser.parseAll(new FileReader(new File("C:/Work/Dev/Projects/Pure_Test/Test/src/cvs")));
Disclosure: I am the author of this library. It's open-source and free (Apache V2.0 license).

com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException UNKNOWN COLUMN

I am currently trying to scan and parse the file that is not in sql format. I am trying to input all the data into the SQL table but for some reason every time i run the program, i get the error saying unknown column 'what' in 'field list.' So the neither of the data goes through. 'what' is one of the names that is on the text. The table currently has 11 columns. I know I am parsing or scanning it wrong but I cannot figure out where. Here is my code:
public class parseTable {
public parseTable (String name) throws FileNotFoundException
{
File file = new File(name);
parse(file);
}
private void parse(File file) throws FileNotFoundException
{
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try{
Class.forName("com.mysql.jdbc.Driver").newInstance();
String connectionUrl = "jdbc:mysql://localhost:3306/";
String connectionUser = "";
String connectionPassword = "";
conn = DriverManager.getConnection(connectionUrl, connectionUser, connectionPassword);
stmt = conn.createStatement();
Scanner scan = new Scanner(file);
String[] rowInfo = new String[11];
int count = 0;
while(scan.hasNextLine()){
//String data = scan.nextLine();
Scanner lineScan = new Scanner(scan.nextLine());
while(lineScan.hasNext()){
String words = lineScan.next();
if(count < 11){
rowInfo[count] = words;
count++;
}
else if(count == 11 && words.equals("States")){
rowInfo[count - 1] = rowInfo[count - 1] + " " + words;
}
else{
String query = "";
for(int i = 0; i < rowInfo.length; i++)
{
if(query.equals(""))
{
query = rowInfo[i];
}
else if(i == 9){
query = query + "," + rowInfo[i];
}
else if(rowInfo[i].equals(null)){
query = query + ", " + "NULL";
}
else
query = query + ", " + "'" + rowInfo[i] + "'";
}
stmt.executeUpdate("INSERT INTO dup VALUES(" + query + ")");
count = 0;
rowInfo = new String[11];
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try { if (rs != null) rs.close(); } catch (SQLException e) { e.printStackTrace(); }
try { if (stmt != null) stmt.close(); } catch (SQLException e) { e.printStackTrace(); }
try { if (conn != null) conn.close(); } catch (SQLException e) { e.printStackTrace(); }
}
}
}
And this is the data I'm trying to input:
1 hello cheese 1111 what#yahoo.com user adm street zip what USA
2 Alex cheese 1111 what#yahoo.com user adm street zip what USA
So this is my new code now, using PrepareStatement. However I still get an error and I looked online for the solution on where I'm making a mistake, but I cant seem to figure out where.
String query = "INSERT INTO mil_table (UserName, NameFirst, NameLast, supportID, EmailAddress, Password,
IDQ, AddressCity, AddressState, AddressZip, AddressCountry) VALUES(?,?,?,?,?,?,?,?,?,?,?)";
pstmt = conn.prepareStatement(query);
Scanner scan = new Scanner(file);
String[] rowInfo = new String[11];
int count = 0;
while(scan.hasNextLine()){
//String data = scan.nextLine();
Scanner lineScan = new Scanner(scan.nextLine());
while(lineScan.hasNext()){
String words = lineScan.next();
if(count < 11){
rowInfo[count] = words;
count++;
}
else if(count == 11 && words.equals("States")){
rowInfo[count - 1] = rowInfo[count - 1] + " " + words;
}
else{
for(int i = 0; i <rowInfo.length; i++)
{
pstmt.setString(i + 1, rowInfo[i]);
}
//stmt.executeUpdate("INSERT INTO mil_table VALUES(" + query + ")");
//System.out.println("#" + query + "#");
pstmt.executeUpdate();
count = 0;
rowInfo = new String[11];
}
}

As you are using MySQL, you will need to enclose the text inputs with quotes. Try enclosing the String values that you are inserting in quotes and then execute your code.

Update MySQL table using data from a text file through Java

I have a text file with four lines, each line contains comma separated values like below file
My file is:
Raj,raj34#myown.com,123455
kumar,kumar#myown.com,23453
shilpa,shilpa#myown.com,765468
suraj,suraj#myown.com,876567
and I have a MySQL table which contains four fields
firstname lastname email phno
---------- ---------- --------- --------
Raj babu raj34#hisown.com 2343245
kumar selva kumar#myown.com 23453
shilpa murali shilpa#myown.com 765468
suraj abd suraj#myown.com 876567
Now I want to update my table using the data in the above text file through Java.
I have tried using bufferedReader to read from the file and used split method using comma as delimiter and stored it in array. But it is not working. Any help appreciated.
This is what I have tried so far
void readingFile()
{
try
{
File f1 = new File("TestFile.txt");
FileReader fr = new FileReader(f1);
BufferedReader br = new BufferedReader(fr);
String strln = null;
strln = br.readLine();
while((strln=br.readLine())!=null)
{
// System.out.println(strln);
arr = strln.split(",");
strfirstname = arr[0];
strlastname = arr[1];
stremail = arr[2];
strphno = arr[3];
System.out.println(strfirstname + " " + strlastname + " " + stremail +" "+ strphno);
}
// for(String i : arr)
// {
// }
br.close();
fr.close();
}
catch(IOException e)
{
System.out.println("Cannot read from File." + e);
}
try
{
st = conn.createStatement();
String query = "update sampledb set email = stremail,phno =strphno where firstname = strfirstname ";
st.executeUpdate(query);
st.close();
System.out.println("sampledb Table successfully updated.");
}
catch(Exception e3)
{
System.out.println("Unable to Update sampledb table. " + e3);
}
}
and the output i got is:
Ganesh Pandiyan ganesh1#myown.com 9591982389
Dass Jeyan jeyandas#myown.com 9689523645
Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: 1
Gowtham Selvan gowthams#myown.com 9894189423
at TemporaryPackages.FileReadAndUpdateTable.readingFile(FileReadAndUpdateTable.java:35)
at TemporaryPackages.FileReadAndUpdateTable.main(FileReadAndUpdateTable.java:72)
Java Result: 1
#varadaraj:
This is the code of yours....
String stremail,strphno,strfirstname,strlastname;
// String[] arr;
Connection conn;
Statement st;
void readingFile()
{
try {
BufferedReader bReader= new BufferedReader(new FileReader("TestFile.txt"));
String fileValues;
while ((fileValues = bReader.readLine()) != null)
{
String[] values=fileValues .split(",");
strfirstname = values[0];
// strlastname = values[1];
stremail = values[1];
strphno = values[2];
System.out.println(strfirstname + " " + strlastname + " " + stremail +" "+ strphno);
}
bReader.close();
} catch (IOException e) {
System.out.println("File Read Error");
}
// for(String i : arr)
// {
// }
try
{
st = conn.createStatement();
String query = "update sampledb set email = stremail,phno =strphno where firstname = strfirstname ";
st.executeUpdate(query);
st.close();
System.out.println("sampledb Table successfully updated.");
}
catch(Exception e3)
{
System.out.println("Unable to Update sampledb table. " + e3);
}
}

What you are having looks like a CSV file, you may consider libraries like Super CSV to help you in reading and parsing the file.

you are getting ArrayIndexOutOfBoundException upon trying to access at index 1 , i.e at lastname field value, so check whether you have no data at index 1 for any of the list elements in your text file

try this
public class FileReaderTesting {
static String stremail;
static String strphno;
static String strfirstname;
static String strlastname;
static Connection conn;
static Statement st;
public static void main(String[] args) {
try {
BufferedReader bReader= new BufferedReader(new FileReader("C:\\fileName.txt"));
String fileValues;
while ((fileValues = bReader.readLine()) != null)
{
String[] values=fileValues .split(",");
strfirstname = values[0];
// strlastname = values[1];
stremail = values[1];
strphno = values[2];
System.out.println(strfirstname + " " + stremail +" "+ strphno);
st = conn.createStatement();
String query = "update sampledb set email = '"+stremail+"',pno = '"+strphno+"' where firstname = '"+strfirstname+"' ";
System.out.println(query);
st.executeUpdate(query);
st.close();
System.out.println("sampledb Table successfully updated.");
}
bReader.close();
} catch (IOException e) {
System.out.println("File Read Error");
}
catch(Exception e3)
{
System.out.println("Unable to Update sampledb table. " + e3);
}
}
}

Parsing empty cells in csv file and storing into mysql

I have a csv file which has few cells in some columns empty.Modifying the csv file is not an option as it has around 50k total records.
Whenever I am executing below code it is throwing error as "Incorrect integer value: '' for column 'ParentId' at row 1".My database table has null value as yes for column 'ParentID'.But still it is throwing error.How can I edit this code so that it fills in correct values without giving error?
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.Date;
import org.apache.commons.lang.StringUtils;
import au.com.bytecode.opencsv.CSVReader;
public class CSVLoader {
static int count;
private static final
//String SQL_INSERT = "INSERT INTO ${table}(${keys}) VALUES(${values})";
String SQL_INSERT = "INSERT INTO ${table} VALUES(${values})";
private static final String TABLE_REGEX = "\\$\\{table\\}";
//private static final String KEYS_REGEX = "\\$\\{keys\\}";
private static final String VALUES_REGEX = "\\$\\{values\\}";
private Connection connection;
private char seprator;
/**
* Public constructor to build CSVLoader object with
* Connection details. The connection is closed on success
* or failure.
* #param connection
*/
public CSVLoader(Connection connection) {
this.connection = connection;
//Set default separator
this.seprator = ',';
}
/**
* Parse CSV file using OpenCSV library and load in
* given database table.
* #param csvFile Input CSV file
* #param tableName Database table name to import data
* #param truncateBeforeLoad Truncate the table before inserting
* new records.
* #throws Exception
*/
public void loadCSV(String csvFile, String tableName,
boolean truncateBeforeLoad) throws Exception {
CSVReader csvReader = null;
if(null == this.connection) {
throw new Exception("Not a valid connection.");
}
try {
csvReader = new CSVReader(new FileReader(csvFile), this.seprator);
} catch (Exception e) {
e.printStackTrace();
throw new Exception("Error occured while executing file. "
+ e.getMessage());
}
//String[] headerRow = csvReader.readNext();
String[] headerRow = csvReader.readNext();
count++;
if (null == headerRow) {
throw new FileNotFoundException(
"No columns defined in given CSV file." +
"Please check the CSV file format.");
}
/*String questionmarks = StringUtils.repeat("?,", headerRow.length);
System.out.println(headerRow.length);
questionmarks = (String) questionmarks.subSequence(0, questionmarks
.length() - 1);
System.out.println(SQL_INSERT);
String query = SQL_INSERT.replaceFirst(TABLE_REGEX, tableName);
//query = query
// .replaceFirst(KEYS_REGEX, StringUtils.join(headerRow, ","));
query = query.replaceFirst(VALUES_REGEX, questionmarks);
System.out.println("Query: " + query);
*/
//Stirng str1 = "insert into posts values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
String[] nextLine;
Connection con = null;
PreparedStatement ps = null;
try {
con = this.connection;
con.setAutoCommit(false);
ps = con.prepareStatement("insert into posts (Id,PostTypeId,AcceptedAnswerId,ParentId,CreationDate,Score,ViewCount,Body,OwnerUserId,OwnerDisplayName,LastEditorUserId,LastEditorDisplayName,LastEditDate,LastActivityDate,Title,Tags,AnswerCount,CommentCount,FavoriteCount,ClosedDate,CommunityOwnedDate,RowNum) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
if(truncateBeforeLoad) {
//delete data from table before loading csv
con.createStatement().execute("DELETE FROM " + tableName);
}
final int batchSize = 1000;
int count = 0;
Date date = null;
while ((nextLine = csvReader.readNext()) != null) {
if (null != nextLine) {
int index = 1;
for (String string : nextLine) {
date = DateUtil.convertToDate(string);
if (null != date) {
ps.setDate(index++, new java.sql.Date(date
.getTime()));
} else {
ps.setString(index++, string);
}
}
System.out.println(count);
ps.addBatch();
System.out.println(count);
}
if (++count % batchSize == 0) {
System.out.println(count);
ps.executeBatch();
}
}
ps.executeBatch(); // insert remaining records
con.commit();
} catch (Exception e) {
con.rollback();
e.printStackTrace();
throw new Exception(
"Error occured while loading data from file to database."
+ e.getMessage());
} finally {
if (null != ps)
ps.close();
if (null != con)
con.close();
csvReader.close();
}
}
public char getSeprator() {
return seprator;
}
public void setSeprator(char seprator) {
this.seprator = seprator;
}
}

It might be the case where int column is actually having blank spaces, in such case you can place manual check as below and still go ahead. Hope this answers your doubt.
// check if value is null or blank spaces
if(certain_value== null || certain.value.trim().length==0){
ps.setNull(4, java.sql.Types.INTEGER); // This will set null value for int type
}