For a simulation/mining on graph data I need to write thousands of data at once into a postgresql 9.5 Database with high performance. Currently I'm doing that with prepared statements and "row by row INSERTs". It works fine - but too slow. I couldn't find anything about how to write data from a collection (array, vector or ArrayList) with the Copy statement via CopyManager into a Postgresql DB Table. I would be grateful to get a simple example. How to deal with data types? CopyManager does not request for any data types.
My current Code:
package io;
import db.PostgresConnector;
import entitiesP.Edge;
import entitiesP.Graph;
import entitiesP.Node;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
public class DbLogger {
public DbLogger(Graph g, int simNr) throws Exception {
Connection conn = new PostgresConnector().getConnection();
//1. Graph Data File:
//-------------------------------------------------
//Filename und Header:
long timestamp = getTimeStamp();
//Daten:
logGraphData(conn, simNr,
g.getNodes().size(),
g.getEdges().size(),
g.getRadiusAndDiameter()[0],
g.getRadiusAndDiameter()[1],
g.getRadiusAndDiameter()[2],
Node.maxDegree,
//new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(g.getSimStartTime()),
g.getSimStartTime(),
timestamp);
//2. Node Data File:
//-------------------------------------------------
//Daten einsammeln:
for (Node n: g.getNodes()){
logNodeData( conn, simNr ,
n.getId(),
n.getDegree(),
n.getClusteringCoefficient(),
timestamp);
}
//3. RelationData File:
//-------------------------------------------------
int[][] distance = g.getDistanceMatrix().clone();
//AdMatrix dist = new AdMatrix(distance);
//dist.printAdMatrix();
Object[] edges = g.getEdges().toArray();
//adjacency nodes
int numNodes = g.getNodes().size();
int numEdges = g.getEdges().size();
//Edges:
for (int i=0; i < edges.length; i++){
Object[] nodes = ((Edge) edges[i]).getNodes().toArray(); //pair of nodes of an edge
//pairs A--B
logRelationData(conn, simNr ,
((Node) nodes[0]).getId() ,
((Node) nodes[1]).getId() ,
distance[((Node) nodes[0]).getId() -1][((Node) nodes[1]).getId() -1] ,
((Edge) edges[i]).getTieStrength() ,
((Edge) edges[i]).getNeighborhoodOverlap(((Node) nodes[0]),((Node) nodes[1])) ,
timestamp
);
//pairs B--A
logRelationData(conn, simNr ,
((Node) nodes[1]).getId() ,
((Node) nodes[0]).getId() ,
distance[((Node) nodes[0]).getId() -1][((Node) nodes[1]).getId() -1] ,
((Edge) edges[i]).getTieStrength() ,
((Edge) edges[i]).getNeighborhoodOverlap(((Node) nodes[0]),((Node) nodes[1])) ,
timestamp
);
//end edges----------------------------
}
//System.out.println("NumNodes: " + numNodes);
//non-adjacent nodes
for (int i=0; i < numNodes; i++){
for (int j=0; j < numNodes; j++){
if (distance[i][j] != 1 && i != j ){
logRelationData(conn, simNr,
(i+1),
(j+1),
(distance[i][j]==0? 999 : distance[i][j]) , //0 auf infinity setzen (999)
-1,
-1,
timestamp
);
}
}
}
conn.close();
}
private static void logGraphData(Connection con, int sim_no, int numberOfNodes, int numberOfEdges, int radius, int diameter, int effDiameter, int maxDegree, long startTime, long timestamp) throws SQLException{
PreparedStatement preStmt = con.prepareStatement("INSERT INTO public.GRAPH_DATA (" + "SIM_NO,NUMBER_OF_NODES,NUMBER_OF_EDGES,RADIUS,DIAMETER,EFF_DIAMETER,MAX_DEGREE,START_TIME,TIME_STAMP) VALUES (?,?,?,?,?,?,?,?,?)");
preStmt.setInt(1, sim_no);
preStmt.setInt(2, numberOfNodes);
preStmt.setInt(3, numberOfEdges);
preStmt.setInt(4, radius);
preStmt.setInt(5, diameter);
preStmt.setInt(6, effDiameter);
preStmt.setInt(7, maxDegree);
preStmt.setTimestamp(8, new Timestamp(startTime));
preStmt.setTimestamp(9, new Timestamp(timestamp));
preStmt.executeUpdate();
preStmt.close();
//con.close();
}
private static void logNodeData(Connection con, int sim_no, int nodeId, int degree, double clusteringCoeff, long timeStamp) throws SQLException{
PreparedStatement preStmt = con.prepareStatement("INSERT INTO public.NODE_DATA (" + "SIM_NO,NODE_ID,DEGREE,CLUSTERING_COEFFICIENT,TIME_STAMP) VALUES (?,?,?,?,?)");
preStmt.setInt(1, sim_no);
preStmt.setInt(2, nodeId);
preStmt.setInt(3, degree);
preStmt.setDouble(4, clusteringCoeff);
preStmt.setTimestamp(5, new Timestamp(timeStamp));
preStmt.executeUpdate();
preStmt.close();
//con.close();
}
private static void logRelationData(Connection con, int sim_no, int nodeId1, int nodeId2, int distance, int tieStrength, double neighborhoodOverlap, long timeStamp) throws SQLException{
PreparedStatement preStmt = con.prepareStatement("INSERT INTO public.RELATION_DATA (" + "SIM_NO,NODE_ID1,NODE_ID2,DISTANCE,TIE_STRENGTH,NEIGHBORHOOD_OVERLAP,TIME_STAMP) VALUES (?,?,?,?,?,?,?)");
preStmt.setInt(1, sim_no);
preStmt.setInt(2, nodeId1);
preStmt.setInt(3, nodeId2);
preStmt.setInt(4, distance);
preStmt.setInt(5, tieStrength);
preStmt.setDouble(6, neighborhoodOverlap);
preStmt.setTimestamp(7, new Timestamp(timeStamp));
preStmt.executeUpdate();
preStmt.close();
//con.close();
}
public static long getTimeStamp(){
long t = System.currentTimeMillis();
//SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//String timestamp = sdf.format(t);
return t; //timestamp;
}
}
I found the solution for my problem: it works very fast :)
package io;
import db.PostgresConnector;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import org.postgresql.copy.CopyManager;
import org.postgresql.core.BaseConnection;
public class DbLogger3 {
public static void main (String args[]) throws Exception {
try {
Connection conn = new PostgresConnector().getConnection();
CopyManager copyManager = new CopyManager((BaseConnection) conn);
String str = new String();
//add 5000 data sets: (important: \r\n after each record to set a line feed (postgres interpretes that as the end of each dataset)
for (int i=0; i<5000; i++){
str += "2;100;99;6.00;12.00;11.00;13;28.06.2016 00:08;28.06.2016 00:08"+"\r\n";
}
//transform the String into bytes:
byte[] bytes = str.getBytes();
//create ByteArrayInputStream object
ByteArrayInputStream input = new ByteArrayInputStream(bytes);
//insert into the database table (in my case: public.graph_data)
copyManager.copyIn("COPY public.graph_data FROM STDIN WITH DELIMITER ';'", input);
} catch (SQLException | IOException e) {
throw new Exception(e);
}
}
}
Related
package javaapplication2;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
//jdbc:derby://localhost:1527/LMS
class abc {
public abc() throws SQLException{
//ConnectionURL, username and password should be specified in getConnection()
//ConnectionURL, username and password should be specified in getConnection()
try {String url = "jdbc:derby://localhost:1527/LMS";
Connection conn = DriverManager.getConnection(url,"zain","12345");
System.out.println("Connected! ");
String sql = "SELECT * FROM BOOK";`enter code here`
Statement st = conn.createStatement();
ResultSet rs=st.executeQuery(sql);
String x;
while(rs.next()){
System.out.println(rs.getString("Password"));
}
}
catch (SQLException ex) {
System.out.println(ex);
}
}
}
I am getting this any lead? I am trying this thing for hours and not getting any lead :/ plz help. I am making a Library Management System and made Db for it with a column book in it and trying to get data from it.
Derby is a tricky beast.
Here is a working example of a data retreivale and prep for insert into a Jtable
public static final String DRIVER= "org.apache.derby.jdbc.EmbeddedDriver";
public static final String JDBC_URL = "jdbc:derby://localhost:1527/LMS";
private void GetData()throws SQLException
{
Connection connection=DriverManager.getConnection(JDBC_URL,"zain","12345");
ResultSet result = null;
String sql2 = "select * from book";
PreparedStatement ps1 = connection.prepareStatement( sql2, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY) ;
result = ps1.executeQuery();
// get number of rows
result.last();
int numRows = result.getRow();
result.first();
//Get metadata object for result
ResultSetMetaData meta = result.getMetaData();
// create an arry of string for the colum names
colNames = new String[meta.getColumnCount()];
// store column names in the new col names array
for( int i = 0; i< meta.getColumnCount(); i++)
{
//get column name
colNames[i] = meta.getColumnLabel(i+1);
}
// create 2 d string array for table data
tableData = new String [numRows][meta.getColumnCount()];
// store columns in the data
for ( int row = 0 ; row < numRows; row++)
{
for (int col = 0; col < meta.getColumnCount(); col++)
{
tableData[row][col]= result.getString(col + 1);
}
result.next();
}
// close statement
ps1.close();
connection.close();
} // end get data
To insert into a a gui...
table = new JTable(tableData,colNames);
JScrollPane scrollPane = new JScrollPane(table);
package Simple;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.text.*;
public class CurrentProg {
//connecting to the database
private static final String DB_DRIVER = "com.mysql.jdbc.Driver";
private static final String DB_CONNECTION ="jdbc:mysql://localhost:3306/db?autoReconnect=true";
private static final String DB_USER = "root";
private static final String DB_PASSWORD = "root";
static Connection dbConnection = null;
static Statement statement = null;
static int total=1;
//Searching between startdate and enddate
public static java.util.LinkedList searchBetweenDates(java.util.Date startDate, java.util.Date endDate) {
java.util.Date begin = new Date(startDate.getTime());
java.util.LinkedList list = new java.util.LinkedList();
list.add(new Date(begin.getTime()));
java.util.Date end = new Date(endDate.getTime());
endDate.setTime(endDate.getTime() + 24*3600*1000);
Calendar cal = Calendar.getInstance();
cal.setTime(begin);
dbConnection = getDBConnection();
while(begin.compareTo(endDate)<0){
begin = new Date(begin.getTime() + 86400000);
list.add(new Date(begin.getTime()));
Timestamp timestamp = new Timestamp(new Date().getTime());
//For a single day calculation: 24hours*60mins=1440 /2 (2 mins time difference as per the requirement) = 720
for (int j = 0; j < 720; j++) {
cal.add(Calendar.MINUTE, 2);
timestamp = new Timestamp(cal.getTime().getTime());
String S = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(timestamp);
String[] parts = S.split(" ");
String date=parts[0];
String time=parts[1];
cal.getTime().toString();
// To create data loop into a List
List<String> records = new ArrayList<String>();
StringBuffer record = new StringBuffer();
for (int i = 1; i <= total; i++) {
records = new ArrayList<String>(total);
int a2 = 220 + j % 31; // 230 - 244 by 1
String wString = Integer.toString(a2);
String a = String.valueOf(a2);
double b2 = 0.00 + j % 3.7 ; // 1.3 - 3.9 by 0.1
String aString = Double.toString(b2);
String b = String.valueOf(b2);
b = b.substring(0, Math.min(b.length(), 5));
record.delete(0, record.length());
record.append(a + "," + b + ",'"+ date + "', '"+ time + "'");
record.append("\t\t");
record.append("\n");
records.add(record.toString());
//Insert Query
String insertTableSQL = "INSERT INTO cmd1"
+ "(a, b, date, time) " + "VALUES"
+ "("+record.toString()+")";
System.out.println("insertTableSQL - " + insertTableSQL); // Statement.executeUpdate(insertTableSQL);
try {
statement = dbConnection.createStatement();
statement.executeUpdate(insertTableSQL);
System.out.println("Record is inserted into Db table!");
} catch (SQLException e) {
System.out.println(e.getMessage());
}
try {
// dbConnection = getDBConnection();
statement = dbConnection.createStatement();
statement.executeUpdate(insertTableSQL);
System.out.println("Record is inserted into Db table!");
} catch (SQLException e) {
System.out.println(e.getMessage());
}
finally {
// httpPost.releaseConnection()
try{
if(statement!=null)
statement.close();
}
finally{
}
try{
if(dbConnection!=null)
dbConnection.close();
}
finally{
}
}
}
}
}
return list;
}
#SuppressWarnings("unused")
public static void main(String[] args) throws Exception {
//To enter startDate and enddate
// EntityManagerFactory.getCache().evictAll;
SimpleDateFormat startDate=new java.text.SimpleDateFormat("yyyy-MM-dd");
SimpleDateFormat endDate=new java.text.SimpleDateFormat("yyyy-MM-dd");
java.util.LinkedList hitList = searchBetweenDates(
startDate.parse("2016-01-01"),
endDate.parse("2016-03-01"));
String[] combo = new String[hitList.size()];
for(int i=0; i<hitList.size(); i++)
combo[i] = new java.text.SimpleDateFormat("yyyy-MM-dd").format(((java.util.Date)hitList.get(i)));
}
private static void insertRecordIntodb() {
//
}
private static Connection getDBConnection() {
Connection dbConnection = null;
try {
Class.forName(DB_DRIVER);
} catch (ClassNotFoundException e) {
System.out.println(e.getMessage());
}
try {
dbConnection = DriverManager.getConnection(DB_CONNECTION, DB_USER, DB_PASSWORD);
return dbConnection;
} catch (SQLException e) {
System.out.println(e.getMessage());
}
return dbConnection;
}
}
Server connection failure during transaction. Due to underlying exception: 'java.net.SocketException: java.net.SocketException: No buffer space available (maximum connections reached?): connect'.
** BEGIN NESTED EXCEPTION **
java.net.SocketException
MESSAGE: java.net.SocketException: No buffer space available (maximum connections reached?): connect
STACKTRACE:
java.net.SocketException: java.net.SocketException: No buffer space available (maximum connections reached?): connect
at com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:156)
at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:276)
at com.mysql.jdbc.Connection.createNewIO(Connection.java:2717)
at com.mysql.jdbc.Connection.<init>(Connection.java:1509)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:266)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at Simple.CurrentProg.getDBConnection(CurrentProg.java:186)
at Simple.CurrentProg.searchBetweenDates(CurrentProg.java:126)
at Simple.CurrentProg.main(CurrentProg.java:164)
** END NESTED EXCEPTION **
Attempted reconnect 3 times. Giving up.
Exception in thread "main" java.lang.NullPointerException
at Simple.CurrentProg.searchBetweenDates(CurrentProg.java:137)
at Simple.CurrentProg.main(CurrentProg.java:164)
Here I am trying to connect java program with database but when i am trying to insert large data say for 1 month so its only fetching 16000 records not more than that i want the data should be inserted as per the given date range what should i do to get that . In stacktrace its showing an exception as no buffer space available maximum connection reached. Thanks In advance
Here is the modified code. I removed a lot of code which is not necessary:
public static LinkedList<Date> searchBetweenDates(Date startDate, Date endDate) throws SQLException {
Date begin = new Date(startDate.getTime());
LinkedList<Date> list = new LinkedList<Date>();
list.add(new Date(begin.getTime()));
endDate.setTime(endDate.getTime() + 24 * 3600 * 1000);
Calendar cal = Calendar.getInstance();
cal.setTime(begin);
dbConnection = getDBConnection();
PreparedStatement ps = dbConnection.prepareStatement("INSERT INTO cmd1(aaaa, bbbb, datee, timee) VALUES(?, ?, ?, ?)");
while (begin.compareTo(endDate) < 0) {
begin = new Date(begin.getTime() + 86400000);
list.add(new Date(begin.getTime()));
Timestamp timestamp = new Timestamp(new Date().getTime());
// For a single day calculation: 24hours*60mins=1440 /2 (2 mins time
// difference as per the requirement) = 720
for (int j = 0; j < 720; j++) {
cal.add(Calendar.MINUTE, 2);
timestamp = new Timestamp(cal.getTime().getTime());
String S = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(timestamp);
String[] parts = S.split(" ");
String date = parts[0];
String time = parts[1];
cal.getTime().toString();
// To create data loop into a List
for (int i = 1; i <= total; i++) {
int a2 = 220 + j % 31; // 230 - 244 by 1
String a = String.valueOf(a2);
double b2 = 0.00 + j % 3.7; // 1.3 - 3.9 by 0.1
String b = String.valueOf(b2);
b = b.substring(0, Math.min(b.length(), 5));
ps.setString(1, a);
ps.setString(2, b);
ps.setString(3, date);
ps.setString(4, time);
ps.execute();
}
}
}
if (ps != null)
ps.close();
if (dbConnection != null)
dbConnection.close();
return list;
}
What I changed:
I removed your try/catch's because I wanted to make the code short so that I can edit it here easily. But you should handle exception correctly. And don't swallow exceptions ever. I mean the following is no go:
} catch (SQLException e) {
System.out.println(e.getMessage());
}
It is very bad Idea because you'll not see the real cause of the problem in this case; at least do e.printStackTrace() eventhough it is not recommended in a real project.
I exchanged the Statement object for PreparedStatement because it is much more efficient.
Removed package names and put import statements instead because it is not necessary to do so unless you have different classes from different packages with the same name.
I changed column names because my DB does not want to accept them. Column names like a are very bad. Choose instead descriptive names so that you might understand what it is for a couple of months later. Don't use column names like date because they are reserved words for some databases systems I know.
Don't create database resources like Connection in a loop unless it is absolutely needed! Otherwise, your program will go out of resources because these resources are very expensive! That is exactly what you are experiencing right now.
Hope it helps, otherwise, drop me a comment.
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 6 years ago.
Improve this question
package Simple;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import com.sun.org.apache.xerces.internal.impl.xpath.regex.ParseException;
import java.util.LinkedList;
public class CheckJdbc {
private static final String DB_DRIVER = "com.mysql.jdbc.Driver";
private static final String DB_CONNECTION = "jdbc:mysql://localhost:3306/db";
private static final String DB_USER = "root";
private static final String DB_PASSWORD = "root";
private static int RECORD_COUNT = 1;
int days=1;
static int total=1;
public static java.util.LinkedList searchBetweenDates(java.util.Date startDate, java.util.Date endDate) {
java.util.Date begin = new Date(startDate.getTime());
java.util.LinkedList list = new java.util.LinkedList();
list.add(new Date(begin.getTime()));
java.util.Date end = new Date(endDate.getTime());
endDate.setTime(endDate.getTime() + 24*3600*1000);
while(begin.compareTo(endDate)<0){
list.add(new Date(begin.getTime()));
Timestamp timestamp = new Timestamp(new Date().getTime());
int total=1;
Calendar cal = Calendar.getInstance();
cal.setTime(startDate);
for(int d=0; d<=total; d++)
{
cal.add(Calendar.MINUTE, 2);
timestamp = new Timestamp(cal.getTime().getTime());
String S = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(timestamp);
String[] parts = S.split(" ");
// System.out.println("Date:" + parts[0]);
// System.out.println("Time:" + parts[1]);
String date=parts[0];
String time=parts[1];
begin = new Date(begin.getTime() + 86400000);
cal.setTime(endDate);
System.out.println(timestamp);
List<String> records = new ArrayList<String>();
StringBuffer record = new StringBuffer();
for (int i = 1; i <= RECORD_COUNT; i++) {
records = new ArrayList<String>(RECORD_COUNT);
int RECORD_COUNT= total;
for (int j = 0; j < total; j++) {
int a2 = 220 + j % 30; // 230 - 244 by 1
String wString = Integer.toString(a2);
String a = String.valueOf(a2);
double b2 = 0.12 + j % 3.9 * 0.01 ; // 1.3 - 3.9 by 0.1
String aString = Double.toString(a2);
String b = String.valueOf(b2);
b = b.substring(0, Math.min(b.length(), 5));
double c2 = 0.01 + j % 49 * 0.01 ; // 0.01 - 0.49 by 0.01
String bString = Double.toString(c2);
String c = String.valueOf(c2);
c = c.substring(0, Math.min(c.length(), 5));
record.append(a + "," + b + "," + c + "," +date+ ","+ time );
record.append("\t\t");
record.append("\n");
records.add(record.toString());
try {
String insertTableSQL = "INSERT INTO cmd1"
+ "(a, b, c ,date, time) " + "VALUES"
+ "("+record.toString()+")";
System.out.println("insertTableSQL - " + insertTableSQL);
insertRecordIntodb();
Connection dbConnection = null;
Statement statement = null;
dbConnection = getDBConnection();
statement = dbConnection.createStatement();
statement.executeUpdate(insertTableSQL);
System.out.println(insertTableSQL);
System.out.println("Record is inserted into Db table!");
} catch (SQLException e) {
System.out.println(e.getMessage());
} finally {
}
}
}
}
}
return list;
}
#SuppressWarnings("unused")
public static void main(String[] args) throws Exception {
//searchBetweenDates(d);
java.util.LinkedList hitList = searchBetweenDates(
new java.text.SimpleDateFormat("MM/dd/yyyy").parse("01/10/2016"),
new java.text.SimpleDateFormat("MM/dd/yyyy").parse("01/15/2016"));
String[] combo = new String[hitList.size()];
for(int i=0; i<hitList.size(); i++)
combo[i] = new java.text.SimpleDateFormat("MM/dd/yyyy").format(((java.util.Date)hitList.get(i)));
}
private static void insertRecordIntodb() {
}
private static Connection getDBConnection() {
Connection dbConnection = null;
try {
Class.forName(DB_DRIVER);
} catch (ClassNotFoundException e) {
System.out.println(e.getMessage());
}
try {
dbConnection = DriverManager.getConnection(DB_CONNECTION, DB_USER, DB_PASSWORD);
return dbConnection;
} catch (SQLException e) {
System.out.println(e.getMessage());
}
return dbConnection;
}
}
I want to call data for date and time which is in searchBetweenDates method into main method. I have created object in main method but still its not fetching. What should i do for that? As its not fetching data from date and time.
I am getting following error:
insertTableSQL - INSERT INTO cmd1(a, b, c ,date, time) VALUES(220,0.12,0.01,01/10/2016,00:02:00
)
You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ':02:00
)' at line 1
First of all, your code is not even valid Java code (i.e. it does not even compile). Secondly, you are calling a static method through an instance, which you shouldn't. Use:
Test.searchBetweenDates(startDate, endDate);
This, however won't make your code work per se, because startDate and endDate is null.
Check your main method:
public static void main(String[] args) throws Exception {
new Test().searchBetweenDates(startDate, endDate);
searchBetweenDates(startDate, endDate);
the parenthesis is missing. It should be:
public static void main(String[] args) throws Exception {
new Test().searchBetweenDates(startDate, endDate);
searchBetweenDates(startDate, endDate);
}
just curious if anyone has any idea for making this program more simple. It reads records from a database into an ArrayList and allows the user to search for records by state. It processes a database of 1 million records in aprox 16000ms.
import java.sql.*;
import java.util.*;
public class ShowEmployeeDB
{
public static void main(String args[])
{
ArrayList <String> Recs = new ArrayList <String>();
String driverName = "sun.jdbc.odbc.JdbcOdbcDriver";
String connectionURL = "jdbc:odbc:CitizensDB";
Connection con = null;
Statement stmt = null;
String sqlStatement = "SELECT * FROM Citizens";
ResultSet rs = null;
int r = 0;
Scanner scan = new Scanner(System.in);
String search = null;
long starttime = System.currentTimeMillis();
try
{
Class.forName(driverName).newInstance();
con = DriverManager.getConnection(connectionURL);
stmt = con.createStatement();
rs = stmt.executeQuery(sqlStatement);
String ID = null;
String Age = null;
String State = null;
String Gender = null;
String Status = null;
String record = null;
while (rs.next())
{
for (int k = 1; k <= 1; ++k)
{
ID = rs.getString(k) + " ";
for (int j = 2; j <= 2; ++j)
Age = rs.getString(j) + " ";
for (int i = 3; i <= 3; ++i)
State = rs.getString(i).toUpperCase() + " ";
for (int h = 4; h <= 4; ++h)
Gender = rs.getString(h) + " ";
for (int g = 5; g <= 5; ++g)
Status = rs.getString(g) + " ";
}//for
record = ID + Age + State + Gender + Status;
Recs.add(record);
++r;
}//while
rs.close();
stmt.close();
con.close();
} catch (Exception ex) { ex.printStackTrace(); }
String endtime = System.currentTimeMillis() - starttime + "ms";
System.out.println(endtime);
System.out.print("Enter A Search State: ");
search = scan.nextLine().toUpperCase();
Iterator<String> iter = Recs.iterator();
while(iter.hasNext())
{
String s = iter.next();
if (s.contains(search))
{
System.out.println(s);
}
}//while
} // main
} // ShowEmployeeBD
Any advice would be greatly appreciated. Thank you in advance!
If search is not often, I would suggest to take the search string input before running the query, so that search results are directly from the DB. I this case you do not have to reiterate all 1 million records.
Perform searching directly on DB rather than fetching all the records and searching through java code.
Also if search is on multiple column, then prepare a meta data in DB at a single place on the basis of IDs, and the meta data can further be used for fetching the required results that match the query.
Separate your logic from the technical stuff. In such a convolut it is difficult to run unit tests or any optimizations.
Why do you need for loops, when only asking one value.
Use StringBuilder instead of String concatenation.
Use either try-with or put your close statements in a finally clause.
Don't initialize variables you don't need (r).
Use for each statements.
Query the database, not the result set.
Tune your database.
If you are only searching for a state, filter only those, so build an object and compare the state instead of a string contains.
Compare the state before storing strings in the list.
Tune your list because it constantly grows with 1Mio records.
Use a hashset instead of an arraylist.
Develop against interfaces.
A better program might look like following:
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Scanner;
import java.util.Set;
public class ShowEmployeeDB {
private static final String DRIVERNAME = "sun.jdbc.odbc.JdbcOdbcDriver";
private static final String CONNECTIONURL = "jdbc:odbc:CitizensDB";
private static final String SELECT_CITIZENS = "SELECT * FROM Citizens";
static {
try {
DriverManager.registerDriver((Driver) Class.forName(DRIVERNAME).newInstance());
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException | SQLException e) {
e.printStackTrace();
}
}
public static void main(final String args[]) {
System.out.print("Enter A Search State: ");
searchRecords();
}
private static void searchRecords() {
try(Scanner scan = new Scanner(System.in);) {
final String state = scan.nextLine();
final long starttime = System.currentTimeMillis();
final Set<Record> records = searchRecordsByState(state);
System.out.println(System.currentTimeMillis() - starttime + "ms");
for(final Record r : records) {
System.out.println(r);
}
} catch (Exception e) {
e.printStackTrace();
}
}
private static Set<Record> searchRecordsByState(final String stateToFilter) {
final Set<Record> records = new HashSet<>();
try(Connection con = DriverManager.getConnection(CONNECTIONURL);
PreparedStatement stmt = con.prepareStatement(SELECT_CITIZENS);
ResultSet rs = stmt.executeQuery(); ) {
while(rs.next()) {
final String state = rs.getString(3);
if(state.equalsIgnoreCase(stateToFilter)) {
final Record r = new Record(rs.getString(1), rs.getString(2), state, rs.getString(4), rs.getString(5));
records.add(r);
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
return records;
}
}
class Record {
String id, age, state, gender, status;
public Record(String id, String age, String state, String gender, String status) {
this.id = id;
this.age = age;
this.state = state;
this.gender = gender;
this.status = status;
}
public String getState() {
return state;
}
#Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(id).append(' ')
.append(age).append(' ')
.append(state).append(' ')
.append(gender).append(' ')
.append(status);
return sb.toString();
}
}
This is untested, because I don't have a database with a million entries by hand.
But the best would be to query the database and catch only those entries you need. So use the WHERE-clause in your statement.
I am new to Java. I have created a generic list of Products type ,how can I add it into the database. The list contains objects of Products, and columns in the database are fields of Products class. Even if I separate the list by listvariable.get(0) and so on, I get the object, not the values inside that objects.
UPDATE : Inserted using for loop and getting fields for each objects. Is there any better way
import java.util.*;
public class Products {
public static List <Products> li = new ArrayList<Products> ();
static
{
Products o = new Products (1,"Milky Way",12.0,7); // Static because
Products o1 = new Products (2,"Dairy Milk",50.0,17); // these entries
Products o2 = new Products (3,"Borunville",70.0,27); // are mandatory
Products o3 = new Products (4,"Lindt",1022.0,107);
li.add(o);
li.add(o1);
li.add(o2);
li.add(o3);
}
int ItemCode;
String ItemName;
double UnitPrice;
int AvailableCount;
public int v=3;
Products()
{}
Products (int x,String y, double c, int d)
{
ItemCode=x;
ItemName=y;
UnitPrice=c;
AvailableCount=d;
}
public String toString ()
{
return ( ItemName+" "+ ItemCode +" "+ UnitPrice + " "+ AvailableCount);
}
void addProduct()
{
li.add(this);
}
public List <Products> initProducts()
{ return li;
}
}
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Iterator;
public class Shopowner {
public static void main (String ...args)
{
Products o = new Products(6,"Eclairs",12.33,5);
o.addProduct();
System.out.println(new Products().initProducts());
try
{
Connectivity.establishConnection(); // static method for db url and drivers
for (int i =0;i<4;i++)
{
Products x=Products.li.get(i);
String name=x.ItemName;
int id= x.ItemCode;
int count =x.AvailableCount;
double price = x.UnitPrice;
PreparedStatement stm = Connectivity.con.prepareStatement("insert into Products_tbl values (?,?,?,?)");
stm.setInt(1, id);
stm.setString(2, name);
stm.setDouble(3,price);
stm.setInt(4, count);
stm.executeUpdate();
}
}
catch(Exception e)
{e.printStackTrace();}
//System.out.println(new Products().li);
}
}
Use batch insert in this manner:
try {
connection con.setAutoCommit(false);
PreparedStatement prepStmt = con.prepareStatement(
"insert into product(code,name,price,available) values (?,?,?,?");
Iterator<Product> it = li.iterator();
while(it.hasNext()){
Product p = it.next();
prepStmt.setString(1,p.getCode());
prepStmt.setString(2,p.getCode());
prepStmt.setInt(3,p.getPrice());
prepStmt.setBoolean(4,p.isAvailable());
prepStmt.addBatch();
}
int [] numUpdates=prepStmt.executeBatch();
for (int i=0; i < numUpdates.length; i++) {
if (numUpdates[i] == -2)
System.out.println("Execution " + i +
": unknown number of rows updated");
else
System.out.println("Execution " + i +
"successful: " + numUpdates[i] + " rows updated");
}
con.commit();
} catch(BatchUpdateException b) {
// process BatchUpdateException
}