Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 4 years ago.
Improve this question
I am pretty new to Object Programming and Java, so I am here to gather your advice and feedback. Basically I am trying to write a background service which performs different tasks at different intervals. I'm just not 100% sure of what I am doing is following the coding standards or is efficient.
Main / Start Class:
public class Start {
public static void main(String[] args) {
Service s = new Service();
s.Start();
}
}
Database Class:
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
public class Database {
/* Database settings */
private final String HOSTNAME = "localhost";
private final String DATABASE = "java_database";
private final String USERNAME = "java_username";
private final String PASSWORD = "java_password";
/* Database connection */
public Connection getConnection() {
try {
return DriverManager.getConnection("jdbc:mysql://" + HOSTNAME + "/" + DATABASE + "?user=" + USERNAME + "&password=" + PASSWORD + "&useSSL=false&useUnicode=true&characterSetResults=utf8");
} catch (SQLException ex) {
ex.printStackTrace();
}
return null;
}
}
Service Class:
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Service {
private int taskId;
private int taskType;
/* Start Service */
public void Start() {
try {
System.out.println("Starting Service...");
while(true) {
System.out.print("Checking for tasks... ");
getNextTask();
if (this.taskId > 0) {
System.out.println("Task ID " + this.taskId + " found.");
switch (this.taskType) {
case 1:
System.out.println("Task 1");
SampleTask s = new SampleTask();
s.Start();
s = null;
break;
default:
System.out.println("Error: Unknown Task");
}
setUsedTask();
} else {
System.out.println("No tasks to perform at this time.");
}
this.taskId = 0;
this.taskType = 0;
Thread.sleep(5000);
}
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
/* Gets the next available task from the database */
public void getNextTask() {
try {
Database db = new Database();
String query = "select taskId, taskType "
+ "from tasks "
+ "where (time_to_sec(timediff(now(), taskLastRun)) > taskFrequency or taskLastRun is null) and taskEnabled = 1 "
+ "limit 1";
Statement stmt = db.getConnection().createStatement();
ResultSet rset = stmt.executeQuery(query);
if (rset.next()) {
this.taskId = rset.getInt(1);
this.taskType = rset.getInt(2);
}
} catch (SQLException ex) {
ex.printStackTrace();
}
}
/* Set task as complete */
public void setUsedTask() {
try {
Database db = new Database();
String query = "update tasks "
+ "set taskLastRun = now() "
+ "where taskId = ? "
+ "limit 1";
PreparedStatement pstmt = db.getConnection().prepareStatement(query);
pstmt.setInt(1, this.taskId);
pstmt.executeUpdate();
} catch (SQLException ex) {
ex.printStackTrace();
}
}
}
Consider replacing your Thread.sleep() approach with a wait() and notify() approach as discussed here.
public class Service {
private int taskId;
private int taskType;
private final Object serviceMonitor;
/* Start Service */
public void Start() {
synchronized(serviceMonitor){
try {
System.out.println("Starting Service...");
while(true) {
System.out.print("Checking for tasks... ");
getNextTask();
if (this.taskId > 0) {
System.out.println("Task ID " + this.taskId + " found.");
switch (this.taskType) {
case 1:
System.out.println("Task 1");
SampleTask s = new SampleTask();
s.Start();
s = null;
break;
default:
System.out.println("Error: Unknown Task");
}
setUsedTask();
} else {
System.out.println("No tasks to perform at this time.");
}
this.taskId = 0;
this.taskType = 0;
serviceMonitor.wait();
}
}
}
catch (InterruptedException ex) {
ex.printStackTrace();
}
}
public void getNextTask() {
synchronized(serviceMonitor){
try {
Database db = new Database();
String query = "select taskId, taskType "
+ "from tasks "
+ "where (time_to_sec(timediff(now(), taskLastRun)) > taskFrequency or taskLastRun is null) and taskEnabled = 1 "
+ "limit 1";
Statement stmt = db.getConnection().createStatement();
ResultSet rset = stmt.executeQuery(query);
if (rset.next()) {
this.taskId = rset.getInt(1);
this.taskType = rset.getInt(2);
serviceMonitor.notifyAll();
}
}
} catch (SQLException ex) {
ex.printStackTrace();
}
}
public class Main {
public static void main(String[] args) {
Service service = new Service(Arrays.asList(new SampleTask(),new AnotherTask()));
service.execute();
}
}
class Service {
private List<Task> taskList;
public Service(List<Task> taskList) {
this.taskList = taskList;
}
public void addTask(Task task) {
taskList.add(task);
}
public void execute() {
for (Task task : taskList) {
new Timer().schedule(task, task.getDelay(), task.getPeriod());
}
}
public void clearTasks() {
taskList.clear();
}
}
abstract class Task extends TimerTask {
abstract long getDelay();
abstract long getPeriod();
}
class SampleTask extends Task {
public void run() {
System.out.println("Sample task executed");
}
long getDelay() {
return 1000;
}
long getPeriod() {
return 60000;
}
}
class AnotherTask extends Task {
public void run() {
System.out.println("Another task is executed");
}
long getDelay() {
return 1000;
}
long getPeriod() {
return 500000;
}
}
Related
Im trying to make a simulation program in which:
A can is passed on to a conveyor
the can is filled
the can is checked
if the can is damaged, it will be removed from the conveyor
errors:
Exception in thread "RealtimeThread[pri=20,aperiodic]"
java.lang.NullPointerException
at assignment$Sensor.run(assignment.java:99)
at javax.realtime.RealtimeThread$Logic.run(RealtimeThread.java:244)
at javax.realtime.MemoryArea.enter(MemoryArea.java)
at javax.realtime.MemoryArea.enterInternal(MemoryArea.java:1472)
at javax.realtime.RealtimeThread$Logic.run(RealtimeThread.java:230)
I want to use the AsyncEventHandler to trigger a code if the can is damaged but it gives a NullPointerException error if the can is found damaged on the first try however the code works fine if the can is not damaged on the first loop.
import java.util.Random;
import javax.realtime.*;
public class realtime{
AsyncEvent damage;
public static void main(String[] args) {
realtime a = new realtime();
}
public realtime() {
Can can = new Can(1);
Conveyer conveyer = new Conveyer(can);
Sensor Sensor = new Sensor(can);
Grabber grabber = new Grabber(can, Sensor);
ReleaseParameters relConveyer = new PeriodicParameters(new RelativeTime(1000,0));
ReleaseParameters relSensor = new PeriodicParameters(new RelativeTime(1000,0));
conveyer.setReleaseParameters(relConveyer);
Sensor.setReleaseParameters(relSensor);
conveyer.start();
Sensor.start();
damage = new AsyncEvent();
damage.setHandler(grabber);
}
class Can {
int id;
boolean filled;
boolean damaged;
public Can(int id) {
this.id = id;
}
public void isFilled(boolean status) {
this.filled = status; //Print if the Can is filled
}
public void isDamaged(boolean status) {
this.damaged = status;
}
}
class Conveyer extends RealtimeThread {
Can can;
Random random = new Random();
public Conveyer(Can can) {
this.can = can;
}
#Override
public void run() { //While loop can be used to repeat
while(true) {
System.out.println("Can " + can.id + " has entered the conveyer");
System.out.println("Can " + can.id + " is being filled");
can.isFilled(true); //Sleep to give time to fill
System.out.println("Can " + can.id + " is filled");
System.out.println("Can " + can.id + " is being scanned");
can.isDamaged(random.nextBoolean());
try {
waitForNextRelease();
}
catch (Exception e) {}
}
}
}
class Sensor extends RealtimeThread{
Can can;
public Sensor(Can can) {
this.can = can;
}
#Override
public void run() { //While loop can be used to repeat
while(true) {
if(can.damaged) {
System.out.println("Can " + can.id + " is Damaged!");
damage.fire();
} else {
System.out.println("Can " + can.id + " is moved to Stage 2");
}
try {
waitForNextRelease();
}
catch (Exception e) {}
}
}
}
class Grabber extends AsyncEventHandler {
Can can;
RealtimeThread rtt;
boolean damaged = false;
public Grabber(Can can, RealtimeThread rtt) {
this.can = can;
this.rtt = rtt;
}
public void handleAsyncEvent() {
System.out.println("Can " + can.id + " is disposed");
}
}
}
You do not have a default constructor for your Can class. So in your Conveyor class you have Can can; This probably gives you the NullPointerException. Just add another constructor to your Can class like so:
public Can() {};
I searched now for a while with google but i could not get it to work. My approach is to have a Multithread software which collects data on one Thread and then execute the batch query for Mysql in a second thread while the first thread is going on collecting data. Goal should it be to use as less RAM as possible for millions of inserts. In My data collection Process i use one Object of MYSQLInsertThread and i insert Data like this:
String[] types = {"Long", "Long"};
int[] pos = {1, 1};
Object[] values = {123, 456};
nodeTagsInsertThread.addBatch(types, pos, values);
The first call works, if I set the batchCount to 100 i got 104 entrys in my Database but that are the only entrys which my class produces (it should import 5 million entrys!
public class MYSQLInsertThread implements Runnable
{
private Thread t;
private String threadName = "MYSQL InsertThread";
private String query;
private PreparedStatement pstmt;
private PreparedStatement pstmt2;
private long batchCount;
private long maxBatchAmount;
private Boolean pstmt1Active = true;
private Boolean isRunning = false;
public MYSQLInsertThread(String name, String query, Connection conn, int maxBatchAmount) throws SQLException
{
threadName = name;
this.pstmt = conn.prepareStatement(query);
this.pstmt2 = conn.prepareStatement(query);
this.query = query;
this.maxBatchCount = maxBatchAmount;
System.out.println("Creating Thread: " + name);
}
public synchronized void addBatch(String[] types, int[] positions, Object[] values) throws SQLException
{
PreparedStatement _pstmt;
if(pstmt1Active)
{
_pstmt = pstmt;
}
else
{
_pstmt = pstmt2;
}
if(_pstmt != null)
{
for(int i=0;i<types.length; i++)
{
switch(types[i])
{
case "string":
case "String":
_pstmt.setString(positions[i], (String)values[i]);
break;
case "long":
case "Long":
_pstmt.setLong(positions[i], (long)values[i]);
break;
case "int":
case "Integer":
_pstmt.setInt(positions[i], (int)values[i]);
break;
case "double":
case "Double":
_pstmt.setDouble(positions[i], (double)values[i]);
break;
default:
break;
}
}
_pstmt.addBatch();
batchCount++;
if(batchCount % maxBatchCount == 0)
{
System.out.println("["+ this.threadName +"]Adding " + batchCount + " Entrys to DB" );
this.executeBatch();
}
}
else
{
System.err.println("[MYSQLInsertThread]Error PreparedStatment is NULL, Parameter could not be added");
}
}
public synchronized void executeBatch()
{
PreparedStatement _pstmt;
if(pstmt1Active)
{
_pstmt = pstmt;
}
else
{
_pstmt = pstmt2;
}
if(_pstmt != null)
{
if(isRunning)System.out.println("Waiting for previous Batch Execution to finish");
while(isRunning)
{
System.out.print(".");
try {
Thread.sleep(100);
} catch (InterruptedException e)
{
e.printStackTrace();
}
}
this.start();
System.out.println("Execution Started Successfully");
}
else
{
System.err.println("[" + this.threadName + "]PSTMT is NULL");
}
}
#Override
public void run()
{
PreparedStatement _pstmt;
if(pstmt1Active)
{
_pstmt = pstmt;
}
else
{
_pstmt = pstmt2;
}
if(_pstmt != null)
{
isRunning = true;
pstmt1Active = !pstmt1Active;
try
{
_pstmt.executeBatch();
}
catch(Exception e)
{
e.printStackTrace();
}
isRunning = false;
}
}
public void start()
{
if(t == null)
{
t = new Thread(this, threadName);
t.start();
}
}
}
I found a solution now by myself searching a bit through youtube videos :D
I made a new Class which hold my PreparedStatements. If one Statement is executing a flag is preivously set. If the second Statemet wants to execute the Batch now but the previous Statement is still running it makes the collecting thread waiting for a notify of the writing process. The writing process notifys after finishing the execution of the statements!
Here is my Code:
public class ThreadPreparedStatement
{
private String query;
private String threadName;
private PreparedStatement pstmt1;
private PreparedStatement pstmt2;
private boolean pstmt1Active;
private boolean isRunning;
private long maxBatchAmount;
private long batchCount;
public ThreadPreparedStatement(String threadName, String query, Connection conn, long maxBatchAmount) throws SQLException
{
this.threadName = threadName;
this.pstmt1 = conn.prepareStatement(query);
this.pstmt2 = conn.prepareStatement(query);
this.maxBatchAmount = maxBatchAmount;
this.query = query;
this.batchCount = 0;
}
public synchronized void addParameters(String[] types, int[] positions, Object[] values) throws SQLException
{
PreparedStatement _pstmt;
if(pstmt1Active)
{
_pstmt = pstmt1;
}
else
{
_pstmt = pstmt2;
}
if(_pstmt != null)
{
for(int i=0;i<types.length; i++)
{
switch(types[i])
{
case "string":
case "String":
_pstmt.setString(positions[i], (String)values[i]);
break;
case "long":
case "Long":
_pstmt.setLong(positions[i], (long)values[i]);
break;
case "int":
case "Integer":
_pstmt.setInt(positions[i], (int)values[i]);
break;
case "double":
case "Double":
_pstmt.setDouble(positions[i], (double)values[i]);
break;
default:
break;
}
}
_pstmt.addBatch();
batchCount++;
if(batchCount % maxBatchAmount == 0)
{
Thread t = new Thread(new Runnable()
{
#Override
public void run()
{
try
{
executeBatch(_pstmt);
}catch(Exception e)
{
e.printStackTrace();
}
}
});
t.start();
}
}
else
{
System.err.print("Error Prepared Statements are both NULL");
}
}
public synchronized void executeAllBatches()
{
try
{
if(isRunning)
{
try
{
wait(); //wait for finish execution
}catch(InterruptedException e)
{
e.printStackTrace();
}
}
isRunning = true;
pstmt1.executeBatch();
pstmt2.executeBatch();
isRunning = false;
notify();
}catch(Exception e)
{
}
}
public synchronized void executeBatch(PreparedStatement _pstmt) throws SQLException
{
System.out.println("["+ this.threadName +"]Adding " + batchCount + " Entrys to DB" );
if(isRunning)
{
try
{
wait(); //wait for finish execution
}catch(InterruptedException e)
{
e.printStackTrace();
}
}
pstmt1Active = !pstmt1Active;
isRunning = true;
_pstmt.executeBatch(); //execute Query Batch
isRunning = false;
notify(); //notify that Batch is executed
}
}
I need some input from you regarding a scenario for which I am doing some POC(Proof of Concept).
I am novice to multithreading in java and trying to some test. My requirement is I want to load millions of records using simple
java and then after doing some conversion with the data will insert in a database table.
For that I want to perform a simple test related to finishing of all task.
Currently I want to try that my main method finishes only after my executor services finishes. Below is the code which I have tried.
Can anyone please help me to know if thats the correct way to finish the main method after finishing the executor threads.
Your suggestion will be highly appreciated.
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class SimpleThreadPool {
public static void main(String[] args) {
ExecutorService executor = Executors.newFixedThreadPool(5);
for (int i = 0; i < 10; i++) {
executor = Executors.newFixedThreadPool(5);
// Runnable worker = new WorkerThread("Thread executor :" + i);
executor.execute(new WorkerThread("Thread executor :" + i));
}
executor.shutdown();
while (!executor.isTerminated()) {
//System.out.println("Waiting");
}
System.out.println("Will start for Executor 1");
ExecutorService executor1 = Executors.newFixedThreadPool(5);
for (int i = 0; i < 10; i++) {
// Runnable worker = new WorkerThread("Thread executor1 :" + i);
executor1.execute(new WorkerThread("Thread executor1 :" + i));
}
executor1.shutdown();
while (!executor1.isTerminated()) {
//System.out.println("Waiting");
}
System.out.println("Finished all threads");
//
String s=null;
s.toUpperCase();
}
}
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
public class WorkerThread implements Runnable {
private String command;
public WorkerThread(String s){
this.command=s;
}
public void run() {
ExecutorService executor2 = Executors.newFixedThreadPool(5);
Future loadData=executor2.submit(new LoadData());
System.out.println(" Start. Command = "+command);
try {
List listOfData=(List) loadData.get();
for(int i=0;i<listOfData.size();i++){
//Thread.sleep(500);
//System.out.println("Printing the value from list:"+listOfData.get(i));
ConversionLogic conversion= new ConversionLogic();
conversion.doConversion(command);
}
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(" End."+command);
}
public String toString(){
return this.command;
}
}
class LoadData implements Callable{
public List call() throws Exception {
List<String> dataList= new ArrayList<String>();
for(int i=0;i<100;i++){
String data="Data_"+i;
//System.out.println("Data Added in List:"+data);
dataList.add(data);
}
Thread.sleep(10000);
return dataList;
}
}
public class ConversionLogic {
public void doConversion(String threadName){
try {
System.out.println("Converting Data for Thread starts:"+threadName);
Thread.sleep(5000);
System.out.println("Converting Data for Thread ends:"+threadName);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
SO this is what I have understood from the answers provided below:
package stackoverflow.test;
import java.util.List;
import java.util.concurrent.*;
class SimpleThreadPool {
public static void main(String[] args) throws InterruptedException, ExecutionException {
ExecutorService executor = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor = new ExecutorCompletionService<List>(executor);
ExecutorService executor2 = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor2 = new ExecutorCompletionService<List>(executor2);
//start loading data
int procCount = 0;
for (int i = 0; i < 10; i++) {
Future loadData = processor.submit(new LoadData("Org"));
procCount++;
}
//now all loading tasks have been submitted and are being executed
System.out.println("All LOADING tasks have been submitted and are being executed");
//new work queue using the same executor (or another one if you want more parallelism)
ExecutorCompletionService<Void> converter = new ExecutorCompletionService<Void>(executor);
while (procCount-- > 0) {
Future<List> listOfDataFuture = processor.take(); //blocks until a 'LoadData' finishes
System.out.println("A loading task just completed");
List listOfData = listOfDataFuture.get();
for (int i = 0; i < listOfData.size(); i++) {
ConversionLogic conversion = new ConversionLogic(procCount + "_" + i);
converter.submit(conversion);
}
}
System.out.println("All LOADING tasks have been COMPLETED for Org");
//now all conversion tasks have been submitted and are being executed
System.out.println("All CONVERSION task have been submitted and are being executed for Org:");
/* You don't need to wait for conversion tasks to complete:
* they will be completed nonetheless. Wait for them (with take())
* if you need the results.
* */
executor.shutdown();
try {
System.out.println("Waiting for finish");
executor.awaitTermination(1000, TimeUnit.SECONDS);
System.out.println("Stopped nicely");
} catch (InterruptedException e) {
System.out.println("Could not stop in alloted time");
}
System.out.println("Fund Data Loading Starts:");
//___________________________________________________________________//
// Some argument to get Fund Data
int procCount1 = 0;
for (int i = 0; i < 5; i++) {
Future loadData = processor2.submit(new LoadData("Fund"));
procCount1++;
}
//now all loading tasks have been submitted and are being executed
System.out.println("All LOADING tasks have been submitted and are being executed for Fund:");
//new work queue using the same executor (or another one if you want more parallelism)
ExecutorCompletionService<Void> converter1 = new ExecutorCompletionService<Void>(executor2);
while (procCount1-- > 0) {
Future<List> listOfDataFuture = processor2.take(); //blocks until a 'LoadData' finishes
System.out.println("A loading task just completed for Fund:");
List listOfData = listOfDataFuture.get();
for (int i = 0; i < listOfData.size(); i++) {
ConversionLogic conversion = new ConversionLogic(procCount + "_" + i);
converter1.submit(conversion);
}
}
System.out.println("All LOADING tasks have been COMPLETED for Org");
//now all conversion tasks have been submitted and are being executed
System.out.println("All CONVERSION task have been submitted and are being executed for Org:");
/* You don't need to wait for conversion tasks to complete:
* they will be completed nonetheless. Wait for them (with take())
* if you need the results.
* */
executor2.shutdown();
try {
System.out.println("Waiting for finish");
executor.awaitTermination(1000, TimeUnit.SECONDS);
System.out.println("Stopped nicely");
} catch (InterruptedException e) {
System.out.println("Could not stop in alloted time");
}
System.out.println("<<<<<<<<<< End>>>>>>>>");
System.exit(0);
}
}
package stackoverflow.test;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
class LoadData implements Callable {
String dataType;
public List call() throws Exception {
List<String> dataList = new ArrayList<String>();
for (int i = 0; i < 20; i++) {
String data = "Data_" + i;
System.out.println("Processing Data of Type :" + dataType + "Data is:"+data);
dataList.add(data);
}
Thread.sleep(2000);
return dataList;
}
LoadData(String type){
this.dataType=type;
}
}
package stackoverflow.test;
import java.util.concurrent.Callable;
class ConversionLogic implements Callable {
private String threadName;
public ConversionLogic(String threadName) {
this.threadName = threadName;
}
public Void call() throws Exception {
try {
System.out.println("Converting Data for Thread starts:" + threadName);
Thread.sleep(1000);
System.out.println("Converting Data for Thread ends:" + threadName);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}
Updating the code for one set of requirement.Any suggestion to improve the performance is welcome.
package stackoverflow.tesst;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.*;
import connection.java.JdbcConnection;
class SimpleThreadPool {
public static void main(String[] args) throws InterruptedException,
ExecutionException {
ExecutorService executor = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor = new ExecutorCompletionService<List>(
executor);
ExecutorService executor2 = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor2 = new ExecutorCompletionService<List>(
executor2);
System.out.println("Connecting to DB...");
try {
System.out.println("Connection is :"
+ JdbcConnection.getConnection());
} catch (ClassNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
// start loading data
int priceRange1 = 200;
int priceRange2 = priceRange1 + 200;
int procCount = 0;
for (int i = 0; i < 10; i++) {
String query = "select code,name,price from Product where price ";
if (i == 0) {
String finalQuery = query + " <= " + priceRange1;
Future loadData = processor.submit(new LoadData("Org",
finalQuery));
} else {
String finalQuery = query + " <= " + priceRange2
+ " and price > " + priceRange1;
Future loadData = processor.submit(new LoadData("Org",
finalQuery));
}
priceRange1 = priceRange2;
priceRange2 = priceRange2 + 200;
procCount++;
}
System.out.println("All LOADING tasks have been COMPLETED for Org");
ExecutorCompletionService<Void> converter = new ExecutorCompletionService<Void>(
executor);
while (procCount-- > 0) {
Future<List> listOfDataFuture = processor.take();
System.out.println("A loading task just completed");
List listOfData = listOfDataFuture.get();
for (int i = 0; i < listOfData.size(); i++) {
ConversionLogic conversion = new ConversionLogic(procCount
+ "_" + i, listOfData);
converter.submit(conversion);
}
}
System.out
.println("All CONVERSION task have been submitted and are being executed for Org:");
executor.shutdown();
try {
System.out.println("Waiting for finish");
executor.awaitTermination(1000, TimeUnit.SECONDS);
System.out.println("Stopped nicely for Org");
} catch (InterruptedException e) {
System.out.println("Could not stop in alloted time");
}
System.out.println("<<<<<<<<<< End>>>>>>>>");
System.exit(0);
}
}
package stackoverflow.tesst;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
----------------------------------------------
import connection.java.JdbcConnection;
class LoadData implements Callable {
String dataType;
Connection conn;
String query;
public List call() throws Exception {
List<Product> dataList = new ArrayList<Product>();
try {
conn=JdbcConnection.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(this.query);
while(rs.next()){
System.out.println(rs.getString("code"));
System.out.println(rs.getString("name"));
System.out.println(rs.getInt("price"));
Product p= new Product(rs.getString("code"),rs.getString("name"),rs.getInt("price"));
dataList.add(p);
}
rs.close();//conn.close();
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Thread.sleep(2000);
return dataList;
}
LoadData(String type,String query){
this.dataType=type;
this.query=query;
}
}
}
}
---------------------------
package stackoverflow.tesst;
import java.util.List;
import java.util.concurrent.Callable;
class ConversionLogic implements Callable {
private String threadName;
List<Product> productList;
public ConversionLogic(String threadName,List<Product> productList) {
this.threadName = threadName;
this.productList=productList;
}
public Void call() throws Exception {
try {
System.out.println("Converting Data for Thread starts:" + threadName);
Thread.sleep(1000);
int listSize=productList.size();
for(int i=0;i<listSize;i++){
//Do conversion for product let say
System.out.println("Print product in Conversion:"+productList.get(i).getPrice());
}
System.out.println("Converting Data for Thread ends:" + threadName);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}
------------------------------------
package connection.java;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class JdbcConnection {
static Connection conn;
static String user;
static String pass;
static String dbURL;
public static Connection getConnection() throws ClassNotFoundException,
SQLException {
Class.forName("org.postgresql.Driver");
dbURL = "jdbc:postgresql://localhost:5433:postgres";
user = "postgres";
pass = "root";
Connection conn = DriverManager.getConnection(dbURL, user, pass);
Statement stmt = conn.createStatement();
System.out.println("Created DB Connection....");
return conn;
}
}
package stackoverflow.tesst;
import java.io.Serializable;
import java.math.BigDecimal;
public class Product implements Serializable {
/**
* Product Class using Annotation
*/
private static final long serialVersionUID = 1L;
private Integer id;
private String code;
private String name;
private int price;
Product(String code,String name,int price){
this.code=code;
this.name=name;
this.price=price;
}
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getPrice() {
return price;
}
public void setPrice(int price) {
this.price = price;
}
#Override
public String toString() {
return "Product [id=" + id + ", code=" + code + ", name="
+ name + ", price=" + price + "]";
}
}
Here is the final version of my POC, which I am actually looking for. Only suggestion required is do I suppose to synchronize the insert query part?
package stackoverflow.tesst;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.*;
import connection.java.JdbcConnection;
class SimpleThreadPool {
public static void main(String[] args) throws InterruptedException,
ExecutionException {
ExecutorService executor = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor = new ExecutorCompletionService<List>(
executor);
ExecutorService executor2 = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor2 = new ExecutorCompletionService<List>(
executor2);
/*System.out.println("Connecting to DB...");
try {
System.out.println("Connection is :"
+ JdbcConnection.getConnection());
} catch (ClassNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}*/
// start loading data
int priceRange1 = 200;
int priceRange2 = priceRange1 + 200;
int procCount = 0;
for (int i = 0; i < 10; i++) {
String query = "select code,name,price from Product where price ";
if (i == 0) {
String finalQuery = query + " <= " + priceRange1 + " order by price";
Future loadData = processor.submit(new LoadData("Org",
finalQuery));
} else {
String finalQuery = query + " <= " + priceRange2
+ " and price > " + priceRange1 + " order by price";
Future loadData = processor.submit(new LoadData("Org",
finalQuery));
}
priceRange1 = priceRange2;
priceRange2 = priceRange2 + 200;
procCount++;
}
System.out.println("All LOADING tasks have been COMPLETED for Org");
ExecutorCompletionService<Void> converter = new ExecutorCompletionService<Void>(
executor);
while (procCount-- > 0) {
Future<List> listOfDataFuture = processor.take();
System.out.println("A loading task just completed");
List listOfData = listOfDataFuture.get();
for (int i = 0; i < listOfData.size(); i++) {
ConversionLogic conversion = new ConversionLogic("<<Org>>"
+ "_" + i, listOfData);
converter.submit(conversion);
}
}
System.out
.println("All CONVERSION task have been submitted and are being executed for Org:");
executor.shutdown();
try {
System.out.println("Waiting for finish");
executor.awaitTermination(1000, TimeUnit.SECONDS);
System.out.println("Stopped nicely for Org");
} catch (InterruptedException e) {
System.out.println("Could not stop in alloted time");
}
System.out.println("Fund Data Loading Starts:");
// ___________________________________________________________________//
int fundRange1 = 200;
int fundRange2 = fundRange1 + 200;
int procCount1 = 0;
for (int i = 0; i < 10; i++) {
String query = "select code,name,price from Product where price ";
if (i == 0) {
String finalQuery = query + " <= " + fundRange1;
Future loadData = processor2.submit(new LoadData("Fund",
finalQuery));
} else {
String finalQuery = query + " <= " + fundRange2
+ " and price > " + fundRange1;
Future loadData = processor2.submit(new LoadData("Fund",
finalQuery));
}
fundRange1 = fundRange2;
fundRange2 = fundRange2 + 200;
procCount1++;
}
System.out.println("All LOADING tasks have been COMPLETED for Fund");
ExecutorCompletionService<Void> converter1 = new ExecutorCompletionService<Void>(
executor2);
while (procCount1-- > 0) {
Future<List> listOfDataFuture = processor2.take();
System.out.println("A loading task just completed");
List listOfData = listOfDataFuture.get();
for (int i = 0; i < listOfData.size(); i++) {
ConversionLogic conversion = new ConversionLogic("<<Fund>>"
+ "_" + i, listOfData);
converter1.submit(conversion);
}
}
System.out
.println("All CONVERSION task have been submitted and are being executed for Fund:");
executor2.shutdown();
try {
System.out.println("Waiting for finish");
executor.awaitTermination(1000, TimeUnit.SECONDS);
System.out.println("Stopped nicely for Fund");
} catch (InterruptedException e) {
System.out.println("Could not stop in alloted time");
}
System.out.println("<<<<<<<<<< End>>>>>>>>");
System.exit(0);
}
}
package stackoverflow.tesst;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import connection.java.JdbcConnection;
class LoadData implements Callable {
String dataType;
Connection conn;
String query;
public List call() throws Exception {
List<Product> dataList = new ArrayList<Product>();
try {
System.out.println("Connection establishing for Loading Org Data:");
conn = JdbcConnection.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(this.query);
while (rs.next()) {
System.out.println(rs.getString("code"));
System.out.println(rs.getString("name"));
System.out.println(rs.getInt("price"));
Product p = new Product(rs.getString("code"),
rs.getString("name"), rs.getInt("price"));
dataList.add(p);
}
rs.close();
conn.close();
System.out.println("Connection Closed While loading for :"+dataType);
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Thread.sleep(2000);
return dataList;
}
LoadData(String type, String query) {
this.dataType = type;
this.query = query;
}
}
package stackoverflow.tesst;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.concurrent.Callable;
import connection.java.JdbcConnection;
class ConversionLogic implements Callable {
private String threadName;
List<Product> productList;
Connection conn;
public ConversionLogic(String threadName, List<Product> productList) {
this.threadName = threadName;
this.productList = productList;
}
public Void call() throws Exception {
int listSize = productList.size();
try {
conn = JdbcConnection.getConnection();
System.out.println("Connection establishing for Converting Org Data:");
String insertTableSQL = "INSERT INTO item"
+ "(code, name, price) VALUES" + "(?,?,?)";
PreparedStatement preparedStatement = conn
.prepareStatement(insertTableSQL);
for (int i = 0; i < listSize; i++) {
preparedStatement.setString(1, productList.get(i)
.getCode());
preparedStatement.setString(2, productList.get(i)
.getName());
preparedStatement.setInt(3, productList.get(i)
.getPrice());
//Guess we suppose to synchronize the insert part in case
// In case mutiple threads trying to insert some records in a table and we might end up loosing data
//
preparedStatement.executeUpdate();
}
preparedStatement.close();
conn.close();
System.out.println("Connection Closed While Converting for Org :");
} catch (ClassNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
return null;
}
}
Instead of that while loop I would advise you to use the built-in await function like this:
executor.shutdown();
try {
System.out.println("Waiting for finish");
executor.awaitTermination(1000, TimeUnit.SECONDS);
System.out.println("Stopped nicely");
} catch (InterruptedException e) {
System.out.println("Could not stop in alloted time");
}
System.exit(0);
Place this code wherever you want to make sure that your executorService finished before you continue any further.
There are way to many thread pools in your code and, in general, spawning new threads inside a spawned thread is not a good idea: it can easily get out of control. In your case, you don't need the WorkerThread: you already have the thread pool provided by the Java framework (ExecutorService).
Since you need the result from a thread (LoadData) to be processed (ConversionLogic) I would use also an ExecutorCompletionService to help with gathering results from LoadData.
Following the refactored code. I've ditched the WorkerThread and used only a threadpool (altough you can use two if you want more parallelism), also the ConversionLogic now implements Callable so that it can be easily processed by the thread pool.
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
class SimpleThreadPool {
public static void main(String[] args) throws InterruptedException, ExecutionException {
ExecutorService executor = Executors.newFixedThreadPool(10);
ExecutorCompletionService<List> processor = new ExecutorCompletionService<List>(executor);
//start loading data
int procCount = 0;
for (int i = 0; i < 10; i++) {
Future loadData = processor.submit(new LoadData());
procCount++;
}
//now all loading tasks have been submitted and are being executed
System.out.println("All LOADING tasks have been submitted and are being executed");
//new work queue using the same executor (or another one if you want more parallelism)
ExecutorCompletionService<Void> converter = new ExecutorCompletionService<Void>(executor);
while (procCount-- > 0) {
Future<List> listOfDataFuture = processor.take(); //blocks until a 'LoadData' finishes
System.out.println("A loading task just completed");
List listOfData = listOfDataFuture.get();
for (int i = 0; i < listOfData.size(); i++) {
ConversionLogic conversion = new ConversionLogic(procCount + "_" + i);
converter.submit(conversion);
}
}
System.out.println("All LOADING tasks have been COMPLETED");
//now all conversion tasks have been submitted and are being executed
System.out.println("All CONVERSION task have been submitted and are being executed");
/* You don't need to wait for conversion tasks to complete:
* they will be completed nonetheless. Wait for them (with take())
* if you need the results.
* */
executor.shutdown();
System.out.println(" End.");
}
}
class LoadData implements Callable {
public List call() throws Exception {
List<String> dataList = new ArrayList<String>();
for (int i = 0; i < 20; i++) {
String data = "Data_" + i;
System.out.println("Data Added in List:" + data);
dataList.add(data);
}
Thread.sleep(2000);
return dataList;
}
}
class ConversionLogic implements Callable {
private String threadName;
public ConversionLogic(String threadName) {
this.threadName = threadName;
}
#Override
public Void call() throws Exception {
try {
System.out.println("Converting Data for Thread starts:" + threadName);
Thread.sleep(1000);
System.out.println("Converting Data for Thread ends:" + threadName);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}
executor.awaitTermination() method sounds convenient, but this is not an efficient one because what if all your executor services finishes before/after 1000 seconds (or any time you mention as an argument in awaitTermination() method).Either you are wasting the time or not letting your services finish their task.
Instead i'd suggest you use CountDownLatch as explained in below example.
public class app {
public static void main(String[] args) throws InterruptedException {
CountDownLatch latch = new CountDownLatch(3); // here assuming you know number of threads
ExecutorService executor = Executors.newFixedThreadPool(3);
for (int i = 0; i < 3; i++) {
executor.submit(new processor(latch));
}
latch.await();
System.out.println("main resume");
}
}
class processor extends Thread{
private CountDownLatch latch;
processor(CountDownLatch latch){
this.latch=latch;
}
public void run(){
System.out.println("in thread");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
latch.countDown();
}
}
Thus, you don't waste single ms of time.
I'm trying to setup a producers/consumers using the selenium webdriver as queries/browsers.
While running the following code, not all the queries are actually entered even though the output says they are. Some queries also either get doubled ( i.e. "Fish" becomes "FisFishh" ).
APP
public class App {
public static void main(String[] args) {
DriverHashMap.init();
Executor exec = new Executor();
// Add queries to list
Query query = new Query(exec);
query.addQuery("Cats");
query.addQuery("Dogs");
query.addQuery("Fish");
// query.addQuery("Bats");
// query.addQuery("Rats");
// query.addQuery("Geese");
ExecutorService threadPool = Executors.newFixedThreadPool(4);
// Submit queries to pool
Future queryStatus = threadPool.submit(query);
// Start browsers
threadPool.execute(new Browser("1", exec));
threadPool.execute(new Browser("2", exec));
// this will wait for the producer to finish its execution.
try {
queryStatus.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// Wait for pool to finish
threadPool.shutdown();
}
}
BROWSER
public class Browser implements Runnable {
private String name;
private Executor exec;
private static WebDriver driver;
private static String baseURL = "http://www.google.com";
private static String query;
public Browser(String name, Executor exec) {
synchronized (this) {
this.name = name;
this.exec = exec;
System.out.println("\tStart Browser-" + this.name);
driver = new FirefoxDriver();
// Wait up to 30 seconds for a response
driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
DriverHashMap.addDriver(name, driver);
System.out.println("\tAdded Browser-" + name + " to hashmap");
}
}
private void enterQuery() {
synchronized (this) {
// Get driver for this browser
driver = DriverHashMap.getDriver(this.name);
System.out.println("Browser " + this.name
+ " entering query from executor: " + query);
// Search field qbqfq
driver.findElement(By.id("gbqfq")).clear();
// Enter query
driver.findElement(By.id("gbqfq")).sendKeys(query);
// Click search button
driver.findElement(By.id("gbqfb")).click();
}
}
public void run() {
try {
synchronized (this) {
// Receive first query
driver = DriverHashMap.getDriver(this.name);
query = exec.get();
System.out.println("Browser " + this.name
+ "\n\rReceived first query from executor: " + query);
}
do {
synchronized (this) {
// Process query
driver = DriverHashMap.getDriver(this.name);
driver.get(baseURL);
enterQuery();
}
synchronized (this) {
// Receive next query
query = exec.get();
driver = DriverHashMap.getDriver(this.name);
System.out.println("Browser " + this.name
+ "\n\rReceived new query from executor: " + query);
}
} while (query != null);
// Close this browser
synchronized (this) {
driver = DriverHashMap.getDriver(this.name);
System.out.println("Browser " + this.name
+ " finished its job; terminating.");
driver.close();
}
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
}
QUERY
public class Query implements Runnable {
private Executor exec;
private List<String> queries = new ArrayList<String>();
public Query(Executor exec) {
this.exec = exec;
}
public void addQuery( String query ) {
System.out.println("Adding " + query + " to queries");
queries.add( query );
}
public void run() {
Iterator it = queries.iterator();
String currQuery = new String();
try {
while( it.hasNext()) {
currQuery = (String)it.next();
System.out.println("Adding " + currQuery + " to Executor");
exec.put(currQuery);
}
this.exec.continueProducing = Boolean.FALSE;
System.out.println("Query has finished its job; terminating.");
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
}
EXECUTOR
public class Executor {
public ArrayBlockingQueue<String> queue = new ArrayBlockingQueue<String>(
100);
public Boolean continueProducing = Boolean.TRUE;
public void put(String query) throws InterruptedException {
this.queue.put(query);
}
public String get() throws InterruptedException {
return this.queue.poll(1, TimeUnit.SECONDS);
}
}
Webdriver is not threadsafe per se. Try wrapping it into a ThreadLocal.
That has helped me a lot in a similar situation.
I am having a run method which tries to override another run method. But its not happening because I am getting a "Class not found Exception" before it passed on to run method.
Here´s my class with run method
public class PollingSynchronizer implements Runnable{
public Collection<KamMessage> incomingQueue,outgoingQueue,fetchedMessages;
private Connection dbConnection;
/**
* Constructor. Requires to provide a reference to the Kam message queue
*
* #param incomingMessages reference to message queue
* #param dbConnection
*
*/
public PollingSynchronizer(Collection<KpiMessage> incomingQueue, Connection dbConnection) {
super();
this.incomingQueue = incomingQueue;
this.dbConnection = dbConnection;
}
private int seqId;
public int getSeqId() {
return seqId;
}
public void setSeqId(int seqId) {
this.seqId = seqId;
}
#Override
/**
* The method which runs Polling action and record the time at which it is done
*
*/
public void run() {
int seqId = 0;
while(true) {
List<KamMessage> list = null;
try {
list = fullPoll(seqId);
if (!list.isEmpty()) {
seqId = list.get(0).getSequence();
incomingQueue.addAll(list);
this.outgoingQueue = incomingQueue;
System.out.println("waiting 3 seconds");
System.out.println("new incoming message");
Thread.sleep(3000);
//when I debug my execution stops here and throws exception
MessageProcessor processor = new MessageProcessor() {
#Override
public void run() {
new MessageProcessor().generate(outgoingQueue);
}
};
}
} catch (Exception e1) {
e1.printStackTrace();
}
}
}
This is the method which I have to call in order to execute.
public abstract class MessageProcessor implements Runnable {
private Collection<KpiMessage> fetchedMessages;
private Connection dbConnection;
Statement st = null;
ResultSet rs = null;
PreparedStatement pstmt = null;
private Collection<KpiMessage> outgoingQueue;
public KpiMsg804 MessageProcessor(Collection<KpiMessage> outgoingQueue, Connection
dbConnection){
this.outgoingQueue = outgoingQueue;
this.dbConnection = dbConnection;
return (KpiMsg804) fetchedMessages;
}
public Collection<KamMessage> generate(Collection<KamMessage> outgoingQueue)
{
while(true){
try {
while (rs.next()) {
KamMessage filedClass = convertRecordsetToPojo(rs);
outgoingQueue.add(filedClass);
}
for (KamMessage pojoClass : outgoingQueue) {
KamMsg804 updatedValue = createKamMsg804(pojoClass);
System.out.print(" " + pojoClass.getSequence());
System.out.print(" " + pojoClass.getTableName());
System.out.print(" " + pojoClass.getAction());
System.out.print(" " + updatedValue.getKeyInfo1());
System.out.print(" " + updatedValue.getKeyInfo2());
System.out.println(" " + pojoClass.getEntryTime());
}
return outgoingQueue;
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
How can I implement this?
Since I am new here, please give a reason for thumbs down. So that I can explain my question.
Try this:
MessageProcessor processor = new MessageProcessor() {
#Override
public void run() {
**new MessageProcessor()**.generate(outgoingQueue);
}
};
MessageProcessor is an abstract class. object creation inside the run method should have failed at compile time.
processor object is created but unused.. you need to create a thread with the processor instace and start the thread.