Develop Reference

Unexpected query result inside each executorservice thread

I am pulling data in batches (chunkSize=1000) for that I have implemented executor Service. In the below loop I am calculating the firstResult and sending to executorService to fetch data and insert into mongodb
for(int i = 1; i <= numOfBatches; i++){
int firstResult = (i -1) * chunkSize;
explicitAudienceCreationExecutorService.fetchFromMartAndInsertIntoMongo(firstResult,chunkSize,query,promotion,
filterKeywords,audienceFilterName,programId,counttoReturn.get(0).intValue());
}
This is my runnable task which is giving unexpected result while executing query
For ex: when I am executing the code without any loop and directly pass firstResult as 302000 it prints in log
firstResult 302000 queryResultSize 1000
But when I do this in loop I saw this in logs. This happens for several values.
firstResult 302000 queryResultSize 899
package com.loylty.campaign.com.loylty.campaign.service;
import javax.persistence.TypedQuery;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import static com.loylty.campaign.com.loylty.campaign.config.MongoTenantTemplate.tenantTemplates;
#Service
public class ExplicitAudienceCreationExecutorService {
static int classCount = 0;
#Autowired
CampaignExecutorService campaignExecutorService;
#Autowired
MongoTemplate mongoTemplate;
#Autowired
AudienceWithCriteria audienceWithCriteria;
private final Logger logger = LoggerFactory.getLogger(this.getClass());
public void fetchFromMartAndInsertIntoMongo(int fr, int cs, TypedQuery<Object[]> qr, Promotion promotion,
FilterKeywords filterKeywords, String audienceFilterName, String programId, int queryrernCont) {
final int firstResult = fr;
final int chunkSize = cs;
final TypedQuery<Object[]> query = qr;
campaignExecutorService.dotask(new Runnable() {
#Override
public void run() {
mongoTemplate = tenantTemplates.get(programId);
final List<Object[]> toReturn = query.setFirstResult(firstResult).setMaxResults(chunkSize).getResultList();
classCount++;
System.out.println("classCount "+ classCount);
logger.info("firstResult "+ firstResult + " queryResultSize " + toReturn.size() );
if (toReturn != null || toReturn.size() > 0) {
List<TGAudience> tgAudienceList = new ArrayList<>();
for (Object[] tuple : toReturn) {
HashMap<String, Object> queryResponseTuple = new HashMap<>();
int index = 0;
for (RequiredKeys selectProperty : promotion.getRequiredKeys()) {
queryResponseTuple.put(filterKeywords.matcher(selectProperty.getKeyName()).iterator().next(), tuple[index++]);
}
if (null != promotion.getAggregation() && promotion.getAggregation().size() > 0) {
for (Aggregation aggregations : promotion.getAggregation()) {
queryResponseTuple.put(filterKeywords.matcher(aggregations.getAggregateOn()).iterator().next() + "_" + aggregations.getAggregateStrategy().name(), tuple[index++]);
}
}
TGAudience tgAudience1 = new TGAudience();
String stringToConvert = String.valueOf(queryResponseTuple.get("CUSTOMER_MOBILE"));
tgAudience1.setMobile(stringToConvert);
tgAudience1.setCustomerId(String.valueOf(queryResponseTuple.get("CUSTOMER_CUSTOMER_ID")));
tgAudienceList.add(tgAudience1);
}
System.out.println("tgAudienceList "+ tgAudienceList.size());
mongoTemplate.insert(tgAudienceList, audienceFilterName);
}
try {
Thread.sleep(50);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
);
}
}
CampaignExecutorService
package com.loylty.campaign.com.loylty.campaign.service;
import org.springframework.stereotype.Service;
import java.util.concurrent.*;
#Service
public class CampaignExecutorService {
private ExecutorService executorService = Executors.newFixedThreadPool(100);
public void dotask(Runnable runnable){
executorService.submit(runnable);
}
}

Java concurrent file writing - should fail

I've been testing to write multiple items to a filesystem, fully expecting to get a failure where by one thread overwrites anthers data, or interleaves with the data from another item.
However the following code unexpectedly passes.
Why is the data from one thread not overwriting the data from another thread? All the threads share one writer. Does the code pass because of a JVM implementation detail, or can it genuinely be expected to not mix up individual items.
I've seen some other quests about multiple threads writing to the same file but these were about performance optimizations. Note the import style is just for brevity when posting.
package com.test;
import static org.junit.Assert.assertEquals;
import java.io.*;
import java.nio.charset.*;
import java.nio.file.*;
import java.util.*;
import org.springframework.boot.CommandLineRunner;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectMapper;
public class DiskWriterApplication implements CommandLineRunner {
public static void main(String[] args) throws Exception {
new DiskWriterApplication().run(args);
}
#Override
public void run(String... args) throws Exception {
Path path = Paths.get(System.getProperty("user.home")+"/java-file.txt");
if (!Files.exists(path)) {
Files.createFile(path);
} else {
Files.delete(path);
Files.createFile(path);
}
BufferedWriter writer = Files.newBufferedWriter(path, Charset.forName("UTF-8"), StandardOpenOption.APPEND);
Thread[] threads = new Thread[4];
for (int i=0; i< 4; i++) {
threads[i] = new Thread(new DataWriter(writer, createDataItems(i)));
}
Arrays.asList(threads).forEach(Thread::start);
Arrays.asList(threads).forEach(t-> {
try {
t.join();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
});
writer.close();
//Verify Lines were written correctly
ObjectMapper mapper = new ObjectMapper();
MappingIterator<Data> valueIterator = mapper.readerFor(Data.class).readValues(Files.newInputStream(path));
Set<String> uniqueItems = new HashSet<>();
int[] groupItemCount = new int[4];
while (valueIterator.hasNext())
{
Data item = valueIterator.next();
assertEquals("First Item and second Item should be equal", item.firstValue, item.secondValue);
assertEquals(10, item.innerObject.size());
assertEquals(20, item.listValues.size());
for (int i = 0 ; i< 10; i++) {
assertEquals(item.firstValue, item.innerObject.get("innerProp"+i));
}
for (int i = 0 ; i< 20; i++) {
assertEquals(item.firstValue, item.listValues.get(i));
}
uniqueItems.add(item.firstValue);
groupItemCount[item.group]++;
}
System.out.println("Got " + uniqueItems.size() + " uniqueItems");
assertEquals("Should be 4000 uniqueItems", 4000, uniqueItems.size());
assertEquals("Should be 1000 items in group[0]", 1000, groupItemCount[0]);
assertEquals("Should be 1000 items in group[1]", 1000, groupItemCount[1]);
assertEquals("Should be 1000 items in group[2]", 1000, groupItemCount[2]);
assertEquals("Should be 1000 items in group[3]", 1000, groupItemCount[3]);
}
private List<Data> createDataItems(int groupNumber) {
List<Data> items = new ArrayList<>();
for (int i =0; i<1000; i++) {
Data item = new Data();
item.group = groupNumber;
item.itemNumber = i;
item.firstValue = "{group" + groupNumber + "item" + i + "}";
item.secondValue = "{group" + groupNumber + "item" + i + "}";
for (int j =0; j< 10; j ++) {
item.addInnerProperty("innerProp"+j , "{group" + groupNumber + "item" + i + "}");
}
for (int j=0; j<20; j++) {
item.addListValue("{group" + groupNumber + "item" + i + "}");
}
items.add(item);
}
return items;
}
private class DataWriter implements Runnable {
private ArrayList<String> data;
private PrintWriter writer;
public DataWriter(BufferedWriter writer, List<Data> items) {
this.writer = new PrintWriter(writer);
this.data = new ArrayList<String>();
ObjectMapper mapper = new ObjectMapper();
for (Data i : items) {
try {
String stringValue = mapper.writeValueAsString(i);
data.add(stringValue);
} catch (JsonProcessingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
#Override
public void run() {
System.out.println("Starting batch");
data.forEach(t -> {
writer.println(t);
writer.flush();
});
System.out.println("finishing batch");
}
}
public static class Data {
public int itemNumber;
public int group;
#JsonProperty
private String firstValue;
#JsonProperty
private String secondValue;
#JsonProperty
private Map<String, String> innerObject = new HashMap<>();
#JsonProperty
private List<String> listValues = new ArrayList<>();
public void addInnerProperty(String key, String value){
this.innerObject.put(key, value);
}
public void addListValue(String value) {
this.listValues.add(value);
}
}
}

As you can see in the others threads asking the same thing :
Writing a file using multiple threads in java
Is writting on file using bufferwriter initialized by filewriter thread safe or not?
the BufferedWriter is synchronized and thread-safe

Wait in ForkJoin Pool ( Java )

I am using Fork join pool in java for multitasking. Now i came across a situation where, for every task, I need to hit a url then wait for 10 minutes and then again hit another url to read the data. Now the problem is that for those 10 minutes my CPU is idle and not starting another tasks ( more than those defined in fork join pool).
static ForkJoinPool pool = new ForkJoinPool(10);
public static void main(String[] args){
List<String> list = new ArrayList<>();
for(int i=1; i<=100; i++){
list.add("Str"+i);
}
final Tasker task = new Tasker(list);
pool.invoke(task);
public class Tasker extends RecursiveAction{
private static final long serialVersionUID = 1L;
List<String> myList;
public Tasker(List<String> checkersList) {
super();
this.myList = checkersList;
}
#Override
protected void compute() {
if(myList.size()==1){
System.out.println(myList.get(0) + "start");
//Date start = new Date();
try {
Thread.sleep(10*60*1000);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(myList.get(0) + "Finished");
}
else{
List<String> temp = new ArrayList<>();
temp.add( myList.get( myList.size()-1 ) );
myList.remove( myList.size()-1 );
Tasker left = new Tasker(myList);
Tasker right = new Tasker(temp);
left.fork();
right.compute();
left.join();
}
}
Now What should I do so that CPU picks all the tasks and then wait parallaly for them.

Unfortunately, ForkJoinPool does not work well in the face of Thread.sleep(), because it designed for many short tasks that finish quickly, rather than tasks that block for a long time.
Instead, for what you are trying to accomplish, I would recommend using ScheduledThreadPoolExecutor and dividing your task into two parts.
import java.util.*;
import java.util.concurrent.*;
public class Main {
static ScheduledThreadPoolExecutor pool = new ScheduledThreadPoolExecutor(10);
public static void main(String[] args){
for(int i=1; i<=100; i++){
pool.schedule(new FirstHalf("Str"+i), 0, TimeUnit.NANOSECONDS);
}
}
static class FirstHalf implements Runnable {
String name;
public FirstHalf(String name) {
this.name = name;
}
public void run() {
System.out.println(name + "start");
pool.schedule(new SecondHalf(name), 10, TimeUnit.MINUTES);
}
}
static class SecondHalf implements Runnable {
String name;
public SecondHalf(String name) {
this.name = name;
}
public void run() {
System.out.println(name + "Finished");
}
}
}
If Java provides a thread pool which allows releasing the underlying resources (that is, the kernel thread participating in the thread pool) during a Thread.sleep(), you should use that instead, but I currently do not know of one.

According to docs forkJoin basic use section tells:
if (my portion of the work is small enough)
do the work directly
else
split my work into two pieces
invoke the two pieces and wait for the results
Hopefully this meets your need if you are using forkjoin
public class Tasker extends RecursiveAction {
static ForkJoinPool pool = new ForkJoinPool(10);
static int threshold = 10;
public static void main(String[] args){
List<String> list = new ArrayList<>();
for(int i=1; i<=100; i++){
list.add("Str"+i);
}
final Tasker task = new Tasker(list);
pool.invoke(task);
}
private static final long serialVersionUID = 1L;
List<String> myList;
public Tasker(List<String> checkersList) {
super();
this.myList = checkersList;
}
void computeDirectly() {
for(String url : myList){
System.out.println(url + " start");
}
//Date start = new Date();
try {
//keep hitting url
while (true) {
for(String url : myList) {
//url hitting code here
System.out.println(url + " hitting");
}
Thread.sleep(10 * 60 * 1000);
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
for(String url : myList){
System.out.println(url + " Finished");
}
}
#Override
protected void compute() {
if (myList.size() <= threshold) {
computeDirectly();
return;
}
//temp list have only one url
//List<String> temp = new ArrayList<>();
//temp.add( myList.get( myList.size()-1 ) );
//myList.remove( myList.size()-1 );
//Tasker left = new Tasker(myList);
//Tasker right = new Tasker(temp);
//left.fork();
//right.compute();
//left.join();
List<String> first = new ArrayList<>();
List<String> second = new ArrayList<>();
//divide list
int len = myList.size();
int smHalf = len / 2;//smaller half
first = myList.subList(0, smHalf);
second = myList.subList(smHalf + 1, len);
invokeAll(new Tasker(first), new Tasker(second));
}
}

Logic behind semaphores for threading

I am expecting 5 results i.e. 1, 2, 3, 4, 5 yet I am getting far more? can someone please help me understand semaphores? surely as each thread calls the run method using "start", it should add 1
to the int aNumber and then print aNumber.
I don't understand why I get more than five results.
// mainClass creates Semaphore, so Copier class no longer needed
import java.util.concurrent.Semaphore;
public class mainClass {
public static void main(String[] args) throws InterruptedException {
Semaphore cp1 = new Semaphore(1, true);
Worker[] workers = new Worker[5];
for (int x=0;x<5;x++) {
workers[x] = new Worker("w" + x, cp1);
}
for (int x=0;x<5;x++) {
workers[x].start();
}
for (int x=0;x<5;x++) {
workers[x].join();
}
}
}
import java.util.concurrent.Semaphore;
public class Worker extends Thread{
int aNumber = 0;
String myName;
Semaphore myCopier;
public Worker(String nameIn, Semaphore copierIn) {
myName = nameIn;
myCopier = copierIn;
}
public void run() {
for (int x=0;x<5;x++) {
try {
sleep((int) (Math.random() * 5000)); // do some filing
myCopier.acquire();
aNumber +=1;
//System.out.println(myName + " doing some copying");
System.out.println(aNumber);
sleep((int) (Math.random() * 1000)); // do some copying
myCopier.release();
} catch (InterruptedException e) { }
}
}
}

I could not clearly get your intention. Are you trying to use semaphores to print the nos ( 1 to 5) in sequence by each thread? In that case, you can try the below:
//mainClass creates Semaphore, so Copier class no longer needed
import java.util.concurrent.Semaphore;
public class mainClass {
public static void main(String[] args) throws InterruptedException {
Semaphore cp1 = new Semaphore(1, true);
Worker[] workers = new Worker[5];
for (int x=0;x<5;x++) {
workers[x] = new Worker("w" + x, cp1, x+1);
}
for (int x=0;x<5;x++) {
workers[x].start();
}
for (int x=0;x<5;x++) {
workers[x].join();
}
}
}
class Worker extends Thread{
int aNumber = 0;
String myName;
Semaphore myCopier;
public Worker(String nameIn, Semaphore copierIn, int no) {
myName = nameIn;
myCopier = copierIn;
aNumber = no;
}
public void run() {
/*for (int x=0;x<5;x++) {*/
try {
//sleep((int) (Math.random() * 5000)); // do some filing
myCopier.acquire();
//System.out.println(myName + " doing some copying");
System.out.println("Name of the thread:" + myName + " Printed No is:" + aNumber);
//sleep((int) (Math.random() * 1000)); // do some copying
myCopier.release();
} catch (InterruptedException e) { }
//}
}
}

cant find my logical error

I have made a simple program with :
working with files(read write)
end class extends
but the program does not work. Netbeans show no errors but when i run it ......some kind of errors show up .....and well i can't understand where is my bug (i think is a logical one).
Here is the simple program:
package detyre_kursi;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Scanner;
public class Detyre_kursi {
public static void main(String[] args) {
LlogariBankare llogaria1 = new LlogariBankare("aaa", 1000);
llogaria1.Balanca();
}
}
class LlogariBankare {
//variablat e instances
private String id;
private int nrLlogarise;
private int vitiHapjes;
private double balanca;
static int nrTranasksioneve = 0;
public LlogariBankare() {
System.out.println("Ju keni harruar te vendosi id dhe nrLlogarise");
}
public LlogariBankare(String id, int nrLlogarise) {
this.id = id;
this.nrLlogarise = nrLlogarise;
vitiHapjes = 0;
balanca = 0;
Lexim(this.id, this.nrLlogarise);
}
public double getBalanca() {
return balanca;
}
public int getVitiHapjes() {
return vitiHapjes;
}
private void Lexim(String s, int llog) {
try {
File file = new File("c:\\java\\balanca.txt");
Scanner scanner = new Scanner(file);
while (scanner.hasNextLine()) {
if (scanner.next().equals(s) && scanner.nextInt() == llog) {
vitiHapjes = scanner.nextInt();
balanca = scanner.nextDouble();
}
}
} catch (IOException e) {
e.getMessage();
}
}
void Balanca() {
try{
File file = new File("c:\\java\\test.txt");
PrintWriter out = new PrintWriter(file);
out.println(this.balanca);
} catch (IOException e) {
e.getMessage();
}
System.out.println(this.id + " , ju keni " + this.balanca +
" lek ne llogarine tuaj te krijuar ne vitin " + vitiHapjes +
" dhe keni kryer " + nrTranasksioneve + " transaksione gjithsej");
}
void Terheqe(double terheqe) {
this.balanca -= terheqe;
System.out.println("Ju sapo keni terhequr " + terheqe + " nga llogaria juaj");
nrTranasksioneve++;
}
void Depozitim(double depozitim) {
this.balanca += depozitim;
System.out.println("Ju sapo keni depozituar " + depozitim + " nga llogaria juaj");
nrTranasksioneve++;
}
}
class Interesi extends LlogariBankare {
int vitiTanishem = 2012;
double interesi = 0;
int diferencaViteve = vitiTanishem - getVitiHapjes();
Interesi(String id, int nrLlogarise) {
super(id,nrLlogarise);
}
void gjejInteresisn() {
interesi = getBalanca() + getBalanca() * diferencaViteve * 0.01;
}
}
The file balanca has this line in it :
aaa 1000 1990 34000
In poor words this is some simple version of a bank.
You read the balance from a file, and
you use the Terheqe() and Depozitim() for - and + the balance.
You use Balance() to see how many $ you have. When I run it, this error show up:
Exception in thread "main" java.util.NoSuchElementException
at java.util.Scanner.throwFor(Scanner.java:907)
at java.util.Scanner.next(Scanner.java:1416)
at detyre_kursi.LlogariBankare.Lexim(Detyre_kursi.java:57)
at detyre_kursi.LlogariBankare.<init>(Detyre_kursi.java:40)
at detyre_kursi.Detyre_kursi.main(Detyre_kursi.java:11)
Java Result: 1

This line causing issue. scanner.nextInt() might not be an int and I feel it is not good to do two next() calls unless you have specific reason.
if(scanner.next().equals(s)&&scanner.nextInt()==llog){

It's just a wild guess, but try replacing:
scanner.next().equals(s)
with:
s.equals(scanner.next())

I think your logical problem is from the constructor of
LlogariBankare llogaria1 = new LlogariBankare("aaa", 1000);
Check it out again.