Develop Reference

ThreadPoolTaskExecutor throws RejectedExecutionException

I want to implement following behaviour:
Read n events from file
Process them i n threads
Go back to to step 1 if any events remains
I wrote following app to test solution but it fails in random moment, eg.
java.lang.IllegalStateException: Failed to execute CommandLineRunner
Caused by: org.springframework.core.task.TaskRejectedException: Executor [java.util.concurrent.ThreadPoolExecutor#62b3df3a[Running, pool size = 5, active threads = 4, queued tasks = 0, completed tasks = 70]] did not accept task: java.util.concurrent.CompletableFuture$AsyncSupply#71ea1fda
What queue capacity should I set if I don't want to put events in queue? I want to process them immediately.
I am using Open JDK 11 and Spring Boot 2.2.2.RELEASE
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
#SpringBootApplication
public class DemoApplication implements CommandLineRunner {
#Autowired
private ThreadPoolTaskExecutor eventExecutor;
public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args);
}
#Bean(name = "eventExecutor")
public ThreadPoolTaskExecutor eventExecutor() {
ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
pool.setCorePoolSize(5);
pool.setMaxPoolSize(5);
pool.setQueueCapacity(0);
pool.setAwaitTerminationSeconds(0);
pool.initialize();
return pool;
}
#Override
public void run(String... args) {
System.out.println("Start events processing");
long start = System.currentTimeMillis();
int result = 0;
for (int i = 0; i < 100; i++) {
List<CompletableFuture<Integer>> completableFutures = getEvents(5).stream()
.map(event -> CompletableFuture.supplyAsync(() -> processEvent(event), eventExecutor))
.collect(Collectors.toList());
result += completableFutures.stream()
.mapToInt(CompletableFuture::join)
.sum();
}
long timeMillis = System.currentTimeMillis() - start;
System.out.println("Took " + timeMillis + "ms, " + result);
}
private List<Event> getEvents(int n) {
List<Event> events = new ArrayList<>();
for (int i = 1; i <= n; i++) {
events.add(new Event(i));
}
return events;
}
private int processEvent(Event event) {
System.out.println("processing event " + event.id);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("processing event " + event.id + " finished");
return 1;
}
private static class Event {
private int id;
private Event(int id) {
this.id = id;
}
}
}

I'll suggest to change pool.setQueueCapacity(0) to use a positive value to allow for tasks to be queued up for processing when there are no threads available in the so-configured fixed-size pool (corePoolSize == maxPoolSize == 5).
The output "pool size = 5, active threads = 4" shows approximate number of active threads.
In theory it could happen that threads are not returned back into the pool before an attempt to process new batch of events.

Streams performance difference

Performance difference on 2 different streams executions
I try to do the same operation for default parallel stream and using custom ForkJoin pool.
I see huge performance difference for the same operation.
94 ms vs ~5341 ms (Time1 and Time2 are almost the same - so I don't blame awaitQuiescence here)
What can be a reason ? Tricky java intrinsic ?
public final class SharedForkJoinExecutor {
private static final Logger LOGGER = LoggerFactory.getLogger(SharedForkJoinExecutor.class);
private static final ForkJoinPool EXEC = new ForkJoinPool(ForkJoinPool.commonPool().getParallelism(),
pool -> {
final ForkJoinWorkerThread aThread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool);
aThread.setName("ForkJoin-Executor-" + aThread.getPoolIndex());
return aThread;
},
(t, e) -> LOGGER.info(e.getMessage(), e),
true);
/**
* Shuts down this executor
*/
public static void shutdown() {
EXEC.shutdown();
}
public static ForkJoinPool get() {
return EXEC;
}
}
package com.stream;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.IntStream;
import com.stream.SharedForkJoinExecutor;
import org.junit.Test;
import static junit.framework.TestCase.assertEquals;
public class ForkJoinTest {
private static final int INT_NUMBERS = 1_000_000;
#Test
public void forEachIntTest() {
final AtomicInteger aEvenCounter = new AtomicInteger(0);
final AtomicInteger aAllCounter = new AtomicInteger(0);
long t = System.currentTimeMillis();
IntStream.range(0, INT_NUMBERS).parallel().forEach(theIndex -> {
if (theIndex % 2 == 0) {
aEvenCounter.incrementAndGet();
}
aAllCounter.incrementAndGet();
});
System.out.println("Time=" + (System.currentTimeMillis() - t));
assertEquals(INT_NUMBERS / 2, aEvenCounter.get());
assertEquals(INT_NUMBERS, aAllCounter.get());
aEvenCounter.set(0);
aAllCounter.set(0);
t = System.currentTimeMillis();
SharedForkJoinExecutor.get().execute(() -> IntStream.range(0, INT_NUMBERS).parallel().forEach(theIndex -> {
if (theIndex % 2 == 0) {
aEvenCounter.incrementAndGet();
}
aAllCounter.incrementAndGet();
}));
System.out.println("Time1=" + (System.currentTimeMillis() - t));
SharedForkJoinExecutor.get().awaitQuiescence(10, TimeUnit.HOURS);
System.out.println("Time2=" + (System.currentTimeMillis() - t));
assertEquals(INT_NUMBERS / 2, aEvenCounter.get());
assertEquals(INT_NUMBERS, aAllCounter.get());
}
}

Disk reading operation is performing very slow | Java Stream

I need to read images from a folder and generate checksum for them. There are about 330760 images. Following is the code:
package com.test;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.apache.commons.codec.digest.DigestUtils;
public class FileTest2 {
private void readFiles() throws IOException {
try (Stream<Path> filePathStream = Files
.walk(Paths.get("d:\\codebase\\images"))) {
filePathStream.parallel().forEach(filePath -> {
String checksumSHA256 = "";
try {
checksumSHA256 = DigestUtils.sha384Hex(new FileInputStream(filePath.toString()));
} catch (IOException e) {
e.printStackTrace();
}
if (Files.isRegularFile(filePath)) {
System.out.println(checksumSHA256);
System.out.println(filePath);
System.out.println("\n");
}
});
}
}
public static void main(String[] args) throws IOException {
long startTime = System.currentTimeMillis();
FileTest2 fileTest = new FileTest2();
fileTest.readFiles();
long endTime = System.currentTimeMillis();
System.out.println("Total Time took: " + (endTime - startTime) / 1000);
}
}
It took about 36 minutes.
System configuration:
Cores: 8
Memory: 32 GB (15-17 GB is free). Rest of the memory is being used by another server.
36 minutes are too much. Is there a way to improve performance?

As others pointed out you do not terminate the executor. To see the actual times run the following
public static void main(String[] args) throws Exception {
long startTime = System.currentTimeMillis();
FileTest fileTest = new FileTest();
fileTest.readFiles();
long endTime = System.currentTimeMillis();
System.out.println("Total Time took: "+ (endTime-startTime)/1000);
}
Note: at least from the bit of code you posted there is no reason to use an ExecutorService

Making a program runnable

Below is my program. I want to use a ExecutorService on it to run once a day. However, the program is not 'runnable' please advice the necessary change to make it so.
package priceCollector;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.text.DateFormat;
import java.util.Date;
public class App extends myTimerTask {
public static void main(String[] args) {
Date now = new Date();
DateFormat df = DateFormat.getDateInstance();
String s = df.format(now);
String fileName = new String();
fileName = "/Users/Desktop/" + s + ".csv";
URL link = null;
try {
link = new URL("http://finance.yahoo.com/d/quotes.csv?s=III.L+ADM.L+AAL.L+ANTO.L+AHT.L+ABF.L+AZN.L+AV.L+BAB.L+BA.L+BARC.L+BDEV.L+BLT.L+BP.L+BATS.L+BLND.L+BTA.L+BNZL.L+BRBY.L+CPI.L+CCL.L+CNA.L+CCH.L+CPG.L+CRH.L+CRDA.L+DCC.L+DGE.L+DLG.L+DC.L+EZJ.L+EXPN.L+FRES.L+GKN.L+GSK.L+GLEN.L+HMSO.L+HL.L+HIK.L+HSBA.L+IMB.L+INF.L+IHG.L+IAG.L+ITRK.L+INTU.L+ITV.L+JMAT.L+KGF.L+LAND.L+LGEN.L+LLOY.L+LSE.L+MKS.L+MDC.L+MERL.L+MCRO.L+MNDI.L+MRW.L+NG.L+NXT.L+OML.L+PPB.L+PSON.L+PSN.L+POLY.L+PFG.L+PRU.L+RRS.L+RB.L+REL.L+RIO.L+RR.L+RBS.L+RDSA.L+RDSB.L+RMG.L+RSA.L+SGE.L+SBRY.L+SDR.L+SVT.L+SHP.L+SKY.L+SN.L+SMIN.L+SSE.L+STJ.L+STAN.L+SL.L+TW.L+TSCO.L+TPK.L+TUI.L+ULVR.L+UU.L+VOD.L+WTB.L+WOS.L+WPG.L+WPP.L&f=np");
InputStream in = new BufferedInputStream(link.openStream());
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buf = new byte[1024];
int n = 0;
while (-1!=(n=in.read(buf))) {
out.write(buf, 0, n);
}
out.close();
in.close();
byte[] response = out.toByteArray();
FileOutputStream fos = new FileOutputStream(fileName);
fos.write(response);
fos.close();
} catch (Exception e) {
System.out.println("not available");
}
}
}
I am guessing you would need to implement the run() instance but not sure how you would write a program into that.

You need to follow the below steps which use ScheduledExecutorService as shown below:
Step (1): You need to move your code into a separate class as shown below:
public TaskRunner implements Runnable {
public void run() {
//your code here (logic froom main method)
}
}
Step(2): Now use ScheduledExecutorService as shown below:
public class MySchduler {
private final ScheduledExecutorService scheduler =
Executors.newScheduledThreadPool(1);
public static void main(String[] args) {
scheduler.scheduleAtFixedRate(new TaskRunner(), 0, 1, TimeUnit.DAYS);
}
}
You can look here

Comparing Fork And Join with Single threaded program

I am trying to get started with the Fork-Join framework for a smaller task. As I start-up example I tried copying mp3 files
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveTask;
public class DeepFileCopier extends RecursiveTask<String>{
/**
*
*/
private static final long serialVersionUID = 1L;
private static Path startingDir = Paths.get("D:\\larsen\\Music\\");
private static List<Path> listOfPaths = new ArrayList<>();
private int start, end;
public static void main(String[] args) throws IOException
{
long startMillis = System.currentTimeMillis();
Files.walkFileTree(startingDir, new CustomFileVisitor());
final DeepFileCopier deepFileCopier = new DeepFileCopier(0,listOfPaths.size());
final ForkJoinPool pool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
pool.invoke(deepFileCopier);
System.out.println("With Fork-Join " + (System.currentTimeMillis() - startMillis));
long secondStartMillis = System.currentTimeMillis();
deepFileCopier.start = 0;
deepFileCopier.end = listOfPaths.size();
deepFileCopier.computeDirectly();
System.out.println("Without Fork-Join " + (System.currentTimeMillis() - secondStartMillis));
}
private static class CustomFileVisitor extends SimpleFileVisitor<Path> {
#Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException
{
if (file.toString().endsWith(".mp3")) {
listOfPaths.add(file);
}
return FileVisitResult.CONTINUE;
}
}
#Override
protected String compute() {
int length = end-start;
if(length < 4) {
return computeDirectly();
}
int split = length / 2;
final DeepFileCopier firstHalfCopier = new DeepFileCopier(start, start + split);
firstHalfCopier.fork();
final DeepFileCopier secondHalfCopier = new DeepFileCopier(start + split, end);
secondHalfCopier.compute();
firstHalfCopier.join();
return null;
}
private String computeDirectly() {
for(int index = start; index< end; index++) {
Path currentFile = listOfPaths.get(index);
System.out.println("Copying :: " + currentFile.getFileName());
Path targetDir = Paths.get("D:\\Fork-Join Test\\" + currentFile.getFileName());
try {
Files.copy(currentFile, targetDir, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
private DeepFileCopier(int start, int end ) {
this.start = start;
this.end = end;
}
}
On comparing the performance I noticed -
With Fork-Join 149714
Without Fork-Join 146590
Am working on a Dual Core machine. I was expecting a 50% reduction in the work time but the portion with Fork-Join takes 3 seconds more than a single threaded approach. Please let me know if some thing is incorrect.

Your problem is not well suited to benefit from multithreading on normal systems. The execution time is spent copying all the files. But this is limited by your hard drive that will process the files in sequence.
If you run a more CPU intense task, you should note a difference. For test purposes you could try the following:
private String computeDirectly() {
Integer nonsense;
for(int index = start; index< end; index++) {
for( int j = 0; j < 1000000; j++ )
nonsense += index*j;
}
return nonsense.toString();
}
On my system (i5-2410M) this will print:
With Fork-Join 2628
Without Fork-Join 6421