Develop Reference

calculating directory file size with multi-Thread

I want to write a program that can calculate the size of a directory and its subdirectories with multi-thread.
I write this:
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadPoolExecutor;
public class Innerclass {
Semaphore semaphore = new Semaphore(1);
private List<String> availableConnections = new ArrayList();
public Innerclass(){
this.availableConnections.add("A");
}
public String acquireConnection() throws InterruptedException {
semaphore.acquire();
System.out.println("Acquiring connection " + Thread.currentThread().getName());
return availableConnections.remove(0);
}
static Long count = 0L;
static File file1 ;
public static void main(String[] args) {
System.out.println(countFilesInDirectory(new File("target directory address")));
}
public static Long countFilesInDirectory(File directory) {
Innerclass connection = new Innerclass();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(20);
Task task1 = new Task();
//Long count = 0L;
for (File file : directory.listFiles()) {
if (file.isFile()) {
executor.execute(new Runnable() {
#Override
public void run() {
try {
String far = connection.acquireConnection();
count += printFileSizeNIO(String.valueOf(file));
connection.acquireConnection();
} catch (InterruptedException e) {
e.printStackTrace();
}
//System.out.println(printFileSizeNIO(String.valueOf(file)));
}
});
}
if (file.isDirectory()) {
count += countFilesInDirectory(file);
}
}
executor.shutdown();
//System.out.println(task1.getCount());
//return task1.getCount();
return count;
}
public static Long printFileSizeNIO(String fileName) {
Path path = Paths.get(fileName);
Long bytes = 0L;
try {
bytes = Files.size(path);
} catch (IOException e) {
e.printStackTrace();
}
return bytes;
}
}
This program gives results close to reality but can not calculate exactly. What do you think is the problem?
and with single thread this program work exactly!
Receives the current path and the number of fragments from the user and performs the calculation. Do you think there is another way besides the thread pool to write this program. Thank you, dear professors.

With executor service, your main thread wont wait for threads inside pool to finish.
See here:
Wait main thread until all the thread pools task complete of ExecutorService?
I have updated your code a little bit. It will give the same answer always.
public class TestSF
{
Semaphore semaphore = new Semaphore(1);
public void acquireConnection() throws InterruptedException
{
semaphore.acquire();
}
public void releaseConnection() throws InterruptedException
{
semaphore.release();
}
static AtomicLong count = new AtomicLong(0);
public static void main(String[] args)
{
long bytes = countFilesInDirectory(new File("C:\\Users\\ashish\\Desktop\\Loader"));
System.out.println(humanReadableByteCountBin(bytes));
}
public static Long countFilesInDirectory(File directory)
{
TestSF connection = new TestSF();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(20);
for (File file : Objects.requireNonNull(directory.listFiles()))
{
executor.execute(() -> {
if (file.isFile())
{
try
{
connection.acquireConnection();
count.addAndGet(printFileSizeNIO(String.valueOf(file)));
connection.releaseConnection();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
}
if (file.isDirectory())
{
countFilesInDirectory(file);
}
});
}
executor.shutdown();
try {
if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException ex) {
executor.shutdownNow();
Thread.currentThread().interrupt();
}
return count.get();
}
public static Long printFileSizeNIO(String fileName)
{
Path path = Paths.get(fileName);
long bytes = 0L;
try
{
bytes = Files.size(path);
}
catch (IOException e)
{
e.printStackTrace();
}
return bytes;
}
public static String humanReadableByteCountBin(long bytes)
{
long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
if (absB < 1024) {
return bytes + " B";
}
long value = absB;
CharacterIterator ci = new StringCharacterIterator("KMGTPE");
for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10)
{
value >>= 10;
ci.next();
}
value *= Long.signum(bytes);
return String.format("%.1f %ciB", value / 1024.0, ci.current());
}
}

how to create loops with threads (creates several new threads in a loop)?

how can I create a loop (or something else if that is a better way) where I can create some new threads.
So far I have 2 producer and consumer threads. But I would like to create, for example, 5 producers and 5 consumers, and each thread produced / consumed a different "product", two threads cannot do the same.
I'd like it to be something like this:
Produced thread0 produce 0
Consume thread0 consume 0
....
Produced thread4 produce 4
Consume thread4 consume 4
Thank you for every hint.
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
public class ProducerConsumer {
public static void main(String[] args) throws InterruptedException {
BlockingQueue<Integer> blockingQueue = new ArrayBlockingQueue<>(2);
Thread producerThread = new Thread(new Runnable() {
#Override
public void run() {
try {
int value = 0;
while (true) {
blockingQueue.put(value);
System.out.println("Produced " + value);
value++;
Thread.sleep(1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
Thread consumerThread = new Thread(new Runnable() {
#Override
public void run() {
try {
while (true) {
int value = blockingQueue.take();
System.out.println("Consume " + value);
Thread.sleep(1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
producerThread.start();
consumerThread.start();
producerThread.join();
consumerThread.join();
}
}

Use a thread pool (Executors.newFixedThreadpool or Executors.newCachedThreadPool)
Don't forget to manage thread synchronization for resources using synchronized blocks.
Use volatile keyword for values that will be written/read simutaneously by several threads (see What is the volatile keyword useful for?)
I've used lambda syntax to redefine your runnables for clarity.
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class ProducerConsumer {
private static volatile int prodValue = 0;
private static final Object valueSync = new Object();
public static void main(String[] args) throws InterruptedException {
final BlockingQueue<Integer> blockingQueue = new ArrayBlockingQueue<>(2);
final ExecutorService threadPool = Executors.newCachedThreadPool();
final Runnable producer = () -> {
try {
while (true) {
synchronized(valueSync) {
blockingQueue.put(prodValue);
System.out.println(Thread.currentThread().getId() + " Produced " + prodValue);
prodValue++;
}
Thread.sleep(1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
};
final Runnable consumer = () -> {
try {
while (true) {
int value = blockingQueue.take();
System.out.println(Thread.currentThread().getId() + " Consumed " + value);
Thread.sleep(1200);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
};
for (int i = 0; i < 5; i++) { //Create 5 threads of each
System.out.println("Loop " + i);
threadPool.execute(producer);
threadPool.execute(consumer);
Thread.sleep(500); //Wait a little
}
System.out.println("Loop done");
//Wait for all threads to complete with timeout
threadPool.awaitTermination(15, TimeUnit.SECONDS);
System.out.println("STOP !");
//Forceful shutdown of all threads (will happen as all threads are in a while(true) loop
threadPool.shutdownNow();
}
}
About synchronization: here you want your value to be added to the queue and incremented seemingly at once (atomically). synchronized around the operations prevents threads from simultaneously running this piece of code, which would result in the same value added multiple times into the queue, and then incremented multiple times (it happens if you decrease the Thread.sleep values to something close to 0 and remove the synchronized block).
I could have used blockingQueue as argument for synchronized but chose to use a dedicated object to make it more obvious.

What concurrently bug is in this program?

I have one program with strange concurrently bug.
What this program does:
Execute event loop each EVENT_LOOP_PAUSE_DURATION_IN_MS.
For each given task execute processor TaskProcessor
Each 500 ms prints queue size of my executor.
I want to have at most one task in queue per taskId. So, when I add task in queue, I check whether tasks has already existed or not. If there is no task, I add it. In the end of task processing, I remove task from activeTasks map.
If you run the program, then you see the following output:
ERROR: 50
ERROR: 70
ERROR: 80
ERROR: 90
ERROR: 110
ERROR: 120
ERROR: 120
ERROR: 140
So, there is a bug. I don't know why, but size of thread pool queue is infinitely increasing.
You can see, that I remove active tasks in 2 point of program:
In finally block of TaskProcessor, when task has processed.
I remove stale tasks in event loop.
So, if I remove code, which removes tasks at point (2), then the bug disappears. I don't understand this behavior.
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class Program {
private static final int NUMBER_OF_TASKS = 40;
private static final int NUMBER_OF_THREADS = 10;
private static final long EVENT_LOOP_PAUSE_DURATION_IN_MS = 40L;
class QueueSizePrinter extends Thread {
private final LinkedBlockingQueue<Runnable> workQueue;
public QueueSizePrinter(LinkedBlockingQueue<Runnable> workQueue) {
this.workQueue = workQueue;
}
#Override
public void run() {
while (true) {
int qSize = workQueue.size();
if (qSize > NUMBER_OF_TASKS) {
System.out.println("ERROR: " + qSize);
}
try {
Thread.sleep(500L);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
class TaskProcessor implements Runnable {
private final String currentTaskId;
private final ConcurrentHashMap<String, Long> activeTasks;
public TaskProcessor(String currentTaskId, ConcurrentHashMap<String, Long> activeTasks) {
this.currentTaskId = currentTaskId;
this.activeTasks = activeTasks;
}
#Override
public void run() {
try {
// emulate of useful work
Thread.sleep(300L);
} catch (Exception e) {
System.out.println("error: " + e.toString());
} finally {
activeTasks.remove(currentTaskId); // (1)
}
}
}
public void program() {
LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();
ExecutorService executor = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS, workQueue);
Set<String> initialTasks = ConcurrentHashMap.newKeySet();
for (int currentTaskIndex = 0; currentTaskIndex < NUMBER_OF_TASKS; currentTaskIndex++) {
initialTasks.add(String.valueOf(currentTaskIndex));
}
new QueueSizePrinter(workQueue).start();
ConcurrentHashMap<String, Long> activeTasks = new ConcurrentHashMap<>();
while (true) {
initialTasks.forEach((currentTaskId) -> {
if (!activeTasks.containsKey(currentTaskId)) {
activeTasks.put(currentTaskId, System.currentTimeMillis());
executor.submit(new TaskProcessor(currentTaskId, activeTasks));
}
});
// (2)
activeTasks.entrySet().removeIf(entry -> {
boolean hasDelete = System.currentTimeMillis() - entry.getValue() > 1000;
if (hasDelete) {
//System.out.println("DELETE id=" + entry.getKey());
}
return hasDelete;
});
try {
Thread.sleep(EVENT_LOOP_PAUSE_DURATION_IN_MS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public static void main(String[] args) {
Program main = new Program();
main.program();
}
}

Problem is at point (2), You are removing stale tasks from activeTasks map. But they are still submitted to ExecutorService. Since You removed it from the map, when while loop execute another cycle, the same task will be resubmitted to ExecutorService. This causes task numbers to increase.

Trigger SheduledExecutor with blockingQueue Java

I'm currently working on java application which has a scenario of multiple producers adding tasks to a queue and whenever queue is not empty tasks should be executed at predefined rate. (using multiple threads to maintain execution rate) After executing the available tasks executor has to wait till tasks available in the queue again.
I know blockingQueue can be used to triggering part in here and ScheduledExecutorService for execute tasks at fixed rate. But I could not find a way to link ability of both of this for my need. So I would be very thankful if you could give me any suggestion to make this happen.

You need the task queue to be accessible by both the producer and consumer threads. I've written a basic program to demonstrate this, but I'll let you play around with the BlockingQueue API and the ScheduledExecutor as per your needs:
import java.util.concurrent.*;
public class ProducerConsumer {
private static final BlockingQueue<Integer> taskQueue = new LinkedBlockingQueue<>();
public static void main(String[] args) {
ExecutorService consumers = Executors.newFixedThreadPool(3);
consumers.submit(new Consumer());
consumers.submit(new Consumer());
consumers.submit(new Consumer());
ExecutorService producers = Executors.newFixedThreadPool(2);
producers.submit(new Producer(1));
producers.submit(new Producer(2));
}
private static class Producer implements Runnable {
private final int task;
Producer(int task) {
this.task = task;
}
#Override
public void run() {
System.out.println("Adding task: " + task);
taskQueue.add(task); // put is better, since it will block if queue is full
}
}
private static class Consumer implements Runnable {
#Override
public void run() {
try {
Integer task = taskQueue.take(); // block if there is no task available
System.out.println("Executing task: " + task);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}

This is the way I could come up with as a solution. It looks little bit rusty but I have tested this and the code is working.
package test;
import java.util.concurrent.*;
public class FixedRateConsumer {
private BlockingQueue<String> queue = new ArrayBlockingQueue<>(20);
private ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(5);
private boolean continueRunning = true;
public void executeInBackGraound() throws InterruptedException, ExecutionException {
while (continueRunning) {
String s = queue.take();
Worker w = new Worker(s);
ScheduledFuture future = executorService.scheduleAtFixedRate(w, 0, 1, TimeUnit.SECONDS);
w.future = future;
try {
if (!future.isDone()) {
future.get();
}
} catch (CancellationException e) {
// Skipping
}
}
}
public void setContinueRunning(boolean state) {
continueRunning = state;
}
public void addConsumableObject(String s) throws InterruptedException {
queue.put(s);
}
private void consumeString(String s) {
System.out.println("Consumed -> " + s + ", ... # -> " + System.currentTimeMillis() + " ms");
}
private class Worker implements Runnable {
String consumableObject;
ScheduledFuture future;
public Worker(String initialConsumableObject) {
this.consumableObject = initialConsumableObject;
}
#Override
public void run() {
try {
if (consumableObject == null) {
consumableObject = queue.take();
}
consumeString(consumableObject);
consumableObject = null;
if (queue.isEmpty()) {
if (future == null) {
while (future == null) {
Thread.sleep(50);
}
}
future.cancel(false);
}
} catch (Exception e) {
System.out.println("Exception : " + e);
}
}
}
}

Working with BlockingQueue and Multithreads. All Threads Stuck in Waiting

I am creating a system that will have multiple suite deployments and each deployment will have a queue of test suites. Since I want the test suites to run concurrently on their individual suite deployment, I need to add concurrency to the code. I have created a simplified version of the code I am using, but the concurrency portion doesn't work when I try to shut it down.
When the Runner.stopEverything() gets called, the result is that the queue gets emptied, and it waits for the threads to complete, but even when the tests all complete, the wait never finishes even with the notifyAll(). The result is that the process just sits there never ending. I go look at it in debug mode and the result is that all 3 threads show waiting.
Main:
public static void main(String args[]) throws Exception {
Runner.queueTestSuites("SD1", Arrays.asList("A", "B", "C"));
Runner.queueTestSuites("SD2", Arrays.asList("D", "E", "F"));
Runner.queueTestSuites("SD3", Arrays.asList("G", "H", "I"));
Thread.sleep(5000);
System.out.println("~~~~~~~~~~~~~~~~~~~~~~~~");
Runner.stopEverything();
}
Runner:
public class Runner {
private static Map<String, TestQueue> runnerQueueMap = new ConcurrentHashMap<>();
public synchronized static void queueTestSuites(String suiteDeployment, List<String> testSuiteQueueAsJSON) throws Exception {
TestQueue queue;
if(runnerQueueMap.containsKey(suiteDeployment)) {
queue = runnerQueueMap.get(suiteDeployment);
} else {
queue = new TestQueue(suiteDeployment);
}
for (int i = 0; i < testSuiteQueueAsJSON.size(); i++) {
String name = testSuiteQueueAsJSON.get(i);
queue.addToQueue(name);
}
runnerQueueMap.put(suiteDeployment,queue);
}
public synchronized static void stopEverything() throws InterruptedException {
for (String s : runnerQueueMap.keySet()) {
TestQueue q = runnerQueueMap.get(s);
q.saveAndClearQueue();
}
for (String s : runnerQueueMap.keySet()) {
TestQueue q = runnerQueueMap.get(s);
q.waitForThread();
}
System.out.println("All done at " + new Date());
}
}
TestQueue:
public class TestQueue {
private Consumer consumer;
private Thread consumerThread;
private java.util.concurrent.BlockingQueue<String> queue;
private String suiteDeployment;
public TestQueue(String suiteDeployment) {
this.suiteDeployment = suiteDeployment;
queue = new ArrayBlockingQueue<>(100);
startConsumer();
}
public void addToQueue(String testSuite) {
try {
queue.put(testSuite);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public synchronized void waitForThread() {
try {
if (consumer.running.get()) {
synchronized (consumerThread) {
System.out.println("Waiting for " + consumerThread.getName());
consumerThread.wait();
}
}
System.out.println("Thread complete at " + new Date());
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void saveAndClearQueue() {
List<String> suiteNames = new ArrayList<>();
for (String suite : queue) {
suiteNames.add(suite);
}
queue.clear();
}
private void startConsumer() {
consumer = new Consumer(queue,suiteDeployment);
consumerThread = new Thread(consumer);
consumerThread.start();
}
private class Consumer implements Runnable{
private BlockingQueue<String> queue;
private String suiteDeployment;
public AtomicBoolean running;
public Consumer(BlockingQueue<String> queue, String suiteDeployment){
this.queue = queue;
this.suiteDeployment = suiteDeployment;
this.running = new AtomicBoolean(false);
}
#Override
public void run() {
try{
while(!Thread.currentThread().isInterrupted()) {
String testSuite = queue.take();
this.running.set(true);
new Test(testSuite, suiteDeployment).run();
this.running.set(false);
}
notifyAll();
}catch(Exception e) {
e.printStackTrace();
}
}
}
}
Test:
public class Test {
String testSuite = "";
String suiteDeployment = "";
public Test(String testSuite, String suiteDeployment) {
this.testSuite = testSuite;
this.suiteDeployment = suiteDeployment;
}
public void run() {
int time = new Random().nextInt() % 10000;
time = Math.max(time, 3000);
System.out.println("Test Started: " + testSuite + " on " + suiteDeployment + " at " + new Date() + " running for " + time + " on thread " + Thread.currentThread().getName());
try {
Thread.sleep(time);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("Test Completed: " + testSuite + " on " + suiteDeployment + " at " + new Date());
}
}

Inside run method of your consumer, you have a blocking call to queue.take() which means it will block until there is an item inside your queue. You run out of elements inside the queue eventually and all your thread are blocked by the queue.take() call waiting for more elements to become available to process.
Although your call is in a while loop where it check if the thread is interrupted, you actually never interrupt the threads so it never gets to the while loop evaluation & blocked at the call to queue.take()
So your threads stay in wait as they are waiting for input to become avilable inside your blocking queue
Also your saveAndClear method must lock on the correct object which is the queue itself, like below:
public void saveAndClearQueue() {
List<String> suiteNames = new ArrayList<String>();
synchronized (queue) {
for (String suite : queue) {
suiteNames.add(suite);
}
queue.clear();
}
System.out.println("Saved(not executed) : "+suiteNames);
}
And your waitForThread method should do sth like below:
public void waitForThread() {
synchronized (consumerThread) {
while (consumer.running.get()) {
try {
consumerThread.wait(100);
} catch (InterruptedException e) {
break;
}
}
}
if (!consumer.running.get()) {
consumerThread.interrupt();
}
System.out.println("Thread complete at " + new Date());
}