Develop Reference

Converting serially reading multiple files to reading them in parallel? [closed]

Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
Following code in Java reads multiple files one after another serially and it works well till here. (The files are JSON and at this step they are stored in a String/Buffer without parsing.)
for (int fileIndex = 0; fileIndex < numberOfFiles; fileIndex++) {
BufferedReader br = new BufferedReader(new FileReader("Files/file" + fileIndex + ".json"));
try {
StringBuilder sb = new StringBuilder();
String line = br.readLine();
while (line != null) {
sb.append(line);
sb.append(System.lineSeparator());
line = br.readLine();
}
String contentJSON = sb.toString();
} finally {
br.close();
}
}
How to read those files in parallel by using Threads ?
I could not match Multithreading to above code and every time got errors.

I've not tested this code directly (as I don't have a bunch of files to read), but the basic idea would be to do something like...
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
public class Test {
public static void main(String[] args) {
new Test();
}
public Test() {
try {
int numberOfFiles = 10;
ExecutorService service = Executors.newFixedThreadPool(10);
List<ReadWorker> workers = new ArrayList<>(numberOfFiles);
for (int fileIndex = 0; fileIndex < numberOfFiles; fileIndex++) {
workers.add(new ReadWorker(fileIndex));
}
List<Future<String>> results = service.invokeAll(workers);
for (Future<String> result : results) {
try {
String value = result.get();
} catch (ExecutionException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
}
} catch (InterruptedException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
}
public class ReadWorker implements Callable<String> {
private int fileIndex;
public ReadWorker(int fileIndex) {
this.fileIndex = fileIndex;
}
#Override
public String call() throws Exception {
try (BufferedReader br = new BufferedReader(new FileReader("Files/file" + fileIndex + ".json"))) {
StringBuilder sb = new StringBuilder();
String line = br.readLine();
while (line != null) {
sb.append(line);
sb.append(System.lineSeparator());
line = br.readLine();
}
return sb.toString();
}
}
}
}
This will basically execute a series of Callables and wait for them all to complete, at which time, you can then read the results (or errors)
See the Executors trail for more details
Tested and verified version...
So, I dumped a series of files into a the Files folder at the root of my working directory, modified the above example to list all the files in that directory and read them....
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
public class Test {
public static void main(String[] args) {
new Test();
}
public Test() {
File files[] = new File("Files").listFiles(new FileFilter() {
#Override
public boolean accept(File pathname) {
return pathname.getName().toLowerCase().endsWith(".svg");
}
});
try {
int numberOfFiles = files.length;
ExecutorService service = Executors.newFixedThreadPool(20);
List<ReadWorker> workers = new ArrayList<>(numberOfFiles);
for (File file : files) {
workers.add(new ReadWorker(file));
}
System.out.println("Execute...");
List<Future<String>> results = service.invokeAll(workers);
System.out.println("Results...");
for (Future<String> result : results) {
try {
String value = result.get();
System.out.println(value);
} catch (ExecutionException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
}
service.shutdownNow();
} catch (InterruptedException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
}
public class ReadWorker implements Callable<String> {
private File file;
public ReadWorker(File file) {
this.file = file;
}
#Override
public String call() throws Exception {
System.out.println("Reading " + file);
try (BufferedReader br = new BufferedReader(new FileReader(file))) {
StringBuilder sb = new StringBuilder();
String line = br.readLine();
while (line != null) {
sb.append(line);
sb.append(System.lineSeparator());
line = br.readLine();
}
return sb.toString();
}
}
}
}
And this works just fine and I have no issue.
java.io.FileNotFoundException: Files\file0.json is a localised issue you are going to have to solve. Does file0.json actually exist? Does it exist in the Files directory? Is the Files directory in the root of the working directory when the program is executed?
None of these issues can be solved by us, as we don't have access to your environment
Test #3
I then renamed all the files in my Files directory to file{x}.json using...
File files[] = new File("Files").listFiles(new FileFilter() {
#Override
public boolean accept(File pathname) {
return pathname.getName().toLowerCase().endsWith(".svg");
}
});
for (int index = 0; index < files.length; index++) {
File source = files[index];
File target = new File(source.getParent(), "file" + index + ".json");
source.renameTo(target);
}
And the modified the example slightly to include a File#exists report...
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
public class Test {
public static void main(String[] args) {
new Test();
}
public Test() {
try {
int numberOfFiles = 10;
ExecutorService service = Executors.newFixedThreadPool(20);
List<ReadWorker> workers = new ArrayList<>(numberOfFiles);
for (int index = 0; index < numberOfFiles; index++) {
workers.add(new ReadWorker(index));
}
System.out.println("Execute...");
List<Future<String>> results = service.invokeAll(workers);
System.out.println("Results...");
for (Future<String> result : results) {
try {
String value = result.get();
System.out.println(value);
} catch (ExecutionException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
}
service.shutdownNow();
} catch (InterruptedException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
}
public class ReadWorker implements Callable<String> {
private int fileIndex;
public ReadWorker(int fileIndex) {
this.fileIndex = fileIndex;
}
#Override
public String call() throws Exception {
System.out.println("Reading " + fileIndex);
File file = new File("Files/file" + fileIndex + ".json");
System.out.println("File " + fileIndex + " exists = " + file.exists());
try (BufferedReader br = new BufferedReader(new FileReader(file))) {
StringBuilder sb = new StringBuilder();
String line = br.readLine();
while (line != null) {
sb.append(line);
sb.append(System.lineSeparator());
line = br.readLine();
}
return sb.toString();
} finally {
System.out.println("All done here");
}
}
}
}
Which prints
Execute...
Reading 8
Reading 1
Reading 2
Reading 4
Reading 6
Reading 9
Reading 3
Reading 7
Reading 0
Reading 5
File 8 exists = true
File 1 exists = true
File 5 exists = true
File 4 exists = true
File 9 exists = true
File 2 exists = true
File 0 exists = true
File 3 exists = true
File 7 exists = true
File 6 exists = true
All done here
All done here
All done here
All done here
All done here
All done here
All done here
All done here
All done here
All done here
Results...
// I won't bore you with the results, as it's a lot of pointless text
which all worked without issues

Socket sends message only once

Code below works and sends message at scheduled time but I think it isn't a good solution to open new socket every time timer executes scheduled task. What I would like is to open socket in run method only once and access it in SendMessage class whenever new instance of class is made in timer. That way it doesn't work, it only sends one message and then stops sending. Also I would be happy for some critics about code or tips for making it thread safe.
public class Client implements Runnable{
// Client Constructor here
#Override
public void run(){
//SENDS ONLY ONE MESSAGE
pitcherSocket = new Socket(InetAddress.getByName(hostname), port);
Timer timer = new Timer();
timer.schedule(new SendMessage(), 0, 1000/mps);
}
private class SendMessage extends TimerTask{
private int id;
#Override
public void run() {
try
{ // THIS WORKS FINE, SENDS MESSAGES AT SCHEDULED TIME
pitcherSocket = new Socket(InetAddress.getByName(hostname), port);
OutputStream outToServer = pitcherSocket.getOutputStream();
DataOutputStream out = new DataOutputStream(outToServer);
out.writeInt(id);
out.flush();
}catch(IOException e)
{
e.printStackTrace();
}
}
}
}
EDIT: WHOLE CODE
CLIENT
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
public class Pitcher implements Runnable{
private int port;
private int mps;
private int size;
private String hostname;
private List<Integer> messageIds = Collections.synchronizedList(new ArrayList<Integer>());
private Socket pitcherSocket;
//constatns, integer is 4 bytes, long is 8 bytes
private static final int INT_SIZE = 4;
private static final int LONG_SIZE = 8;
public Pitcher(int port, int mps, int size, String hostname) {
this.port = port;
this.mps = mps;
this.size = size;
this.hostname = hostname;
}
#Override
public void run(){
System.out.println("Pitcher running...");
System.out.println();
Timer timer = new Timer();
timer.schedule(new SendMessage(), 0, 1000/mps);
timer.schedule(new DisplayStatistics(), 0, 1000/mps);
}
//Nested class that sends messages
private class SendMessage extends TimerTask{
private int numberOfSentMessages = 0;
private int id;
#Override
public void run() {
try {
pitcherSocket = new Socket(InetAddress.getByName(hostname), port);
OutputStream outToServer = pitcherSocket.getOutputStream();
DataOutputStream out = new DataOutputStream(outToServer);
//send message size
out.writeInt(size);
//message id is same as number of the sent message
id = numberOfSentMessages + 1;
out.writeInt(id);
messageIds.add(id);
//get system timestamp
long currentTimestamp = System.currentTimeMillis();
out.writeLong(currentTimestamp);
//fill in the rest-
byte[] rest = new byte[size - 2 * INT_SIZE - LONG_SIZE]; //message size(default 300 bytes) - size(4 bytes) - message id(4 bytse) - timestamp(8 bytes)
out.write(rest);
out.flush();
numberOfSentMessages++;
InputStream inFromServer = pitcherSocket.getInputStream();
DataInputStream in = new DataInputStream(inFromServer);
Integer catcherMessageSize = in.readInt();
Integer catcherId = in.readInt();
long catcherTimestamp = in.readLong();
System.out.println("Sent message: " + size + " " + id + " " + currentTimestamp + "...");
System.out.println("Received message: " + catcherMessageSize + " " + catcherId + " " + catcherTimestamp + "...");
System.out.println();
}catch(IOException e)
{
e.printStackTrace();
}
}
}
}
SERVER
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
public class Catcher implements Runnable{
private int port;
private String bind;
private ServerSocket serverSocket;
//constatns, integer is 4 bytes, long is 8 bytes
private static final int INT_SIZE = 4;
private static final int LONG_SIZE = 8;
public Catcher(int port, String bind) {
this.port = port;
this.bind = bind;
}
#Override
public void run() {
System.out.println("Catcher running...");
System.out.println();
try {
serverSocket = new ServerSocket(port, 100, InetAddress.getByName(bind));
}
catch (IOException e1) {
e1.printStackTrace();
}
while(true){
try
{
Socket server = serverSocket.accept();
DataInputStream in = new DataInputStream(server.getInputStream());
Integer pitcherMessageSize = in.readInt();
Integer pitcherId = in.readInt();
long pitcherTimestamp = in.readLong();
DataOutputStream out = new DataOutputStream(server.getOutputStream());
//message id and size are sent back
out.writeInt(pitcherMessageSize);
out.writeInt(pitcherId);
//send back current time
long currentTimestamp = System.currentTimeMillis();
out.writeLong(currentTimestamp);
//fill in the rest
byte[] rest = new byte[pitcherMessageSize - 2 * INT_SIZE - LONG_SIZE]; //message size(default 300 bytes) - size(4 bytes) - message id(4 bytes) - timestamp(8 bytes)
out.write(rest);
out.flush();
System.out.println("Received message: " + pitcherMessageSize + " " + pitcherId + " " + pitcherTimestamp + "...");
System.out.println("Sent message: " + pitcherMessageSize + " " + pitcherId + " " + currentTimestamp + "...");
System.out.println();
//server.close();
}
catch(SocketTimeoutException s){
System.out.println("Socket timed out!");
break;
}
catch(IOException e){
e.printStackTrace();
break;
}
}
}
}

Have you thought about making both socket and the DataOutputStream member variables of SendMessage. This is some code to give you a rough start. You will probably want to put some enhancements like checking whether the socket is open and being able to create a new one if the current one is closed...
private class SendMessage extends TimerTask {
private int id = 10;
private Socket pitchSocket;
private DataOutputStream out;
public SendMessage(Socket socket) {
this.pitchSocket = socket;
try{
out = new DataOutputStream(pitchSocket.getOutputStream());
} catch(IOException e) {
e.printStackTrace();
}
}
#Override
public void run() {
try {
out.writeInt(id);
out.flush();
} catch(IOException e) {
e.printStackTrace();
}
}
}

After being able to view the whole code I think that you definitely have some threading issues, though I think they are more heavily on the server side rather than the client side. Your server is single threaded. That means that you can only deal with one request at a time. You want a multithreaded server. I've refactored your code to create an example of Catcher which is multithreaded. I'm using the Thead class to do all of this which may be a little old fashioned. You might want to take a look at java.util.concurrent, they will probably have a more up to date.
package clientserver;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
public class Catcher implements Runnable{
private int port;
private String bind;
private ServerSocket serverSocket;
public Catcher(int port, String bind) {
this.port = port;
this.bind = bind;
}
#Override
public void run() {
System.out.println("Catcher running...");
System.out.println();
try {
serverSocket = new ServerSocket(port, 100, InetAddress.getByName(bind));
}
catch (IOException e1) {
e1.printStackTrace();
}
while(true){
try
{
new Thread(new CatcherHandler(serverSocket.accept())).start();
Thread.sleep(1000);
}
catch(SocketTimeoutException s){
System.out.println("Socket timed out!");
break;
}
catch(IOException e){
e.printStackTrace();
break;
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static void main(String[] argv){
new Thread( new Catcher(8093, "localhost")).start();;
}
}
class CatcherHandler implements Runnable{
Socket server;
DataOutputStream out;
DataInputStream in;
private static final int INT_SIZE = 4;
private static final int LONG_SIZE = 8;
public CatcherHandler(Socket server) {
super();
this.server = server;
try {
in = new DataInputStream(server.getInputStream());
out = new DataOutputStream(server.getOutputStream());
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
#Override
public void run() {
try{
if(in.available() > 0){
Integer pitcherMessageSize = in.readInt();
Integer pitcherId = in.readInt();
long pitcherTimestamp = in.readLong();
//message id and size are sent back
out.writeInt(pitcherMessageSize);
out.writeInt(pitcherId);
//send back current time
long currentTimestamp = System.currentTimeMillis();
out.writeLong(currentTimestamp);
//fill in the rest
byte[] rest = new byte[pitcherMessageSize - 2 * INT_SIZE - LONG_SIZE]; //message size(default 300 bytes) - size(4 bytes) - message id(4 bytes) - timestamp(8 bytes)
out.write(rest);
out.flush();
System.out.println("Received message: " + pitcherMessageSize + " " + pitcherId + " " + pitcherTimestamp + "...");
System.out.println("Sent message: " + pitcherMessageSize + " " + pitcherId + " " + currentTimestamp + "...");
System.out.println();
Thread.sleep(1000);
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{}
//server.close();
}
}
Additionally I refactored your client to be able to use one socket and be tread safe. Now SendMessage takes in a DataInputStream and a DataOutputSteam as it's arguments.
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
public class Pitcher implements Runnable{
private int port;
private int mps;
private int size;
private String hostname;
private List<Integer> messageIds = Collections.synchronizedList(new ArrayList<Integer>());
private Socket pitcherSocket;
private DataOutputStream out;
private DataInputStream in;
//constatns, integer is 4 bytes, long is 8 bytes
private static final int INT_SIZE = 4;
private static final int LONG_SIZE = 8;
public Pitcher(int port, int mps, int size, String hostname) {
this.port = port;
this.mps = mps;
this.size = size;
this.hostname = hostname;
try {
this.pitcherSocket = new Socket(InetAddress.getByName(hostname), port);
out = new DataOutputStream(pitcherSocket.getOutputStream());
in = new DataInputStream(pitcherSocket.getInputStream());
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String[] argv) throws Exception{
for(int i = 0; i < 10; i++){
new Thread(new Pitcher(8093, 1, 200, "localhost")).start();
Thread.sleep(1000);
}
Thread.sleep(10000);
}
#Override
public void run(){
System.out.println("Pitcher running...");
System.out.println();
Timer timer = new Timer();
timer.schedule(new SendMessage(out, in), 0, 1000);
//timer.schedule(new DisplayStatistics(), 0, 1000);
}
//Nested class that sends messages
private class SendMessage extends TimerTask{
private int numberOfSentMessages = 0;
private int id;
private DataOutputStream out;
private DataInputStream in;
public SendMessage(DataOutputStream out, DataInputStream in){
this.out = out;
this.in = in;
}
#Override
public void run() {
try {
long currentTimestamp = 0L;
synchronized(out){
//send message size
out.writeInt(size);
//message id is same as number of the sent message
id = numberOfSentMessages + 1;
out.writeInt(id);
messageIds.add(id);
//get system timestamp
currentTimestamp = System.currentTimeMillis();
out.writeLong(currentTimestamp);
//fill in the rest-
byte[] rest = new byte[size - 2 * INT_SIZE - LONG_SIZE]; //message size(default 300 bytes) - size(4 bytes) - message id(4 bytse) - timestamp(8 bytes)
out.write(rest);
out.flush();
}
numberOfSentMessages++;
long catcherTimestamp = 0L;
Integer catcherMessageSize;
Integer catcherId;
synchronized(in){
catcherMessageSize = in.readInt();
catcherId = in.readInt();
catcherTimestamp = in.readLong();
}
System.out.println("Sent message: " + size + " " + id + " " + currentTimestamp + "...");
System.out.println("Received message: " + catcherMessageSize + " " + catcherId + " " + catcherTimestamp + "...");
System.out.println();
Thread.sleep(1000);
}catch(IOException e)
{
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}

The Java Socket class is not thread safe. To have multiple threads access the same Socket object you would need to synchronize their actions. This could be done by providing all your SendMessage-threads with a common object that would then act as a lock. You would need an object for each socket operation you are planning to use (e.g. read and write). Then, refactor every action that does a call to the Socket object into separate methods and synchronize them around that object. E.g. for the read operation you could have a method called read() inside SendMessage that calls Socket.read and synchronize this method around the lock-object for read.
private class SendMessage extends TimerTask{
private Object readLock;
private Socket socket;
public SendMessage(Object readLock, Socket socket) {
this.readLock = readLock;
this.socket = socket;
}
public void readFromSocket() {
synchronized(readLock) {
socket.read();
}
}
#Override
public void run() {
readFromSocket();
// do other stuff
}
}

How to create a java Server that accepts client connections and then build a relay connection for a client pair

I want to create a server that can accept multiple connections and then bind 2 clients as a pair and forward the data between these 2 clients. But it is about multiple pairs of clients. I already have multithread server that can create a new thread for each new connected client. The problem for me is that these threads dont know of each other and somehow I have to connect 2 clients to a connection pair.
For now I just create these pair connection as this: I wait for the first client, then I wait for the second client and then open a thread for the input of client 1 that gets forwarded to client 2 and the other way around. This is not usable for multiple clients.
How can I do this decent?

The way I see it, a client would need to
establish a TCP(?) connection with your server,
identify itself
give the ID of the other client it wishes to talk to
The first that connects would have to be kept on hold (in some global table in your server) until the second client connects.
Once a pair of clients would have been recognized as interlocutors, you would create a pair of threads to forward the data sent by each client to the other one.
UPDATE: Example
ClientSocket.java
package matchmaker;
import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
public class ClientSocket implements Closeable {
private final Socket socket;
private final InputStream in;
private final OutputStream out;
private final String ownId;
private final String peerId;
public ClientSocket(Socket socket) throws IOException {
this.socket = socket;
this.in = socket.getInputStream();
this.out = socket.getOutputStream();
DataInputStream din = new DataInputStream(in);
this.ownId = din.readUTF();
this.peerId = din.readUTF();
}
public ClientSocket(String server, int port, String ownId, String peerId)
throws IOException {
this.socket = new Socket(server, port);
this.socket.setTcpNoDelay(true);
this.in = socket.getInputStream();
this.out = socket.getOutputStream();
this.ownId = ownId;
this.peerId = peerId;
DataOutputStream dout = new DataOutputStream(out);
dout.writeUTF(ownId);
dout.writeUTF(peerId);
}
public String getOwnId() {
return ownId;
}
public String getPeerId() {
return peerId;
}
public InputStream getInputStream() {
return in;
}
public OutputStream getOutputStream() {
return out;
}
#Override
public void close() throws IOException {
socket.close();
}
}
Matchmaker.java: the server
package matchmaker;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.ServerSocket;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
public class Matchmaker extends Thread {
private static final Logger LOG
= Logger.getLogger(Matchmaker.class.getName());
private final int port;
private final Map<ClientPair,ClientSocket> waiting = new HashMap<>();
public static void main(String[] args) {
try {
int port = 1234;
int st = 0;
for (String arg: args) {
switch (st) {
case 0:
switch (arg) {
case "-p":
st = 1;
break;
default:
System.out.println("Unknown option: " + arg);
return;
}
break;
case 1:
port = Integer.parseInt(arg);
st = 0;
break;
}
}
Matchmaker server = new Matchmaker(port);
server.start();
server.join();
} catch (InterruptedException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
private Matchmaker(int port) {
this.port = port;
setDaemon(true);
}
#Override
public void run() {
try {
ServerSocket server = new ServerSocket(port);
while (true) {
ClientSocket socket = new ClientSocket(server.accept());
ClientPair pair = new ClientPair(
socket.getOwnId(), socket.getPeerId());
ClientSocket other;
synchronized(this) {
other = waiting.remove(pair.opposite());
if (other == null) {
waiting.put(pair, socket);
}
}
if (other != null) {
LOG.log(Level.INFO, "Establishing connection for {0}",
pair);
establishConnection(socket, other);
} else {
LOG.log(Level.INFO, "Waiting for counterpart {0}", pair);
}
}
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
private void establishConnection(ClientSocket socket, ClientSocket other)
throws IOException {
Thread thread = new StreamCopier(
socket.getInputStream(), other.getOutputStream());
thread.start();
thread = new StreamCopier(
other.getInputStream(), socket.getOutputStream());
thread.start();
}
}
StreamCopier.java: a thread that reads from an InputStream and writes to an OutputStream
package matchmaker;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.logging.Level;
import java.util.logging.Logger;
public class StreamCopier extends Thread {
private static final Logger LOG
= Logger.getLogger(StreamCopier.class.getName());
private final InputStream in;
private final OutputStream out;
public StreamCopier(InputStream in, OutputStream out) {
this.in = in;
this.out = out;
setDaemon(true);
}
#Override
public void run() {
LOG.info("Start stream copier");
try {
for (int b = in.read(); b != -1; b = in.read()) {
out.write(b);
}
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
} finally {
LOG.info("End stream copier");
try {
out.close();
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
}
}
ClientPair.java: a pair of client IDs
package matchmaker;
public class ClientPair {
private final String client1;
private final String client2;
public ClientPair(String client1, String client2) {
this.client1 = client1;
this.client2 = client2;
}
public String getClient1() {
return client1;
}
public String getClient2() {
return client2;
}
public ClientPair opposite() {
return new ClientPair(client2, client1);
}
#Override
public int hashCode() {
int hash = 5;
hash = 73 * hash + client1.hashCode();
hash = 73 * hash + client2.hashCode();
return hash;
}
#Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final ClientPair other = (ClientPair) obj;
return client1.equals(other.client1) && client2.equals(other.client2);
}
#Override
public String toString() {
return "[" + client1 + "," + client2 + "]";
}
}
ReaderClient.java: a sample client that reads from the socket and writes to standard output
package matchmaker;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.logging.Level;
import java.util.logging.Logger;
public class ReaderClient {
private static final Logger LOG = Logger.getLogger(ReaderClient.class.getName());
public static void main(String[] args) {
try (ClientSocket client
= new ClientSocket("localhost", 1234, "reader", "writer")) {
Reader reader
= new InputStreamReader(client.getInputStream(), "UTF-8");
BufferedReader in = new BufferedReader(reader);
for (String s = in.readLine(); s != null; s = in.readLine()) {
System.out.println(s);
}
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
}
WriterClient.java: a sample client that writes to the socket
package matchmaker;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.logging.Level;
import java.util.logging.Logger;
public class WriterClient {
private static final Logger LOG = Logger.getLogger(ReaderClient.class.getName());
public static void main(String[] args) {
try (ClientSocket client
= new ClientSocket("localhost", 1234, "writer", "reader")) {
Writer writer
= new OutputStreamWriter(client.getOutputStream(), "UTF-8");
PrintWriter out = new PrintWriter(writer);
for (int i = 0; i < 30; ++i) {
out.println("Message line " + i);
}
out.flush();
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
}

Multithreading, Read multiple files and write to new Files

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class Test6 implements Runnable {
private File file;
private int totalNumberOfFiles = 0;
private static int nextFile = -1;
private static ArrayList<String> allFilesArrayList = new ArrayList<String>();
private static ExecutorService executorService = null;
public Test6(File file) {
this.file = file;
}
private String readFileToString(String fileAddress) {
FileInputStream stream = null;
MappedByteBuffer bb = null;
String stringFromFile = "";
try {
stream = new FileInputStream(new File(fileAddress));
FileChannel fc = stream.getChannel();
bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
/* Instead of using default, pass in a decoder. */
stringFromFile = Charset.defaultCharset().decode(bb).toString();
} catch (IOException e) {
System.out.println("readFileToString IOException");
e.printStackTrace();
} finally {
try {
stream.close();
} catch (IOException e) {
System.out.println("readFileToString IOException");
e.printStackTrace();
}
}
return stringFromFile;
}
private void toFile(String message, String fileName) {
try {
FileWriter fstream = new FileWriter("C:/Users/Nomi/Desktop/Workspace2/Test6/TestWritten/" + fileName);
System.out.println("printing to file: ".concat(fileName));
BufferedWriter out = new BufferedWriter(fstream);
out.write(message);
out.close();
} catch (Exception e) {
System.out.println("toFile() Exception");
System.err.println("Error: " + e.getMessage());
}
}
// private void listFilesForFolder(final File fileOrFolder) {
// String temp = "";
// if (fileOrFolder.isDirectory()) {
// for (final File fileEntry : fileOrFolder.listFiles()) {
// if (fileEntry.isFile()) {
// temp = fileEntry.getName();
// toFile(readFileToString(temp), "Copy".concat(temp));
// }
// }
// }
// if (fileOrFolder.isFile()) {
// temp = fileOrFolder.getName();
// toFile(readFileToString(temp), "Copy".concat(temp));
// }
// }
public void getAllFilesInArrayList(final File fileOrFolder) {
String temp = "";
System.out.println("getAllFilesInArrayList fileOrFolder.getAbsolutePath()" + fileOrFolder.getAbsolutePath());
if (fileOrFolder.isDirectory()) {
for (final File fileEntry : fileOrFolder.listFiles()) {
if (fileEntry.isFile()) {
temp = fileEntry.getAbsolutePath();
allFilesArrayList.add(temp);
}
}
}
if (fileOrFolder.isFile()) {
temp = fileOrFolder.getAbsolutePath();
allFilesArrayList.add(temp);
}
totalNumberOfFiles = allFilesArrayList.size();
for (int i = 0; i < allFilesArrayList.size(); i++) {
System.out.println("getAllFilesInArrayList path: " + allFilesArrayList.get(i));
}
}
public synchronized String getNextFile() {
nextFile++;
if (nextFile < allFilesArrayList.size()) {
// File tempFile = new File(allFilesArrayList.get(nextFile));
return allFilesArrayList.get(nextFile);
} else {
return null;
}
}
#Override
public void run() {
getAllFilesInArrayList(file);
executorService = Executors.newFixedThreadPool(allFilesArrayList.size());
while(nextFile < totalNumberOfFiles)
{
String tempGetFile = getNextFile();
File tempFile = new File(allFilesArrayList.get(nextFile));
toFile(readFileToString(tempFile.getAbsolutePath()), "Copy".concat(tempFile.getName()));
}
}
public static void main(String[] args) {
Test6 test6 = new Test6(new File("C:/Users/Nomi/Desktop/Workspace2/Test6/Test Files/"));
Thread thread = new Thread(test6);
thread.start();
// executorService.execute(test6);
// test6.listFilesForFolder(new File("C:/Users/Nomi/Desktop/Workspace2/Test6/"));
}
}
The programs' doing what's expected. It goes into the folder, grabs a file, reads it into a string and then writes the contents to a new file.
I would like to do this multi threaded. If the folder has N number of files, I need N number of threads. Also I would like to use executor framework if possible. I'm thinking that there can be a method along this line:
public synchronized void getAllFilesInArrayList() {
return nextFile;
}
So each new thread could pick the next file.
Thank you for your help.
Error:
Exception in thread "Thread-0" java.lang.IllegalArgumentException
at java.util.concurrent.ThreadPoolExecutor.<init>(ThreadPoolExecutor.java:589)
at java.util.concurrent.ThreadPoolExecutor.<init>(ThreadPoolExecutor.java:480)
at java.util.concurrent.Executors.newFixedThreadPool(Executors.java:59)
at Test6.run(Test6.java:112)
at java.lang.Thread.run(Thread.java:662)

Firstly, your approach to the problem will result in more synchronization and race condition worries than seems necessary. A simple strategy to keep your threads from racing would be this:
1) Have a dispatcher thread read all the file names in your directory.
2) For each file, have the dispatcher thread spawn a worker thread and hand off the file reference
3) Have the worker thread process the file
4) Make sure you have some sane naming convention for your output file names so that you don't get threads overwriting each other.
As for using an executor, a ThreadPoolExecutor would probably work well. Go take a look at the javadoc: http://docs.oracle.com/javase/1.5.0/docs/api/java/util/concurrent/ThreadPoolExecutor.html

Communicate with external C program with Java using stdin and stdout

What I'm trying to do is launch the C program executable inside the Java application and allow them to communicate with each other using stdin and stdout. The C program will wait for a command from the java app and echo it back. I've tested the java code with "gnugo --mode gtp" (gnugo with in gtp mode communicates with stdin and stdout) and it works fine but my doesn't work with my C code. Any suggestion would greatly appreciated.
C code
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(void) {
unsigned int byte_read;
char *string, *tok;
int cmd_id;
int len = 64;
string = (char *) malloc(len + 1);
while (1) {
byte_read = getline(&string,&byte_read, stdin);
if (byte_read == -1) {
printf("Error reading input\n");
free(string);
exit(0);
//
} else {
printf("Got command: %s\n", string);
}
}
return EXIT_SUCCESS;
}
Java code
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
public class Test {
private BlockingQueue<String> m_queue;
private PrintWriter print_out;
private BufferedReader bufIn;
private InputThread inputThread;
private PrintWriter printOut;
private Process p;
public static void main(String[] args) {
Test test = new Test();
test.start();
}
public void start(){
try
{
Runtime rt = Runtime.getRuntime() ;
p = rt.exec("path/to/the/c/program") ;
InputStream in = p.getInputStream() ;
OutputStream out = p.getOutputStream ();
InputStream err = p.getErrorStream();
printOut = new PrintWriter(out);
m_queue = new ArrayBlockingQueue<String>(10);
inputThread = new InputThread(in, m_queue);
inputThread.start();
//send a command to
printOut.println("sample command");
printOut.flush();
//p.destroy() ;
}catch(Exception exc){
System.out.println("Err " + exc.getMessage());
}
}
private void mainLoop(){
String line;
while (true){
try
{
System.out.println("Before");
line = bufIn.readLine();
System.out.println("After");
if (line != null)
System.out.println(line);
}
catch (IOException e)
{
System.out.println("Error readline " + e.getMessage());
return;
}
}
}
private class InputThread extends Thread
{
InputThread(InputStream in, BlockingQueue<String> queue)
{
bufIn = new BufferedReader(new InputStreamReader(in));
m_queue = queue;
}
public void run()
{
try
{
mainLoop();
}
catch (Throwable t)
{
}
}
}
}

Try flushing stdout before you exit, that might do better. Or at least explain in more detail what does happen.