Develop Reference

parallel file downloads using nio without creating threads per file download

I have tried a program which download files parallely using java.nio by creating a thread per file download.
package com.java.tftp.nio;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.DatagramChannel;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
/**
* This class is used to download files concurrently from tftp server by
* configuring the filenames, no of files.
*
* #author SHRIRAM
*
*/
public class TFTP_NIO_Client {
/**
* destination folder
* */
private String destinationFolder;
/**
* list of files names to download
* */
private List<String> fileNames;
/**
* integer indicates the number of files to download concurrently
* */
private int noOfFilesToDownload;
public TFTP_NIO_Client(List<String> fileNames, String destinationFolder,
int noOfFilesToDownload) {
this.destinationFolder = destinationFolder;
this.fileNames = fileNames;
this.noOfFilesToDownload = noOfFilesToDownload;
initializeHandlers();
}
/**
* This method creates threads to register the channel to process download
* files concurrently.
*
* #param noOfFilesToDownload
* - no of files to download
*/
private void initializeHandlers() {
for (int i = 0; i < noOfFilesToDownload; i++) {
try {
Selector aSelector = Selector.open();
SelectorHandler theSelectionHandler = new SelectorHandler(
aSelector, fileNames.get(i));
theSelectionHandler.start();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* Setup RRQ/WRQ packet Packet : | Opcode | FileName | 0 | mode | 0 |
* Filename -> Filename in array of bytes. 0 -> indicates end of file mode
* -> string in byte array 'netascii' or 'octet'
*
* #param aOpcode
* #param aMode
* #param aFileName
* #throws IOException
*/
private void sendRequest(int aOpcode, int aMode, String aFileName,
DatagramChannel aChannel, InetSocketAddress aAddress)
throws IOException {
// Read request packet
TFTPReadRequestPacket theRequestPacket = new TFTPReadRequestPacket();
aChannel.send(
theRequestPacket.constructReadRequestPacket(aFileName, aMode),
aAddress);
}
/**
* sends TFTP ACK Packet Packet : | opcode | Block# | opcode -> 4 -> 2 bytes
* Block -> block number -> 2bytes
*
* #param aBlock
*/
private ByteBuffer sendAckPacket(int aBlockNumber) {
// acknowledge packet
TFTPAckPacket theAckPacket = new TFTPAckPacket();
return theAckPacket.getTFTPAckPacket(aBlockNumber);
}
/**
* This class is used to handle concurrent downloads from the server.
*
* */
public class SelectorHandler extends Thread {
private Selector selector;
private String fileName;
/**
* flag to indicate the file completion.
* */
private boolean isFileReadFinished = false;
public SelectorHandler(Selector aSelector, String aFileName)
throws IOException {
this.selector = aSelector;
this.fileName = aFileName;
registerChannel();
}
private void registerChannel() throws IOException {
DatagramChannel theChannel = DatagramChannel.open();
theChannel.configureBlocking(false);
selector.wakeup();
theChannel.register(selector, SelectionKey.OP_READ);
sendRequest(Constants.OP_READ, Constants.ASCII_MODE, fileName,
theChannel, new InetSocketAddress(Constants.HOST,
Constants.TFTP_PORT));
}
#Override
public void run() {
process();
}
private void process() {
System.out.println("Download started for " + fileName + " ");
File theFile = new File(destinationFolder
+ fileName.substring(fileName.lastIndexOf("/")));
FileOutputStream theFout = null;
try {
theFout = new FileOutputStream(theFile);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
while (!isFileReadFinished) {
try {
if (selector.select() == 0) {
try {
// sleep 2sec was introduced because selector is
// thread safe but keys are not thread safe
Thread.sleep(2000);
} catch (InterruptedException e) {
continue;
}
continue;
}
Set<SelectionKey> theSet = selector.selectedKeys();
Iterator<SelectionKey> theSelectedKeys = theSet.iterator();
synchronized (theSelectedKeys) {
while (theSelectedKeys.hasNext()) {
SelectionKey theKey = theSelectedKeys.next();
theSelectedKeys.remove();
if (theKey.isReadable()) {
isFileReadFinished = read(theKey, theFout,
fileName);
if (!isFileReadFinished) {
theKey.interestOps(SelectionKey.OP_READ);
}
} else if (theKey.isWritable()) {
// there is no implementation for file write to
// server.
theKey.interestOps(SelectionKey.OP_READ);
}
}
}
} catch (IOException ie) {
ie.printStackTrace();
}
}
System.out.println("Download finished for " + fileName);
try {
if (selector.isOpen()) {
selector.close();
}
if (theFout != null) {
theFout.close();
}
} catch (IOException ie) {
}
}
}
/**
* #param aKey
* registered key for the selector
* #param aOutStream
* - file output stream to write the file contents.
* #return boolean
* #throws IOException
*/
private boolean read(SelectionKey aKey, OutputStream aOutStream,
String aFileName) throws IOException {
DatagramChannel theChannel = (DatagramChannel) aKey.channel();
// data packet
TFTPDataPacket theDataPacket = new TFTPDataPacket();
ByteBuffer theReceivedBuffer = theDataPacket.constructTFTPDataPacket();
SocketAddress theSocketAddress = theChannel.receive(theReceivedBuffer);
theReceivedBuffer.flip();
byte[] theBuffer = theReceivedBuffer.array();
byte[] theDataBuffer = theDataPacket.getDataBlock();
if (theDataPacket.getOpCode() == Constants.OP_DATA) {
int theLimit = theDataPacket.getLimit();
// checks the limit of the buffer because a packet with data less
// than 512 bytes of content signals that it is the last packet in
// transmission for this particular file
if (theLimit != Constants.MAX_BUFFER_SIZE
&& theLimit < Constants.MAX_BUFFER_SIZE) {
byte[] theLastBlock = new byte[theLimit];
System.arraycopy(theBuffer, 0, theLastBlock, 0, theLimit);
// writes the lastblock
aOutStream.write(theLastBlock);
// sends an acknowledgment to the server using TFTP packet
// block number
theChannel
.send(sendAckPacket((((theBuffer[2] & 0xff) << 8) | (theBuffer[3] & 0xff))),
theSocketAddress);
if (theChannel.isOpen()) {
theChannel.close();
}
return true;
} else {
aOutStream.write(theDataBuffer);
// sends an acknowledgment to the server using TFTP packet
// block number
theChannel
.send(sendAckPacket((((theBuffer[2] & 0xff) << 8) | (theBuffer[3] & 0xff))),
theSocketAddress);
return false;
}
} else if (Integer.valueOf(theBuffer[1]) == Constants.OP_ERROR) {
System.out.println("File : " + aFileName + " not found ");
handleError(theReceivedBuffer);
}
return false;
}
/**
* This method handles the error packet received from Server.
*
* #param aBuffer
*/
private void handleError(ByteBuffer aBuffer) {
// Error packet
new TFTPErrorPacket(aBuffer);
}
}
Is it possible to download multiple files in parallel using java.nio by not creating a thread per file download? If yes can anybody suggest a solution to proceed further.

I would provide an approach to achieve what you are aiming for :
Let L the list of files to be downloaded.
Create a Map M which will hold the mapping of File name to be downloaded and the corresponding Selector instance.
For each file F in L
Get Selector SK from M corresponding to F
Process the state of the Selector by checking for any of the events being ready.
If processing is complete then set the Selector corresponding to F as null. This will help in identifying files
whose
processing is completed.Alternatively, you can remove F from
L; so that the next time you are looping you only process files that are not yet completely downloaded.
The above being said, I am curious to understand why you would want to attempt such a feat? If the thought process behind this requirement is to reduce the number of threads to 1 then it is not correct. Remember, you would end up really taxing the single thread running and for sure your throughput would not necessarily be optimal since the single thread would be dealing with both network as well as disk I/O. Also, consider the case of encountering an exception while writing one of the several files to the disk - you would end up aborting the transfer for all the files; something I am sure you do not want.
A better and more scalable approach would be to poll selectors on a single thread, but hand off any I/O activity to a worker thread. A better approach still would be to read the techniques presented in Doug Lea's paper and implement them. In fact Netty library already implements this pattern and is widely used in production.

Permanent and persistent Socket connection in java

I've created a client-server connection, something like a chat system. Previously I was using a while loop on the client side, and it was waiting to read a message from the console every time (of course server has a while loop as well to serve forever). But now, I'm trying to first create a connection at the beginning of the session, and then occasionally send a message during the session, so to maintain a permanent and persistent connection.
Currently, without the while loop, the client closes the connection and I don't know how to find a workaround.
Here is the client code:
import java.net.*;
import java.io.*;
public class ControlClientTest {
private Socket socket = null;
// private BufferedReader console = null;
private DataOutputStream streamOut = null;
public static void main(String args[]) throws InterruptedException {
ControlClientTest client = null;
String IP="127.0.0.1";
client = new ControlClientTest(IP, 5555);
}
public ControlClientTest(String serverName, int serverPort) throws InterruptedException {
System.out.println("Establishing connection. Please wait ...");
try {
socket = new Socket(serverName, serverPort);
System.out.println("Connected: " + socket);
start();
} catch (UnknownHostException uhe) {
System.out.println("Host unknown: " + uhe.getMessage());
} catch (IOException ioe) {
System.out.println("Unexpected exception: " + ioe.getMessage());
}
String line = "";
// while (!line.equals(".bye")) {
try {
Thread.sleep(1000);
//TODO get data from input
// line = console.readLine();
line="1";
if(line.equals("1"))
line="1,123";
streamOut.writeUTF(line);
streamOut.flush();
} catch (IOException ioe) {
System.out.println("Sending error: " + ioe.getMessage());
}
// }
}
public void start() throws IOException {
// console = new BufferedReader(new InputStreamReader(System.in));
streamOut = new DataOutputStream(socket.getOutputStream());
}
}
And here is the Server code:
import java.awt.*;
import java.io.*;
import java.net.ServerSocket;
import java.net.Socket;
public class ControlServer {
private Socket socket = null;
private ServerSocket server = null;
private DataInputStream streamIn = null;
public static void main(String args[]) {
ControlServer server = null;
server = new ControlServer(5555);
}
public ControlServer(int port) {
try {
System.out
.println("Binding to port " + port + ", please wait ...");
server = new ServerSocket(port);
System.out.println("Server started: " + server);
System.out.println("Waiting for a client ...");
socket = server.accept();
System.out.println("Client accepted: " + socket);
open();
boolean done = false;
while (!done) {
try {
String line = streamIn.readUTF();
// TODO get the data and do something
System.out.println(line);
done = line.equals(".bye");
} catch (IOException ioe) {
done = true;
}
}
close();
} catch (IOException ioe) {
System.out.println(ioe);
}
}
public void open() throws IOException {
streamIn = new DataInputStream(new BufferedInputStream(
socket.getInputStream()));
}
public void close() throws IOException {
if (socket != null)
socket.close();
if (streamIn != null)
streamIn.close();
}
}

I would like to summarize some good practices regarding the stability of TCP/IP connections which I apply on a daily basis.
Good practice 1 : Built-in Keep-Alive
socket.setKeepAlive(true);
It automatically sends a signal after a period of inactivity and checks for a reply. The keep-alive interval is operating system dependent though, and has some shortcomings. But all by all, it could improve the stability of your connection.
Good practice 2 : SoTimeout
Whenver you perform a read (or readUTF in your case), your thread will actually block forever. In my experience this is bad practice for the following reasons: It's difficult to close your application. Just calling socket.close() is dirty.
A clean solution, is a simple read time-out (e.g. 200ms). You can do this with the setSoTimeoutmethod. When the read() method timeouts it will throw a SocketTimeoutException. (which is a subclass of IOException).
socket.setSoTimeout(timeoutInterval);
Here is an example to implement the loop. Please note the shutdown condition. Just set it to true, and your thread will die peacefully.
while (!shutdown)
{
try
{
// some method that calls your read and parses the message.
code = readData();
if (code == null) continue;
}
catch (SocketTimeoutException ste)
{
// A SocketTimeoutExc. is a simple read timeout, just ignore it.
// other IOExceptions will not be stopped here.
}
}
Good practice 3 : Tcp No-Delay
Use the following setting when you are often interfacing small commands that need to be handled quickly.
try
{
socket.setTcpNoDelay(true);
}
catch (SocketException e)
{
}
Good practice 4 : A heartbeat
Actually there are a lot of side scenario's that are not covered yet.
One of them for example are server applications that are designed to only communicate with 1 client at a time. Sometimes they accept connections and even accept messages, but never reply to them.
Another one: sometimes when you lose your connection it actually can take a long time before your OS notices this. Possibly due to the shortcomings described in good practice 3, but also in more complex network situations (e.g. using RS232-To-Ethernet converters, VMware servers, etc) this happens often.
The solution here is to create a thread that sends a message every x seconds and then waits for a reply. (e.g. every 15 seconds). For this you need to create a second thread that just sends a message every 15 seconds. Secondly, you need to expand the code of good practice 2 a little bit.
try
{
code = readData();
if (code == null) continue;
lastRead = System.currentTimeMillis();
// whenever you receive the heart beat reply, just ignore it.
if (MSG_HEARTBEAT.equals(code)) continue;
// todo: handle other messages
}
catch (SocketTimeoutException ste)
{
// in a typical situation the soTimeout is about 200ms
// the heartbeat interval is usually a couple of seconds.
// and the heartbeat timeout interval a couple of seconds more.
if ((heartbeatTimeoutInterval > 0) &&
((System.currentTimeMillis() - lastRead) > heartbeatTimeoutInterval))
{
// no reply to heartbeat received.
// end the loop and perform a reconnect.
break;
}
}
You need to decide if your client or server should send the message. That decision is not so important. But e.g. if your client sends the message, then your client will need an additional thread to send the message. Your server should send a reply when it receives the message. When your client receives the answer, it should just continue (i.e. see code above). And both parties should check: "how long has it been?" in a very similar way.

You could wrap a thread around the connection and have it periodically send a status to keep the line open, say every 30 seconds or whatever. Then, when it actually has data to send it would reset the keep alive to be 30 seconds after the last transmission. The status could be helpful to see if the client is still alive anyway, so at least it can be a useful ping.
Also, you should change your server code, you appear to only handle one connection at the moment. You should loop and when a socket connection comes in spawn a thread to handle the client request and go back to listening. I may be reading to much into what may just be your test code, though.

Make the client socket connection wrapped around a thread. Use a blocking queue to wait for messages. There should only be a single sender queue throughout your application, so use a singleton pattern.
e.g.
QueueSingleton queue = QueueSingleton.getSenderQueue();
Message message = queue.take() // blocks thread
send(message); //send message to server
When you need to send a message to the server, you can use the blocking queue to send the message.
QueueSingleton queue = QueueSingleton.getSenderQueue();
queue.put(message)
The client thread will wake up and process the message.
For maintaining the connection, use a timer task. This is special type of thread that calls a run method repetitively at specified periods. You can use this to post a message, a ping message, every so often.
For processing the received message, you could have another thread, waiting for messages on another blocking queue (receiver queue). The client thread will put the received message on this queue.

Async reading in server-client: separate client messages

I'm implementing a simple server using AsynchronousServerSocketChannel. For testing purposes, I created a tiny client prototype that sends two messages, "hi" and "stackoverflow", then disconnects. On server side, I read the arrived messages and print them to standard output. When the client executed, I'm expecting to receive:
message [hi], bytecount 2
message [stackoverflow], bytecount 13
The problem is, that sometimes both messages already arrived when server invokes reading callback so I get
message [histackoverflow], bytecount 15
instead.
The question is, if it is possible to ensure on server side that the messages arrive separately and if yes, how to do it?
Here's my CompletionHandler prototype that handles client connections:
class CommunicationHandler implements CompletionHandler<AsynchronousSocketChannel, Void> {
private final AsynchronousServerSocketChannel server;
public CommunicationHandler(final AsynchronousServerSocketChannel server) {
this.server = server;
}
#Override
public void failed(Throwable ex, Void attachment) {}
#Override
public void completed(final AsynchronousSocketChannel client, Void attachment) {
// handle client messages
final ByteBuffer buffer = ByteBuffer.allocateDirect(Server.BUFFER_SIZE);
final Session session = new Session();
try {
client.read(buffer, session, new CompletionHandler<Integer, Session>() {
#Override
public void completed(Integer byteCount, final Session currSession) {
if (byteCount == -1) {
return;
}
buffer.flip();
// TODO forward buffer to message handler (probably protocol?)
System.out.println("message [" + convertToString(buffer) + "], byteCount " + byteCount);
buffer.clear();
// read next message
client.read(buffer, currSession, this);
}
#Override
public void failed(Throwable ex, final Session currSession) {}
});
}
// accept the next connection
server.accept(null, this);
}
ByteBuffer to String conversion:
public static String convertToString(ByteBuffer bb) {
final byte[] bytes = new byte[bb.remaining()];
bb.duplicate().get(bytes);
return new String(bytes);
}
Here is a test client prototype:
public class Client {
public final void start() {
try (AsynchronousSocketChannel client = AsynchronousSocketChannel.open();) {
Future<Void> connCall = client.connect(InetAddress.getByName("127.0.0.1"), 8060));
connCall.get();
// client is now connected
// send greeting message
Future<Integer> writeCall = client.write(Charset.forName("utf-8").encode(CharBuffer.wrap("hi")));
writeCall.get();
// Thread.sleep(5000L);
writeCall = client.write(Charset.forName("utf-8").encode(CharBuffer.wrap("stackoverflow")));
writeCall.get();
client.close();
} catch (IOException e) {
} catch (InterruptedException ex) {
} catch (ExecutionException ex) {
}
}

In addition to the possibility of getting two (or even more) writes in one read, for larger messages (usually about 3k or more) you can get one write split over several reads. TCP is a stream protocol and does not preserve record boundaries, unless by chance: What is a message boundary? There are two solutions that work in general, although with async channel I think you'll need to do your own buffer management which may be confusing and hard to test:
add an explicit length field before each record
add a delimiter after each record when there is a byte not otherwise used, or an escape can be used to distinguish data from the delimiter
and several others that have been tried:
as your comment suggests, wait long enough that the first request has always been read before the second is sent. On the local networks and test systems used by developers this usually is a few milliseconds or even less; on the real Internet it is fairly often several seconds, sometimes minutes, and in theory can be hours or even days.
if records are never longer than a few fragments (maybe 10k or so) use UDP (available in Java as DatagramSocket but not as a NIO channel AFAICS) and implement your own protocols to handle message loss, duplication and reordering (which is hard to do and often ends up failing in some obscure cases that were discovered and avoided or fixed in TCP 30 years ago)
use SCTP (not available in Java at all AFAICS, and not too many other systems either)
Aside: your test client sends data in UTF-8, but new String (byte[]) uses the default encoding which is platform-dependent and not necessarily UTF-8. I'm not sure it's guaranteed but in practice all usable encodings include ASCII as a subset, and your example data is ASCII. But if you want to support actual UTF-8 data code for it.

Configuring JMS Message Size in WebLogic: weblogic.socket.MaxMessageSizeExceededException

I currently have two clients (Producer/Consumer), and I am trying to send a large file via JMS. I am successfully producing and sending the file to the JMS Server without any problems. The problem is when I try to consume the message, and I get the following exception:
Aug 24, 2012 11:25:37 AM client.Client$1 onException
SEVERE: Connection to the Server has been lost, will retry in 30 seconds. weblogic.jms.common.LostServerException: java.lang.Exception: weblogic.rjvm.PeerGoneException: ; nested exception is:
weblogic.socket.MaxMessageSizeExceededException: Incoming message of size: '10000080' bytes exceeds the configured maximum of: '10000000' bytes for protocol: 't3'
<Aug 24, 2012 11:25:37 AM CDT> <Error> <Socket> <BEA-000403> <IOException occurred on socket: Socket[addr=127.0.0.1/127.0.0.1,port=7001,localport=51764]
weblogic.socket.MaxMessageSizeExceededException: Incoming message of size: '10000080' bytes exceeds the configured maximum of: '10000000' bytes for protocol: 't3'.
weblogic.socket.MaxMessageSizeExceededException: Incoming message of size: '10000080' bytes exceeds the configured maximum of: '10000000' bytes for protocol: 't3'
at weblogic.socket.BaseAbstractMuxableSocket.incrementBufferOffset(BaseAbstractMuxableSocket.java:174)
at weblogic.rjvm.t3.MuxableSocketT3.incrementBufferOffset(MuxableSocketT3.java:351)
at weblogic.socket.SocketMuxer.readFromSocket(SocketMuxer.java:983)
at weblogic.socket.SocketMuxer.readReadySocketOnce(SocketMuxer.java:922)
I believe this has to do with my MaxMessage size setting in WebLogic and not a code problem (but I could of course be wrong). Here are my settigns for Maximum Message Size
I am not sure why I am getting this exception since the Maximum message size for this protocol is larger than what the exception is claiming... Any thoughts?
I have also tried adding the server start argument -Dweblogic.MaxMessageSize=200000000, but to no avail.
Here is some code of my where I set the MessageListener, and the message consumer itself.
public boolean setClient(MessageListener listener) {
try {
Properties parm = new Properties();
parm.setProperty("java.naming.factory.initial",
"weblogic.jndi.WLInitialContextFactory");
parm.setProperty("java.naming.provider.url", iProps.getURL());
parm.setProperty("java.naming.security.principal", iProps.getUser());
parm.setProperty("java.naming.security.credentials",
iProps.getPassword());
ctx = new InitialContext(parm);
final QueueConnectionFactory connectionFactory = (QueueConnectionFactory) ctx
.lookup(iProps.getConFactory());
connection = connectionFactory.createQueueConnection();
((WLConnection) connection)
.setReconnectPolicy(JMSConstants.RECONNECT_POLICY_ALL);
((WLConnection) connection).setReconnectBlockingMillis(30000L);
((WLConnection) connection).setTotalReconnectPeriodMillis(-1L);
session = connection.createQueueSession(false,
Session.AUTO_ACKNOWLEDGE);
queue = (Queue) ctx.lookup(iProps.getQueue());
// The following code in the switch statement is the only code that
// differs for the producer and consumer.
switch (cType)
{
case PRODUCER: {
producer = (WLMessageProducer) session
.createProducer(queue);
// Setting to send large files is done in WebLogic
// Administration Console.
// Set
producer.setSendTimeout(60000L);
break;
}
case CONSUMER: {
consumer = session.createConsumer(queue);
if (listener != null) {
consumer.setMessageListener(listener);
}else{
log.warning("A Message listener was not set for the consumer, messages will not be acknowledged!");
}
break;
}
default:
log.info("The client type " + cType.toString()
+ " is not currently supported!");
return false;
}
connection.setExceptionListener(new ExceptionListener() {
#Override
public void onException(JMSException arg0) {
Logger log2 = new MyLogger().getLogger("BPEL Client");
if (arg0 instanceof LostServerException) {
log2.severe("Connection to the Server has been lost, will retry in 30 seconds. "
+ arg0.toString());
} else {
log2.severe(arg0.toString());
}
}
});
shutdownListener = new ShutdownListener(connection, session, producer, consumer);
connection.start();
log.info("Successfully connected to " + iProps.getURL());
return true;
} catch (JMSException je) {
log.severe("Could not connect to the WebLogic Server, will retry in 30 seconds. "
+ je.getMessage());
try {
Thread.sleep(30000L);
} catch (InterruptedException e) {
log.warning(e.toString());
}
return setClient(listener);
} catch (Exception e) {
log.severe("Could not connect to the WebLogic Server, will retry in 30 seconds. "
+ e.toString());
try {
Thread.sleep(30000L);
} catch (InterruptedException ie) {
log.warning(ie.toString());
}
return setClient(listener);
}
}
Here's the MessageListener:
public class ConsumerListener implements MessageListener {
private Logger log;
private File destination;
private Execute instructions;
public ConsumerListener(Execute instructions, File destination) {
this.instructions = instructions;
this.destination = destination;
log = new MyLogger().getLogger("BPEL Client");
}
#Override
public void onMessage(Message arg0) {
try {
if (arg0.getJMSRedelivered()) {
log.severe("A re-delivered message has been received, and it has been ignored!"
+ arg0.toString());
} else {
try {
if (arg0 instanceof TextMessage) {
consumeMessage((TextMessage) arg0);
} else if (arg0 instanceof BytesMessage) {
consumeMessage((BytesMessage) arg0);
} else {
log.warning("Currently, only TextMessages and BytesMessages are supported!");
}
} catch (JMSException e) {
log.severe(e.toString());
} catch (IOException e) {
log.severe(e.toString());
} catch (Throwable t) {
log.severe(t.toString());
}
}
} catch (JMSException e) {
log.severe(e.toString());
}
}
/**
* Unwraps the JMS message received and creates a file and a control file if
* there are instructions present.
*
* #param textMessage
* JMS message received to be consumed.
* #throws JMSException
* #throws IOException
*/
protected void consumeMessage(TextMessage textMessage) throws JMSException,
IOException {
// ***All properties should be lowercase. for example fileName
// should be
// filename.***
String fileName = textMessage.getStringProperty("filename");
if (fileName == null || fileName.isEmpty()) {
fileName = textMessage.getStringProperty("fileName");
}
if (fileName != null && !fileName.isEmpty()) {
// Check if the
// file name is equal to the shutdown file. If it
// is, shutdown the consumer. This is probably not a good way to
// do this, as the program can no longer be shutdown locally!
// We have a file in the queue, need to create the file.
createFile(destination.getAbsolutePath() + "\\" + fileName,
textMessage.getText());
log.info("Done creating the file");
String inst = textMessage.getStringProperty("instructions");
// If there are instructions included, then create the
// instruction file, and route the message based on this file.
if (inst != null && !inst.isEmpty()) {
// We need to rout the file.
log.info("Instructions found, executing instructions");
String[] tokens = fileName.split("\\.");
String instFileName = "default.ctl";
if (tokens.length == 2) {
instFileName = tokens[0] + ".ctl";
}
File controlFile = createFile(destination.getAbsolutePath()
+ "\\" + instFileName, inst);
Control control = new Control(controlFile);
instructions.execute(control);
log.info("Done executing instructions");
} else {
log.info("No instructions were found");
}
log.info("Done consuming message: " + textMessage.getJMSMessageID());
}
}
/**
* Unwraps the JMS message received and creates a file and a control file if
* there are instructions present.
*
* #param bytesMessage
* The bytes payload of the message.
* #throws JMSException
* #throws IOException
*/
protected void consumeMessage(BytesMessage bytesMessage)
throws JMSException, IOException {
// ***All properties should be lowercase. for example fileName
// should be
// filename.***
log.info("CONSUME - 1");
String fileName = bytesMessage.getStringProperty("filename");
if (fileName == null || fileName.isEmpty()) {
fileName = bytesMessage.getStringProperty("fileName");
}
if (fileName != null && !fileName.isEmpty()) {
// Check if the
// file name is equal to the shutdown file. If it
// is, shutdown the consumer. This is probably not a good way to
// do this, as the program can no longer be shutdown locally!
// We have a file in the queue, need to create the file.
byte[] payload = new byte[(int) bytesMessage.getBodyLength()];
bytesMessage.readBytes(payload);
createFile(destination.getAbsolutePath() + "\\" + fileName, payload);
log.info("Done creating the file");
String inst = bytesMessage.getStringProperty("instructions");
// If there are instructions included, then create the
// instruction file, and route the message based on this file.
if (inst != null && !inst.isEmpty()) {
// We need to rout the file.
log.info("Instructions found, executing instructions");
String[] tokens = fileName.split("\\.");
String instFileName = "default.ctl";
if (tokens.length == 2) {
instFileName = tokens[0] + ".ctl";
}
File controlFile = createFile(destination.getAbsolutePath()
+ "\\" + instFileName, inst);
Control control = new Control(controlFile);
instructions.execute(control);
log.info("Done executing instructions");
} else {
log.info("No instructions were found");
}
log.info("Done consuming message: "
+ bytesMessage.getJMSMessageID());
}
}
/**
* Creates a file with the given filename (this should be an absolute path),
* and the text that is to be contained within the file.
*
* #param fileName
* The filename including the absolute path of the file.
* #param fileText
* The text to be contained within the file.
* #return The newly created file.
* #throws IOException
*/
protected File createFile(String fileName, String fileText)
throws IOException {
File toCreate = new File(fileName);
FileUtils.writeStringToFile(toCreate, fileText);
return toCreate;
}
/**
* Creates a file with the given filename (this should be an absolute path),
* and the text that is to be contained within the file.
*
* #param fileName
* The filename including the absolute path of the f ile.
* #param fileBytes
* The bytes to be contained within the file.
* #return The newly created file.
* #throws IOException
*/
protected File createFile(String fileName, byte[] fileBytes)
throws IOException {
File toCreate = new File(fileName);
FileUtils.writeByteArrayToFile(toCreate, fileBytes);
return toCreate;
}
}

You will also have to increase the Maximum Message Size from the WLS console as shown in the screenshot for all the managed servers.
Afterwards perform a restart and the problem will be solved.
Furthermore there is a second alternative solution. According the Oracle Tuning WebLogic JMS Doc:
Tuning MessageMaximum Limitations
If the aggregate size of the messages pushed to a consumer is larger than the current protocol's maximum message size (default size is 10 MB and is configured on a per WebLogic Server instance basis using the console and on a per client basis using the Dweblogic.MaxMessageSize command line property), the message delivery fails.
Setting Maximum Message Size on a Client
You may need to configure WebLogic clients in addition to the WebLogic Server instance, when sending and receiving large messages. To set the maximum message size on a client, use the following command line property:
-Dweblogic.MaxMessageSize
Note: This setting applies to all WebLogic Server network packets delivered to the client, not just JMS related packets.
EDITED:
This issue can be resolved by following one or more of the below actions.
Configure the System Property -Dweblogic.MaxMessageSize
Increase the max message size using the WLS console for the admin and all the managed servers. The steps in the WLS console are: server / Protocols / General
Increase the Maximum Message Size from WLS console. The steps in the WLS console are: Queue / Configuration / Threshholds and Quotas / Maximum Message Size
PROCEDURE for applying the weblogic.MaxMessageSize Property
Keep a backup of the setDomainEnv files. Stop all the servers. Add the -Dweblogic.MaxMessageSize=yourValue in each setDomainEnv file and more specifically in the EXTRA_JAVA_PROPERTIES line. Afterwards start the ADMIN first and when the ADMIN is in status RUNNING, then start the MANAGED servers.
I hope this helps.

In my case setting the -Dweblogic.MaxMessageSize solved the issue. My questions is what should be the maximum limit of the message size? We just can not keep on increasing the
message size to resolve this issue. Is there any way to optimise this value in addition
with certain other values?

ConcurrentModificationException exception with iterator in multithread chat server

I'm creating a multithread chat server in java.
When user u1 logs in and sends a message to user u2, if user u2 is not connected the message is sent to the server and put in an ArrayList of pending messages. When user u2 connects, he receive the message from the server and send a message to user u1 as a receipt.
This is my code:
if (pendingmsgs.size()>0) {
for(Iterator<String> itpendingmsgs = pendingmsgs.iterator(); itpendingmsgs.hasNext();) {
//....parsing of the message to get the recipient, sender and text
String pendingmsg = itpendingmsgs.next();
if (protocol.author != null && protocol.author.equals(recipient)) {
response+=msg;
protocol.sendMsg(sender, "Msg "+text+" sent to "+recipient);
itpendingmsgs.remove();
}
}
}
out.write(response.getBytes(), 0, response.length());
This is the ServerProtocol sendMsg() method:
private boolean sendMsg(String recip, String msg) throws IOException {
if (nicks.containsKey(recip)) { //if the recipient is logged in
ClientConnection c = nick.get(recipient); //get the client connection
c.sendMsg(msg); //sends the message
return true;
} else {
/* if the recipient is not logged in I save the message in the pending messages list */
pendingmsgs.add("From: "+nick+" to: "+recip+" text: "+msg);
return false;
}
}
and this is the ClientConnection sendMsg() method:
public void sendMsg(String msg) throws IOException {
out.write(msg.getBytes(), 0, msg.length());
}
where out is an OutputStream.
When user u1 logs in, sends a message to user u2 who is not logged in and then user u1 leaves, when user u2 logs in he doesn't receive the message and I get this exception:
Exception in thread "Thread-2" java.util.ConcurrentModificationException
at java.util.AbstractList$Itr.checkForComodification(Unknown Source)
at java.util.AbstractList$Itr.remove(Unknown Source)
at ChatServer$ClientConnection.run(ChatServer.java:400)
at java.lang.Thread.run(Unknown Source)
Line 400 is
itpendingmsgs.remove();
I've tried using a CopyOnWriteArrayList but it still doesn't work.

CopyOnWriteArrayList.iterator() doesn't support remove(). You should probably use a Collections.synchronizedList(ArrayList) (properly locked during iteration as specified in the Javadoc).
That's really the simplest way to allow one thread to add to the list and the other to iterate through removing elements.

Most probably after looking at your code, issue seems to be that while you are looping through your iterator you add new content to the ArrayList in sendMsg method
protocol.sendMsg(sender, "Msg "+text+" sent to "+recipient); // this line invokes the code which adds
pendingmsgs.add("From: "+nick+" to: "+recip+" text: "+msg); // this line adds a new item
See this discussion for reason why this happened last time around.
Edit: As per comment
line 400 is itpendingmsgs.remove();
This is definitely because of addition in the list, as when you reach itpendingmsgs.remove();, you have already added a new entry in the list which makes your iterator complain.
Update:
Appraches to fix this issue:
Instead of Iterator use ListIterator and add, remove objects from the ListIterator and not underlying List.
Update Sample Code :
package com.mumz.test.listiterator;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.Random;
/**
* Test Class to show case List Iterator.
*/
public class TestListIterator {
/** The pendingmsgs. */
List<String> pendingmsgs = new ArrayList<String>();
/**
* Add some content to the list and then start processing the same list.
*/
private void init() {
addContentToList();
doProcessing();
}
/**
* Add test content to list.
*/
private void addContentToList() {
for (int iDx = 0; iDx < 10; iDx++) {
pendingmsgs.add("Message " + iDx);
}
}
/**
* Randomly decide if message should be added or removed, showcase iteration using list iterator.
*/
private void doProcessing() {
if (pendingmsgs.size() > 0) {
for(ListIterator<String> listIterator = pendingmsgs.listIterator(); listIterator.hasNext();){
String currentMessage = listIterator.next();
Random random = new Random();
int nextInt = random.nextInt(100);
if((nextInt % 2) == 0){
sendMsg(currentMessage, listIterator);
} else {
listIterator.remove();
}
}
}
}
/**
* Add new content to the list using listIterator of the underlying list.
*
* #param msg
* the msg
* #param listIterator
* the list iterator
* #return true, if successful
*/
private boolean sendMsg(String msg, ListIterator<String> listIterator) {
Random random = new Random();
int nextInt = random.nextInt(10);
// Randomly add new message to list
if ((nextInt % 2) == 0) {
listIterator.add("New Messsage : " + msg);
return false;
}
return true;
}
/**
* The main method.
*
* #param args
* the arguments
*/
public static void main(String[] args) {
try {
TestListIterator testListIterator = new TestListIterator();
testListIterator.init();
System.out.println(testListIterator);
} catch (Exception e) {
e.printStackTrace();
}
}
/* (non-Javadoc)
* #see java.lang.Object#toString()
*/
#Override
public String toString() {
return String.format("TestListIterator [pendingmsgs=%s]", pendingmsgs);
}
}
Instead of using Iterator or ListIterator just use normal for or while loop, in this case you can directly modify your collection (list in this case) without getting this exception.
Use Iterator itself but dont add new elements into the list while you are looping.
Add your messages to another list say tempMessageHolder so sendMsg will add message to this list.
Once your loop is complete, add all the messages from tempMessageHolder to your main list pendingmsgs