Develop Reference

How to read Meta data in gRPC using Java at client side

I am using Java and Protoc 3.0 compiler and my proto file is mention below.
https://github.com/openconfig/public/blob/master/release/models/rpc/openconfig-rpc-api.yang
syntax = "proto3";
package Telemetry;
// Interface exported by Agent
service OpenConfigTelemetry {
// Request an inline subscription for data at the specified path.
// The device should send telemetry data back on the same
// connection as the subscription request.
rpc telemetrySubscribe(SubscriptionRequest) returns (stream OpenConfigData) {}
// Terminates and removes an exisiting telemetry subscription
rpc cancelTelemetrySubscription(CancelSubscriptionRequest) returns (CancelSubscriptionReply) {}
// Get the list of current telemetry subscriptions from the
// target. This command returns a list of existing subscriptions
// not including those that are established via configuration.
rpc getTelemetrySubscriptions(GetSubscriptionsRequest) returns (GetSubscriptionsReply) {}
// Get Telemetry Agent Operational States
rpc getTelemetryOperationalState(GetOperationalStateRequest) returns (GetOperationalStateReply) {}
// Return the set of data encodings supported by the device for
// telemetry data
rpc getDataEncodings(DataEncodingRequest) returns (DataEncodingReply) {}
}
// Message sent for a telemetry subscription request
message SubscriptionRequest {
// Data associated with a telemetry subscription
SubscriptionInput input = 1;
// List of data models paths and filters
// which are used in a telemetry operation.
repeated Path path_list = 2;
// The below configuration is not defined in Openconfig RPC.
// It is a proposed extension to configure additional
// subscription request features.
SubscriptionAdditionalConfig additional_config = 3;
}
// Data associated with a telemetry subscription
message SubscriptionInput {
// List of optional collector endpoints to send data for
// this subscription.
// If no collector destinations are specified, the collector
// destination is assumed to be the requester on the rpc channel.
repeated Collector collector_list = 1;
}
// Collector endpoints to send data specified as an ip+port combination.
message Collector {
// IP address of collector endpoint
string address = 1;
// Transport protocol port number for the collector destination.
uint32 port = 2;
}
// Data model path
message Path {
// Data model path of interest
// Path specification for elements of OpenConfig data models
string path = 1;
// Regular expression to be used in filtering state leaves
string filter = 2;
// If this is set to true, the target device will only send
// updates to the collector upon a change in data value
bool suppress_unchanged = 3;
// Maximum time in ms the target device may go without sending
// a message to the collector. If this time expires with
// suppress-unchanged set, the target device must send an update
// message regardless if the data values have changed.
uint32 max_silent_interval = 4;
// Time in ms between collection and transmission of the
// specified data to the collector platform. The target device
// will sample the corresponding data (e.g,. a counter) and
// immediately send to the collector destination.
//
// If sample-frequency is set to 0, then the network device
// must emit an update upon every datum change.
uint32 sample_frequency = 5;
}
// Configure subscription request additional features.
message SubscriptionAdditionalConfig {
// limit the number of records sent in the stream
int32 limit_records = 1;
// limit the time the stream remains open
int32 limit_time_seconds = 2;
}
// Reply to inline subscription for data at the specified path is done in
// two-folds.
// 1. Reply data message sent out using out-of-band channel.
// 2. Telemetry data send back on the same connection as the
// subscription request.
// 1. Reply data message sent out using out-of-band channel.
message SubscriptionReply {
// Response message to a telemetry subscription creation or
// get request.
SubscriptionResponse response = 1;
// List of data models paths and filters
// which are used in a telemetry operation.
repeated Path path_list = 2;
}
// Response message to a telemetry subscription creation or get request.
message SubscriptionResponse {
// Unique id for the subscription on the device. This is
// generated by the device and returned in a subscription
// request or when listing existing subscriptions
uint32 subscription_id = 1;
}
// 2. Telemetry data send back on the same connection as the
// subscription request.
message OpenConfigData {
// router name:export IP address
string system_id = 1;
// line card / RE (slot number)
uint32 component_id = 2;
// PFE (if applicable)
uint32 sub_component_id = 3;
// Path specification for elements of OpenConfig data models
string path = 4;
// Sequence number, monotonically increasing for each
// system_id, component_id, sub_component_id + path.
uint64 sequence_number = 5;
// timestamp (milliseconds since epoch)
uint64 timestamp = 6;
// List of key-value pairs
repeated KeyValue kv = 7;
}
// Simple Key-value, where value could be one of scalar types
message KeyValue {
// Key
string key = 1;
// One of possible values
oneof value {
double double_value = 5;
int64 int_value = 6;
uint64 uint_value = 7;
sint64 sint_value = 8;
bool bool_value = 9;
string str_value = 10;
bytes bytes_value = 11;
}
}
// Message sent for a telemetry subscription cancellation request
message CancelSubscriptionRequest {
// Subscription identifier as returned by the device when
// subscription was requested
uint32 subscription_id = 1;
}
// Reply to telemetry subscription cancellation request
message CancelSubscriptionReply {
// Return code
ReturnCode code = 1;
// Return code string
string code_str = 2;
};
// Result of the operation
enum ReturnCode {
SUCCESS = 0;
NO_SUBSCRIPTION_ENTRY = 1;
UNKNOWN_ERROR = 2;
}
// Message sent for a telemetry get request
message GetSubscriptionsRequest {
// Subscription identifier as returned by the device when
// subscription was requested
// --- or ---
// 0xFFFFFFFF for all subscription identifiers
uint32 subscription_id = 1;
}
// Reply to telemetry subscription get request
message GetSubscriptionsReply {
// List of current telemetry subscriptions
repeated SubscriptionReply subscription_list = 1;
}
// Message sent for telemetry agent operational states request
message GetOperationalStateRequest {
// Per-subscription_id level operational state can be requested.
//
// Subscription identifier as returned by the device when
// subscription was requested
// --- or ---
// 0xFFFFFFFF for all subscription identifiers including agent-level
// operational stats
// --- or ---
// If subscription_id is not present then sent only agent-level
// operational stats
uint32 subscription_id = 1;
// Control verbosity of the output
VerbosityLevel verbosity = 2;
}
// Verbosity Level
enum VerbosityLevel {
DETAIL = 0;
TERSE = 1;
BRIEF = 2;
}
// Reply to telemetry agent operational states request
message GetOperationalStateReply {
// List of key-value pairs where
// key = operational state definition
// value = operational state value
repeated KeyValue kv = 1;
}
// Message sent for a data encoding request
message DataEncodingRequest {
}
// Reply to data encodings supported request
message DataEncodingReply {
repeated EncodingType encoding_list = 1;
}
// Encoding Type Supported
enum EncodingType {
UNDEFINED = 0;
XML = 1;
JSON_IETF = 2;
PROTO3 = 3;
}
In order to do the service call (rpc TelemetrySubscribe) first i need to read header which have subscription id and then start reading messages. Now, using Java i am able to connect with the service, i did introduce the interceptor but when i print/retrieve header it is null. My code of calling interceptor is below,
ClientInterceptor interceptor = new HeaderClientInterceptor();
originChannel = OkHttpChannelBuilder.forAddress(host, port)
.usePlaintext(true)
.build();
Channel channel = ClientInterceptors.intercept(originChannel, interceptor);
telemetryStub = OpenConfigTelemetryGrpc.newStub(channel);
This is interceptor code to read meta Data.
#Override
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(MethodDescriptor<ReqT, RespT> method,
CallOptions callOptions, Channel next) {
return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, callOptions)) {
#Override
public void start(Listener<RespT> responseListener, Metadata headers) {
super.start(new SimpleForwardingClientCallListener<RespT>(responseListener) {
#Override
public void onHeaders(Metadata headers) {
Key<String> CUSTOM_HEADER_KEY = Metadata.Key.of("responseKEY", Metadata.ASCII_STRING_MARSHALLER);
System.out.println("Contains Key?? "+headers.containsKey(CUSTOM_HEADER_KEY));
Wondering is there any other way to read meta data or first message which have subscription ID in it? All i need to read first message which have subscription Id, and return the same subscription id to server so that streaming can start I have equivalent Python code using same proto file and it is communicating with server by code mention below for reference only:
sub_req = SubscribeRequestMsg("host",port)
data_itr = stub.telemetrySubscribe(sub_req, _TIMEOUT_SECONDS)
metadata = data_itr.initial_metadata()
if metadata[0][0] == "responseKey":
metainfo = metadata[0][1]
print metainfo
subreply = agent_pb2.SubscriptionReply()
subreply.SetInParent()
google.protobuf.text_format.Merge(metainfo, subreply)
if subreply.response.subscription_id:
SUB_ID = subreply.response.subscription_id
From the python code above i can easily retrieve meta data object, not sure how to retrieve same using Java?
After reading metaData all i am getting is: Metadata({content-type=[application/grpc], grpc-encoding=[identity], grpc-accept-encoding=[identity,deflate,gzip]})
But i know there is one more line from meta data to it, which is
response {
subscription_id: 2
}
How can i extract last response from Header which have subscription id in it. I did try many options and i am lost here.

The method you used is for request metadata, not response metadata:
public void start(Listener<RespT> responseListener, Metadata headers) {
For response metadata, you will need a ClientCall.Listener and wait for the onHeaders callback:
public void onHeaders(Metadata headers)
I do feel like the usage of metadata you mention seems strange. Metadata is generally for additional error details or cross-cutting features that aren't specific to the RPC method (like auth, tracing, etc.).

Often times using the ClientInterceptor is inconvenient because you need to maintain a reference to it in order to pull the data back out. In your case, the data is actually Metadata. One way you can get access to the Metadata easier is by putting it inside of the Context.
For example, you could create a Context.Key for the subscription id. In your client interceptor, you could extract the Metadata header that you want, and put it inside the Context, using Context.current().withValue(key, metadata). Inside your StreamObserver, you can extract this This by calling key.get(Context.current()). This assumes you are using the Async API, rather than the blocking API.
The reason it is more difficult is because usually metadata is information about a call, but not directly related to the call itself. It is for things like tracing, encoding, stats, cancellation and things like that. If something changes the way you handle the request, it probably needs to go directly into the request itself, rather than being on the side.

In case this helps anyone else out:
I needed to write a specific response header in my gRPC-java server based upon the request/response.
What I ended up doing was storing the response header value in a Context using the Context::withValue (which doesn't modify the existing context but instead creates a new Context actually), and then calling the request service handler method's StreamObserver::onNext inside of the Context::run callback. StremableObserver::onNext calls the ServerCall::sendHeaders I have in my set from my ServerInterceptor. There in the sendHeaders, it can read the value in the Context I stored and set the response header value.
I think this is similar to #carl-mastrangelo's approach, just may be spelled out a little bit more.
public enum MyServerInterceptor implements ServerInterceptor {
INSTANCE;
public static final Metadata.Key<String> METADATA_KEY =
Metadata.Key.of("fish", ASCII_STRING_MARSHALLER);
public static final Context.Key<String> CONTEXT_KEY = Context.key("dog");
#Override
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
Metadata requestHeaders,
ServerCallHandler<ReqT, RespT> next) {
ServerCall<ReqT, RespT> myServerCall = new ForwardingServerCall.SimpleForwardingServerCall<ReqT, RespT>(call) {
#Override
public void sendHeaders(Metadata responseHeaders) {
String value = CONTEXT_KEY.get(Context.current());
responseHeaders.put(METADATA_KEY, value);
super.sendHeaders(responseHeaders);
}
};
return next.startCall(myServerCall, requestHeaders);
}
}
public class MyService extends MyServiceGrpc.MyServiceyImplBase {
#Override
public void serviceMethod(MyRequest request, StreamObserver<MyResponse> responseObserver) {
MyResponse response = new MyResponse();
Context.current().withValue(Context.key("dog"), "cat").run(() -> {
responseObserver.onNext(response);
});
responseObserver.onCompleted();
}
}
The again the critical piece here is that responseObserver::onNext calls into ForwardingServerCall.SimpleForwardingServerCall::sendHeaders.
Please let me know if there is a better way. This all seems more complex than I'd like.

Delete top message from MQQueue

I am constructing a messaging system using MQSeries. For some reason, when I perform q.get(...), I am getting an exception thrown (I don't know the specific MQException). Below is the code causing the error:
private static MQGetMessageOptions GMO = new MQGetMessageOptions();
private static int GMO_OPTIONS = MQC.MQGMO_SYNCPOINT | MQC.MQGMO_WAIT;
GMO.options = GMO.options | GMO_OPTIONS;
GMO.waitInterval = MQC.MQWI_UNLIMITED;
MQEnvironment.hostname = args[0];
MQEnvironment.channel = args[2];
MQEnvironment.port = Integer.parseInt(args[1]);
MQQueueManager queueManager = new MQQueueManager(args[3])
MQMessage msg = new MQMessage();
MQQueue q = queueManager.accessQueue("qName1",MQC.MQOO_OUTPUT);
q.get(msg, GMO);
My plan is, when this error occurs, skip the message and delete it. To perform the delete I will call the following function:
private void deleteMsg(MQQueueManager queueManager, String queueName) throws MQException {
MQGetMessageOptions tempGmo = new MQGetMessageOptions();
tempGmo.options |= MQC.MQGMO_WAIT;
tempGmo.waitInterval = 1000;
MQQueue remover = queueManager.accessQueue(queueName, MQC.MQOO_INPUT_AS_Q_DEF);
remover.get(new MQMessage(), tempGmo);
queueManager.commit();
}
Would the remover.get() in my deleteMsg function also, in this specific scenario, fail for the same reason? Or does the option used to construct the MQQueue(MQC.MQOO_INPUT_AS_Q_DEF vs MQC.MQOO_OUTPUT) prevent it from also failing? If I am having trouble accessing my queue's message, how do I discard the top message and move to the next?
To shorten my question:
If I am unable to perform a get() on a given queue to retrieve a message, how can we delete that corrupt message on the same queue?
Thank you!

OMG!
MQQueue q = queueManager.accessQueue("qName1",MQC.MQOO_OUTPUT);
q.get(msg, GMO);
Your are opening a queue for output (writing) but you are trying to get a message. You have your shoes on the wrong feet!! Secondly, why aren't you catching the MQException that MQ would be throwing?? The exception would have included the reason code which would have given you the exact explanation to your issue.
Here's how you should be opening the queue for reading:
try
{
int oo = MQC.MQOO_INPUT_SHARED + MQC.MQOO_FAIL_IF_QUIESCING;
MQQueue q = queueManager.accessQueue("qName1",oo);
MQGetMessageOptions gmo = new MQGetMessageOptions();
gmo.options = MQC.MQGMO_NO_WAIT + MQC.MQGMO_FAIL_IF_QUIESCING;
q.get(msg, gmo);
}
catch (MQException e)
{
System.err.println(e.getLocalizedMessage() );
System.err.println("CC = " + e.completionCode + " - RC = " + e.reasonCode);
}
Also, make sure you use the appropriate "Fail if quiescing" option for the particular MQ API call.
Finally, look up "backout queue". If your application is having an issue with a message then the message should be moved to a backout queue and not simply deleted.

I do not why what you are doing does not work for you but I wonder why are you using proprietary API of MQ Series instead of using JMS API. In JMS terms remove top message just means receive the message, so call of session.receieve() does the work.
Using common JMS API has a lot of advantages. The main of them is that you can easily move from MQ Series to any other messaging solution without changing even one line of your code.

I wonder if the program compiled because there is no option called GMO_OPTIONS. All MQ constants are prefixed MQC

How can I create a node recursively using zookeeper client library on Java?

I know this question has been already asked and answered for a zookeeper using python. The answer was good, however, I want something more related with the code. I've already implemented a method to create a node, but I want to do it recursively. The structure for my nodes will be like this:
ZOOKEEPER
WEB SERVER
SERVER1
SERVER2
MODULE CONNECTED
DATABASE MODULE
COMPUTER1
COMPUTER2
SERVICE MODULE
COMPUTER3
SEARCH MODULE
COMPUTER4
I have something like:
Zookeeper zk = new Zookeeper(...);
public void createNodeRecursively(String type) {
final String node = "/" + type + "/" + info.getIP() + ":" + info.getPort(); // Correct line
if (zk.exists("/" + type, null) == null) {
Object ctx = new Object();
StringCallback cb = new StringCallback() {
public void processResult(int rc, String path,
Object ctx, String name) {
if (name.equals("/" + type))// just in case
try {
zk.create(node, info.getBytes(),
Ids.OPEN_ACL_UNSAFE,CreateMode.EPHEMERAL);
} catch (Exception e) {
e.printStackTrace();
}
}
};
zk.create("/" + type, info.getBytes(), Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT, cb, ctx);
} else
zk.create(node, info.getBytes(), Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL);
}
}
As you can see I am using zk.create many times, so I want to make the method recursive in order to gain performance and have a better code, but I don't know how to start, I'll be very grateful if somebody can help me with this. Thank you very much in advance.

Zookeeper has useful properties:
Total order of (write) requests
Its asynchronous nature.
You can put on use that.
Simply issue whole tree as a bunch of asynchronous requests in correct order and then wait, until all of them successfully execute. Of course, you can ignore 'NodeExists' exceptions (but it is not good, due of the fact, that such errors will be written to logs).

I managed to achieve a better performance:
public void createNode(NodePath nodePath, NodeData nodeData, NodeRights nodeRights, NodeCreationHandler nodeCreationHandler) throws KeeperException, InterruptedException, ZookeeperCreationException {
if (zk == null) {
throw new ZookeeperCreationException("The zookeeper client has not been instanced.");
}
String targetPath = nodePath.getFullNodePath();
targetPath = targetPath.substring(1, targetPath.length());
byte[] serializedData = nodeData.serialize(new Object());
String[] array = targetPath.split(ICoordinationConstants.BASE_ROOT_SPTR);
String acum="";
for (int i = 0; i < array.length-1; i++) {
acum+=(ICoordinationConstants.BASE_ROOT_SPTR+array[i]);
if (zk.exists(acum, null) == null) {
zk.create(acum, serializedData, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
}
zk.create(acum+ICoordinationConstants.BASE_ROOT_SPTR+array[array.length-1], serializedData, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
}

java websphere MQ

My aim is to put n number of messages in a for loop to a WebSphere MQ queue using WebSphere MQ java programming.
My java program will run as a standalone program.
If any exception in between , I need to rollback all the messages.
If no exception then I should commit all the messages .
The outside world should not see my messages in the queue until I complete fully.
How do I achieve this?
Updated with sample code as per reply from T.Rob:
Please check if sample code is fine ?
Does setting MQGMO_SYNCPOINT is only related to my program's invocation ?
(because similar programs running parallely will also be putting messages on the same queue and those messages should not gett affected by my program's SYNCPOINT.)
public void sendMsg() {
MQQueue queue = null;
MQQueueManager queueManager = null;
MQMessage mqMessage = null;
MQPutMessageOptions pmo = null;
System.out.println("Entering..");
try {
MQEnvironment.hostname = "x.x.x.x";
MQEnvironment.channel = "xxx.SVRCONN";
MQEnvironment.port = 9999;
queueManager = new MQQueueManager("XXXQMANAGER");
int openOptions = MQConstants.MQOO_OUTPUT;
queue = queueManager.accessQueue("XXX_QUEUENAME", openOptions, null, null, null);
pmo = new MQPutMessageOptions();
pmo.options = CMQC.MQGMO_SYNCPOINT;
String input = "testing";
System.out.println("sending messages....");
for (int i = 0; i < 10; i++) {
input = input + ": " + i;
mqMessage = new MQMessage();
mqMessage.writeString(input);
System.out.println("Putting message: " + i);
queue.put(mqMessage, pmo);
}
queueManager.commit();
System.out.println("Exiting..");
} catch (Exception e) {
e.printStackTrace();
try {
System.out.println("rolling back messages");
if (queueManager != null)
queueManager.backout();
} catch (MQException e1) {
e1.printStackTrace();
}
} finally {
try {
if (queue != null)
queue.close();
if (queueManager != null)
queueManager.close();
} catch (MQException e) {
e.printStackTrace();
}
}
}

WMQ supports both local and global (XA) units of work. The local units of work are available simply by specifying the option. Global XA transactions require a transaction manager, as mentioned by keithkreissl in another answer.
For what you described, a POJO doing messaging under syncpoint, specify MQC.MQGMO_SYNCPOINT in your MQGetMessageOptions. When you are ready to commit, issue the MQQManager.commit() or MQQManager.backout() call.
Note that the response and doc provided by ggrandes refers to the JMS and not Java classes. The Java classes use Java equivalents of the WMQ procedural API, can support many threads (doc) and even provide connection pooling (doc). Please refer to the Java documentation rather than the JMS documentation for the correct behavior. Also, I've linked to the WMQ V7.5 documentation which goes with the latest WMQ Java V7.5 client. The later clients have a lot more local functionality (tracing, flexible install path, MQClient.ini, etc.) and work with back-level QMgrs. It is highly recommended to be using the latest client and the download is free.

you only need to create a session with transaction enabled.
Session session;
// ...
boolean transacted = true;
session = connection.createSession(transacted, Session.AUTO_ACKNOWLEDGE);
try {
// ...do things...
session.commit();
} catch (Exception e) {
session.rollback();
}
// ...
WARN-NOTE: Sessions are not thread-safe ;-)
Doc Websphere MQ/JMS

If you have access to a transaction manager and more importantly an XATransaction wired up to your MQ access, you can start a transaction at the beginning of your message processing put all the messages on the queue then commit the transaction. Using the XATransactions it will not put any messages until the transaction commits. If you don't have access to that, you can do a little more plumbing by placing your messages in a local data object, wrap your code in a try/catch if no exceptions iterate through the local data object sending the messages. The issue with the later approach is that it will commit all your other processing but if a problem occurs in the sending of messages your other processing will not be rolled back.

JavaMail IMAP over SSL quite slow - Bulk fetching multiple messages

I am currently trying to use JavaMail to get emails from IMAP servers (Gmail and others). Basically, my code works: I indeed can get the headers, body contents and so on. My problem is the following: when working on an IMAP server (no SSL), it basically takes 1-2ms to process a message. When I go on an IMAPS server (hence with SSL, such as Gmail) I reach around 250m/message. I ONLY measure the time when processing the messages (the connection, handshake and such are NOT taken into account).
I know that since this is SSL, the data is encrypted. However, the time for decryption should not be that important, should it?
I have tried setting a higher ServerCacheSize value, a higher connectionpoolsize, but am seriously running out of ideas. Anyone confronted with this problem? Solved it one might hope?
My fear is that the JavaMail API uses a different connection each time it fetches a mail from the IMAPS server (involving the overhead for handshake...). If so, is there a way to override this behavior?
Here is my code (although quite standard) called from the Main() class:
public static int connectTest(String SSL, String user, String pwd, String host) throws IOException,
ProtocolException,
GeneralSecurityException {
Properties props = System.getProperties();
props.setProperty("mail.store.protocol", SSL);
props.setProperty("mail.imaps.ssl.trust", host);
props.setProperty("mail.imaps.connectionpoolsize", "10");
try {
Session session = Session.getDefaultInstance(props, null);
// session.setDebug(true);
Store store = session.getStore(SSL);
store.connect(host, user, pwd);
Folder inbox = store.getFolder("INBOX");
inbox.open(Folder.READ_ONLY);
int numMess = inbox.getMessageCount();
Message[] messages = inbox.getMessages();
for (Message m : messages) {
m.getAllHeaders();
m.getContent();
}
inbox.close(false);
store.close();
return numMess;
} catch (MessagingException e) {
e.printStackTrace();
System.exit(2);
}
return 0;
}
Thanks in advance.

after a lot of work, and assistance from the people at JavaMail, the source of this "slowness" is from the FETCH behavior in the API. Indeed, as pjaol said, we return to the server each time we need info (a header, or message content) for a message.
If FetchProfile allows us to bulk fetch header information, or flags, for many messages, getting contents of multiple messages is NOT directly possible.
Luckily, we can write our own IMAP command to avoid this "limitation" (it was done this way to avoid out of memory errors: fetching every mail in memory in one command can be quite heavy).
Here is my code:
import com.sun.mail.iap.Argument;
import com.sun.mail.iap.ProtocolException;
import com.sun.mail.iap.Response;
import com.sun.mail.imap.IMAPFolder;
import com.sun.mail.imap.protocol.BODY;
import com.sun.mail.imap.protocol.FetchResponse;
import com.sun.mail.imap.protocol.IMAPProtocol;
import com.sun.mail.imap.protocol.UID;
public class CustomProtocolCommand implements IMAPFolder.ProtocolCommand {
/** Index on server of first mail to fetch **/
int start;
/** Index on server of last mail to fetch **/
int end;
public CustomProtocolCommand(int start, int end) {
this.start = start;
this.end = end;
}
#Override
public Object doCommand(IMAPProtocol protocol) throws ProtocolException {
Argument args = new Argument();
args.writeString(Integer.toString(start) + ":" + Integer.toString(end));
args.writeString("BODY[]");
Response[] r = protocol.command("FETCH", args);
Response response = r[r.length - 1];
if (response.isOK()) {
Properties props = new Properties();
props.setProperty("mail.store.protocol", "imap");
props.setProperty("mail.mime.base64.ignoreerrors", "true");
props.setProperty("mail.imap.partialfetch", "false");
props.setProperty("mail.imaps.partialfetch", "false");
Session session = Session.getInstance(props, null);
FetchResponse fetch;
BODY body;
MimeMessage mm;
ByteArrayInputStream is = null;
// last response is only result summary: not contents
for (int i = 0; i < r.length - 1; i++) {
if (r[i] instanceof IMAPResponse) {
fetch = (FetchResponse) r[i];
body = (BODY) fetch.getItem(0);
is = body.getByteArrayInputStream();
try {
mm = new MimeMessage(session, is);
Contents.getContents(mm, i);
} catch (MessagingException e) {
e.printStackTrace();
}
}
}
}
// dispatch remaining untagged responses
protocol.notifyResponseHandlers(r);
protocol.handleResult(response);
return "" + (r.length - 1);
}
}
the getContents(MimeMessage mm, int i) function is a classic function that recursively prints the contents of the message to a file (many examples available on the net).
To avoid out of memory errors, I simply set a maxDocs and maxSize limit (this has been done arbitrarily and can probably be improved!) used as follows:
public int efficientGetContents(IMAPFolder inbox, Message[] messages)
throws MessagingException {
FetchProfile fp = new FetchProfile();
fp.add(FetchProfile.Item.FLAGS);
fp.add(FetchProfile.Item.ENVELOPE);
inbox.fetch(messages, fp);
int index = 0;
int nbMessages = messages.length;
final int maxDoc = 5000;
final long maxSize = 100000000; // 100Mo
// Message numbers limit to fetch
int start;
int end;
while (index < nbMessages) {
start = messages[index].getMessageNumber();
int docs = 0;
int totalSize = 0;
boolean noskip = true; // There are no jumps in the message numbers
// list
boolean notend = true;
// Until we reach one of the limits
while (docs < maxDoc && totalSize < maxSize && noskip && notend) {
docs++;
totalSize += messages[index].getSize();
index++;
if (notend = (index < nbMessages)) {
noskip = (messages[index - 1].getMessageNumber() + 1 == messages[index]
.getMessageNumber());
}
}
end = messages[index - 1].getMessageNumber();
inbox.doCommand(new CustomProtocolCommand(start, end));
System.out.println("Fetching contents for " + start + ":" + end);
System.out.println("Size fetched = " + (totalSize / 1000000)
+ " Mo");
}
return nbMessages;
}
Do not that here I am using message numbers, which is unstable (these change if messages are erased from the server). A better method would be to use UIDs! Then you would change the command from FETCH to UID FETCH.
Hope this helps out!

You need to add a FetchProfile to the inbox before you iterate through the messages.
Message is a lazy loading object, it will return to the server for each message and for each
field that doesn't get provided with the default profile.
e.g.
for (Message message: messages) {
message.getSubject(); //-> goes to the imap server to fetch the subject line
}
If you want to display like an inbox listing of say just From, Subject, Sent, Attachement etc.. you would use something like the following
inbox.open(Folder.READ_ONLY);
Message[] messages = inbox.getMessages(start + 1, total);
FetchProfile fp = new FetchProfile();
fp.add(FetchProfile.Item.ENVELOPE);
fp.add(FetchProfileItem.FLAGS);
fp.add(FetchProfileItem.CONTENT_INFO);
fp.add("X-mailer");
inbox.fetch(messages, fp); // Load the profile of the messages in 1 fetch.
for (Message message: messages) {
message.getSubject(); //Subject is already local, no additional fetch required
}
Hope that helps.

The total time includes the time required in cryptographic operations. The cryptographic operations need a random seeder. There are different random seeding implementations which provide random bits for use in the cryptography. By default, Java uses /dev/urandom and this is specified in your java.security as below:
securerandom.source=file:/dev/urandom
On Windows, java uses Microsoft CryptoAPI seed functionality which usually has no problems. However, on unix and linux, Java, by default uses /dev/random for random seeding. And read operations on /dev/random sometimes block and takes long time to complete. If you are using the *nix platforms then the time spent in this would get counted in the overall time.
Since, I dont know what platform you are using, I can't for sure say that this could be your problem. But if you are, then this could be one of reasons why your operations are taking long time. One of the solution to this could be to use /dev/urandom instead of /dev/random as your random seeder, which does not block. This can be specified with the system property "java.security.egd". For example,
-Djava.security.egd=file:/dev/urandom
Specifying this system property will override the securerandom.source setting in your java.security file. You can give it a try. Hope it helps.