Develop Reference

Ignite thin client does not reconnect if all nodes in a cluster go down & come back up

I am evaluating Ignite as a caching layer for our architecture. When trying out Ignite Java thin client for the use case mentioned below, I do not find any pointer(s) in ignite doc/any forum as to how this is being tackled by the ignite community. Any pointer(s) will be helpful before I go ahead and use my custom solution.
Use case: All nodes in an ignite cluster go down and come back up. Basically, thin client loses connection to all cluster nodes for some time.
What I was expecting
I am using continuous query and register for disconnect events. Hence, I was expecting some disconnect event which I never got. Reference code below.
public static QueryCursor<Cache.Entry<String, String>> subscribeForDataUpdates(ClientCache<String, String> entityCache,
AtomicLong totalUpdatesTracker) {
ClientDisconnectListener disconnectListener = reason ->
System.out.printf("Client: %s received disconnect event with reason:%s %n",
getClientIpAddr(),
reason.getMessage());
ContinuousQuery<String, String> continuousQuery = new ContinuousQuery<>();
continuousQuery.setLocalListener(new CacheUpdateListener(entityCache.getName(), totalUpdatesTracker));
QueryCursor<Cache.Entry<String, String>> queryCursor = entityCache.query(continuousQuery, disconnectListener);
System.out.printf("Client: %s - subscribed for change notification(s) for entity cache: %s %n",
getClientIpAddr(),
entityCache.getName());
return queryCursor;
}
What I ended up doing
Writing my own checker to re-initialize the thin client connection to ignite cluster and re-subscribing for continuous query updates.
import io.vavr.control.Try;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.client.IgniteClient;
import javax.cache.Cache;
import javax.inject.Inject;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static com.cisco.ignite.consumer.CacheChangeSubscriber.subscribeForDataUpdates;
import static com.cisco.ignite.consumer.Utils.addShutDownHookToCloseCacheUpdates;
import static com.cisco.ignite.consumer.Utils.getClientIpAddr;
public class ClusterConnectionChecker implements Runnable {
private static final List<QueryCursor<Cache.Entry<String, String>>> querySubscriptions = new ArrayList<>();
#Inject
private CacheChangeSubscriber cacheChangeSubscriber;
private IgniteClient thinClientInstance;
private final long secondsDelayBetweenChecks;
private final List<String> cacheNames;
private final AtomicLong totalUpdatesTracker;
private boolean needsReSubscription = false;
public ClusterConnectionChecker(IgniteClient client, long delayBetweenChecks,
List<String> cacheNames, AtomicLong totalUpdatesTracker) {
this.thinClientInstance = client;
this.secondsDelayBetweenChecks = delayBetweenChecks;
this.cacheNames = cacheNames;
this.totalUpdatesTracker = totalUpdatesTracker;
}
#Override
public void run() {
while(!Thread.interrupted()) {
try {
Thread.sleep(TimeUnit.SECONDS.toMillis(secondsDelayBetweenChecks));
boolean isClusterConnectionActive = isConnectionToClusterActive();
if (!isClusterConnectionActive) {
needsReSubscription = true;
System.out.printf("Time: %s | Connection to ignite cluster is not active !!! %n",
LocalDateTime.now());
reInitializeThinClient();
reSubscribeForUpdates();
} else {
// we only need to conditionally re-subscribe
if (needsReSubscription) {
reSubscribeForUpdates();
}
}
} catch (InterruptedException ie) {
// do nothing - just reset the interrupt flag.
Thread.currentThread().interrupt();
}
}
}
private boolean isConnectionToClusterActive() {
return Try.of(() -> {
return thinClientInstance.cluster().state().active();
}).recover(ex -> {
return false;
}).getOrElse(false);
}
private void reInitializeThinClient() {
Try.of(() -> {
thinClientInstance = cacheChangeSubscriber.createThinClientInstance();
if (thinClientInstance.cluster().state().active()) {
System.out.printf("Client: %s | Thin client instance was re-initialized since it was not active %n",
getClientIpAddr());
}
return thinClientInstance;
}).onFailure(th -> System.out.printf("Client: %s | Failed to re-initialize ignite cluster connection. " +
"Will re-try after:%d seconds %n", getClientIpAddr(),secondsDelayBetweenChecks));
}
private void reSubscribeForUpdates() {
if (isConnectionToClusterActive()) {
System.out.printf("Client: %s | Re-subscribing for cache updates after cluster connection re-init... %n",
getClientIpAddr());
// re-set the counter to 0 since we are re-subscribing fresh
totalUpdatesTracker.set(0);
cacheNames.forEach(name -> querySubscriptions.add(subscribeForDataUpdates(
thinClientInstance.getOrCreateCache(name),
totalUpdatesTracker)));
addShutDownHookToCloseCacheUpdates(querySubscriptions, thinClientInstance);
needsReSubscription = false;
}
}
}

How to check is a Websocket connection is alive

I have a websocket connection to a server:
import javax.websocket.*;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
#ClientEndpoint
public class WebsocketExample {
private Session userSession;
private void connect() {
try {
WebSocketContainer container = ContainerProvider.getWebSocketContainer();
container.connectToServer(this, new URI("someaddress"));
} catch (DeploymentException | URISyntaxException | IOException e) {
e.printStackTrace();
}
}
#OnOpen
public void onOpen(Session userSession) {
// Set the user session
this.userSession = userSession;
System.out.println("Open");
}
#OnClose
public void onClose(Session userSession, CloseReason reason) {
this.userSession = null;
System.out.println("Close");
}
#OnMessage
public void onMessage(String message) {
// Do something with the message
System.out.println(message);
}
}
After some time, it seems I don't receive any more messages from the server but the onClose method has not been called.
I would like to have a sort of timer that would at least log an error (and at best try to reconnect) if I did not receive any message during the last five minutes for instance. The timer would be reset when I receive a new message.
How can I do this?

Here is what I did. I changed javax.websocket by jetty and implemented a ping call:
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.eclipse.jetty.websocket.client.WebSocketClient;
import java.io.IOException;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
#WebSocket
public class WebsocketExample {
private Session userSession;
private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
private void connect() {
try {
SslContextFactory sslContextFactory = new SslContextFactory();
WebSocketClient client = new WebSocketClient(sslContextFactory);
client.start();
client.connect(this, new URI("Someaddress"));
} catch (Exception e) {
e.printStackTrace();
}
}
#OnWebSocketConnect
public void onOpen(Session userSession) {
// Set the user session
this.userSession = userSession;
System.out.println("Open");
executorService.scheduleAtFixedRate(() -> {
try {
String data = "Ping";
ByteBuffer payload = ByteBuffer.wrap(data.getBytes());
userSession.getRemote().sendPing(payload);
} catch (IOException e) {
e.printStackTrace();
}
},
5, 5, TimeUnit.MINUTES);
}
#OnWebSocketClose
public void onClose(int code, String reason) {
this.userSession = null;
System.out.println("Close");
}
#OnWebSocketMessage
public void onMessage(String message) {
// Do something with the message
System.out.println(message);
}
}
Edit: This is just a ping example... I don't know if all servers are supposed to answer by a pong...
Edit2: Here is how to deal with the pong message. The trick was not to listen for String messages, but to Frame messages:
#OnWebSocketFrame
#SuppressWarnings("unused")
public void onFrame(Frame pong) {
if (pong instanceof PongFrame) {
lastPong = Instant.now();
}
}
To manage server time out, I modified the scheduled task as follows:
scheduledFutures.add(executorService.scheduleAtFixedRate(() -> {
try {
String data = "Ping";
ByteBuffer payload = ByteBuffer.wrap(data.getBytes());
userSession.getRemote().sendPing(payload);
if (lastPong != null
&& Instant.now().getEpochSecond() - lastPong.getEpochSecond() > 60) {
userSession.close(1000, "Timeout manually closing dead connection.");
}
} catch (IOException e) {
e.printStackTrace();
}
},
10, 10, TimeUnit.SECONDS));
... and handle the reconnection in the onClose method

You should work around this problem by implementing a heartbeat system which one side sends ping and one side answers with pong. Almost every websocket client and server (as far as I know) support this feature internally. This ping/pong frames could be sent from both sides. I usually implement it on server side because I usually know it has better chance to stay alive than clients (my opinion). If clients dont send back pong for long time, I know the connection is dead. On client side, I check the same: If server has not sent ping messages for a long time, I know connection is dead.
If ping/pong are not implemented in libraries you use (which I think javax websocket has it) you could make your own protocol for that.

The accepted answer uses Jetty specific API. There's a standard API for this:
to send ping: session.getAsyncRemote().sendPing(data)
to send pong (just keep-alive, without answer) session.getAsyncRemote().sendPong(data)
to react to pongs either session.addMessageHandler(handler) where handler implements MessageHandler.Whole<PongMessage> or create a method that is annotated with #OnMessage and has PongMessage param:
#OnMessage
public void onMessage(PongMessage pong) {
// check if the pong has the same payload as ping that was sent etc...
}
Periodic ping/keep-alive sending can be scheduled for example using ScheduledExecutorService just as the accepted answer does, but proper care of synchronization must be taken: if session.getBasicRemote() is used then all calls to the remote need to be synchronized. In case of session.getAsyncRemote() probably all containers except Tomcat handle synchronization automatically: see the discussion in this bug report.
Finally, it's important to cancel the pinging task (ScheduledFuture obtained from executor.scheduleAtFixedRate(...)) in onClose(...).
I've developed a simple WebsocketPingerService to ease up things (available in maven central). Create an instance and store it somewhere as a static var:
public Class WhicheverClassInYourApp {
public static WebsocketPingerService pingerService = new WebsocketPingerService();
// more code here...
}
You can configure ping interval, ping size, failure limit after which sessions should be closed, etc by passing arguments to the constructor.
After that register your endpoints for pinging in onOpen(...) and deregister in onClose(...):
#ClientEndpoint // or #ServerEndpoint -> pinging can be done from both ends
public class WebsocketExample {
private Session userSession;
#OnOpen
public void onOpen(Session userSession) {
this.userSession = userSession;
WhicheverClassInYourApp.pingerService.addConnection(userSession);
}
#OnClose
public void onClose(Session userSession, CloseReason reason) {
WhicheverClassInYourApp.pingerService.removeConnection(userSession);
}
// other methods here
}

No EntityManager Error in Play Framework 2.5 recieving data in another thread

using the play Framework 2.5 for Java
I want to create an application that consumes data from ActiveMQ and from this message received executes one hibernate query to database and store one object User.
I had simplified my consumer class in order to create one object there an make the call to the persistence at the same time.
my Problem comes when I call the persistence from the thread (or invoke another class) that receives the measurement. Then the EntityManager is not present there and I can not create a new one, or be frank I dont know to make it in Play.
package controllers;
import javax.jms.Connection;
import javax.jms.Destination;
import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.apache.activemq.ActiveMQConnectionFactory;
import models.User;
import play.Logger;
import play.db.jpa.JPA;
public class MessageConsumerController implements Runnable, ExceptionListener {
private static final String TOPIC_NAME = "miguelTopic";
private static Thread consumerService;
public static synchronized void initService() {
Logger.info("Message Consumer initialized");
MessageConsumerController MessageConsumer = new MessageConsumerController();
if (consumerService != null) {
Logger.info("STOPPING MessageConsumer thread.");
consumerService.interrupt();
}
Logger.info("Starting MessageConsumer thread.");
consumerService = new Thread(MessageConsumer);
consumerService.setDaemon(true);
consumerService.setName("MessageConsumer Service");
consumerService.start();
Logger.info("MessageConsumer thread started.");
}
#Override
public void run() {
try {
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("admin", "admin",
"tcp://localhost:61616");
Logger.info("Creating ActiveMQ connection");
Connection connection = connectionFactory.createConnection();
connection.start();
connection.setExceptionListener(this);
Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
Logger.info("Connecting to topic " + TOPIC_NAME);
Destination destination = session.createTopic(TOPIC_NAME);
Logger.info("Creating consumer");
MessageConsumer consumer = session.createConsumer(destination);
while (!Thread.currentThread().isInterrupted()) {
Logger.info("Wait for messages...");
Message message = consumer.receive();
if (message instanceof TextMessage) {
TextMessage textMessage = (TextMessage) message;
String text = textMessage.getText();
Logger.info("Received: " + text);
User user = new User();
user.setId(555);
user.setName("miguel");
user.setTime_of_start("time");
Logger.info("insert user");
//JPAApi jpa = Play.current().injector().instanceOf(JPAApi.class);
JPA.em().getTransaction().begin();
JPA.em().persist(user); //
JPA.em().getTransaction().commit();
} else {
Logger.info("Received: " + message.getClass().getSimpleName());
}
}
Logger.info("Message consumer interrupted.");
consumer.close();
session.close();
connection.close();
} catch (Exception e) {
if (e instanceof InterruptedException) {
Logger.info("Message Consumer thread interrupted.");
} else {
Logger.error(e.getLocalizedMessage(), e);
}
}
}
public synchronized void onException(JMSException ex) {
Logger.error("JMS Exception occured. Shutting down client.");
Logger.error("ErrorCode=" + ex.getErrorCode() + " , " + ex.getMessage(), ex);
}
}
I had tried injecting JPaapi , and many more different things but I am nearly new to Play Framework

First of all, in Play Framework I recommend using Akka actors instead of the traditional java Threads. It may seem complicated at first, however it will save a lot of debugging time once you get the hang of it. Akka is already integrated in Play so it should not be a problem.
Now, regarding your Entity Manager. First, make sure you have the java JDBC in your sbt build file: libraryDependencies += javaJdbc
Next, in your configuration file make sure you have the default DS configured: db.default.jndiName=DefaultDS as well as a persistance unit (more here: https://www.playframework.com/documentation/2.5.x/JavaJPA )
Now, your controller (I don't see the actual controller in the code you posted) must have #Transactional annotation.

Fire and forget for HTTP in Java

We're implementing our own analytics for that we've exposed a web service which needs to be invoked which will capture the data in our DB.
The problem is that as this is analytics we would be making lot of calls (like for every page load, call after each js, CSS loads etc...) so there'll be many many such calls. So I don want the server to be loaded with lots of requests to be more precise pending for response. Because the response we get back will hardly be of any use to us.
So is there any way to just fire the web service request and forget that I've fired it?
I understand that every HTTP request will have as response as well.
So one thing that ticked my mind was what if we make the request timeout to zero second? But I'm pretty not sure if this is the right way of doing this.
Please provide me with more suggestions

You might find following AsyncRequestDemo.java useful:
import java.net.URI;
import java.net.URISyntaxException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.http.client.fluent.Async;
import org.apache.http.client.fluent.Content;
import org.apache.http.client.fluent.Request;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.concurrent.FutureCallback;
/**
* Following libraries have been used:
*
* 1) httpcore-4.4.5.jar
* 2) httpclient-4.5.2.jar
* 3) commons-logging-1.2.jar
* 4) fluent-hc-4.5.2.jar *
*
*/
public class AsyncRequestDemo {
public static void main(String[] args) throws Exception {
URIBuilder urlBuilder = new URIBuilder()
.setScheme("http")
.setHost("stackoverflow.com")
.setPath("/questions/38277471/fire-and-forget-for-http-in-java");
final int nThreads = 3; // no. of threads in the pool
final int timeout = 0; // connection time out in milliseconds
URI uri = null;
try {
uri = urlBuilder.build();
} catch (URISyntaxException use) {
use.printStackTrace();
}
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Async async = Async.newInstance().use(executorService);
final Request request = Request.Get(uri).connectTimeout(timeout);
Future<Content> future = async.execute(request, new FutureCallback<Content>() {
public void failed(final Exception e) {
System.out.println("Request failed: " + request);
System.exit(1);
}
public void completed(final Content content) {
System.out.println("Request completed: " + request);
System.out.println(content.asString());
System.exit(0);
}
public void cancelled() {
}
});
System.out.println("Request submitted");
}
}

I used this:
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
URL url = new URL(YOUR_URL_PATH, "UTF-8"));
ExecutorService executor = Executors.newFixedThreadPool(1);
Future<HttpResponse> response = executor.submit(new HttpRequest(url));
executor.shutdown();
for HttpRequest,HttpResponse
public class HttpRequest implements Callable<HttpResponse> {
private URL url;
public HttpRequest(URL url) {
this.url = url;
}
#Override
public HttpResponse call() throws Exception {
return new HttpResponse(url.openStream());
}
}
public class HttpResponse {
private InputStream body;
public HttpResponse(InputStream body) {
this.body = body;
}
public InputStream getBody() {
return body;
}
}
that is.

Yes, you could initiate the request and break the connection without waiting for a response... But you probably don't want to do that. The overhead of the server-side having to deal with ungracefully broken connections will far outweigh letting it proceed with returning a response.
A better approach to solving this kind of performance problem in a Java servlet would bet to shove all the data from the requests into a queue, respond immediately, and have one or more worker threads pick up items out of the queue for processing (such as writing it into a database).

Hazelcast queue stops working, where to look for the error?

I use Hazelcast as a non-persistent queue between two applications running in a Tomcat.
Problem: QueueListener stops listening to its queue. This means, until a certain point, the following line appears periodically in the log, then it disappears:
LOGGER.debug("No messages on {}, {}", queueName, QueueListener.this.getClass().getSimpleName());
There is no error in the logs. I have several class that extends the QueueListener, all of them listen to a different named queue. One of them just stops and I have no clue why, except one thing: it happens right after handling an item. The descendant class's handle method logs the item I can see that in the logs. Then the "No messages on {queuename}" loglines just disappear. The executor had 2 threads. Both stopped, not sure if at once.
The descendant class's handle method executes a Http request and logs the response. Note that the response did not appear in the logs for the previous two handle call, before the listener stopped.
The descendant class's handle method does not have any catch block so it will not swallow any Exceptions. No exception was logged in the QueueListener.
My question, how to proceed to find the cause of this? Where to look for it?
The application that send messages into this queue runs in the same Tomcat as the one that listens to this queue. Multicast is enabled (see full HazelCast config bellow). There is an other Tomcat that runs on the same host and some other Tomcats running on different hosts, all connecting to this same Hazelcast instance. They're using the same confing.
Hazelcast version: 2.6
QueueListener.java:
package com.mi6.publishers;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
public abstract class QueueListener<T> {
private static final long TIMEOUT = 10000L;
private static final Logger LOGGER = LoggerFactory.getLogger(QueueListener.class);
/**
* queue which is processed
*/
private IQueue<T> queue;
private final String queueName;
#Autowired
private HazelcastInstance instance;
private ExecutorService svc;
private final int threadCount;
private volatile boolean shutdown = false;
/**
* Constructor
*
* #param queueName
* #param threadCount
*/
public QueueListener(String queueName, int threadCount) {
this.queueName = queueName;
this.threadCount = threadCount;
}
/**
* #PostConstuct Start background threads
*/
#PostConstruct
public void init() {
LOGGER.info("Constructing hazelcast listener for {}", getClass().getSimpleName());
if (instance != null) {
queue = instance.getQueue(queueName);
svc = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
svc.submit(new Runnable() {
#Override
public void run() {
while (!shutdown) {
try {
T item = queue.poll(TIMEOUT, TimeUnit.MILLISECONDS);
if (item != null) {
handle(item);
} else {
LOGGER.debug("No messages on {}, {}", queueName, QueueListener.this.getClass().getSimpleName());
}
} catch (InterruptedException ex) {
// do nothing if interrupted
} catch (Exception ex) {
LOGGER.error("Error while receiving messages from queue:{}", queueName);
LOGGER.error("Error while receiving messages", ex);
}
}
}
});
}
} else {
throw new IllegalStateException("Hazelcast instance cannot be null");
}
}
/**
* call before stop
*/
#PreDestroy
public void destroy() {
shutdown = true;
if (svc != null) {
svc.shutdown();
}
}
/**
* Event handler
*
* #param item
*/
public abstract void handle(T item);
public String getQueueName() {
return queueName;
}
}
This is how Hazelcast is configured:
#Value("${hazelcast.multicast:True}")
private Boolean hazelcastMulticast;
#Value("${hazelcast.group:groupNameNotSet}")
private String hazelcastGroup;
#Bean(destroyMethod = "shutdown")
public HazelcastInstance hazelcastInstance() {
Config cfg = new Config();
cfg.setInstanceName(hazelcastGroup);
NetworkConfig network = cfg.getNetworkConfig();
network.setPortAutoIncrement(true);
Join join = network.getJoin();
join.getMulticastConfig().setEnabled(hazelcastMulticast);
cfg.getGroupConfig().setName(hazelcastGroup);
cfg.getGroupConfig().setPassword(hazelcastGroup);
QueueConfig sms = new QueueConfig();
sms.setName("some-queue-name1");
cfg.addQueueConfig(sms);
QueueConfig flash = new QueueConfig();
flash.setName("some-queue-name2");
cfg.addQueueConfig(flash);
QueueConfig apns = new QueueConfig();
apns.setName("some-queue-name3");
cfg.addQueueConfig(apns);
QueueConfig gcm = new QueueConfig();
gcm.setName("some-queue-name4");
cfg.addQueueConfig(gcm);
return Hazelcast.newHazelcastInstance(cfg);
}