Develop Reference

Does ActiveMQ support max message processing time in consumer

I have a ActiveMQ consumer script running in Java where I am calling consumer.receive() in a while(true) loop.
I need to implement timeout for each message processed (eg: if a message process goes beyond 15 seconds I have to receive the next one).
I have given the client acknowledge mode for ACK.
Please look at the consumeMessage method where I have implemented the consume.
Desired outcome:
After 15 seconds the first message needs to be discarded (i.e. it should not invoke acknowledge()). The next message needs to be processed instead.
//package consumer;
import org.apache.activemq.ActiveMQConnectionFactory;
import javax.jms.Connection;
import javax.jms.DeliveryMode;
import javax.jms.Destination;
import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
public class ActivmqConsumer implements ExceptionListener {
ActiveMQConnectionFactory connectionFactory = null;
Connection connection = null;
Session session = null;
public ActivmqConsumer() throws Exception{
String USERNAME = "admin";
String PASSWORD = "admin";
this.connectionFactory = new ActiveMQConnectionFactory(USERNAME, PASSWORD, "tcp://192.168.56.101:61616?jms.prefetchPolicy.all=1");
// Create a Connection
this.connection = connectionFactory.createConnection();
connection.start();
connection.setExceptionListener(this);
// Create a Session
this.session = connection.createSession(false, Session.CLIENT_ACKNOWLEDGE);
}
public void consumeMessage(String destinationName, EventProcesser eventprocess){
Destination destination = null;
MessageConsumer consumer = null;
try{
// Create the destination (Topic or Queue)
destination = session.createQueue(destinationName);
// Create a MessageConsumer from the Session to the Topic or Queue
consumer = session.createConsumer(destination);
// Wait for a message
while(true){
Message message = consumer.receive(2);
if(message==null){
continue;
}
else if(message instanceof TextMessage) {
TextMessage textMessage = (TextMessage) message;
String text = textMessage.getText();
System.out.println("Received: " + text);
eventprocess.processEvent(text);
message.acknowledge();
} else{
System.out.println("Received: " + message);
}
}
}catch(Exception ex){
ex.printStackTrace();
}finally{
try{
consumer.close();
}catch(Exception ex){
ex.printStackTrace();
}
}
}
}

There is no "max message processing time" or equivalent feature in ActiveMQ. You'll need to monitor the processing yourself. Maybe take a look at this question/answer for ideas on how to do that. An alternative would be to use a JTA transaction manager and consume the message in a transaction with a timeout of 15 seconds. Using an MDB in a Java EE container would be a simple way to get the transaction timeout functionality.

Twitter Java project : failed to establish connection properly

I am trying to consume Twitter streams with the help of a Java Kafka application.
I have created a Twitter developer account and a Twitter application and generated all the 4 keys required.
Please find the code below:
package com.github.simpleanand.kafkabeginner.tutorial2;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpHost;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.twitter.hbc.ClientBuilder;
import com.twitter.hbc.core.Client;
import com.twitter.hbc.core.Constants;
import com.twitter.hbc.core.Hosts;
import com.twitter.hbc.core.HttpHosts;
import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;
import com.twitter.hbc.core.processor.StringDelimitedProcessor;
import com.twitter.hbc.httpclient.auth.Authentication;
import com.twitter.hbc.httpclient.auth.OAuth1;
public class TwitterProducer {
private Logger logger = LoggerFactory.getLogger(TwitterProducer.class);
private String consumerKey = "<consumer-key>";
private String consumerSecret = "<consumerSecret>";
private String token = "<token value>";
private String secret = "<secret>";
public TwitterProducer() {
}
public static void main(String[] args) {
new TwitterProducer().run();
}
public void run() {
logger.info("inside run........");
// create a twitter client
/**
* Set up your blocking queues: Be sure to size these properly based on
* expected TPS of your stream
*/
BlockingQueue<String> msgQueue = new LinkedBlockingQueue<String>(100000);
Client client = createTwitterClient(msgQueue);
client.connect();
// create a kafka producer
// loop to send tweets to kafka
// on a different thread, or multiple different threads....
while (!client.isDone()) {
String msg = null;
try {
msg = msgQueue.poll(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
client.stop();
}
if (null != msg) {
logger.info("Msg -> " + msg);
}
}
logger.info("end of application........");
}
public Client createTwitterClient(BlockingQueue<String> msgQueue) {
HttpHost proxy = new HttpHost("<compayny proxy value>", 8080);
DefaultHttpClient httpClient = new DefaultHttpClient();
httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
/**
* Declare the host you want to connect to, the endpoint, and
* authentication (basic auth or oauth)
*/
Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();
// Optional: set up some followings and track terms
// List<Long> followings = Lists.newArrayList(1234L, 566788L);
List<String> terms = Lists.newArrayList("bitcoin");
// hosebirdEndpoint.followings(followings);
hosebirdEndpoint.trackTerms(terms);
// These secrets should be read from a config file
Authentication hosebirdAuth = new OAuth1(consumerKey, consumerSecret, token, secret);
hosebirdAuth.setupConnection(httpClient);
// Creating a client:
ClientBuilder builder = new ClientBuilder().name("Hosebird-Client-01") // optional:
// mainly
// for
// the
// logs
.hosts(hosebirdHosts).authentication(hosebirdAuth).endpoint(hosebirdEndpoint)
.processor(new StringDelimitedProcessor(msgQueue));
Client hosebirdClient = builder.build();
// Attempts to establish a connection.
return hosebirdClient;
}
}
I'm getting the following error:
[hosebird-client-io-thread-0] INFO com.twitter.hbc.httpclient.ClientBase - Hosebird-Client-01 Establishing a connection
[hosebird-client-io-thread-0] WARN com.twitter.hbc.httpclient.ClientBase - Hosebird-Client-01 Unknown host - stream.twitter.com
[hosebird-client-io-thread-0] WARN com.twitter.hbc.httpclient.ClientBase - Hosebird-Client-01 failed to establish connection properly
[hosebird-client-io-thread-0] INFO com.twitter.hbc.httpclient.ClientBase - Hosebird-Client-01 Done processing, preparing to close connection
Please advise on how to solve this error.
It seems there is an issue of proxy object being sent.It that correct?

MessageDrivenBean isnt handle messages [Wildfly]

Im trying to develop a "Message Driven Bean" to handle all the local ActiveMQ messages, but it's the first time that i try to do something like this.
The most part of the material that i found explain how to write a MDB using JBOSS server, in this case there's a xml file with some queue information, but in all wildfly tutorials there's no mention to any kind of configuration like that.
I have the following scenario:
A simple java project like message producer
An ActiveMQ instance running local
An EJB project deployed into Wildfly 10
My producer project is able to send messages to ActiveMQ queue, this part its working,but my EJB project just have a single class called TestMDBHandle with #MessageDriven annotation. Is this enough to receive my queue messages? Because the MDB isnt working, i imagine must be a kind of configuration or property in EJB to specify the host of the message-broker.
My message producer:
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import javax.naming.InitialContext;
import javax.naming.NamingException;
public class MessageSender {
public static void main(String args[]) throws NamingException, JMSException {
MessageSender sender = new MessageSender();
sender.sender();
}
public void sender() throws NamingException, JMSException {
InitialContext jndi = null;
Session session = null;
Connection connection = null;
try {
jndi = new InitialContext();
ConnectionFactory factory = (ConnectionFactory)jndi.lookup("connectionFactory");
connection = factory.createConnection();
connection.start();
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
Destination destination = (Destination)jndi.lookup("MyQueue");
MessageProducer producer = session.createProducer(destination);
TextMessage mensagem = session.createTextMessage("Eu enviei uma mensagem!");
producer.send(mensagem);
} catch (Exception e) {
e.printStackTrace();
} finally {
session.close();
connection.close();
jndi.close();
}
}
}
My jms properties located inside my producer project
java.naming.factory.initial=org.apache.activemq.jndi.ActiveMQInitialContextFactory
java.naming.provider.url=tcp://localhost:61616
connectionFactoryNames = connectionFactory, queueConnectionFactory, topicConnectionFactory
queue.MyQueue=jms/myqueue
Finally, my ejb project have this single class, without any kind of property file or xml.
package br.com.jms.mdb;
import javax.annotation.Resource;
import javax.ejb.ActivationConfigProperty;
import javax.ejb.EJB;
import javax.ejb.MessageDriven;
import javax.ejb.MessageDrivenContext;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageListener;
import javax.jms.TextMessage;
#MessageDriven(name = "meuHandler", activationConfig = {
#ActivationConfigProperty(propertyName = "destinationType", propertyValue = "javax.jms.Queue"),
#ActivationConfigProperty(propertyName = "destination", propertyValue = "jms/myqueue") })
public class Teste implements MessageListener {
#Resource
private MessageDrivenContext mdctx;
public Teste() {
}
#Override
public void onMessage(Message message) {
TextMessage objectMessage = null;
try {
objectMessage = (TextMessage)message;
System.out.println("Achei a mensagem : " + objectMessage.getText().toString());
}catch(JMSException e) {
e.printStackTrace();
}
}
}

Maybe you can provide a little more information such as the xml file with the queue information and the annotation properties of the MDB? Because it sounds you are heading in the right direction. The two main things:
You have to specify the exact queue that the MDB is listening to, for example through the properties of the #MessageDriven annotation (such as "name", "mappedName", "activationConfig"). And of course override the onMessage() method to process the messages.
You also have to make sure that this specific queue is available as a resource for your application. You have to provide jms configuration for this, which also defines the resource type (Queue or Topic). From your question I can't tell which of these steps you have (partly) completed.

Qpid and JNDI for encrypted messages

I'm currently working on a JMS project and I have created 2 keys and 2 certificates as well as a TrustStorage, mytruststore which I created through Qpid's UI.
In my jndi.properties file I have the following code:
//Set the InitialContextFactory class to use
java.naming.factory.initial = org.apache.qpid.jms.jndi.JmsInitialContextFactory
//Define the required ConnectionFactory instances
//connectionfactory.<JNDI-lookup-name> = <URI>
connectionfactory.myFactoryLookup = amqp://localhost:5672
connectionfactory.producerConnectionFactory = amqp://admin:admin#?brokerlist='tcp://localhost:5672?encryption_remote_trust_store='$certificates%255c/mytruststore''
connectionfactory.consumer1ConnectionFactory = amqp://admin:admin#?brokerlist='tcp://localhost:5672?encryption_key_store='C:\OpenSSL-Win64\bin\mytruststorage.jks'&encryption_key_store_password='thanos''
connectionfactory.consumer2ConnectionFactory = amqp://admin:admin#?brokerlist='tcp://localhost:5672?encryption_key_store='C:\OpenSSL-Win64\bin\mytruststorage.jks'&encryption_key_store_password='thanos''
//Configure the necessary Queue and Topic objects
//queue.<JNDI-lookup-name> = <queue-name>
//topic.<JNDI-lookup-name> = <topic-name>
queue.myQueueLookup = queue
topic.myTopicLookup = topic
queue.myTestQueue = queue
In my EncryptionExample.java class I have the following code:
package org.apache.qpid.jms.example;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import javax.jms.BytesMessage;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.DeliveryMode;
import javax.jms.Destination;
import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import javax.naming.Context;
import javax.naming.InitialContext;
import javax.naming.NamingException;
public class EncryptionExample {
public EncryptionExample() {
}
public static void main(String[] args) throws Exception {
EncryptionExample encryptionExampleApp = new EncryptionExample();
encryptionExampleApp.runProducerExample();
encryptionExampleApp.runReceiverExample();
}
private void runProducerExample() throws Exception
{
Connection connection = createConnection("producerConnectionFactory");
try {
Session session = connection.createSession(true, Session.SESSION_TRANSACTED);
Destination destination = createDesination("myTestQueue");
MessageProducer messageProducer = session.createProducer(destination);
TextMessage message = session.createTextMessage("Hello world!");
// ============== Enable encryption for this message ==============
message.setBooleanProperty("x-qpid-encrypt", true);
// ============== Configure recipients for encryption ==============
message.setStringProperty("x-qpid-encrypt-recipients", "CN=client1, OU=Qpid, O=Apache, C=US");
messageProducer.send(message);
session.commit();
}
finally {
connection.close();
}
}
private void runReceiverExample() throws Exception
{
Connection connection = createConnection("consumer1ConnectionFactory");
try {
connection.start();
Session session = connection.createSession(true, Session.SESSION_TRANSACTED);
Destination destination = createDesination("myTestQueue");
MessageConsumer messageConsumer = session.createConsumer(destination);
Message message = messageConsumer.receive();
if (message instanceof TextMessage) {
// application logic
System.out.println(((TextMessage) message).getText());
} else if (message instanceof BytesMessage) {
// handle potential decryption failure
System.out.println("Potential decryption problem. Application not in list of intended recipients?");
}
session.commit();
}
finally {
connection.close();
}
}
///////////////////////////////////////
// The following is boilerplate code //
///////////////////////////////////////
private Connection createConnection(final String connectionFactoryName) throws JMSException, IOException, NamingException
{
try (InputStream resourceAsStream = getResourceAsStream("jndi.properties")) {
Properties properties = new Properties();
properties.load(resourceAsStream);
Context context = new InitialContext(properties);
ConnectionFactory connectionFactory = (ConnectionFactory) context.lookup(connectionFactoryName);
final Connection connection = connectionFactory.createConnection();
context.close();
return connection;
}
}
private InputStream getResourceAsStream(String string) {
// TODO Auto-generated method stub
return null;
}
private Destination createDesination(String desinationJndiName) throws IOException, NamingException
{
try (InputStream resourceAsStream = this.getClass().getResourceAsStream("example.properties")) {
Properties properties = new Properties();
properties.load(resourceAsStream);
Context context = new InitialContext(properties);
Destination destination = (Destination) context.lookup(desinationJndiName);
context.close();
return destination;
}
}
}
When I'm trying to build it I get the following exceptions.
Exception in thread "main" java.lang.NullPointerException
at java.util.Properties$LineReader.readLine(Unknown Source)
at java.util.Properties.load0(Unknown Source)
at java.util.Properties.load(Unknown Source)
at
org.apache.qpid.jms.example.EncryptionExample.createConnection(EncryptionExample.java:106)
at
org.apache.qpid.jms.example.EncryptionExample.runProducerExample(EncryptionExample.java:54)
at org.apache.qpid.jms.example.EncryptionExample.main(EncryptionExample.java:48)
I assume that something's wrong with the following code in jndi.properties file:
connectionfactory.myFactoryLookup = amqp://localhost:5672
connectionfactory.producerConnectionFactory = amqp://admin:admin#?brokerlist='tcp://localhost:5672?encryption_remote_trust_store='$certificates%255c/mytruststore''
connectionfactory.consumer1ConnectionFactory = amqp://admin:admin#?brokerlist='tcp://localhost:5672?encryption_key_store='C:\OpenSSL-Win64\bin\mytruststorage.jks'&encryption_key_store_password='thanos''
connectionfactory.consumer2ConnectionFactory = amqp://admin:admin#?brokerlist='tcp://localhost:5672?encryption_key_store='C:\OpenSSL-Win64\bin\mytruststorage.jks'&encryption_key_store_password='thanos''
This is my solution Explorer:

This first and biggest problem you have is that you are trying to use connection URIs and client features from a client other than the one you have configured you project to use. You seem to be using Qpid JMS which is the new AMQP 1.0 client developed at the Qpid project. This client uses a different URI syntax that the previous AMQP 0.x clients and you will get exception from the connection factory when passing in these invalid URIs.
The other problem you will have (which was called out in the comments on your post) is that there is not message encryption feature in the AMQP 1.0 JMS client so that would be your next problem once you got the URIs correctly defined.
The documentation for the newer AMQP 1.0 JMS client is here.

You cannot use // for comments in Properties files. Use # or ! instead.
See: https://docs.oracle.com/javase/8/docs/api/java/util/Properties.html#load-java.io.Reader-

kafka java consumer not reading data

I am trying to write a simple java kafka consumer to read data using similar code as in https://github.com/bkimminich/apache-kafka-book-examples/blob/master/src/test/kafka/consumer/SimpleHLConsumer.java.
Looks like my app is able to connect, but its not fetching any data. Please suggest.
import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
//import scala.util.parsing.json.JSONObject
import scala.util.parsing.json.JSONObject;
public class SimpleHLConsumer {
private final ConsumerConnector consumer;
private final String topic;
public SimpleHLConsumer(String zookeeper, String groupId, String topic) {
Properties props = new Properties();
props.put("zookeeper.connect", zookeeper);
props.put("group.id", groupId);
// props.put("zookeeper.session.timeout.ms", "5000");
// props.put("zookeeper.sync.time.ms", "250");
// props.put("auto.commit.interval.ms", "1000");
consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
this.topic = topic;
}
public void testConsumer() {
Map<String, Integer> topicCount = new HashMap<>();
topicCount.put(topic, 1);
Map<String, List<KafkaStream<byte[], byte[]>>> consumerStreams = consumer.createMessageStreams(topicCount);
System.out.println(consumerStreams);
List<KafkaStream<byte[], byte[]>> streams = consumerStreams.get(topic);
System.out.println(streams);
System.out.println(consumer);
for (final KafkaStream stream : streams) {
ConsumerIterator<byte[], byte[]> it = stream.iterator();
System.out.println("for loop");
System.out.println(it);
System.out.println("Message from Single Topic: " + new String(it.next().message()));
//System.out.println("Message from Single Topic: " + new String(it.message()));
while (it.hasNext()) {
System.out.println("in While");
System.out.println("Message from Single Topic: " + new String(it.next().message()));
}
}
// if (consumer != null) {
// consumer.shutdown();
// }
}
public static void main(String[] args) {
String topic = "test";
SimpleHLConsumer simpleHLConsumer = new SimpleHLConsumer("localhost:2181", "testgroup", topic);
simpleHLConsumer.testConsumer();
}
}
Here is the output i see in eclipse. It does seem to connect to my zookeeper , but it just hangs there, it does not display any message at all.
log4j:WARN No appenders could be found for logger (kafka.utils.VerifiableProperties).
log4j:WARN Please initialize the log4j system properly.
SLF4J: The requested version 1.6 by your slf4j binding is not compatible with [1.5.5, 1.5.6]
SLF4J: See http://www.slf4j.org/codes.html#version_mismatch for further details.
{test=[testgroup kafka stream]}
[testgroup kafka stream]
kafka.javaapi.consumer.ZookeeperConsumerConnector#6200f9cb
for loop

Consumer iterator hasNext is blocking call. It will block indefinitely if no new message is available for consumption.
To verify this, change your code to
// Comment 2 lines below
// System.out.println(it);
// System.out.println("Message from Single Topic: " + new String(it.next().message()));
// Line below is blocking. Your code will hang till next message in topic.
// Add new message in topic using producer, message will appear in console
while (it.hasNext()) {
Better way is to execute code in separate thread. Use consumer.timeout.ms to specify time in ms, after which consumer will throw timeout exception
// keepRunningThread is flag to control when to exit consumer loop
while(keepRunningThread)
{
try
{
if(it.hasNext())
{
System.out.println(new String(it.next().message()));
}
}
catch(ConsumerTimeoutException ex)
{
// Timeout exception waiting for kafka message
// Wait for 5 (or t) seconds before checking for message again
Thread.sleep(5000);
}
}‍‍‍‍‍‍‍‍‍‍