I am not able to receive notifications using HCATALOG using JMS. I wrote simple producer consumer program.
Apache MQ service is running in background. I am able to send simple text messages using ApacheMQ. But "markPartitionForEvent()" is not able to send event to consumer's "onMessage()" call.
I refered following link :
https://cwiki.apache.org/confluence/display/Hive/HCatalog+Notification
Please guide
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.thrift.TException;
import javax.jms.MessageConsumer;
import javax.management.*;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hcatalog.common.HCatConstants;
import java.util.Properties;
import javax.jms.*;
import javax.jdo.*;
import javax.naming.*;
import java.io.*;
import java.io.InputStreamReader;
import java.util.*;
import javax.jms.MessageConsumer;
import org.slf4j.LoggerFactory;
import org.datanucleus.api.jdo.*;
import javax.jdo.metadata.JDOMetadata;
import org.datanucleus.store.rdbms.RDBMSStoreManager;
import org.datanucleus.properties.PropertyStore;
import org.datanucleus.store.AbstractStoreManager;
import org.apache.hadoop.hive.metastore.api.Table;
class Consumer implements MessageListener
{
public void start(){
try
{
HiveConf hiveConf;
ActiveMQConnectionFactory connFac = new ActiveMQConnectionFactory("tcp://localhost:61616");
Connection conn = connFac.createConnection();
conn.start();
hiveConf = new HiveConf(Consumer.class);
HiveMetaStoreClient msc = new HiveMetaStoreClient(hiveConf);
Table table = msc.getTable("mydb","myTbl");
table.getParameters().put(HCatConstants.HCAT_MSGBUS_TOPIC_NAME, "TOPIC"+ ".HCATALOG");
System.out.println("TABLE = " + table.toString());
Map<String,String> map = table.getParameters();
System.out.println("MAP= " + map.toString());
String fetchTopic = map.get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME);
System.out.println("fetchTopic = " + fetchTopic);
String topicName = msc.getTable("mydb",
"mytbl").getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME);
System.out.println("TOPICNAME = " + topicName);
Session session = conn.createSession(false, Session.AUTO_ACKNOWLEDGE);
if (session == null)
System.out.println("NULL");
System.out.println(session.toString());
Destination hcatTopic = session.createTopic(fetchTopic);
MessageConsumer consumer = session.createConsumer(hcatTopic);
consumer.setMessageListener(this);
}
catch(Exception e){
System.out.println("ERROR");
e.printStackTrace();
}
}
#Override
public void onMessage(Message message){
try
{
if(message.getStringProperty(HCatConstants.HCAT_EVENT).equals(HCatConstants.HCAT_PARTITION_DONE_EVENT)){
MapMessage mapMsg = (MapMessage)message;
Enumeration<String> keys = mapMsg.getMapNames();
while(keys.hasMoreElements())
{
String key = keys.nextElement();
System.out.println(key + " : " + mapMsg.getString(key));
}
System.out.println("Message: "+message);
}
}
catch(Exception e){
System.out.println("ERROR");
}
}
};
class Producer extends Thread
{
private HiveConf hiveConf;
public Producer()
{
}
#Override
public void run() {
try
{
hiveConf = new HiveConf(this.getClass());
HiveMetaStoreClient msc = new HiveMetaStoreClient(hiveConf);
HashMap<String,String> partMap = new HashMap<String, String>();
partMap.put("date","20110711");
partMap.put("date","20110712");
partMap.put("date","20110714");
while(true)
{
msc.markPartitionForEvent("mydb", "mytbl", partMap, PartitionEventType.LOAD_DONE);
Thread.sleep(1000);
}
}
catch(Exception e){
System.out.println("ERROR");
}
}
};
Related
I use Java mail APIs from a spring web application to send a weekly email to an outlook mail.
The feature was behaving normally for the first couple of weeks, then without any changes outlook received two emails, the next week three emails were received, then four, then five emails.
The logs set in the Java code indicates that only one email is being sent from the application.
I can't replicate the issue by changing the schedule to send each 15 minutes, or hour, or any shorter interval.
Email controller class
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
#Component
public class WeeklyReportScheduler {
#Autowired
private WeeklyReportService weeklyReportService;
#Scheduled(cron = "${cron.expression}")
public void sendWeeklyReport(){
weeklyReportService.sendWeeklyReport();
}
}
Email service class:
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.mail.MailException;
import org.springframework.stereotype.Service;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
#Service
public class WeeklyReportService {
#Value("${weekly.report.mail.subject}")
private String subject;
#Value("${weekly.report.mail.body}")
private String mailBody;
#Value("${mail.body.empty.report}")
private String emptyReportMailBody;
#Value("${receiver}")
private String receiver;
#Autowired
private MailService mailService;
#Autowired
private WeeklyReportLogDao weeklyReportLogDao;
#Autowired
private ProjectService projectService;
#Value("${export.path}")
private String exportDir;
protected final Log logger = LogFactory.getLog(getClass());
public void sendWeeklyReport(){
boolean emptyReport = true;
//retrieving attachment data 'projects'
if(projects.size() != 0){
emptyReport = false;
}
String body = "";
if(emptyReport){
body = emptyReportMailBody;
} else {
body = mailBody;
}
SimpleDateFormat format = new SimpleDateFormat("MM/dd/YYYY");
String dateString = format.format(new Date());
String mailSubject = subject + " " + dateString;
List recipients = new ArrayList<String>();
recipients.add(receiver);
String fileName = mailSubject.replace(" ", "_").replace("/", "_");
WeeklyReportExcelExport excelExport = new WeeklyReportExcelExport(exportDir, fileName);
excelExport.createReport(projects);
File excelFile = excelExport.saveToFile();
File[] attachments = new File[1];
attachments[0] = excelFile;
boolean sent = false;
String exceptionMessage = "";
for(int i=0; i<3; i++){
try {
logger.info("Sending Attempt: " + i+1);
Thread.sleep(10000);
mailService.mail(recipients, mailSubject, body, attachments);
sent = true;
break;
} catch (Exception ex) {
logger.info("sending failed because: " + ex.getMessage() + " \nRe-attempting in 10 seconds");
exceptionMessage = ex.getMessage();
sent = false;
}
}
if(!sent){
weeklyReportLogDao.logFailedReporting(dateString, exceptionMessage);
}
//re-try 3 times in case of mail sending failure
}
MailService class:
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Map;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.mail.MailException;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.mail.javamail.MimeMessageHelper;
public class MailServiceImpl implements MailService {
/** The From address for the e-mail. read from ant build properties file */
private String fromAddress;
/** The mail sender. */
private JavaMailSender mailSender;
/** Logger for this class and subclasses */
protected final Log logger = LogFactory.getLog(getClass());
public void mail(List<String> emailAddresses, String subject, String text, File[] attachments) throws MailException {
//System.out.println("mail: "+subject);
MimeMessage message = null;
// Fill in the From, To, and Subject fields.
try {
message = mailSender.createMimeMessage();
MimeMessageHelper messageHelper = new MimeMessageHelper(message, true);
messageHelper.setFrom(fromAddress);
for (String emailAddress : emailAddresses) {
messageHelper.addTo(emailAddress);
}
messageHelper.setSubject(subject);
// Fill in the body with the message text, sending it in HTML format.
messageHelper.setText(text, true);
// Add any attachments to the message.
if ((attachments != null) && (attachments.length != 0)) {
for (File attachment : attachments) {
messageHelper.addAttachment(attachment.getName(), attachment);
}
}
}
catch(MessagingException mse) {
String warnMessage = "Error creating message.";
logger.warn(warnMessage);
throw (new RuntimeException(warnMessage, mse));
}
try {
mailSender.send(message);
} catch (Exception ex){
logger.info("Exception sending message: " + ex.getMessage());
}
}
}
you might have duplicate entries on your argument emailAddresses, try moving it to treeset if you cant ensure duplicate entries from the repository layer
I am looking for reference where I can get simple program to send a SNMP trap to Apache Kafka topic using Apache Camel.
Please help me if someone can explain the it using simple java program.
My RouteBuilder configuration
import org.apache.camel.builder.RouteBuilder;
public class SimpleRouteBuilder extends RouteBuilder{
#Override
public void configure() throws Exception {
String topicName = "topic=first_topic";
String kafkaServer = "kafka:localhost:9092";
String zooKeeperHost = "zookeeperHost=localhost&zookeeperPort=2181";
String serializerClass = "serializerClass=kafka.serializer.StringEncoder";
String toKafka = new StringBuilder().append(kafkaServer).append("?").append(topicName).append("&")
.append(zooKeeperHost).append("&").append(serializerClass).toString();
System.out.println(toKafka);
from("snmp:127.0.0.1:161?protocol=udp&type=POLL&oids=1.3.6.1.2.1.1.5.0").split().tokenize("\n").to(toKafka);
}
}
Main Method
import org.apache.camel.CamelContext;
import org.apache.camel.impl.DefaultCamelContext;
import org.snmp4j.Snmp;
public class MainApp {
public static void main(String[] args) {
SimpleRouteBuilder routeBuilder = new SimpleRouteBuilder();
CamelContext ctx = new DefaultCamelContext();
try {
ctx.addRoutes(routeBuilder);
ctx.start();
Thread.sleep(5 * 60 * 1000);
ctx.stop();
}
catch (Exception e) {
e.printStackTrace();
}
}
}
I was in wrong direction. The write direction is as below -
Create a Trap sender program.
Create Trap receiver/listener program.
Inside Trap receiver or listener, receive trap and send it to Apache Kafka topic through Apache camel.
POM.XML
add below dependencies -
camel-core
snmp4j
camel-kafka
Trap Sender Program
package <>;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.snmp4j.*;
import org.snmp4j.event.ResponseEvent;
import org.snmp4j.mp.MPv2c;
import org.snmp4j.mp.MPv3;
import org.snmp4j.mp.SnmpConstants;
import org.snmp4j.security.*;
import org.snmp4j.smi.*;
import org.snmp4j.transport.DefaultUdpTransportMapping;
import java.util.Date;
public class Trapsender {
public static final String community = "public";
public static final String Oid = ".1.3.6.1.2.1.1.8";
public static final String ipAddress = "127.0.0.1";
public static final int port = 162;
public static void main(String[] args) {
Trapsender trapv3 = new Trapsender();
trapv3.sendTrap_Version3();
}
public void sendTrap_Version3() {
try {
// Create Transport Mapping
TransportMapping transport = new DefaultUdpTransportMapping();
transport.listen();
// Create Target
CommunityTarget cTarget = new CommunityTarget();
cTarget.setCommunity(new OctetString(community));
cTarget.setVersion(SnmpConstants.version2c);
cTarget.setAddress(new UdpAddress(ipAddress + "/" + port));
cTarget.setRetries(2);
cTarget.setTimeout(10000);
// Create PDU for V3
PDU pdu = new PDU();
pdu.setType(PDU.TRAP);
// need to specify the system up time
pdu.add(new VariableBinding(SnmpConstants.sysUpTime, new OctetString(new Date().toString())));
pdu.add(new VariableBinding(SnmpConstants.snmpTrapOID, new OID(Oid)));
pdu.add(new VariableBinding(new OID(Oid), new OctetString("Major")));
// Send the PDU
Snmp snmp = new Snmp(transport);
System.out.println("Sending V2 Trap... Check Wheather NMS is Listening or not? ");
ResponseEvent send = snmp.send(pdu, cTarget);
snmp.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Receiver Trap with Apache Camel
package <>;
import org.apache.camel.CamelContext;
import org.apache.camel.ProducerTemplate;
import org.apache.camel.RoutesBuilder;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.snmp4j.*;
import org.snmp4j.mp.MPv1;
import org.snmp4j.mp.MPv2c;
import org.snmp4j.security.Priv3DES;
import org.snmp4j.security.SecurityProtocols;
import org.snmp4j.smi.OctetString;
import org.snmp4j.smi.TcpAddress;
import org.snmp4j.smi.TransportIpAddress;
import org.snmp4j.smi.UdpAddress;
import org.snmp4j.transport.AbstractTransportMapping;
import org.snmp4j.transport.DefaultTcpTransportMapping;
import org.snmp4j.transport.DefaultUdpTransportMapping;
import org.snmp4j.util.MultiThreadedMessageDispatcher;
import org.snmp4j.util.ThreadPool;
import java.io.IOException;
public class Trapreceiver implements CommandResponder {
public static CamelContext ctx=null;
public static ProducerTemplate producer=null;
public static void main(String[] args) {
Trapreceiver snmp4jTrapReceiver = new Trapreceiver();
SimpleRouteBuilder routeBuilder = new SimpleRouteBuilder();
ctx = new DefaultCamelContext();
producer = ctx.createProducerTemplate();
try {
ctx.addRoutes(routeBuilder);
ctx.start();
}
catch (Exception e) {
e.printStackTrace();
}
// producer.sendBody("direct:start", snmp);
try {
snmp4jTrapReceiver.listen(new UdpAddress("localhost/162"), producer);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Trap Listner
*/
public synchronized void listen(TransportIpAddress address, ProducerTemplate producer)
throws IOException {
AbstractTransportMapping transport;
if (address instanceof TcpAddress) {
transport = new DefaultTcpTransportMapping((TcpAddress) address);
} else {
transport = new DefaultUdpTransportMapping((UdpAddress) address);
}
ThreadPool threadPool = ThreadPool.create("DispatcherPool", 10);
MessageDispatcher mDispathcher = new MultiThreadedMessageDispatcher(
threadPool, new MessageDispatcherImpl());
// add message processing models
mDispathcher.addMessageProcessingModel(new MPv1());
mDispathcher.addMessageProcessingModel(new MPv2c());
// add all security protocols
SecurityProtocols.getInstance().addDefaultProtocols();
SecurityProtocols.getInstance().addPrivacyProtocol(new Priv3DES());
// Create Target
CommunityTarget target = new CommunityTarget();
target.setCommunity(new OctetString("public"));
Snmp snmp = new Snmp(mDispathcher, transport);
snmp.addCommandResponder(this);
transport.listen();
System.out.println("Listening on " + address);
try {
this.wait();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
/**
* This method will be called whenever a pdu is received on the given port
* specified in the listen() method
*/
public synchronized void processPdu(CommandResponderEvent cmdRespEvent) {
System.out.println("Received PDU...");
PDU pdu = cmdRespEvent.getPDU();
if (pdu != null) {
System.out.println("Trap Type = " + pdu.getType());
System.out.println("Variables = " + pdu.getVariableBindings());
producer.sendBody("direct:start","Variables = " + pdu.getVariableBindings() );
}
}
}
I have been looking for some documentation/example for checking if a dynamically created topic exist and if it does, how to get the subscriber count for the topic.
I use following code for sending out message to a topic -
jmsTemplate.send(destination, new MessageCreator() {
#Override
public Message createMessage(Session session) throws JMSException {
TextMessage message = session.createTextMessage();
message.setText(commandStr);
return message;
}
});
This code seems to create the topic and publish message to topic.
I need to check if the topic exists before creating it.
Check if the topic have any subscriber.
Thanks in advance
i was able to find the solution to (1) problem (Hope this helps)-
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://localhost:61616");
ActiveMQConnection connection = (ActiveMQConnection)connectionFactory.createConnection();
connection.start();
DestinationSource ds = connection.getDestinationSource();
Set<ActiveMQTopic> topics = ds.getTopics();
To get the destination names, as you did it is correct, you can do it by JMX too specifically to get statistical information like subscriber count ...
import java.util.HashMap;
import java.util.Map;
import javax.management.MBeanServerConnection;
import javax.management.MBeanServerInvocationHandler;
import javax.management.ObjectName;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
import org.apache.activemq.broker.jmx.BrokerViewMBean;
import org.apache.activemq.broker.jmx.TopicViewMBean;
public class JMXGetDestinationInfos {
public static void main(String[] args) throws Exception {
JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi");
Map<String, String[]> env = new HashMap<>();
String[] creds = { "admin", "admin" };
env.put(JMXConnector.CREDENTIALS, creds);
JMXConnector jmxc = JMXConnectorFactory.connect(url, env);
MBeanServerConnection conn = jmxc.getMBeanServerConnection();
ObjectName activeMq = new ObjectName("org.apache.activemq:type=Broker,brokerName=localhost");
BrokerViewMBean mbean = MBeanServerInvocationHandler.newProxyInstance(conn, activeMq, BrokerViewMBean.class,
true);
for (ObjectName name : mbean.getTopics()) {
if (("YOUR_TOPIC_NAME".equals(name.getKeyProperty("destinationName")))) {
TopicViewMBean topicMbean = MBeanServerInvocationHandler.newProxyInstance(conn, name,
TopicViewMBean.class, true);
System.out.println(topicMbean.getConsumerCount());
}
}
}
}
or
import java.util.Set;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.JMSException;
import org.apache.activemq.ActiveMQConnection;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.advisory.DestinationSource;
import org.apache.activemq.command.ActiveMQQueue;
import org.apache.activemq.command.ActiveMQTopic;
public class AdvisorySupportGetAllDestinationsNames {
public static void main(String[] args) throws JMSException {
Connection conn = null;
try {
ConnectionFactory cf = new ActiveMQConnectionFactory("tcp://localhost:61616");
conn = cf.createConnection();
conn.start();
DestinationSource destinationSource = ((ActiveMQConnection) conn).getDestinationSource();
Set<ActiveMQQueue> queues = destinationSource.getQueues();
Set<ActiveMQTopic> topics = destinationSource.getTopics();
System.out.println(queues);
System.out.println(topics);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (conn != null) {
try {
conn.close();
} catch (Exception e) {
}
}
}
}
}
UPDATE
You can use AdvisorySupport.getConsumerAdvisoryTopic()
Note that the consumer start/stop advisory messages also have a
consumerCount header to indicate the number of active consumers on the
destination when the advisory message was sent.
So I have a simple JMS app, using a topic, powered by activeMQ. It works, but only 1 message is being sent (even though I am writing more lines in the console and so trying to send more stuff).
When I check the web console of ActiveMq, only 1 message is being sent (I also get this message in the ReceiverTopic class)...Why is this happening?
Below you can see my sender code:
package topic;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageListener;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.apache.activemq.ActiveMQConnection;
import org.apache.activemq.ActiveMQConnectionFactory;
public class SenderTopic {
private ConnectionFactory factory = null;
private Connection connection = null;
private Session session = null;
private Destination destination = null;
private MessageProducer producer = null;
private boolean jmsInitialized = false;
public SenderTopic() {
}
private void initJMS() throws JMSException {
factory = new ActiveMQConnectionFactory(ActiveMQConnection.DEFAULT_BROKER_URL);
connection = factory.createConnection();
connection.start();
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
destination = session.createTopic("SAMPLE_TOPIC");
producer = session.createProducer(destination);
jmsInitialized = true;
}
private void sendMessage(String message) {
if (!jmsInitialized) {
try {
initJMS();
sendTextMessage(message);
} catch (JMSException e) {
jmsInitialized = false;
e.printStackTrace();
}
}
}
private void sendTextMessage(String message) throws JMSException {
TextMessage textMessage = session.createTextMessage(message);
producer.send(textMessage);
}
public static void main(String[] args) throws IOException {
SenderTopic receiver = new SenderTopic();
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
while (true) {
String message = reader.readLine();
receiver.sendMessage(message);
}
}
}
Intially the value of jmsInitialized is false, so your if condition (!jmsInitialized) will be true.
On second call of sendMessage the value of jmsInitialized will be true and the if condition fails because you are using not on boolean value.
You can add a else condition with only call to sendTextMessage.
try out this
private void sendMessage(String message) {
try {
if (!jmsInitialized) {
initJMS();
sendTextMessage(message);
}else{
sendTextMessage(message);
}
} catch (JMSException e) {
jmsInitialized = false;
e.printStackTrace();
}
}
}
Here's my server's code
package local.xx.mavenws;
import java.io.IOException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.websocket.OnClose;
import javax.websocket.OnError;
import javax.websocket.OnMessage;
import javax.websocket.OnOpen;
import javax.websocket.Session;
import javax.websocket.server.ServerEndpoint;
import javax.enterprise.context.ApplicationScoped;
import org.joda.time.DateTime;
#ApplicationScoped
#ServerEndpoint("/")
public class Server {
private final ArrayList<Session> sessions;
public Server() {
this.sessions = new ArrayList<>();
}
#OnOpen
public void onOpen(Session session) {
this.sessions.add(session);
this.echo("Client connected!");
}
#OnClose
public void onClose(Session session) {
this.sessions.remove(session);
this.echo("Client disconnected!");
}
#OnError
public void onError(Throwable error) {
this.echo("Error occured!");
this.echo(error.getLocalizedMessage());
}
#OnMessage
public void onMessage(String message, Session session) {
try {
message = "[" + this.currentDate() + "] " + message;
this.echo(message);
for( Session sess : this.sessions ) {
sess.getBasicRemote().sendText(message);
}
} catch (IOException ex) {
Logger.getLogger(Server.class.getName()).log(Level.SEVERE, null, ex);
}
}
private void echo(String info) {
System.out.println(info);
}
private String currentDate() {
String dateArray[] = (new DateTime()).toString().split("T");
String date = dateArray[0] + " " + (dateArray[1].split("\\.")[0]);
return date;
}
}
I want it to send received message to all the users connected. The problem is, it treats every connection individually like each one of them had it's own instance of the server. When I connect in two browser windows, messages show separately. Does anybody have any ideas on this?
Finally got this working! The solution is that the sessions variable must be static and I had to call it always by Server scope, not this. That implicates the fact that, despite there's a new instance of Server created for every user connected, the variable is mutual for everyone.