Let say we have MBean on the server side with predefined API. I would like to call that MBean from client machine using API directly. How do I create simple JMX client to get remote java objects via RMI?
Usage
private CacheMBean<String, Message> messageStorage = newJmxClient(
CacheMBean.class,
"com.my.company:type=cache,name=MessageCacheImpl",
"service:jmx:rmi:///jndi/rmi://<host>:<port>/karaf-root",
"smx",
"smx");
...
Message message = messageStorage.get(messageId);
Implementation
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Map;
import javax.management.InstanceAlreadyExistsException;
import javax.management.JMX;
import javax.management.MBeanServer;
import javax.management.MBeanServerConnection;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectInstance;
import javax.management.ObjectName;
import javax.management.StandardMBean;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
public final class JmxUtils {
private static final Logger LOG = LoggerFactory.getLogger(JmxUtils.class);
private JmxUtils() {
}
public static <T> T newJmxClient(Class<T> clazz, String objectName, String serviceUrl) {
return newJmxClient(clazz, objectName, serviceUrl, null, null);
}
public static <T> T newJmxClient(Class<T> clazz, String objectName, String serviceUrl, final String user, final String pass) {
try {
JMXServiceURL jmxServiceUrl = new JMXServiceURL(serviceUrl);
Map<String, ?> env = user == null ? null : ImmutableMap.of(JMXConnector.CREDENTIALS, new String[] {user, pass});
JMXConnector jmxc = JMXConnectorFactory.connect(jmxServiceUrl, env);
MBeanServerConnection mbsc = jmxc.getMBeanServerConnection();
ObjectName mbeanName = new ObjectName(objectName);
return JMX.newMBeanProxy(mbsc, mbeanName, clazz, true);
} catch (IOException | MalformedObjectNameException e) {
throw new RuntimeException("Can not create client for remote JMX " + serviceUrl, e);
}
}
/**
* #param objName
* domain:type=value[,name=value]
* #param implementation
* #param mbeanInterface
* #see ObjectName
* #see StandardMBean
*/
public static <I> ObjectInstance registerMBean(String objName, I implementation, Class<I> mbeanInterface) {
int counter = 0;
String uniqueSuffix = "";
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
while (true) {
try {
ObjectName name = new ObjectName(objName + uniqueSuffix);
StandardMBean mbean = new StandardMBean(implementation, mbeanInterface);
return mbs.registerMBean(mbean, name);
} catch (InstanceAlreadyExistsException e) {
LOG.trace("JMX instance exists, trying next available", e);
uniqueSuffix = "" + ++counter;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
/**
* #param objName
* #see ObjectName
*/
public static void unregisterMBean(String objName) {
try {
final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
final ObjectName name = new ObjectName(objName);
mbs.unregisterMBean(name);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
Related
When I'm running a Java WebSocketStompClient, I got below error:
org.eclipse.jetty.websocket.api.MessageTooLargeException: Text message size [73728] exceeds maximum size [65536]
Sample code:
import org.apache.log4j.Logger;
import org.springframework.messaging.simp.stomp.StompFrameHandler;
import org.springframework.messaging.simp.stomp.StompHeaders;
import org.springframework.messaging.simp.stomp.StompSession;
import org.springframework.messaging.simp.stomp.StompSessionHandlerAdapter;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.web.socket.WebSocketHttpHeaders;
import org.springframework.web.socket.client.WebSocketClient;
import org.springframework.web.socket.client.standard.StandardWebSocketClient;
import org.springframework.web.socket.messaging.WebSocketStompClient;
import org.springframework.web.socket.sockjs.client.SockJsClient;
import org.springframework.web.socket.sockjs.client.Transport;
import org.springframework.web.socket.sockjs.client.WebSocketTransport;
import org.springframework.web.socket.sockjs.frame.Jackson2SockJsMessageCodec;
import java.lang.reflect.Type;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
public class HelloClient {
private static Logger logger = Logger.getLogger(HelloClient.class);
StompSession session;
private final static WebSocketHttpHeaders headers = new WebSocketHttpHeaders();
public ListenableFuture<StompSession> connect() {
Transport webSocketTransport = new WebSocketTransport(new StandardWebSocketClient());
List<Transport> transports = Collections.singletonList(webSocketTransport);
SockJsClient sockJsClient = new SockJsClient(transports);
sockJsClient.setMessageCodec(new Jackson2SockJsMessageCodec());
WebSocketStompClient stompClient = new WebSocketStompClient(sockJsClient);
long[] hb = stompClient.getDefaultHeartbeat();
boolean en = stompClient.isDefaultHeartbeatEnabled();
long timeout = stompClient.getReceiptTimeLimit();
String url = "https://www.test.com";
return stompClient.connect(url, headers, new MyHandler());
}
public void subscribeMsg(StompSession stompSession) throws ExecutionException, InterruptedException {
stompSession.subscribe("/topic/test", new StompFrameHandler() {
public Type getPayloadType(StompHeaders stompHeaders) {
return byte[].class;
}
public void handleFrame(StompHeaders stompHeaders, Object o) {
logger.info("Received message " + new String((byte[]) o));
String response = new String((byte[]) o);
}
});
}
private class MyHandler extends StompSessionHandlerAdapter {
public void afterConnected(StompSession stompSession, StompHeaders stompHeaders) {
logger.info("Now connected");
session = stompSession;
}
}
public boolean isConnected() {
try {
Thread.sleep(500);
return session != null && session.isConnected();
} catch (Exception e) {
logger.warn("Error happens when checking connection status, ", e);
return false;
}
}
public static void main(String[] args) throws Exception {
HelloClient helloClient = new HelloClient();
ListenableFuture<StompSession> f = helloClient.connect();
StompSession stompSession = f.get();
helloClient.subscribeMsg(stompSession);
while (true) {
if (!helloClient.isConnected()) {
logger.info("wss diconnected ");
logger.info("need re-create ");
}
}
}
}
How to increase the limitation for a Java stomp websocket client? I found some not related answers How can I set max buffer size for web socket client(Jetty) in Java which are not suitable for stomp websocket client.
Also tried stompClient.setInboundMessageSizeLimit(Integer.MAX_VALUE); which doesn't work.
Hi i have created a handler in java for getting the events from dynamo DB
Here is my code
package com.Lambda.dynamodb;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
public class DDBEventProcessor implements
RequestHandler<DynamodbEvent, String> {
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()){
System.out.println(record.getEventID());
System.out.println(record.getEventName());
System.out.println(record.getDynamodb().toString());
}
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
}
}
Lambda function able to write the events in cloudwatch but the challenge is i have to index all the streamed records to the AWS elasticsearch service endpoint and index it.
while search through blogs i got few code samples in python and node.js but my requirement is i have to build this lambda function in java
Could anyone please suggest how to achieve this in java lambda function?
Hi i have included the code below may helpful to some one. Dynamo DB streams to index the document in elasticsearch both inside AWS and outside AWS
package com.Firstlambda;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemUtils;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.json.JSONObject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HelloWorld implements RequestHandler<DynamodbEvent, String> {
private static String serviceName = "es";
private static String region = "us-east-1";
private static String aesEndpoint = ""
private static String index = "";
private static String type = "_doc";
static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()) {
System.out.println("EventName : " + record.getEventName());
System.out.println("EventName : " + record.getDynamodb());
//AWS outside
RestHighLevelClient esClient = esClient();
//AWS outside
//AWS Inside
//RestHighLevelClient esClient = esClient(serviceName, region);
//AWS Inside
if (record.getEventName().toLowerCase().equals("insert")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
IndexRequest indexRequest = new IndexRequest(index, type, JsonUniqueId);
indexRequest.source(JsonString, XContentType.JSON);
try {
IndexResponse indexResponse = esClient.index(indexRequest, RequestOptions.DEFAULT);
System.out.println(indexResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else if (record.getEventName().toLowerCase().equals("modify")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
UpdateRequest request = new UpdateRequest(index, type, JsonUniqueId);
String jsonString = JsonString;
request.doc(jsonString, XContentType.JSON);
try {
UpdateResponse updateResponse = esClient.update(
request, RequestOptions.DEFAULT);
System.out.println(updateResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else {
System.out.println("remove");
System.out.println("KEYID : " + record.getDynamodb().getKeys().get("ID").getN());
String deletedId = record.getDynamodb().getKeys().get("ID").getN();
DeleteRequest request = new DeleteRequest(index, type, deletedId);
try {
DeleteResponse deleteResponse = esClient.delete(
request, RequestOptions.DEFAULT);
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
}
return "Successfullyprocessed";
}
public String getJsonstring(Map<String, AttributeValue> newIma) {
String json = null;
Map<String, AttributeValue> newImage = newIma;
List<Map<String, AttributeValue>> listOfMaps = new ArrayList<Map<String, AttributeValue>>();
listOfMaps.add(newImage);
List<Item> itemList = ItemUtils.toItemList(listOfMaps);
for (Item item : itemList) {
json = item.toJSON();
}
return json;
}
public String GetIdfromJsonString(String Json) {
JSONObject jsonObj = new JSONObject(Json);
return String.valueOf(jsonObj.getInt("ID"));
}
// Adds the interceptor to the ES REST client
// public static RestHighLevelClient esClient(String serviceName, String region) {
// AWS4Signer signer = new AWS4Signer();
// signer.setServiceName(serviceName);
// signer.setRegionName(region);
// HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
// return new RestHighLevelClient(RestClient.builder(HttpHost.create(aesEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
// }
public static RestHighLevelClient esClient() {
String host = "d9bc7cbca5ec49ea96a6ea683f70caca.eastus2.azure.elastic-cloud.com";
int port = 9200;
String userName = "elastic";
String password = "L4Nfnle3wxLmV95lffwsf$Ub46hp";
String protocol = "https";
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocol))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
}
This is just a sample code has to be modified based on our requirements
I am looking for reference where I can get simple program to send a SNMP trap to Apache Kafka topic using Apache Camel.
Please help me if someone can explain the it using simple java program.
My RouteBuilder configuration
import org.apache.camel.builder.RouteBuilder;
public class SimpleRouteBuilder extends RouteBuilder{
#Override
public void configure() throws Exception {
String topicName = "topic=first_topic";
String kafkaServer = "kafka:localhost:9092";
String zooKeeperHost = "zookeeperHost=localhost&zookeeperPort=2181";
String serializerClass = "serializerClass=kafka.serializer.StringEncoder";
String toKafka = new StringBuilder().append(kafkaServer).append("?").append(topicName).append("&")
.append(zooKeeperHost).append("&").append(serializerClass).toString();
System.out.println(toKafka);
from("snmp:127.0.0.1:161?protocol=udp&type=POLL&oids=1.3.6.1.2.1.1.5.0").split().tokenize("\n").to(toKafka);
}
}
Main Method
import org.apache.camel.CamelContext;
import org.apache.camel.impl.DefaultCamelContext;
import org.snmp4j.Snmp;
public class MainApp {
public static void main(String[] args) {
SimpleRouteBuilder routeBuilder = new SimpleRouteBuilder();
CamelContext ctx = new DefaultCamelContext();
try {
ctx.addRoutes(routeBuilder);
ctx.start();
Thread.sleep(5 * 60 * 1000);
ctx.stop();
}
catch (Exception e) {
e.printStackTrace();
}
}
}
I was in wrong direction. The write direction is as below -
Create a Trap sender program.
Create Trap receiver/listener program.
Inside Trap receiver or listener, receive trap and send it to Apache Kafka topic through Apache camel.
POM.XML
add below dependencies -
camel-core
snmp4j
camel-kafka
Trap Sender Program
package <>;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.snmp4j.*;
import org.snmp4j.event.ResponseEvent;
import org.snmp4j.mp.MPv2c;
import org.snmp4j.mp.MPv3;
import org.snmp4j.mp.SnmpConstants;
import org.snmp4j.security.*;
import org.snmp4j.smi.*;
import org.snmp4j.transport.DefaultUdpTransportMapping;
import java.util.Date;
public class Trapsender {
public static final String community = "public";
public static final String Oid = ".1.3.6.1.2.1.1.8";
public static final String ipAddress = "127.0.0.1";
public static final int port = 162;
public static void main(String[] args) {
Trapsender trapv3 = new Trapsender();
trapv3.sendTrap_Version3();
}
public void sendTrap_Version3() {
try {
// Create Transport Mapping
TransportMapping transport = new DefaultUdpTransportMapping();
transport.listen();
// Create Target
CommunityTarget cTarget = new CommunityTarget();
cTarget.setCommunity(new OctetString(community));
cTarget.setVersion(SnmpConstants.version2c);
cTarget.setAddress(new UdpAddress(ipAddress + "/" + port));
cTarget.setRetries(2);
cTarget.setTimeout(10000);
// Create PDU for V3
PDU pdu = new PDU();
pdu.setType(PDU.TRAP);
// need to specify the system up time
pdu.add(new VariableBinding(SnmpConstants.sysUpTime, new OctetString(new Date().toString())));
pdu.add(new VariableBinding(SnmpConstants.snmpTrapOID, new OID(Oid)));
pdu.add(new VariableBinding(new OID(Oid), new OctetString("Major")));
// Send the PDU
Snmp snmp = new Snmp(transport);
System.out.println("Sending V2 Trap... Check Wheather NMS is Listening or not? ");
ResponseEvent send = snmp.send(pdu, cTarget);
snmp.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Receiver Trap with Apache Camel
package <>;
import org.apache.camel.CamelContext;
import org.apache.camel.ProducerTemplate;
import org.apache.camel.RoutesBuilder;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.snmp4j.*;
import org.snmp4j.mp.MPv1;
import org.snmp4j.mp.MPv2c;
import org.snmp4j.security.Priv3DES;
import org.snmp4j.security.SecurityProtocols;
import org.snmp4j.smi.OctetString;
import org.snmp4j.smi.TcpAddress;
import org.snmp4j.smi.TransportIpAddress;
import org.snmp4j.smi.UdpAddress;
import org.snmp4j.transport.AbstractTransportMapping;
import org.snmp4j.transport.DefaultTcpTransportMapping;
import org.snmp4j.transport.DefaultUdpTransportMapping;
import org.snmp4j.util.MultiThreadedMessageDispatcher;
import org.snmp4j.util.ThreadPool;
import java.io.IOException;
public class Trapreceiver implements CommandResponder {
public static CamelContext ctx=null;
public static ProducerTemplate producer=null;
public static void main(String[] args) {
Trapreceiver snmp4jTrapReceiver = new Trapreceiver();
SimpleRouteBuilder routeBuilder = new SimpleRouteBuilder();
ctx = new DefaultCamelContext();
producer = ctx.createProducerTemplate();
try {
ctx.addRoutes(routeBuilder);
ctx.start();
}
catch (Exception e) {
e.printStackTrace();
}
// producer.sendBody("direct:start", snmp);
try {
snmp4jTrapReceiver.listen(new UdpAddress("localhost/162"), producer);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Trap Listner
*/
public synchronized void listen(TransportIpAddress address, ProducerTemplate producer)
throws IOException {
AbstractTransportMapping transport;
if (address instanceof TcpAddress) {
transport = new DefaultTcpTransportMapping((TcpAddress) address);
} else {
transport = new DefaultUdpTransportMapping((UdpAddress) address);
}
ThreadPool threadPool = ThreadPool.create("DispatcherPool", 10);
MessageDispatcher mDispathcher = new MultiThreadedMessageDispatcher(
threadPool, new MessageDispatcherImpl());
// add message processing models
mDispathcher.addMessageProcessingModel(new MPv1());
mDispathcher.addMessageProcessingModel(new MPv2c());
// add all security protocols
SecurityProtocols.getInstance().addDefaultProtocols();
SecurityProtocols.getInstance().addPrivacyProtocol(new Priv3DES());
// Create Target
CommunityTarget target = new CommunityTarget();
target.setCommunity(new OctetString("public"));
Snmp snmp = new Snmp(mDispathcher, transport);
snmp.addCommandResponder(this);
transport.listen();
System.out.println("Listening on " + address);
try {
this.wait();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
/**
* This method will be called whenever a pdu is received on the given port
* specified in the listen() method
*/
public synchronized void processPdu(CommandResponderEvent cmdRespEvent) {
System.out.println("Received PDU...");
PDU pdu = cmdRespEvent.getPDU();
if (pdu != null) {
System.out.println("Trap Type = " + pdu.getType());
System.out.println("Variables = " + pdu.getVariableBindings());
producer.sendBody("direct:start","Variables = " + pdu.getVariableBindings() );
}
}
}
I am using httpcore libraries 4.3.2 to make a custom http server to communicate with some clients (android devices).
The clients manage to communicate (post an xml) with the server but the response of the server is very slow.
My code is based on the ElementalHttpServer of apache samples. Can i find somewhere else better samples for the httpcore libraries?
Also i can not find how to set the parameteres or to finetune the httpserver because the HttParams lib is deprecated.
If i use the com.sun.net.httpserver.HttpServer httpserver my clients are connecting without problem. It is better to stay with suns libs or to use apaches?
Thanks in advance.
This is the code i used:
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ServerSocket;
import java.net.Socket;
import javax.net.ssl.SSLServerSocketFactory;
import org.apache.http.ConnectionClosedException;
import org.apache.http.HttpConnectionFactory;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpServerConnection;
import org.apache.http.HttpStatus;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.DefaultBHttpServerConnection;
import org.apache.http.impl.DefaultBHttpServerConnectionFactory;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpProcessor;
import org.apache.http.protocol.HttpProcessorBuilder;
import org.apache.http.protocol.HttpRequestHandler;
import org.apache.http.protocol.HttpService;
import org.apache.http.protocol.ResponseConnControl;
import org.apache.http.protocol.ResponseContent;
import org.apache.http.protocol.ResponseDate;
import org.apache.http.protocol.ResponseServer;
import org.apache.http.protocol.UriHttpRequestHandlerMapper;
import org.apache.http.util.EntityUtils;
public class Thread_Main_httpcore2 implements Runnable {
public Thread_Main_httpcore2() {
}//Thread_Main()
#Override
public void run() {
int port = 20010;
// Set up the HTTP protocol processor
HttpProcessor httpproc = HttpProcessorBuilder.create()
.add(new ResponseDate())
.add(new ResponseServer("Test/1.1"))
.add(new ResponseContent())
.add(new ResponseConnControl()).build();
// Set up request handlers
UriHttpRequestHandlerMapper reqistry = new UriHttpRequestHandlerMapper();
//// Set up the HTTP service
HttpService httpService = new HttpService(httpproc, reqistry);
reqistry.register("*", new Thread_Main_httpcore2.HttpReqHandler());
SSLServerSocketFactory sf = null;
try {
Thread t = new RequestListenerThread(port, httpService, sf);
t.setDaemon(false);
t.start();
} catch (Exception ex) {
ex.printStackTrace();
}
}
static class HttpReqHandler implements HttpRequestHandler {
public HttpReqHandler() {
super();
}
public void handle(
final HttpRequest request,
final HttpResponse response,
final HttpContext context) throws HttpException, IOException {
String xml_stream="";
if (request instanceof HttpEntityEnclosingRequest) {
HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
byte[] entityContent;
if (entity == null) {
entityContent = new byte[0];
} else {
entityContent = EntityUtils.toByteArray(entity);
}
//byte[] entityContent = EntityUtils.toByteArray(entity);
System.out.println("Incoming entity content (bytes): " + entityContent.length);
xml_stream = new String(entityContent);
}
StringBuilder resp_xml = new StringBuilder();
resp_xml.append("Returns an xml to the client");
StringEntity entity;
try {
response.setStatusCode(HttpStatus.SC_OK);
entity = new StringEntity(resp_xml.toString(), ContentType.create("text/html", "UTF-8"));
response.setEntity(entity);
} catch (Exception ex_send_resp) {
System.out.println("HttpFileHandler.ex_send_resp "+ ex_send_resp.toString());
}
}
}
static class RequestListenerThread extends Thread {
private final HttpConnectionFactory<DefaultBHttpServerConnection> connFactory;
private final ServerSocket serversocket;
private final HttpService httpService;
public RequestListenerThread(
final int port,
final HttpService httpService,
final SSLServerSocketFactory sf) throws IOException {
this.connFactory = DefaultBHttpServerConnectionFactory.INSTANCE;
this.serversocket = sf != null ? sf.createServerSocket(port) : new ServerSocket(port);
this.httpService = httpService;
}
#Override
public void run() {
System.out.println("Listening on port " + this.serversocket.getLocalPort());
while (!Thread.interrupted()) {
try {
// Set up HTTP connection
Socket socket = this.serversocket.accept();
System.out.println("Incoming connection from " + socket.getInetAddress());
HttpServerConnection conn = this.connFactory.createConnection(socket);
// Start worker thread
Thread t = new WorkerThread(this.httpService, conn);
t.setDaemon(true);
t.start();
} catch (InterruptedIOException ex) {
break;
} catch (IOException e) {
System.err.println("I/O error initialising connection thread: "
+ e.getMessage());
break;
}
}
}
}
static class WorkerThread extends Thread {
private final HttpService httpservice;
private final HttpServerConnection conn;
public WorkerThread(
final HttpService httpservice,
final HttpServerConnection conn) {
super();
this.httpservice = httpservice;
this.conn = conn;
}
#Override
public void run() {
System.out.println("New connection thread");
HttpContext context = new BasicHttpContext(null);
try {
while (!Thread.interrupted() && this.conn.isOpen()) {
this.httpservice.handleRequest(this.conn, context);
}
} catch (ConnectionClosedException ex) {
System.err.println("Client closed connection");
} catch (IOException ex) {
System.err.println("I/O error: " + ex.getMessage());
} catch (HttpException ex) {
System.err.println("Unrecoverable HTTP protocol violation: " + ex.getMessage());
} finally {
try {
this.conn.shutdown();
} catch (IOException ignore) {
}
}
}
}
}
I am using java spark API to write some test application . I am using a class which doesn't extends serializable interface . So to make the application work I am using kryo serializer to serialize the class . But the problem which I observed while debugging was that during the de-serialization the returned class object becomes null and in turn throws a null pointer exception . It seems to be closure problem where things are going wrong but not sure.Since I am new to this kind of serialization I don't know where to start digging.
Here is the code I am testing :
package org.apache.spark.examples;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
/**
* Spark application to test the Serialization issue in spark
*/
public class Test {
static PrintWriter outputFileWriter;
static FileWriter file;
static JavaSparkContext ssc;
public static void main(String[] args) {
String inputFile = "/home/incubator-spark/examples/src/main/scala/org/apache/spark/examples/InputFile.txt";
String master = "local";
String jobName = "TestSerialization";
String sparkHome = "/home/test/Spark_Installation/spark-0.7.0";
String sparkJar = "/home/test/TestSerializationIssesInSpark/TestSparkSerIssueApp/target/TestSparkSerIssueApp-0.0.1-SNAPSHOT.jar";
SparkConf conf = new SparkConf();
conf.set("spark.closure.serializer","org.apache.spark.serializer.KryoSerializer");
conf.set("spark.kryo.registrator", "org.apache.spark.examples.MyRegistrator");
// create the Spark context
if(master.equals("local")){
ssc = new JavaSparkContext("local", jobName,conf);
//ssc = new JavaSparkContext("local", jobName);
} else {
ssc = new JavaSparkContext(master, jobName, sparkHome, sparkJar);
}
JavaRDD<String> testData = ssc.textFile(inputFile).cache();
final NotSerializableJavaClass notSerializableTestObject= new NotSerializableJavaClass("Hi ");
#SuppressWarnings({ "serial", "unchecked"})
JavaRDD<String> classificationResults = testData.map(
new Function<String, String>() {
#Override
public String call(String inputRecord) throws Exception {
if(!inputRecord.isEmpty()) {
//String[] pointDimensions = inputRecord.split(",");
String result = "";
try {
FileWriter file = new FileWriter("/home/test/TestSerializationIssesInSpark/results/test_result_" + (int) (Math.random() * 100));
PrintWriter outputFile = new PrintWriter(file);
InetAddress ip;
ip = InetAddress.getLocalHost();
outputFile.println("IP of the server: " + ip);
result = notSerializableTestObject.testMethod(inputRecord);
outputFile.println("Result: " + result);
outputFile.flush();
outputFile.close();
file.close();
} catch (UnknownHostException e) {
e.printStackTrace();
}
catch (IOException e1) {
e1.printStackTrace();
}
return result;
} else {
System.out.println("End of elements in the stream.");
String result = "End of elements in the input data";
return result;
}
}
}).cache();
long processedRecords = classificationResults.count();
ssc.stop();
System.out.println("sssssssssss"+processedRecords);
}
}
Here is the KryoRegistrator class
package org.apache.spark.examples;
import org.apache.spark.serializer.KryoRegistrator;
import com.esotericsoftware.kryo.Kryo;
public class MyRegistrator implements KryoRegistrator {
public void registerClasses(Kryo kryo) {
kryo.register(NotSerializableJavaClass.class);
}
}
Here is the class I am serializing :
package org.apache.spark.examples;
public class NotSerializableJavaClass {
public String testVariable;
public NotSerializableJavaClass(String testVariable) {
super();
this.testVariable = testVariable;
}
public String testMethod(String vartoAppend){
return this.testVariable + vartoAppend;
}
}
This is because spark.closure.serializer only supports the Java serializer. See http://spark.apache.org/docs/latest/configuration.html about spark.closure.serializer