Push data from Dynamo DB to elasticsearch using java - java

Hi i have created a handler in java for getting the events from dynamo DB
Here is my code
package com.Lambda.dynamodb;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
public class DDBEventProcessor implements
RequestHandler<DynamodbEvent, String> {
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()){
System.out.println(record.getEventID());
System.out.println(record.getEventName());
System.out.println(record.getDynamodb().toString());
}
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
}
}
Lambda function able to write the events in cloudwatch but the challenge is i have to index all the streamed records to the AWS elasticsearch service endpoint and index it.
while search through blogs i got few code samples in python and node.js but my requirement is i have to build this lambda function in java
Could anyone please suggest how to achieve this in java lambda function?

Hi i have included the code below may helpful to some one. Dynamo DB streams to index the document in elasticsearch both inside AWS and outside AWS
package com.Firstlambda;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemUtils;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.json.JSONObject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HelloWorld implements RequestHandler<DynamodbEvent, String> {
private static String serviceName = "es";
private static String region = "us-east-1";
private static String aesEndpoint = ""
private static String index = "";
private static String type = "_doc";
static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()) {
System.out.println("EventName : " + record.getEventName());
System.out.println("EventName : " + record.getDynamodb());
//AWS outside
RestHighLevelClient esClient = esClient();
//AWS outside
//AWS Inside
//RestHighLevelClient esClient = esClient(serviceName, region);
//AWS Inside
if (record.getEventName().toLowerCase().equals("insert")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
IndexRequest indexRequest = new IndexRequest(index, type, JsonUniqueId);
indexRequest.source(JsonString, XContentType.JSON);
try {
IndexResponse indexResponse = esClient.index(indexRequest, RequestOptions.DEFAULT);
System.out.println(indexResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else if (record.getEventName().toLowerCase().equals("modify")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
UpdateRequest request = new UpdateRequest(index, type, JsonUniqueId);
String jsonString = JsonString;
request.doc(jsonString, XContentType.JSON);
try {
UpdateResponse updateResponse = esClient.update(
request, RequestOptions.DEFAULT);
System.out.println(updateResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else {
System.out.println("remove");
System.out.println("KEYID : " + record.getDynamodb().getKeys().get("ID").getN());
String deletedId = record.getDynamodb().getKeys().get("ID").getN();
DeleteRequest request = new DeleteRequest(index, type, deletedId);
try {
DeleteResponse deleteResponse = esClient.delete(
request, RequestOptions.DEFAULT);
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
}
return "Successfullyprocessed";
}
public String getJsonstring(Map<String, AttributeValue> newIma) {
String json = null;
Map<String, AttributeValue> newImage = newIma;
List<Map<String, AttributeValue>> listOfMaps = new ArrayList<Map<String, AttributeValue>>();
listOfMaps.add(newImage);
List<Item> itemList = ItemUtils.toItemList(listOfMaps);
for (Item item : itemList) {
json = item.toJSON();
}
return json;
}
public String GetIdfromJsonString(String Json) {
JSONObject jsonObj = new JSONObject(Json);
return String.valueOf(jsonObj.getInt("ID"));
}
// Adds the interceptor to the ES REST client
// public static RestHighLevelClient esClient(String serviceName, String region) {
// AWS4Signer signer = new AWS4Signer();
// signer.setServiceName(serviceName);
// signer.setRegionName(region);
// HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
// return new RestHighLevelClient(RestClient.builder(HttpHost.create(aesEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
// }
public static RestHighLevelClient esClient() {
String host = "d9bc7cbca5ec49ea96a6ea683f70caca.eastus2.azure.elastic-cloud.com";
int port = 9200;
String userName = "elastic";
String password = "L4Nfnle3wxLmV95lffwsf$Ub46hp";
String protocol = "https";
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocol))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
}
This is just a sample code has to be modified based on our requirements

Related

Why does MSAL4J call ends with Error 404 ResourceNotFound?

I'm using msl4j to interact with microsoft products, e.g. emails, calendars. When I call the user informations (without parameters) this will works fine. But, when I try to read messages from the inbox, the call ended with error 404 ("code":"ResourceNotFound","message":"Resource could not be discovered."). I don't know why. The API Permissions seems correct.
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import com.microsoft.aad.msal4j.ClientCredentialFactory;
import com.microsoft.aad.msal4j.ClientCredentialParameters;
import com.microsoft.aad.msal4j.ConfidentialClientApplication;
import com.microsoft.aad.msal4j.DeviceCode;
import com.microsoft.aad.msal4j.DeviceCodeFlowParameters;
import com.microsoft.aad.msal4j.IAccount;
import com.microsoft.aad.msal4j.IAuthenticationResult;
import com.microsoft.aad.msal4j.IClientCredential;
import com.microsoft.aad.msal4j.MsalException;
import com.microsoft.aad.msal4j.OnBehalfOfParameters;
import com.microsoft.aad.msal4j.PublicClientApplication;
import com.microsoft.aad.msal4j.SilentParameters;
import com.microsoft.aad.msal4j.UserAssertion;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
public class AppMSAL4J {
private static String userId;
private static String authority;
private static String clientId;
private static String clientSecret;
private static String tenantId;
private static Set<String> scopes;
public static void main(String args[]) throws Exception {
setUpSampleData();
try {
IAuthenticationResult result = acquireToken();
OkHttpClient client = new OkHttpClient().newBuilder().build();
MediaType mediaType = MediaType.parse("application/json");
String bodyString = "";
RequestBody body = RequestBody.create(bodyString, mediaType);
String baseUrl = "https://graph.microsoft.com/v1.0/users/" + userId;
String parameters = "";
// parameters = "/mailfolders('Inbox')/messages";
// parameters = "/messages";
Request request = new Request.Builder()
.url(baseUrl + parameters)
.method("GET", null)
.addHeader("Accept", "application/json")
.addHeader("Authorization", "Bearer " + result.accessToken())
.build();
Response response = client.newCall(request).execute();
System.out.println(response.body().string());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private static IAuthenticationResult acquireToken() throws Exception {
IClientCredential credential = ClientCredentialFactory.createFromSecret(clientSecret);
ConfidentialClientApplication cca = ConfidentialClientApplication
.builder(clientId, credential)
.authority(authority)
.build();
ClientCredentialParameters parameters = ClientCredentialParameters
.builder(scopes)
.build();
return cca.acquireToken(parameters).join();
}
private static void setUpSampleData() {
userId = "b0f***";
tenantId = "fc2***";
authority = "https://login.microsoftonline.com/" + tenantId;
clientId = "b1a***";
clientSecret = "KJ***";
scopes = Collections.singleton("https://graph.microsoft.com/.default");
}
}

email receives duplicates from SPRING JAVA Mail APIs

I use Java mail APIs from a spring web application to send a weekly email to an outlook mail.
The feature was behaving normally for the first couple of weeks, then without any changes outlook received two emails, the next week three emails were received, then four, then five emails.
The logs set in the Java code indicates that only one email is being sent from the application.
I can't replicate the issue by changing the schedule to send each 15 minutes, or hour, or any shorter interval.
Email controller class
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
#Component
public class WeeklyReportScheduler {
#Autowired
private WeeklyReportService weeklyReportService;
#Scheduled(cron = "${cron.expression}")
public void sendWeeklyReport(){
weeklyReportService.sendWeeklyReport();
}
}
Email service class:
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.mail.MailException;
import org.springframework.stereotype.Service;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
#Service
public class WeeklyReportService {
#Value("${weekly.report.mail.subject}")
private String subject;
#Value("${weekly.report.mail.body}")
private String mailBody;
#Value("${mail.body.empty.report}")
private String emptyReportMailBody;
#Value("${receiver}")
private String receiver;
#Autowired
private MailService mailService;
#Autowired
private WeeklyReportLogDao weeklyReportLogDao;
#Autowired
private ProjectService projectService;
#Value("${export.path}")
private String exportDir;
protected final Log logger = LogFactory.getLog(getClass());
public void sendWeeklyReport(){
boolean emptyReport = true;
//retrieving attachment data 'projects'
if(projects.size() != 0){
emptyReport = false;
}
String body = "";
if(emptyReport){
body = emptyReportMailBody;
} else {
body = mailBody;
}
SimpleDateFormat format = new SimpleDateFormat("MM/dd/YYYY");
String dateString = format.format(new Date());
String mailSubject = subject + " " + dateString;
List recipients = new ArrayList<String>();
recipients.add(receiver);
String fileName = mailSubject.replace(" ", "_").replace("/", "_");
WeeklyReportExcelExport excelExport = new WeeklyReportExcelExport(exportDir, fileName);
excelExport.createReport(projects);
File excelFile = excelExport.saveToFile();
File[] attachments = new File[1];
attachments[0] = excelFile;
boolean sent = false;
String exceptionMessage = "";
for(int i=0; i<3; i++){
try {
logger.info("Sending Attempt: " + i+1);
Thread.sleep(10000);
mailService.mail(recipients, mailSubject, body, attachments);
sent = true;
break;
} catch (Exception ex) {
logger.info("sending failed because: " + ex.getMessage() + " \nRe-attempting in 10 seconds");
exceptionMessage = ex.getMessage();
sent = false;
}
}
if(!sent){
weeklyReportLogDao.logFailedReporting(dateString, exceptionMessage);
}
//re-try 3 times in case of mail sending failure
}
MailService class:
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Map;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.mail.MailException;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.mail.javamail.MimeMessageHelper;
public class MailServiceImpl implements MailService {
/** The From address for the e-mail. read from ant build properties file */
private String fromAddress;
/** The mail sender. */
private JavaMailSender mailSender;
/** Logger for this class and subclasses */
protected final Log logger = LogFactory.getLog(getClass());
public void mail(List<String> emailAddresses, String subject, String text, File[] attachments) throws MailException {
//System.out.println("mail: "+subject);
MimeMessage message = null;
// Fill in the From, To, and Subject fields.
try {
message = mailSender.createMimeMessage();
MimeMessageHelper messageHelper = new MimeMessageHelper(message, true);
messageHelper.setFrom(fromAddress);
for (String emailAddress : emailAddresses) {
messageHelper.addTo(emailAddress);
}
messageHelper.setSubject(subject);
// Fill in the body with the message text, sending it in HTML format.
messageHelper.setText(text, true);
// Add any attachments to the message.
if ((attachments != null) && (attachments.length != 0)) {
for (File attachment : attachments) {
messageHelper.addAttachment(attachment.getName(), attachment);
}
}
}
catch(MessagingException mse) {
String warnMessage = "Error creating message.";
logger.warn(warnMessage);
throw (new RuntimeException(warnMessage, mse));
}
try {
mailSender.send(message);
} catch (Exception ex){
logger.info("Exception sending message: " + ex.getMessage());
}
}
}
you might have duplicate entries on your argument emailAddresses, try moving it to treeset if you cant ensure duplicate entries from the repository layer

SSL handshake Android studio java problem for okhttp ok_get

I face a problem to get Json data into my application. On get action. I have solve the problem for POST. We have a ssl certificate but seems for get and post the way is little different. I can't figure out. So if people can give me an help here. I will appreciate.
package com.example.administrator.superclass.Utils;
import android.app.ProgressDialog;
import android.content.Context;
import android.net.ConnectivityManager;
import android.os.Looper;
import android.util.Log;
import com.example.administrator.superclass.MyApp;
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;
import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.FormBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
public class Okhttp_Util {
public static String Ok_Get(Context context,String url, HashMap<String, String> hashMap) {
String data = "";
// OkHttpClient okHttpClient = new OkHttpClient.Builder()
// .connectTimeout(10, TimeUnit.SECONDS)
// .readTimeout(10,TimeUnit.SECONDS)
// .build();
OkHttpClient okHttpClient= MyApp.handleSSLHandshake();
FormBody.Builder builder = new FormBody.Builder();
for (String key : hashMap.keySet()) {
builder.add(key, hashMap.get(key));
}
builder.build();
RequestBody requestBody = builder.build();
Request request = new Request.Builder().url(url).build();
try {
Response response = okHttpClient.newCall(request).execute();
data = response.body().string();
} catch (IOException e) {
if(e instanceof SocketTimeoutException){//判断超时异常
Looper.prepare();
if (ProgressDialog_Util.isshow){
ProgressDialog_Util.Dialog_dismiss();
}
Toast_util.Toast_show(context,"连接超时!请更换网络后重试!");
Log.e("geterr1: ",e.toString() );
Looper.loop();
}
if(e instanceof ConnectException){//判断连接异常,我这里是报Failed to connect to 10.7.5.144
Looper.prepare();
if (ProgressDialog_Util.isshow){
ProgressDialog_Util.Dialog_dismiss();
}
Toast_util.Toast_show(context,e.toString());
Log.e( "geterr2: ",e.toString() );
Looper.loop();
}
}
return data;
}
public static String Ok_Post(Context context,String url, HashMap<String, String> hashMap) {
String data = "";
// OkHttpClient okHttpClient = new OkHttpClient.Builder()
// .connectTimeout(10, TimeUnit.SECONDS)
// .readTimeout(10,TimeUnit.SECONDS)
// .build();
OkHttpClient okHttpClient= MyApp.handleSSLHandshake();
FormBody.Builder builder = new FormBody.Builder();
for (String key : hashMap.keySet()) {
builder.add(key, hashMap.get(key));
}
builder.build();
RequestBody requestBody = builder.build();
Request request = new Request.Builder().post(requestBody).url(url).build();
try {
Response response = okHttpClient.newCall(request).execute();
data = response.body().string();
} catch (IOException e) {
if(e instanceof SocketTimeoutException){//判断超时异常
Looper.prepare();
if (ProgressDialog_Util.isshow){
ProgressDialog_Util.Dialog_dismiss();
}
Toast_util.Toast_show(context,"连接超时!");
Log.e("posterr1: ",e.toString() );
Looper.loop();
}
if(e instanceof ConnectException){//判断连接异常,我这里是报Failed to connect to 10.7.5.144
Looper.prepare();
if (ProgressDialog_Util.isshow){
ProgressDialog_Util.Dialog_dismiss();
}
Toast_util.Toast_show(context,e.toString());
Log.e( "posterr2: ",e.toString() );
Looper.loop();
}
}
return data;
}
}
From the compilation i get an error :
enter image description here
The error indicates that you are calling your get request without passing a HashMap.
My suggestion would be to either pass an empty HashMap if you have some URLs that require a body. Else remove it entirely:
public static String Ok_Get(Context context,String url) {
String data = "";
OkHttpClient okHttpClient= MyApp.handleSSLHandshake();
FormBody.Builder builder = new FormBody.Builder();
builder.build();
Request request = new Request.Builder().url(url).build();
If you still need it, when you call it make sure you include the HashMap:
Ok_Get(context, "your/url/string/here", Collections.<String, String>emptyMap())

REST Streaming JSON Output

We have JAX RS implementation which needs to send back JSON output. But the response size is huge. And the client expects the same synchronously.
Hence I tried to use StreamingOutput... but the client is not really getting the data in chunks.
Below is sample snippet:
Server Side
streamingOutput = new StreamingOutput() {
#Override
public void write(OutputStream out) throws IOException, WebApplicationException {
JsonGenerator jsonGenerator = mapper.getFactory().createGenerator(out);
jsonGenerator.writeStartArray();
for(int i=0; i < 10; i++) {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField("Response_State", "Response State - " + i);
jsonGenerator.writeStringField("Response_Report", "Response Report - " + i);
jsonGenerator.writeStringField("Error_details", "Error Details - " + i);
jsonGenerator.writeEndObject();;
jsonGenerator.flush();
try {
Thread.currentThread().sleep(2000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
jsonGenerator.writeEndArray();
jsonGenerator.close();
}
};
return Response.status(200).entity(streamingOutput).build();
Client
HttpClient client = HttpClientBuilder.create().build();
HttpPost post = new HttpPost("http://localhost:8080/AccessData/FetchReport");
post.setHeader("Content-type", "application/json");
ResponseHandler<HttpResponse> responseHandler = new BasicResponseHandler();
StringEntity entity = new StringEntity(jsonRequest); //jsonRequest is request string
post.setEntity(entity);
HttpResponse response = client.execute(post);
BufferedReader buffReader = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
JsonParser jsonParser = new JsonFactory().createParser(buffReader);
while(jsonParser.nextToken() != JsonToken.END_OBJECT) {
System.out.println(jsonParser.getCurrentName() + ":" + jsonParser.getCurrentValue());
}
String output;
while((output = buffReader.readLine()) != null) {
System.out.println(output);
}
In the server side code, I am putting sleep call just to simulate a gap between chunks of data. What I need is that the client should receive chunks of data as and when it is thrown back by the server.
But here the client gets the response in entirety always.
Any possible solution?
Thanks in advance.
It looks like the client side is not implemented correctly: reading the array of the objects using the parser.
Also, I would like to recommend reading and writing a data transfer object instead of low level field-by-field reading and writing.
For the sake of completeness, here is a complete draft example that uses: Jersey 2.25.1, Jetty 9.2.14.v20151106.
Common
ResponseData class
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class ResponseData {
private final String responseState;
private final String responseReport;
private final String errorDetails;
#JsonCreator
public ResponseData(
#JsonProperty("Response_State") final String responseState,
#JsonProperty("Response_Report") final String responseReport,
#JsonProperty("Error_details") final String errorDetails) {
this.responseState = responseState;
this.responseReport = responseReport;
this.errorDetails = errorDetails;
}
public String getResponseState() {
return this.responseState;
}
public String getResponseReport() {
return this.responseReport;
}
public String getErrorDetails() {
return this.errorDetails;
}
#Override
public String toString() {
return String.format(
"ResponseData: responseState: %s; responseReport: %s; errorDetails: %s",
this.responseState,
this.responseReport,
this.errorDetails
);
}
}
Service
ServerProgram class
import java.net.URI;
import org.glassfish.jersey.jackson.JacksonFeature;
import org.glassfish.jersey.jetty.JettyHttpContainerFactory;
import org.glassfish.jersey.server.ResourceConfig;
public class ServerProgram {
public static void main(final String[] args) {
final URI uri = URI.create("http://localhost:8080/");
final ResourceConfig resourceConfig = new ResourceConfig(TestResource.class);
resourceConfig.register(JacksonFeature.class);
JettyHttpContainerFactory.createServer(uri, resourceConfig);
}
}
TestResource class
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.OutputStream;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
#Path("/")
public class TestResource {
#GET
#Produces(MediaType.APPLICATION_JSON)
public Response getData() {
final StreamingOutput streamingOutput = new JsonStreamingOutput();
return Response.status(200).entity(streamingOutput).build();
}
private static class JsonStreamingOutput implements StreamingOutput {
#Override
public void write(final OutputStream outputStream) throws IOException, WebApplicationException {
final ObjectMapper objectMapper = new ObjectMapper();
final JsonFactory jsonFactory = objectMapper.getFactory();
try (final JsonGenerator jsonGenerator = jsonFactory.createGenerator(outputStream)) {
jsonGenerator.writeStartArray();
for (int i = 0; i < 10; i++) {
final ResponseData responseData = new ResponseData(
"Response State - " + i,
"Response Report - " + i,
"Error Details - " + i
);
jsonGenerator.writeObject(responseData);
jsonGenerator.flush();
try {
Thread.currentThread().sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
jsonGenerator.writeEndArray();
}
}
}
}
Client
ClientProgram class
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.core.MediaType;
import org.glassfish.jersey.client.ClientProperties;
public class ClientProgram {
public static void main(final String[] args) throws IOException {
Client client = null;
try {
client = ClientBuilder.newClient();
client.property(ClientProperties.READ_TIMEOUT, 10000);
try (final InputStream inputStream = client
.target("http://localhost:8080/")
.request(MediaType.APPLICATION_JSON)
.get(InputStream.class);
final BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream)) {
processStream(bufferedInputStream);
}
} finally {
if (client != null) {
client.close();
}
}
}
private static void processStream(final InputStream inputStream) throws IOException {
final ObjectMapper objectMapper = new ObjectMapper();
final JsonFactory jsonFactory = objectMapper.getFactory();
try (final JsonParser jsonParser = jsonFactory.createParser(inputStream)) {
final JsonToken arrayToken = jsonParser.nextToken();
if (arrayToken == null) {
// TODO: Return or throw exception.
return;
}
if (!JsonToken.START_ARRAY.equals(arrayToken)) {
// TODO: Return or throw exception.
return;
}
// Iterate through the objects of the array.
while (JsonToken.START_OBJECT.equals(jsonParser.nextToken())) {
final ResponseData responseData = jsonParser.readValueAs(ResponseData.class);
System.out.println(responseData);
}
}
}
}
Hope this helps.

Extended Html Unit Driver with screenshot capabilities

I am working on code (java) that will open up a selenium headless browser with HTML Unit webdriver and then take a screenshot. Unfortunately, HTML Unit does not support screenshots on its own, so I had to download an extended version:
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.internal.Base64Encoder;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements TakesScreenshot {
private static Map<String, byte[]> imagesCache = Collections.synchronizedMap(new HashMap<String, byte[]>());
private static Map<String, String> cssjsCache = Collections.synchronizedMap(new HashMap<String, String>());
// http://stackoverflow.com/questions/4652777/java-regex-to-get-the-urls-from-css
private final static Pattern cssUrlPattern = Pattern.compile("background(-image)?[\\s]*:[^url]*url[\\s]*\\([\\s]*([^\\)]*)[\\s]*\\)[\\s]*");// ?<url>
public ScreenCaptureHtmlUnitDriver() {
super();
}
public ScreenCaptureHtmlUnitDriver(boolean enableJavascript) {
super(enableJavascript);
}
public ScreenCaptureHtmlUnitDriver(Capabilities capabilities) {
super(capabilities);
}
public ScreenCaptureHtmlUnitDriver(BrowserVersion version) {
super(version);
DesiredCapabilities var = ((DesiredCapabilities) getCapabilities());
var.setCapability(CapabilityType.TAKES_SCREENSHOT, true);
}
//#Override
#SuppressWarnings("unchecked")
public <X> X getScreenshotAs(OutputType<X> target) throws WebDriverException {
byte[] archive = new byte[0];
try {
archive = downloadCssAndImages(getWebClient(), (HtmlPage) getCurrentWindow().getEnclosedPage());
} catch (Exception e) {
}
if(target.equals(OutputType.BASE64)){
return target.convertFromBase64Png(new Base64Encoder().encode(archive));
}
if(target.equals(OutputType.BYTES)){
return (X) archive;
}
return (X) archive;
}
// http://stackoverflow.com/questions/2244272/how-can-i-tell-htmlunits-webclient-to-download-images-and-css
protected byte[] downloadCssAndImages(WebClient webClient, HtmlPage page) throws Exception {
WebWindow currentWindow = webClient.getCurrentWindow();
Map<String, String> urlMapping = new HashMap<String, String>();
Map<String, byte[]> files = new HashMap<String, byte[]>();
WebWindow window = null;
try {
window = webClient.getWebWindowByName(page.getUrl().toString()+"_screenshot");
webClient.getPage(window, new WebRequest(page.getUrl()));
} catch (Exception e) {
window = webClient.openWindow(page.getUrl(), page.getUrl().toString()+"_screenshot");
}
String xPathExpression = "//*[name() = 'img' or name() = 'link' and (#type = 'text/css' or #type = 'image/x-icon') or #type = 'text/javascript']";
List<?> resultList = page.getByXPath(xPathExpression);
Iterator<?> i = resultList.iterator();
while (i.hasNext()) {
try {
HtmlElement el = (HtmlElement) i.next();
String resourceSourcePath = el.getAttribute("src").equals("") ? el.getAttribute("href") : el
.getAttribute("src");
if (resourceSourcePath == null || resourceSourcePath.equals(""))
continue;
URL resourceRemoteLink = page.getFullyQualifiedUrl(resourceSourcePath);
String resourceLocalPath = mapLocalUrl(page, resourceRemoteLink, resourceSourcePath, urlMapping);
urlMapping.put(resourceSourcePath, resourceLocalPath);
if (!resourceRemoteLink.toString().endsWith(".css")) {
byte[] image = downloadImage(webClient, window, resourceRemoteLink);
files.put(resourceLocalPath, image);
} else {
String css = downloadCss(webClient, window, resourceRemoteLink);
for (String cssImagePath : getLinksFromCss(css)) {
URL cssImagelink = page.getFullyQualifiedUrl(cssImagePath.replace("\"", "").replace("\'", "")
.replace(" ", ""));
String cssImageLocalPath = mapLocalUrl(page, cssImagelink, cssImagePath, urlMapping);
files.put(cssImageLocalPath, downloadImage(webClient, window, cssImagelink));
}
files.put(resourceLocalPath, replaceRemoteUrlsWithLocal(css, urlMapping)
.replace("resources/", "./").getBytes());
}
} catch (Exception e) {
}
}
String pagesrc = replaceRemoteUrlsWithLocal(page.getWebResponse().getContentAsString(), urlMapping);
files.put("page.html", pagesrc.getBytes());
webClient.setCurrentWindow(currentWindow);
return createZip(files);
}
String downloadCss(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (cssjsCache.get(resourceUrl.toString()) == null) {
cssjsCache.put(resourceUrl.toString(), webClient.getPage(window, new WebRequest(resourceUrl))
.getWebResponse().getContentAsString());
}
return cssjsCache.get(resourceUrl.toString());
}
byte[] downloadImage(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (imagesCache.get(resourceUrl.toString()) == null) {
imagesCache.put(
resourceUrl.toString(),
IOUtils.toByteArray(webClient.getPage(window, new WebRequest(resourceUrl)).getWebResponse()
.getContentAsStream()));
}
return imagesCache.get(resourceUrl.toString());
}
public static byte[] createZip(Map<String, byte[]> files) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipOutputStream zipfile = new ZipOutputStream(bos);
Iterator<String> i = files.keySet().iterator();
String fileName = null;
ZipEntry zipentry = null;
while (i.hasNext()) {
fileName = i.next();
zipentry = new ZipEntry(fileName);
zipfile.putNextEntry(zipentry);
zipfile.write(files.get(fileName));
}
zipfile.close();
return bos.toByteArray();
}
List<String> getLinksFromCss(String css) {
List<String> result = new LinkedList<String>();
Matcher m = cssUrlPattern.matcher(css);
while (m.find()) { // find next match
result.add( m.group(2));
}
return result;
}
String replaceRemoteUrlsWithLocal(String source, Map<String, String> replacement) {
for (String object : replacement.keySet()) {
// background:url(http://org.com/images/image.gif)
source = source.replace(object, replacement.get(object));
}
return source;
}
String mapLocalUrl(HtmlPage page, URL link, String path, Map<String, String> replacementToAdd) throws Exception {
String resultingFileName = "resources/" + FilenameUtils.getName(link.getFile());
replacementToAdd.put(path, resultingFileName);
return resultingFileName;
}
}
Anyway, I found this code, but I am unsure how to use this code to actually take a screenshot.
I would like the screenshot to be a .jpg and to be able to be stored in a certain folder.
In the long haul, I am also going to need to run the screenshot code repeatedly.
Any help would be appreciated.

Categories

Resources