Develop Reference

Push data from Dynamo DB to elasticsearch using java

Hi i have created a handler in java for getting the events from dynamo DB
Here is my code
package com.Lambda.dynamodb;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
public class DDBEventProcessor implements
RequestHandler<DynamodbEvent, String> {
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()){
System.out.println(record.getEventID());
System.out.println(record.getEventName());
System.out.println(record.getDynamodb().toString());
}
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
}
}
Lambda function able to write the events in cloudwatch but the challenge is i have to index all the streamed records to the AWS elasticsearch service endpoint and index it.
while search through blogs i got few code samples in python and node.js but my requirement is i have to build this lambda function in java
Could anyone please suggest how to achieve this in java lambda function?

Hi i have included the code below may helpful to some one. Dynamo DB streams to index the document in elasticsearch both inside AWS and outside AWS
package com.Firstlambda;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemUtils;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.json.JSONObject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HelloWorld implements RequestHandler<DynamodbEvent, String> {
private static String serviceName = "es";
private static String region = "us-east-1";
private static String aesEndpoint = ""
private static String index = "";
private static String type = "_doc";
static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()) {
System.out.println("EventName : " + record.getEventName());
System.out.println("EventName : " + record.getDynamodb());
//AWS outside
RestHighLevelClient esClient = esClient();
//AWS outside
//AWS Inside
//RestHighLevelClient esClient = esClient(serviceName, region);
//AWS Inside
if (record.getEventName().toLowerCase().equals("insert")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
IndexRequest indexRequest = new IndexRequest(index, type, JsonUniqueId);
indexRequest.source(JsonString, XContentType.JSON);
try {
IndexResponse indexResponse = esClient.index(indexRequest, RequestOptions.DEFAULT);
System.out.println(indexResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else if (record.getEventName().toLowerCase().equals("modify")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
UpdateRequest request = new UpdateRequest(index, type, JsonUniqueId);
String jsonString = JsonString;
request.doc(jsonString, XContentType.JSON);
try {
UpdateResponse updateResponse = esClient.update(
request, RequestOptions.DEFAULT);
System.out.println(updateResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else {
System.out.println("remove");
System.out.println("KEYID : " + record.getDynamodb().getKeys().get("ID").getN());
String deletedId = record.getDynamodb().getKeys().get("ID").getN();
DeleteRequest request = new DeleteRequest(index, type, deletedId);
try {
DeleteResponse deleteResponse = esClient.delete(
request, RequestOptions.DEFAULT);
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
}
return "Successfullyprocessed";
}
public String getJsonstring(Map<String, AttributeValue> newIma) {
String json = null;
Map<String, AttributeValue> newImage = newIma;
List<Map<String, AttributeValue>> listOfMaps = new ArrayList<Map<String, AttributeValue>>();
listOfMaps.add(newImage);
List<Item> itemList = ItemUtils.toItemList(listOfMaps);
for (Item item : itemList) {
json = item.toJSON();
}
return json;
}
public String GetIdfromJsonString(String Json) {
JSONObject jsonObj = new JSONObject(Json);
return String.valueOf(jsonObj.getInt("ID"));
}
// Adds the interceptor to the ES REST client
// public static RestHighLevelClient esClient(String serviceName, String region) {
// AWS4Signer signer = new AWS4Signer();
// signer.setServiceName(serviceName);
// signer.setRegionName(region);
// HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
// return new RestHighLevelClient(RestClient.builder(HttpHost.create(aesEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
// }
public static RestHighLevelClient esClient() {
String host = "d9bc7cbca5ec49ea96a6ea683f70caca.eastus2.azure.elastic-cloud.com";
int port = 9200;
String userName = "elastic";
String password = "L4Nfnle3wxLmV95lffwsf$Ub46hp";
String protocol = "https";
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocol))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
}
This is just a sample code has to be modified based on our requirements

I can't use the "print/println" functions

I'm using IntelliJ IDEA.
I've done all that my Googling has provided me with as potential solutions.
I've tried doing the File -> Invalidate Caches/Restart, it still doesn't fix it.
This is the entire code;
This was copied from the OCI documentation, and I've already made some amendments to the code, but I don't know how to fix this.
I'm a junior yes, but am I wrong to be annoyed over how much of the documentation through the use of which I'm to be developing myself has these kinds of issues..
This is the entire code.
package com;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.commons.codec.binary.Base64;
import static java.lang.System.out;
import static com.sun.org.apache.xalan.internal.xsltc.cmdline.Compile.printUsage;
public class ociAuth {
private static String server;
private static String user;
private static String password;
private static String port = "8443";
private static String response_format = "json";
private static String server_url;
public static void main(String[] args) {
if(args.length < 3 || args.length > 4) {
printUsage();
System.exit(1);
}
setUserArguments(args);
server_url = "https://" + server + ":" + port + "/rest/v1/assets/storages";
try {
HttpsURLConnection connection =
getAllTrustingHttpsUrlConnection();
if(connection == null) {
System.err.println("FATAL: Failed to create HTTPS connection to URL: " + server_url);
System.exit(1);
}
System.out.println("Invoking API: " + server_url);
connection.setRequestMethod("GET");
connection.setRequestProperty("Accept", "application/" + response_format);
String authString = getAuthorizationString();
connection.setRequestProperty("Authorization", "Basic " +
authString);
if (connection.getResponseCode() != 200) {
System.err.println("API Invocation Failed : HTTP error code : "
+ connection.getResponseCode());
System.exit(1);
}
BufferedReader br = new BufferedReader(new InputStreamReader(
(connection.getInputStream())));
String response;
System.out.println("Response:");
while ((response = br.readLine()) != null) {
System.out.println(response);
}
connection.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
}
//THESE PRINTS HERE BELLOW
//THESE PRINTS HERE BELLOW
//THESE PRINTS HERE BELLOW
System.out.print("\nUsage:\n\tHelloApiServices <api-server host[:port]> <user> <password> [json|xml]\n");
System.out.print("\nExamples:\n\tHelloApiServices localhost admin mypassword");
System.out.print("\tHelloApiServices 10.22.12.34:8320 admin password");
System.out.print("\tHelloApiServices 10.22.12.34 admin password xml");
System.out.print("\tHelloApiServices 10.22.12.34:8212 admin password xml\n");
System.out.print("\nNote:\n\t(1) When port number is not provided, 8443 is chosen by default.");
System.out.print("\t(2) When response format (json or xml) is not provided, json is chosen by default. \n");
}
//THESE PRINTS HERE ABOVE
//THESE PRINTS HERE ABOVE
//THESE PRINTS HERE ABOVE
private static void setUserArguments(String[] args) {
server = args[0];
user = args[1];
password = args[2];
if(args.length == 4) {
response_format = args[3];
if(!response_format.equals("json") && ! response_format.equals("xml")) {
printUsage();
System.exit(1);
}
}
if(server.contains(":")) {
String[] parts = server.split(":");
server = parts[0];
port = parts[1];
}
}
private static HttpsURLConnection getAllTrustingHttpsUrlConnection() {
HttpsURLConnection conn = null;
try {
TrustManager[] trustAllCertificatesManager = new
TrustManager[]{new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() {
return
null;
}
public void checkClientTrusted(X509Certificate[]
certs, String authType) {
}
public void checkServerTrusted(X509Certificate[]
certs, String authType) {
}
}};
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustAllCertificatesManager, new
SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
URL url = new URL(server_url);
conn = (HttpsURLConnection) url.openConnection();
conn.setHostnameVerifier(new HostnameVerifier() {
public boolean verify(String host, SSLSession
session) {
return true;
}
});
} catch (Exception e) {
e.printStackTrace();
}
return conn;
}
private static String getAuthorizationString() {
String userPassword = user + ":" + password;
byte[] authEncodedBytes =
Base64.encodeBase64(userPassword.getBytes());
String ajdeovako = new String(authEncodedBytes);
return ajdeovako;
}
}

You have an extra bracket. The System.out.print calls are not inside you main method, but are instead between methods and the closing bracket after the print calls shouldn't be there. Move the print calls up inside your main method and it should compile fine.

You have one extra }, removing it will fix your error.
} <-- remove this
//THESE PRINTS HERE BELLOW
//THESE PRINTS HERE BELLOW
//THESE PRINTS HERE BELLOW
System.out.print("\nUsage:\n\tHelloApiServices <api-server host[:port]> <user> <password> [json|xml]\n");
System.out.print("\nExamples:\n\tHelloApiServices localhost admin mypassword");
System.out.print("\tHelloApiServices 10.22.12.34:8320 admin password");
System.out.print("\tHelloApiServices 10.22.12.34 admin password xml");
System.out.print("\tHelloApiServices 10.22.12.34:8212 admin password xml\n");
System.out.print("\nNote:\n\t(1) When port number is not provided, 8443 is chosen by default.");
System.out.print("\t(2) When response format (json or xml) is not provided, json is chosen by default. \n");
}

Java. Server don't send proper HTML

I wrote little app in Java but it don't serv HTML properly.
Problem occurs when method checkCookies return false. Normally an "else" clause should be executed but this doesn't happen. Some intresting thing - the app is going 3 times from start of handle() method to first condition (counter is increasing). I don't know how to repair it. If i got a cookie with name "Login" or i manually modify method to always return true it's working good.
Main file:
Loginpage.java
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
public class Loginpage {
public static void main (String[] args) throws Exception{
HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0);
server.createContext("/form", new Form());
server.setExecutor(null);
server.start();
}
}
Form.java
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import java.io.*;
import java.net.HttpCookie;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Form implements HttpHandler {
int counter = 0;
#Override
public void handle(HttpExchange httpExchange) throws IOException {
counter++;
System.out.println(counter);
List<String> cookies = httpExchange.getRequestHeaders().get("Cookie");
HttpCookie cookie;
String response = "<html><body>system</body></html>";
String method = httpExchange.getRequestMethod();
if (checkCookies(cookies)) {
response = "<html><body> Hello! </body></html>";
} else {
if (method.equals("GET")) {
response = "<html><body><form method='post'>" +
"<input type='text' name='username'><br>" +
"<input type='password' name='pass'><br>" +
"<input type='submit' value='login'<br>" +
"</form></body></html>";
} else if (method.equals("POST")) {
InputStreamReader isr = new InputStreamReader(httpExchange.getRequestBody(), "utf-8");
BufferedReader br = new BufferedReader(isr);
String formData = br.readLine();
Map data = parseFormData(formData);
if (checkData(data)) {
cookie = new HttpCookie("Login", String.valueOf(counter + (counter * 12)));
httpExchange.getResponseHeaders().add("Set-Cookie", cookie.toString());
}
}
}
httpExchange.sendResponseHeaders(200, response.length());
OutputStream os = httpExchange.getResponseBody();
os.write(response.getBytes());
os.close();
}
private boolean checkCookies(List<String> cookies) {
boolean isValid = false;
for (String s : cookies) {
System.out.println(s);
if (s.matches("Login.*")) {
isValid = true;
}
}
System.out.println(isValid);
return isValid;
}
private boolean checkData(Map<String, String> data) {
DAO dao = new DAO();
String username = data.get("username");
System.out.println(username);
String password = data.get("pass");
System.out.println(password);
if (dao.checkData(username, password)) {
return true;
} else return false;
}
private static Map<String, String> parseFormData(String formData) throws UnsupportedEncodingException {
Map<String, String> map = new HashMap<>();
String[] pairs = formData.split("&");
for (String pair : pairs) {
String[] keyValue = pair.split("=");
String value = new URLDecoder().decode(keyValue[1], "UTF-8");
map.put(keyValue[0], value);
}
return map;
}
}
UPDATE
I found the cause of my problem. If I send a empty List to the checkCookies() method the program "hangs". Adding the condition
if (cookies == null) return false;
in checkCookies() method solved the problem but I do not know why it was not just return false before fix.
Can anyone explain it?

Getting the roles of a user in wso2

I've created users and assigned them different roles using wso2im. Using these I managed to restrict access to a .jsp file so roles seem to be working right.
The problem lies when I need to display different things to different roles within the same JSP (For instance, role AAA can do xxx and yyy, role BBB can do zzz), I'm trying to check for roles using request.isUserInRole("role") but it always returns null, both when trying from the .jsp itself and the servlet that handles authentication.

Finally managed to get it to work. Getting the roles with the servlet and storing them in a cookie. Neither safe nor pretty but does the job:
package foo;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.PrintWriter;
import javax.servlet.RequestDispatcher;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpSession;
import org.apache.axis2.transport.http.HttpTransportProperties;
import org.apache.axis2.client.Options;
import org.apache.axis2.transport.http.HTTPConstants;
import org.wso2.carbon.um.ws.api.stub.RemoteUserStoreManagerServiceStub;
/**
* Servlet implementation class LoginServlet
*/
#WebServlet("/LoginServlet")
public class LoginServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private final String basicAuthUserID = "admin";
private final String basicAuthPassword = "admin";
private final String serverUrl = "https://localhost:9444/services/";
private RemoteUserStoreManagerServiceStub stub = null;
protected void doPost(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
// get request parameters for userID and password
String user = request.getParameter("user");
String pwd = request.getParameter("pwd");
try {
if(authenticate(user,pwd)){
HttpSession session = request.getSession();
session.setAttribute("user", user);
//setting session to expiry in 30 mins
session.setMaxInactiveInterval(30*60);
Cookie userName = new Cookie("user", user);
userName.setMaxAge(30*60);
String[] roles = getRoleListOfUser(user);
String rolesTodos = null;
for (String s: roles){
if (!s.equals("Internal/everyone")) {
if (rolesTodos == null){
rolesTodos = s;
} else {
//System.out.println("Rol: " + s);
rolesTodos = rolesTodos + "," + s;
}
}
}
//System.out.println("Roles: " + rolesTodos);
Cookie rolesCookie = new Cookie("roles", rolesTodos);
rolesCookie.setMaxAge(30*60);
response.addCookie(userName);
response.addCookie(rolesCookie);
response.sendRedirect("index.jsp");
}else{
RequestDispatcher rd = getServletContext().getRequestDispatcher("/login.html");
PrintWriter out= response.getWriter();
out.println("<font color=red>Either user name or password is wrong.</font>");
rd.include(request, response);
}
} catch (Exception e) {
e.printStackTrace();
}
}
private boolean authenticate(String userName, Object credential) throws Exception {
if (!(credential instanceof String)) {
throw new Exception("Unsupported type of password");
}
try {
if(stub == null) {
stub = new RemoteUserStoreManagerServiceStub(null, serverUrl
+ "RemoteUserStoreManagerService");
HttpTransportProperties.Authenticator basicAuth = new HttpTransportProperties.Authenticator();
basicAuth.setUsername(basicAuthUserID);
basicAuth.setPassword(basicAuthPassword);
basicAuth.setPreemptiveAuthentication(true);
final Options clientOptions = stub._getServiceClient().getOptions();
clientOptions.setProperty(HTTPConstants.AUTHENTICATE, basicAuth);
stub._getServiceClient().setOptions(clientOptions);
}
return stub.authenticate(userName, (String) credential);
} catch (Exception e) {
handleException(e.getMessage(), e);
}
return false;
}
private String[] handleException(String msg, Exception e) throws Exception {
System.out.println(e.getMessage() + e);
throw new Exception(msg, e);
}
public String[] getRoleListOfUser(String userName) throws Exception {
try {
return stub.getRoleListOfUser(userName);
} catch (Exception e) {
System.out.println(e.getMessage() + e);
}
return null;
}
}

Apache HttpCore, simple server to echo received post data

Using the ElementalHttpServer example class found here:
https://hc.apache.org/httpcomponents-core-4.3.x/httpcore/examples/org/apache/http/examples/ElementalHttpServer.java
I am able to successfully receive post data, my goal is to convert the received post data into a string I can print. I've modified the HttpFileHandler as follows, using eneity.getContent() to get the inputStream, but i'm not sure how I can convert the inputStream into a String.
static class HttpFileHandler implements HttpRequestHandler {
private final String docRoot;
public HttpFileHandler(final String docRoot) {
super();
this.docRoot = docRoot;
}
public void handle(
final HttpRequest request,
final HttpResponse response,
final HttpContext context) throws HttpException, IOException {
String method = request.getRequestLine().getMethod().toUpperCase(Locale.ENGLISH);
if (!method.equals("GET") && !method.equals("HEAD") && !method.equals("POST")) {
throw new MethodNotSupportedException(method + " method not supported");
}
String target = request.getRequestLine().getUri();
if (request instanceof HttpEntityEnclosingRequest) {
HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
byte[] entityContent = EntityUtils.toByteArray(entity);
InputStream inputStream = entity.getContent();
String str= inputStream.toString();
byte[] b3=str.getBytes();
String st = new String(b3);
System.out.println(st);
for(int i=0;i<b3.length;i++) {
System.out.print(b3[i]+"\t");
}
System.out.println("Incoming entity content (bytes): " + entityContent.length);
}
}
}
Thanks for any ideas

Here is simple console logging handler; it logs every request (not only POST) - both headers and payload:
package com.mycompany;
import org.apache.http.*;
import org.apache.http.entity.StringEntity;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpRequestHandler;
import org.apache.http.util.EntityUtils;
import org.omg.CORBA.Request;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
/**
* Created by IntelliJ IDEA.
* User: Piotrek
* To change this template use File | Settings | File Templates.
*/
public class LoggingHandler implements HttpRequestHandler {
public void handle(HttpRequest httpRequest, HttpResponse httpResponse, HttpContext httpContext) throws HttpException, IOException {
System.out.println(""); // empty line before each request
System.out.println(httpRequest.getRequestLine());
System.out.println("-------- HEADERS --------");
for(Header header: httpRequest.getAllHeaders()) {
System.out.println(header.getName() + " : " + header.getValue());
}
System.out.println("--------");
HttpEntity entity = null;
if (httpRequest instanceof HttpEntityEnclosingRequest)
entity = ((HttpEntityEnclosingRequest)httpRequest).getEntity();
// For some reason, just putting the incoming entity into
// the response will not work. We have to buffer the message.
byte[] data;
if (entity == null) {
data = new byte [0];
} else {
data = EntityUtils.toByteArray(entity);
}
System.out.println(new String(data));
httpResponse.setEntity(new StringEntity("dummy response"));
}
}
Registration of handler using org.apache.http.localserver.LocalTestServer (with ElementalHttpServer it is similar - you also have HttpRequestHandler implementation above):
public static void main(String[] args) throws Exception {
LocalTestServer server = new LocalTestServer(null, null);
try {
server.start();
server.register("/*", new LoggingHandler());
server.awaitTermination(3600 * 1000);
} catch (Exception e) {
e.printStackTrace();
} finally {
server.stop();
}
}