Bulk upload CSV file into Elasticsearch using JavaAPI - java

I want to bulk upload the csv file into Elasticsearch using JAVA API (without using logstash).
Elasticsearch version - 6.6
I have tried the below program using Jackson format to get source Map for IndexRequest. Because I can't predefined the POJO variables. So I used dynamic Map from CSV file
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.logging.Logger;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import com.zoho.dedupe.connection.DukeESConnection;
public class BulkImport {
private static Logger logger = Logger.getLogger(BulkImport.class.getName());
public static void main(String args[]) {
long starttime = System.currentTimeMillis();
logger.info("ElasticSearchServiceImpl => bulkInsert Service Started");
FileInputStream fis = null;
BulkRequest request;
RestHighLevelClient client;
//elastic Search Index Name
String esIndex = "post";
try {
boolean isHeaderSet = false;
Set<String> header = new HashSet<String>();
fis = new FileInputStream("/Users/test/Documents/Test.csv");
request = new BulkRequest();
MappingIterator<Map<String, Object>> data = parse(fis);
while (data.hasNext()) {
Map<?,?> value = data.next();
if(!isHeaderSet) {
header.add("id");
header = (Set<String>) value.keySet();
isHeaderSet= true;
}
System.out.println(value);
request.add(getIndexRequest(value, esIndex));
}
fis.close();
if(request.numberOfActions()>0) {
String hostsInString = "localhost";
List<HttpHost> httpHosts = new ArrayList<HttpHost> ( );
String[] hosts = hostsInString.split (",");
for (String host : hosts)
{
HttpHost httpHost = new HttpHost (host, 9200, "http");
httpHosts.add (httpHost);
}
client = client = new RestHighLevelClient (RestClient.builder (
httpHosts.toArray(new HttpHost[]{})).setMaxRetryTimeoutMillis (10 * 60000).setRequestConfigCallback(
new RestClientBuilder.RequestConfigCallback() {
#Override
public RequestConfig.Builder customizeRequestConfig(
RequestConfig.Builder requestConfigBuilder) {
return requestConfigBuilder
.setConnectTimeout (60000)
.setSocketTimeout (10 * 60000);
}
}));
CreateIndexRequest crrequest = new CreateIndexRequest(esIndex);
Map<String, Object> jsonMap = new HashMap<>();
Map<String, Object> message = new HashMap<>();
message.put("type", "text");
Map<String, Object> keyword = new HashMap<>();
Map<String, Object> type = new HashMap<>();
type.put("type", "keyword");
type.put("ignore_above", 256);
keyword.put("keyword", type);
message.put("fields", keyword);
Map<String, Object> properties = new HashMap<>();
for (Object hdr :header) {
properties.put(hdr.toString(), message);
}
Map<String, Object> mapping = new HashMap<>();
mapping.put("properties", properties);
jsonMap.put("_doc", mapping);
crrequest.mapping("_doc", jsonMap);
CreateIndexResponse createIndexResponse = client.indices().create(crrequest, RequestOptions.DEFAULT);
boolean acknowledged = createIndexResponse.isAcknowledged();
System.out.println(acknowledged);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if(bulkResponse.hasFailures()) {
logger.info("ElasticSearchServiceImpl => bulkInsert : Some of the record has failed.Please reinitiate the process");
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : Success");
}
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : No request for BulkInsert ="+request.numberOfActions());
}
} catch (Exception e) {
logger.info("ElasticSearchServiceImpl => bulkInsert : Exception =" + e.getMessage());
e.printStackTrace();
}
long endTime = System.currentTimeMillis();
logger.info("ElasticSearchServiceImpl => bulkInsert End " + (endTime - starttime));
}
public static MappingIterator<Map<String, Object>> parse(FileInputStream input) throws Exception {
MappingIterator<Map<String, Object>> map = readObjectsFromCsv(input);
return map;
//writeAsJson(data);
}
public static MappingIterator<Map<String, Object>> readObjectsFromCsv(FileInputStream file) throws IOException {
CsvSchema bootstrap = CsvSchema.emptySchema().withHeader().withColumnSeparator(',');
CsvMapper csvMapper = new CsvMapper();
MappingIterator<Map<String, Object>> mappingIterator = csvMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).reader(Map.class).with(bootstrap).readValues(file);
// System.out.println("Column names: " + mappingIterator.next().keySet());
return mappingIterator;
}
public static void writeAsJson(List<Map<?, ?>> data) throws IOException, JSONException {
ObjectMapper mapper = new ObjectMapper();
String value = mapper.writeValueAsString(data);
JSONArray json = new JSONArray(value);
System.out.println(json);
}
public static IndexRequest getIndexRequest(Map data,String index)throws Exception {
IndexRequest indexRequest = null;
indexRequest = new IndexRequest(index).id(UUID.randomUUID().toString()).source(data);
System.out.println(indexRequest.toString());
return indexRequest;
}
}
I got the below exception while running the program
{Document Name=dhjajga, Title=sdas, Name=asd, DOB=14-43-22}
index {[post][null][c2148857-87e0-4407-b5f5-b4f5f52c40d2], source[{"Document Name":"dhjajga","Title":"sdas","Name":"asd","DOB":"14-43-22"}]}
Jun 11, 2020 4:06:18 PM com.zoho.dedupe.connection.DukeESConnection connect
INFO: Client org.elasticsearch.client.RestHighLevelClient#7c51f34b
true
Jun 11, 2020 4:06:18 PM BulkImport main
INFO: ElasticSearchServiceImpl => bulkInsert : Exception =Validation Failed: 1: type is missing;2: type is missing;3: type is missing;
org.elasticsearch.action.ActionRequestValidationException: Validation Failed: 1: type is missing;2: type is missing;3: type is missing;
at org.elasticsearch.action.bulk.BulkRequest.validate(BulkRequest.java:612)
at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1728)
at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1694)
at org.elasticsearch.client.RestHighLevelClient.bulk(RestHighLevelClient.java:470)
at BulkImport.main(BulkImport.java:85)
Jun 11, 2020 4:06:18 PM BulkImport main
INFO: ElasticSearchServiceImpl => bulkInsert End 1432
When I try to insert the same above indexrequest its working fine.
curl -X POST "localhost:9200/post/_doc/?pretty" -H 'Content-Type: application/json' -d'
{
"Document Name":"dhjajga","Title":"sdas","Name":"asd","DOB":"14-43-22"
}
'
{
"_index" : "post",
"_type" : "_doc",
"_id" : "jBPronIB0Wb3XTTasBjG",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 0,
"_primary_term" : 1
}
Please help to fix the issue in java program. Thanks in advance

Before Elasticsearch version 7 you have to specify a type with your Indexrequest. It is recommended to use the type "_doc".

Related

How to reorder fields in json String in Java

I receive a String with json from an abstract source and I want to reorder the fields in that json so that certain fields come first in specified order and other, uspecified fields can come in any order. Is there a library method that would allow me to do sth like this:
// String reorderFields(String json, String[] orderedFields);
String res = reorderFields("{\"b\":2, \"c\": 3, \"a\":1}", new String[] {"a", "b", "c"});
Assertions.assertEquals("{\"a\":1,\"b\":2,\"c\":3}", res);
Below is solution that should do the trick. It works for first-level fields only. It uses jackson for json handling. Since it operates on basic types only (no custom serializers/deserializers) it should be safe to use for all possible json types.
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
public class ReorderFieldsTest {
public static String reorderFieldsInJson(String json, String[] fields) {
ObjectMapper objectMapper = new ObjectMapper();
Map<String, Object> jsonMap;
try {
jsonMap = objectMapper.readValue(json, new TypeReference<Map<String, Object>>() {
});
} catch (Exception e) {
throw new RuntimeException("Error converting json to map", e);
}
Map<String, Object> resMap = new LinkedHashMap<>();
Set<String> orderedFields = new HashSet<>(Arrays.asList(fields));
for (String fieldName : fields) {
Object val = jsonMap.get(fieldName);
if (val != null) {
resMap.put(fieldName, val);
}
}
for (Entry<String, Object> entry : jsonMap.entrySet()) {
if (orderedFields.contains(entry.getKey())) {
continue;
}
resMap.put(entry.getKey(), entry.getValue());
}
try {
return objectMapper.writeValueAsString(resMap);
} catch (JsonProcessingException e) {
throw new RuntimeException("Error converting map to json", e);
}
}
#Test
public void jsonReorder_validInput_reorderedOutput() {
String src = "{\"b\":2, \"c\": 3, \"a\":1}";
String res = reorderFieldsInJson(src, new String[] {"a", "b"});
Assertions.assertEquals("{\"a\":1,\"b\":2,\"c\":3}", res);
}
}
Result on Java 11 and jackson 2.11.3:
org:
{"b":2, "c": 3, "a":1}
ordered:
{"a":1,"b":2,"c":3}

Push data from Dynamo DB to elasticsearch using java

Hi i have created a handler in java for getting the events from dynamo DB
Here is my code
package com.Lambda.dynamodb;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
public class DDBEventProcessor implements
RequestHandler<DynamodbEvent, String> {
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()){
System.out.println(record.getEventID());
System.out.println(record.getEventName());
System.out.println(record.getDynamodb().toString());
}
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
}
}
Lambda function able to write the events in cloudwatch but the challenge is i have to index all the streamed records to the AWS elasticsearch service endpoint and index it.
while search through blogs i got few code samples in python and node.js but my requirement is i have to build this lambda function in java
Could anyone please suggest how to achieve this in java lambda function?
Hi i have included the code below may helpful to some one. Dynamo DB streams to index the document in elasticsearch both inside AWS and outside AWS
package com.Firstlambda;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemUtils;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.json.JSONObject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HelloWorld implements RequestHandler<DynamodbEvent, String> {
private static String serviceName = "es";
private static String region = "us-east-1";
private static String aesEndpoint = ""
private static String index = "";
private static String type = "_doc";
static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()) {
System.out.println("EventName : " + record.getEventName());
System.out.println("EventName : " + record.getDynamodb());
//AWS outside
RestHighLevelClient esClient = esClient();
//AWS outside
//AWS Inside
//RestHighLevelClient esClient = esClient(serviceName, region);
//AWS Inside
if (record.getEventName().toLowerCase().equals("insert")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
IndexRequest indexRequest = new IndexRequest(index, type, JsonUniqueId);
indexRequest.source(JsonString, XContentType.JSON);
try {
IndexResponse indexResponse = esClient.index(indexRequest, RequestOptions.DEFAULT);
System.out.println(indexResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else if (record.getEventName().toLowerCase().equals("modify")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
UpdateRequest request = new UpdateRequest(index, type, JsonUniqueId);
String jsonString = JsonString;
request.doc(jsonString, XContentType.JSON);
try {
UpdateResponse updateResponse = esClient.update(
request, RequestOptions.DEFAULT);
System.out.println(updateResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else {
System.out.println("remove");
System.out.println("KEYID : " + record.getDynamodb().getKeys().get("ID").getN());
String deletedId = record.getDynamodb().getKeys().get("ID").getN();
DeleteRequest request = new DeleteRequest(index, type, deletedId);
try {
DeleteResponse deleteResponse = esClient.delete(
request, RequestOptions.DEFAULT);
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
}
return "Successfullyprocessed";
}
public String getJsonstring(Map<String, AttributeValue> newIma) {
String json = null;
Map<String, AttributeValue> newImage = newIma;
List<Map<String, AttributeValue>> listOfMaps = new ArrayList<Map<String, AttributeValue>>();
listOfMaps.add(newImage);
List<Item> itemList = ItemUtils.toItemList(listOfMaps);
for (Item item : itemList) {
json = item.toJSON();
}
return json;
}
public String GetIdfromJsonString(String Json) {
JSONObject jsonObj = new JSONObject(Json);
return String.valueOf(jsonObj.getInt("ID"));
}
// Adds the interceptor to the ES REST client
// public static RestHighLevelClient esClient(String serviceName, String region) {
// AWS4Signer signer = new AWS4Signer();
// signer.setServiceName(serviceName);
// signer.setRegionName(region);
// HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
// return new RestHighLevelClient(RestClient.builder(HttpHost.create(aesEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
// }
public static RestHighLevelClient esClient() {
String host = "d9bc7cbca5ec49ea96a6ea683f70caca.eastus2.azure.elastic-cloud.com";
int port = 9200;
String userName = "elastic";
String password = "L4Nfnle3wxLmV95lffwsf$Ub46hp";
String protocol = "https";
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocol))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
}
This is just a sample code has to be modified based on our requirements

Parse JSON into Object

I have a User class defined as:
User.java
package model;
import java.util.List;
import java.util.Map;
public class User {
private final Map<String, List<String>> accountTransactionsMap;
public User(final Map<String, List<String>> accountTransactionsMap) {
this.accountTransactionsMap = accountTransactionsMap;
}
public Map<String, List<String>> getAccountTransactionsMap() {
return accountTransactionsMap;
}
}
I am calling a REST API that returns the following response:
{
"username1":{
"456":[
],
"123":[
],
"789":[
]
},
"username2":{
"123":[
],
"456":[
],
"789":[
]
},
"username3":{
"789":[
],
"123":[
],
"456":[
"transaction10",
"transaction6",
"transaction9",
"transaction3"
]
}
}
I would like to be able to parse through the response and store it in a User object.
I have tried the following:
Test.java
public class Test {
public static void main(final String[] args) {
final String response = "{\"username1\":{\"456\":[],\"123\":[],\"789\":[]},\"username2\":{\"123\":[],\"456\":[],\"789\":[]},\"username3\":{\"789\":[],\"123\":[],\"456\":[\"transaction10\",\"transaction6\",\"transaction9\",\"transaction3\"]}}";
final Gson gson = new Gson();
final Type map = new TypeToken<Map<String, User>>(){}.getType();
final Map<String, User> result = gson.fromJson(response, map);
System.out.println(result);
if (result != null) {
for (final Map.Entry<String, User> entry : result.entrySet()) {
System.out.println("username: " + entry.getKey());
final User user = entry.getValue();
System.out.println("transactions: " + user.getAccountTransactionsMap());
}
}
}
}
This yields output:
{username1=model.User#80ec1f8, username2=model.User#1445d7f, username3=model.User#6a396c1e}
username: username1
transactions: null
username: username2
transactions: null
username: username3
transactions: null
I expect output:
{username1=model.User#80ec1f8, username2=model.User#1445d7f, username3=model.User#6a396c1e}
username: username1
transactions: {123=[],456=[],789=[]}
username: username2
transactions: {123=[],456=[],789=[]}
username: username3
transactions: {123=[],456=["transaction10", "transaction6", "transaction9", "transaction3"],789=[]}
How can I parse the accountId and the list of transactionIds into its own map as a variable in my User class?
Edit: I suppose the question really becomes, how can I create a custom TypeToken for my User class?
Instead of User class you need to use Map<String, Map<String, List<String>>>:
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.io.File;
import java.io.FileReader;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
public class GsonApp {
public static void main(String[] args) throws Exception {
File jsonFile = new File("./resource/test.json").getAbsoluteFile();
final Gson gson = new Gson();
final Type map = new TypeToken<Map<String, Map<String, List<String>>>>(){}.getType();
final Map<String, Map<String, List<String>>> result = gson.fromJson(new FileReader(jsonFile), map);
System.out.println(result);
if (result != null) {
for (final Map.Entry<String, Map<String, List<String>>> entry : result.entrySet()) {
System.out.println("username: " + entry.getKey());
final Map<String, List<String>> user = entry.getValue();
System.out.println("transactions: " + user);
}
}
}
}
Above code prints:
{username1={456=[], 123=[], 789=[]}, username2={123=[], 456=[], 789=[]}, username3={789=[], 123=[], 456=[transaction10, transaction6, transaction9, transaction3]}}
username: username1
transactions: {456=[], 123=[], 789=[]}
username: username2
transactions: {123=[], 456=[], 789=[]}
username: username3
transactions: {789=[], 123=[], 456=[transaction10, transaction6, transaction9, transaction3]}
If you really need, you can create User object after parsing.

Can we use multiple sessions in a JUnit testcase?

I have been trying to write a Junit testcase for the following class. I am able to run one testcase successfully. I am using session to get the data from the database through server.
I am trying to use the session object as different instance. Its working for the first method but not for any other. I know I ust be doing some silly mistake.
Here is my class:
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import com.google.gson.Gson;
import com.hp.legacyloader.service.RetrievePublicationService;
#Controller
public class DashboardController {
/* private RetrievePublicationService retrievePublicationService;
#Autowired
public DashboardController(RetrievePublicationService retrievePublicationService) {
this.retrievePublicationService = retrievePublicationService;
}*/
#Autowired
RetrievePublicationService retrievePublicationService;
private static final Logger logger = Logger.getLogger(DashboardController.class);
#RequestMapping(value = "/viewDashboard")
public String getPaginationDataPage() {
logger.info("Log4j info is working");
logger.warn("Log4j warn is working");
logger.debug("Log4j debug is working");
logger.error("Log4j error is working");
return "ViewDashboard";
}
#RequestMapping(value = "/getPaginationData/{pageSize}/{page}")
//public #ResponseBody List<Map<String, Object>> getPaginationData(#PathVariable String pageSize, #PathVariable String page) {
public #ResponseBody String getPaginationData(#PathVariable String pageSize, #PathVariable String page) throws JsonGenerationException, JsonMappingException, IOException {
// Map<String, Object> map = new HashMap<String, Object>();
System.out.println("pageSize======"+pageSize);
System.out.println("page======"+page);
String json = null;
List<Object[]> list = retrievePublicationService.listJob();
List<Map<String, Object>> activeTeamMap = new ArrayList<Map<String,Object>>();
if (list != null) {
for (Object[] obj : list) {
Map<String, Object> dropDownData = new HashMap<String, Object>();
Gson gson = new Gson();
//System.out.println("obj.getPUBLICATION_ID()====" + ((EventLog) obj).getEventLogPid());
dropDownData.put("publicationId",obj[0]);
dropDownData.put("documentId",obj[1]);
dropDownData.put("documentType",obj[2]);
dropDownData.put("languageCode",obj[3]);
dropDownData.put("processStartDate",obj[4]);
dropDownData.put("publicationType",obj[5]);
activeTeamMap.add(dropDownData);
System.out.println("activeTeamMap==vbnbvn=="+activeTeamMap);
json = gson.toJson(activeTeamMap);
}
}
else {
}
return json;
}
#RequestMapping(value = "/getSearchData/{fdate}/{tdate}", method = RequestMethod.GET)
public #ResponseBody List<Map<String, Object>> getSearchData(#PathVariable String fdate,
#PathVariable String tdate) {
System.out.println("fdate============" + fdate);
System.out.println("tdate============" + tdate);
String frdate = fdate.substring(2, 4) + "/" + fdate.substring(0, 2) + "/" + fdate.substring(4, 8);
String todate = tdate.substring(2, 4) + "/" + tdate.substring(0, 2) + "/" + tdate.substring(4, 8);
String json = null;
List<Object[]> list = retrievePublicationService.getsearchData(frdate, todate);
List<Map<String, Object>> activeTeamMap = new ArrayList<Map<String, Object>>();
if (list != null) {
for (Object[] obj : list) {
Map<String, Object> dropDownData = new HashMap<String, Object>();
dropDownData.put("publicationId", obj[0]);
dropDownData.put("documentId", obj[1]);
dropDownData.put("documentType", obj[2]);
dropDownData.put("languageCode", obj[3]);
dropDownData.put("processStartDate", obj[4]);
dropDownData.put("publicationType", obj[5]);
activeTeamMap.add(dropDownData);
System.out.println("activeTeamMap==vbnbvn==" + activeTeamMap);
}
} else {
}
return activeTeamMap;
}
#RequestMapping(value = "/getEventLogDetails/{eventId}", method = RequestMethod.GET)
public #ResponseBody List<Map<String, Object>> getEventLogDetails(#PathVariable String eventId) {
System.out.println("fdate============"+eventId);
String json = null;
List<Object[]> list = retrievePublicationService.getEventLogDetails(eventId);
List<Map<String, Object>> activeTeamMap = new ArrayList<Map<String,Object>>();
if (list != null) {
for (Object[] obj : list) {
Map<String, Object> dropDownData = new HashMap<String, Object>();
dropDownData.put("eventDetailLogPid",obj[0]);
dropDownData.put("documentId",obj[1]);
dropDownData.put("docLoadStatus",obj[2]);
dropDownData.put("processStartDate",obj[3]);
activeTeamMap.add(dropDownData);
System.out.println("activeTeamMap==vbnbvn=="+activeTeamMap);
}
}
else {
/*Map<String, Object> dropDownData = new HashMap<String, Object>();
dropDownData.put("status", "404");
dropDownData.put("message", "Data not found");
activeTeamMap.add(dropDownData);*/
}
return activeTeamMap;
}
}
FOllowing is my JUnit:
package com.hp.legacyloader.controller;
import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.imageio.spi.ServiceRegistry;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
import org.hibernate.cfg.Configuration;
import org.hibernate.service.ServiceRegistryBuilder;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.ResponseBody;
import com.google.gson.Gson;
import com.hp.legacyloader.controller.DashboardController;
import com.hp.legacyloader.dao.RetrievePublicationDAO;
import com.hp.legacyloader.dao.RetrievePublicationDAOImpl;
import com.hp.legacyloader.service.RetrievePublicationService;
import com.hp.legacyloader.service.RetrievePublicationServiceImpl;
public class DashboardControllerTest {
DashboardController dashboardController=new DashboardController();
RetrievePublicationDAOImpl retrievePublicationDAOImpl=new RetrievePublicationDAOImpl();
RetrievePublicationDAO retrievePublicationDAO;
private static ServiceRegistry serviceRegistry;
private static Configuration configuration;
SessionFactory sessionFactory;
Session session;
RetrievePublicationService retrievePublicationService;
#Before
public void getConnection(){
try{
configuration = new Configuration().addResource("hibernate.cfg.xml").configure();
sessionFactory=configuration.buildSessionFactory(new ServiceRegistryBuilder().applySettings(configuration.getProperties()).build());
}
catch(Throwable ex){
System.err.println("Failed to create sessionFactory object." + ex);
throw new ExceptionInInitializerError(ex);
}
session=sessionFactory.openSession();
}
#Test
public void testGetPaginationDataPage() {
assertEquals("ViewDashboard", dashboardController.getPaginationDataPage());
}
//------------------------------------------------//
#Test
public void testGetPaginationData() {
System.out.println("testGetPaginationData>>>>>>>>>>>");
/* try{
configuration = new Configuration().addResource("hibernate.cfg.xml").configure();
sessionFactory=configuration.buildSessionFactory(new ServiceRegistryBuilder().applySettings(configuration.getProperties()).build());
}
catch(Throwable ex){
System.err.println("Failed to create sessionFactory object." + ex);
throw new ExceptionInInitializerError(ex);
}*/
System.out.println(session);
Transaction tx=null;
// System.out.println("this===");
// RetrievePublicationService retrievePublicationService=Mockito.mock(RetrievePublicationService.class);
// System.out.println("pageSize======"+pageSize);
// System.out.println("page======"+page);
String json = null;
try{
tx=session.beginTransaction();
List<Object[]> list= sessionFactory.openSession().createQuery("select publicationId,documentId,documentType,languageCode,processStartDate,eventType from EventLog").list();
tx.commit();
System.out.println(list.get(0));
List<Map<String, Object>> activeTeamMap = new ArrayList<Map<String,Object>>();
if (list != null) {
for (Object[] obj : list) {
System.out.println(list.get(1));
Map<String, Object> dropDownData = new HashMap<String, Object>();
Gson gson = new Gson();
dropDownData.put("publicationId",obj[0]);
dropDownData.put("documentId", obj[1]);
dropDownData.put("documentType", obj[2]);
dropDownData.put("languageCode", obj[3]);
dropDownData.put("processStartDate", obj[4]);
dropDownData.put("publicationType", obj[5]);
activeTeamMap.add(dropDownData);
System.out.println("activeTeamMap==vbnbvn=="+dropDownData.get(obj));
json = gson.toJson(activeTeamMap);
System.out.println(json);
}
}
else{
System.out.println("return null");
}
}
catch(Exception e){
e.printStackTrace();
}
tx.commit();
session.close();
}
/*-------------------------------------------*/
#Test
public void testGetSearchData() {
System.out.println("testGetSearchData>>>");
Transaction tx=null;
System.out.println("testGetSearchData");
String fdate="12/05/2016";
String tdate="12/05/2016";
System.out.println("fdate============" + fdate);
System.out.println("tdate============" + tdate);
try{
tx=session.beginTransaction();
List<Object[]> list =sessionFactory.getCurrentSession().createQuery
("select publicationId,documentId,documentType,languageCode,processStartDate,eventLogPid,eventType from EventLog where processStartDate between "+"'"+fdate+"'"+" and "+"'"+tdate+"'" ).list();
tx.commit();
List<Map<String, Object>> activeTeamMap = new ArrayList<Map<String, Object>>();
if (list != null) {
for (Object[] obj : list) {
Map<String, Object> dropDownData = new HashMap<String, Object>();
dropDownData.put("publicationId", obj[0]);
dropDownData.put("documentId", obj[1]);
dropDownData.put("documentType", obj[2]);
dropDownData.put("languageCode", obj[3]);
dropDownData.put("processStartDate", obj[4]);
dropDownData.put("publicationType", obj[5]);
activeTeamMap.add(dropDownData);
System.out.println("activeTeamMap==vbnbvn==" + activeTeamMap);
}
} else {
}
}
catch(Exception ex){
ex.printStackTrace();
}
tx.commit();
session.close();
}
#Test
public void testGetEventLogDetails() {
Transaction tx=null;
String eventId="";
System.out.println(">>>>>>>>>eventId>>>>testGetEventLogDetails>>>>>>"+eventId);
String json=null;
try{
tx=session.beginTransaction();
#SuppressWarnings("unchecked")
List<Object[]> list= sessionFactory.getCurrentSession().createQuery
("select eventDetailLogPid,documentId,docLoadStatus,processStartDate from EventDetailLog where eventLogPid="+eventId).list();
Gson gson = new Gson();
List<Map<String, Object>> activeTeamMap = new ArrayList<Map<String,Object>>();
if (list != null) {
for (Object[] obj : list) {
Map<String, Object> dropDownData = new HashMap<String, Object>();
dropDownData.put("eventDetailLogPid",obj[0]);
dropDownData.put("documentId",obj[1]);
dropDownData.put("docLoadStatus",obj[2]);
dropDownData.put("processStartDate",obj[3]);
activeTeamMap.add(dropDownData);
System.out.println("activeTeamMap==vbnbvn=="+activeTeamMap);
json = gson.toJson(activeTeamMap);
System.out.println(json);
}
}
else{
}
System.out.println("acticeteammap>>>>>"+activeTeamMap);
}
catch(Exception ex){
ex.printStackTrace();
}
}
}
The test case for testGetPaginationData is working fine. But as I am trying to reuse the session object for the other methods I am getiing the following error:
org.hibernate.HibernateException: No CurrentSessionContext configured!
at org.hibernate.internal.SessionFactoryImpl.getCurrentSession(SessionFactoryImpl.java:1012)
at com.hp.legacyloader.controller.DashboardControllerTest.testGetSearchData(DashboardControllerTest.java:140)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
at org.junit.runners.BlockJUnit4ClassRunner.runNotIgnored(BlockJUnit4ClassRunner.java:79)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:71)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:49)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:193)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:52)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:191)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:42)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:184)
at org.junit.runners.ParentRunner.run(ParentRunner.java:236)
at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
Can anyone tell me what am I doing wrong here? I know its not at all a clean code, but i am new to Java....
You no need to do session=sessionFactory.openSession(); in your before statement.
In all your test just use
sessionFactory.getCurrentSession(). and I think that should fix your problem.
Also replace your openSession calls in your first test with getCurrentSession

Extended Html Unit Driver with screenshot capabilities

I am working on code (java) that will open up a selenium headless browser with HTML Unit webdriver and then take a screenshot. Unfortunately, HTML Unit does not support screenshots on its own, so I had to download an extended version:
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.internal.Base64Encoder;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements TakesScreenshot {
private static Map<String, byte[]> imagesCache = Collections.synchronizedMap(new HashMap<String, byte[]>());
private static Map<String, String> cssjsCache = Collections.synchronizedMap(new HashMap<String, String>());
// http://stackoverflow.com/questions/4652777/java-regex-to-get-the-urls-from-css
private final static Pattern cssUrlPattern = Pattern.compile("background(-image)?[\\s]*:[^url]*url[\\s]*\\([\\s]*([^\\)]*)[\\s]*\\)[\\s]*");// ?<url>
public ScreenCaptureHtmlUnitDriver() {
super();
}
public ScreenCaptureHtmlUnitDriver(boolean enableJavascript) {
super(enableJavascript);
}
public ScreenCaptureHtmlUnitDriver(Capabilities capabilities) {
super(capabilities);
}
public ScreenCaptureHtmlUnitDriver(BrowserVersion version) {
super(version);
DesiredCapabilities var = ((DesiredCapabilities) getCapabilities());
var.setCapability(CapabilityType.TAKES_SCREENSHOT, true);
}
//#Override
#SuppressWarnings("unchecked")
public <X> X getScreenshotAs(OutputType<X> target) throws WebDriverException {
byte[] archive = new byte[0];
try {
archive = downloadCssAndImages(getWebClient(), (HtmlPage) getCurrentWindow().getEnclosedPage());
} catch (Exception e) {
}
if(target.equals(OutputType.BASE64)){
return target.convertFromBase64Png(new Base64Encoder().encode(archive));
}
if(target.equals(OutputType.BYTES)){
return (X) archive;
}
return (X) archive;
}
// http://stackoverflow.com/questions/2244272/how-can-i-tell-htmlunits-webclient-to-download-images-and-css
protected byte[] downloadCssAndImages(WebClient webClient, HtmlPage page) throws Exception {
WebWindow currentWindow = webClient.getCurrentWindow();
Map<String, String> urlMapping = new HashMap<String, String>();
Map<String, byte[]> files = new HashMap<String, byte[]>();
WebWindow window = null;
try {
window = webClient.getWebWindowByName(page.getUrl().toString()+"_screenshot");
webClient.getPage(window, new WebRequest(page.getUrl()));
} catch (Exception e) {
window = webClient.openWindow(page.getUrl(), page.getUrl().toString()+"_screenshot");
}
String xPathExpression = "//*[name() = 'img' or name() = 'link' and (#type = 'text/css' or #type = 'image/x-icon') or #type = 'text/javascript']";
List<?> resultList = page.getByXPath(xPathExpression);
Iterator<?> i = resultList.iterator();
while (i.hasNext()) {
try {
HtmlElement el = (HtmlElement) i.next();
String resourceSourcePath = el.getAttribute("src").equals("") ? el.getAttribute("href") : el
.getAttribute("src");
if (resourceSourcePath == null || resourceSourcePath.equals(""))
continue;
URL resourceRemoteLink = page.getFullyQualifiedUrl(resourceSourcePath);
String resourceLocalPath = mapLocalUrl(page, resourceRemoteLink, resourceSourcePath, urlMapping);
urlMapping.put(resourceSourcePath, resourceLocalPath);
if (!resourceRemoteLink.toString().endsWith(".css")) {
byte[] image = downloadImage(webClient, window, resourceRemoteLink);
files.put(resourceLocalPath, image);
} else {
String css = downloadCss(webClient, window, resourceRemoteLink);
for (String cssImagePath : getLinksFromCss(css)) {
URL cssImagelink = page.getFullyQualifiedUrl(cssImagePath.replace("\"", "").replace("\'", "")
.replace(" ", ""));
String cssImageLocalPath = mapLocalUrl(page, cssImagelink, cssImagePath, urlMapping);
files.put(cssImageLocalPath, downloadImage(webClient, window, cssImagelink));
}
files.put(resourceLocalPath, replaceRemoteUrlsWithLocal(css, urlMapping)
.replace("resources/", "./").getBytes());
}
} catch (Exception e) {
}
}
String pagesrc = replaceRemoteUrlsWithLocal(page.getWebResponse().getContentAsString(), urlMapping);
files.put("page.html", pagesrc.getBytes());
webClient.setCurrentWindow(currentWindow);
return createZip(files);
}
String downloadCss(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (cssjsCache.get(resourceUrl.toString()) == null) {
cssjsCache.put(resourceUrl.toString(), webClient.getPage(window, new WebRequest(resourceUrl))
.getWebResponse().getContentAsString());
}
return cssjsCache.get(resourceUrl.toString());
}
byte[] downloadImage(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (imagesCache.get(resourceUrl.toString()) == null) {
imagesCache.put(
resourceUrl.toString(),
IOUtils.toByteArray(webClient.getPage(window, new WebRequest(resourceUrl)).getWebResponse()
.getContentAsStream()));
}
return imagesCache.get(resourceUrl.toString());
}
public static byte[] createZip(Map<String, byte[]> files) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipOutputStream zipfile = new ZipOutputStream(bos);
Iterator<String> i = files.keySet().iterator();
String fileName = null;
ZipEntry zipentry = null;
while (i.hasNext()) {
fileName = i.next();
zipentry = new ZipEntry(fileName);
zipfile.putNextEntry(zipentry);
zipfile.write(files.get(fileName));
}
zipfile.close();
return bos.toByteArray();
}
List<String> getLinksFromCss(String css) {
List<String> result = new LinkedList<String>();
Matcher m = cssUrlPattern.matcher(css);
while (m.find()) { // find next match
result.add( m.group(2));
}
return result;
}
String replaceRemoteUrlsWithLocal(String source, Map<String, String> replacement) {
for (String object : replacement.keySet()) {
// background:url(http://org.com/images/image.gif)
source = source.replace(object, replacement.get(object));
}
return source;
}
String mapLocalUrl(HtmlPage page, URL link, String path, Map<String, String> replacementToAdd) throws Exception {
String resultingFileName = "resources/" + FilenameUtils.getName(link.getFile());
replacementToAdd.put(path, resultingFileName);
return resultingFileName;
}
}
Anyway, I found this code, but I am unsure how to use this code to actually take a screenshot.
I would like the screenshot to be a .jpg and to be able to be stored in a certain folder.
In the long haul, I am also going to need to run the screenshot code repeatedly.
Any help would be appreciated.

Categories

Resources