Develop Reference

Bulk upload CSV file into Elasticsearch using JavaAPI

I want to bulk upload the csv file into Elasticsearch using JAVA API (without using logstash).
Elasticsearch version - 6.6
I have tried the below program using Jackson format to get source Map for IndexRequest. Because I can't predefined the POJO variables. So I used dynamic Map from CSV file
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.logging.Logger;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import com.zoho.dedupe.connection.DukeESConnection;
public class BulkImport {
private static Logger logger = Logger.getLogger(BulkImport.class.getName());
public static void main(String args[]) {
long starttime = System.currentTimeMillis();
logger.info("ElasticSearchServiceImpl => bulkInsert Service Started");
FileInputStream fis = null;
BulkRequest request;
RestHighLevelClient client;
//elastic Search Index Name
String esIndex = "post";
try {
boolean isHeaderSet = false;
Set<String> header = new HashSet<String>();
fis = new FileInputStream("/Users/test/Documents/Test.csv");
request = new BulkRequest();
MappingIterator<Map<String, Object>> data = parse(fis);
while (data.hasNext()) {
Map<?,?> value = data.next();
if(!isHeaderSet) {
header.add("id");
header = (Set<String>) value.keySet();
isHeaderSet= true;
}
System.out.println(value);
request.add(getIndexRequest(value, esIndex));
}
fis.close();
if(request.numberOfActions()>0) {
String hostsInString = "localhost";
List<HttpHost> httpHosts = new ArrayList<HttpHost> ( );
String[] hosts = hostsInString.split (",");
for (String host : hosts)
{
HttpHost httpHost = new HttpHost (host, 9200, "http");
httpHosts.add (httpHost);
}
client = client = new RestHighLevelClient (RestClient.builder (
httpHosts.toArray(new HttpHost[]{})).setMaxRetryTimeoutMillis (10 * 60000).setRequestConfigCallback(
new RestClientBuilder.RequestConfigCallback() {
#Override
public RequestConfig.Builder customizeRequestConfig(
RequestConfig.Builder requestConfigBuilder) {
return requestConfigBuilder
.setConnectTimeout (60000)
.setSocketTimeout (10 * 60000);
}
}));
CreateIndexRequest crrequest = new CreateIndexRequest(esIndex);
Map<String, Object> jsonMap = new HashMap<>();
Map<String, Object> message = new HashMap<>();
message.put("type", "text");
Map<String, Object> keyword = new HashMap<>();
Map<String, Object> type = new HashMap<>();
type.put("type", "keyword");
type.put("ignore_above", 256);
keyword.put("keyword", type);
message.put("fields", keyword);
Map<String, Object> properties = new HashMap<>();
for (Object hdr :header) {
properties.put(hdr.toString(), message);
}
Map<String, Object> mapping = new HashMap<>();
mapping.put("properties", properties);
jsonMap.put("_doc", mapping);
crrequest.mapping("_doc", jsonMap);
CreateIndexResponse createIndexResponse = client.indices().create(crrequest, RequestOptions.DEFAULT);
boolean acknowledged = createIndexResponse.isAcknowledged();
System.out.println(acknowledged);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if(bulkResponse.hasFailures()) {
logger.info("ElasticSearchServiceImpl => bulkInsert : Some of the record has failed.Please reinitiate the process");
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : Success");
}
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : No request for BulkInsert ="+request.numberOfActions());
}
} catch (Exception e) {
logger.info("ElasticSearchServiceImpl => bulkInsert : Exception =" + e.getMessage());
e.printStackTrace();
}
long endTime = System.currentTimeMillis();
logger.info("ElasticSearchServiceImpl => bulkInsert End " + (endTime - starttime));
}
public static MappingIterator<Map<String, Object>> parse(FileInputStream input) throws Exception {
MappingIterator<Map<String, Object>> map = readObjectsFromCsv(input);
return map;
//writeAsJson(data);
}
public static MappingIterator<Map<String, Object>> readObjectsFromCsv(FileInputStream file) throws IOException {
CsvSchema bootstrap = CsvSchema.emptySchema().withHeader().withColumnSeparator(',');
CsvMapper csvMapper = new CsvMapper();
MappingIterator<Map<String, Object>> mappingIterator = csvMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).reader(Map.class).with(bootstrap).readValues(file);
// System.out.println("Column names: " + mappingIterator.next().keySet());
return mappingIterator;
}
public static void writeAsJson(List<Map<?, ?>> data) throws IOException, JSONException {
ObjectMapper mapper = new ObjectMapper();
String value = mapper.writeValueAsString(data);
JSONArray json = new JSONArray(value);
System.out.println(json);
}
public static IndexRequest getIndexRequest(Map data,String index)throws Exception {
IndexRequest indexRequest = null;
indexRequest = new IndexRequest(index).id(UUID.randomUUID().toString()).source(data);
System.out.println(indexRequest.toString());
return indexRequest;
}
}
I got the below exception while running the program
{Document Name=dhjajga, Title=sdas, Name=asd, DOB=14-43-22}
index {[post][null][c2148857-87e0-4407-b5f5-b4f5f52c40d2], source[{"Document Name":"dhjajga","Title":"sdas","Name":"asd","DOB":"14-43-22"}]}
Jun 11, 2020 4:06:18 PM com.zoho.dedupe.connection.DukeESConnection connect
INFO: Client org.elasticsearch.client.RestHighLevelClient#7c51f34b
true
Jun 11, 2020 4:06:18 PM BulkImport main
INFO: ElasticSearchServiceImpl => bulkInsert : Exception =Validation Failed: 1: type is missing;2: type is missing;3: type is missing;
org.elasticsearch.action.ActionRequestValidationException: Validation Failed: 1: type is missing;2: type is missing;3: type is missing;
at org.elasticsearch.action.bulk.BulkRequest.validate(BulkRequest.java:612)
at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1728)
at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1694)
at org.elasticsearch.client.RestHighLevelClient.bulk(RestHighLevelClient.java:470)
at BulkImport.main(BulkImport.java:85)
Jun 11, 2020 4:06:18 PM BulkImport main
INFO: ElasticSearchServiceImpl => bulkInsert End 1432
When I try to insert the same above indexrequest its working fine.
curl -X POST "localhost:9200/post/_doc/?pretty" -H 'Content-Type: application/json' -d'
{
"Document Name":"dhjajga","Title":"sdas","Name":"asd","DOB":"14-43-22"
}
'
{
"_index" : "post",
"_type" : "_doc",
"_id" : "jBPronIB0Wb3XTTasBjG",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 0,
"_primary_term" : 1
}
Please help to fix the issue in java program. Thanks in advance

Before Elasticsearch version 7 you have to specify a type with your Indexrequest. It is recommended to use the type "_doc".

Push data from Dynamo DB to elasticsearch using java

Hi i have created a handler in java for getting the events from dynamo DB
Here is my code
package com.Lambda.dynamodb;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
public class DDBEventProcessor implements
RequestHandler<DynamodbEvent, String> {
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()){
System.out.println(record.getEventID());
System.out.println(record.getEventName());
System.out.println(record.getDynamodb().toString());
}
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
}
}
Lambda function able to write the events in cloudwatch but the challenge is i have to index all the streamed records to the AWS elasticsearch service endpoint and index it.
while search through blogs i got few code samples in python and node.js but my requirement is i have to build this lambda function in java
Could anyone please suggest how to achieve this in java lambda function?

Hi i have included the code below may helpful to some one. Dynamo DB streams to index the document in elasticsearch both inside AWS and outside AWS
package com.Firstlambda;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemUtils;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.json.JSONObject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class HelloWorld implements RequestHandler<DynamodbEvent, String> {
private static String serviceName = "es";
private static String region = "us-east-1";
private static String aesEndpoint = ""
private static String index = "";
private static String type = "_doc";
static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
public String handleRequest(DynamodbEvent ddbEvent, Context context) {
for (DynamodbStreamRecord record : ddbEvent.getRecords()) {
System.out.println("EventName : " + record.getEventName());
System.out.println("EventName : " + record.getDynamodb());
//AWS outside
RestHighLevelClient esClient = esClient();
//AWS outside
//AWS Inside
//RestHighLevelClient esClient = esClient(serviceName, region);
//AWS Inside
if (record.getEventName().toLowerCase().equals("insert")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
IndexRequest indexRequest = new IndexRequest(index, type, JsonUniqueId);
indexRequest.source(JsonString, XContentType.JSON);
try {
IndexResponse indexResponse = esClient.index(indexRequest, RequestOptions.DEFAULT);
System.out.println(indexResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else if (record.getEventName().toLowerCase().equals("modify")) {
String JsonString = getJsonstring(record.getDynamodb().getNewImage());
String JsonUniqueId = GetIdfromJsonString(JsonString);
UpdateRequest request = new UpdateRequest(index, type, JsonUniqueId);
String jsonString = JsonString;
request.doc(jsonString, XContentType.JSON);
try {
UpdateResponse updateResponse = esClient.update(
request, RequestOptions.DEFAULT);
System.out.println(updateResponse.toString());
return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
} catch (IOException e) {
System.out.println(e.getMessage());
}
} else {
System.out.println("remove");
System.out.println("KEYID : " + record.getDynamodb().getKeys().get("ID").getN());
String deletedId = record.getDynamodb().getKeys().get("ID").getN();
DeleteRequest request = new DeleteRequest(index, type, deletedId);
try {
DeleteResponse deleteResponse = esClient.delete(
request, RequestOptions.DEFAULT);
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
}
return "Successfullyprocessed";
}
public String getJsonstring(Map<String, AttributeValue> newIma) {
String json = null;
Map<String, AttributeValue> newImage = newIma;
List<Map<String, AttributeValue>> listOfMaps = new ArrayList<Map<String, AttributeValue>>();
listOfMaps.add(newImage);
List<Item> itemList = ItemUtils.toItemList(listOfMaps);
for (Item item : itemList) {
json = item.toJSON();
}
return json;
}
public String GetIdfromJsonString(String Json) {
JSONObject jsonObj = new JSONObject(Json);
return String.valueOf(jsonObj.getInt("ID"));
}
// Adds the interceptor to the ES REST client
// public static RestHighLevelClient esClient(String serviceName, String region) {
// AWS4Signer signer = new AWS4Signer();
// signer.setServiceName(serviceName);
// signer.setRegionName(region);
// HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
// return new RestHighLevelClient(RestClient.builder(HttpHost.create(aesEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
// }
public static RestHighLevelClient esClient() {
String host = "d9bc7cbca5ec49ea96a6ea683f70caca.eastus2.azure.elastic-cloud.com";
int port = 9200;
String userName = "elastic";
String password = "L4Nfnle3wxLmV95lffwsf$Ub46hp";
String protocol = "https";
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocol))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
}
This is just a sample code has to be modified based on our requirements

Java: Thread-Safe Dataset implementation

I have created a custom SQLDataset implementation where it takes a SQL query and returns a List of LinkedHashmap back to the requestcontroller to be displayed in JSP or download in Excel format.
Could you please let me know if the approach is thread safe?
SqlDataset.java
package com.sqle.core;
import com.util.QueryProcessor;
import java.io.Serializable;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class SqlDataset implements Serializable {
private String query;
private QueryProcessor qp;
private ResultSet rsSet;
private List<LinkedHashMap<String, Object>> rsList = new ArrayList<>();
private ArrayList<String> dataHeader = new ArrayList<>();
public SqlDataset() {}
public SqlDataset(String uquery) {
this.query = uquery;
}
private ResultSet getQueryResult() throws Exception {
qp = new QueryProcessor(query);
this.rsSet = qp.getQueryResultSet();
return this.rsSet;
}
public List<LinkedHashMap<String, Object>> getResult() throws Exception {
return this.getValues(this.getQueryResult());
}
public List<LinkedHashMap<String, Object>> getResult(String query) throws Exception {
this.query = query;
return this.getValues(this.getQueryResult());
}
public int getRowCount() {
return this.rsList.size();
}
public ArrayList getHeaders() {
for (LinkedHashMap<String, Object> aRsList : this.rsList) {
for (Map.Entry<String, Object> dh : aRsList.entrySet()) {
if (!this.dataHeader.contains(dh.getKey()))
this.dataHeader.add(dh.getKey());
}
}
return this.dataHeader;
}
private List<LinkedHashMap<String, Object>> getValues(ResultSet rs) throws SQLException {
ResultSetMetaData rmd = rs.getMetaData();
int columns = rmd.getColumnCount();
while (rs.next()) {
LinkedHashMap<String, Object> row = new LinkedHashMap<>(columns);
for (int i = 1; i <= columns; ++i) {
row.put(rmd.getColumnName(i), rs.getObject(i));
}
this.rsList.add(row);
}
return this.rsList;
}
}
Below is the code written in request controller:
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
String act = request.getParameter("act").toLowerCase();
RequestDispatcher rd = request.getRequestDispatcher("sqleditor.jsp");
try {
if (act.equalsIgnoreCase("exec")) {
String uqry = request.getParameter("isql");
if (!uqry.isEmpty()) {
SqlDataset sd = new SqlDataset(uqry);
rslist = sd.getResult();
if (sd.getRowCount() > 0) {
headRow = sd.getHeaders();
request.setAttribute("resHead", headRow);
request.setAttribute("result", rslist);
} else {
throw new NoDataException("No data found to display");
}
} else {
throw new NoDataException("Please enter a query");
}
rd.forward(request, response);
} else if (act.equalsIgnoreCase("excel")) {
String uqry = request.getParameter("isql");
if (!uqry.isEmpty()) {
try {
SqlDataset sd = new SqlDataset();
rslist = sd.getResult(uqry);
if (sd.getRowCount() > 0) {
headRow = sd.getHeaders();
response.reset();
response.setContentType("application/vnd.ms-excel");
response.setHeader("Content-Disposition", "attachment; filename=\"" + FILENAME + "\"");
ExcelWriter ew = new ExcelWriter();
ew.initExcelfile(rslist, headRow, response.getOutputStream());
} else {
throw new NoDataException("No data found to download");
}
} catch (Exception evar1) {
throw new AppException(evar1.getMessage());
}
} else {
throw new NoDataException("Please enter a query");
}
}
} catch (SQLException evar2) {
request.setAttribute("errormsg", evar2.getMessage());
rd.forward(request, response);
} catch (NullPointerException evar3) {
request.setAttribute("errormsg", evar3.getMessage());
rd.forward(request, response);
} catch (Exception evar4) {
request.setAttribute("errormsg", evar4.getMessage());
rd.forward(request, response);
}
}
Will this code work is multiple users use this application and running different queries successively?
Modified SQLdataset class:
package com.sqle.core;
import com.util.QueryProcessor;
import java.io.Serializable;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class SqlDataset implements Serializable {
private List<LinkedHashMap<String, Object>> rsList = new ArrayList<>();
public SqlDataset() {}
private ResultSet getQueryResult(String query) throws Exception {
QueryProcessor qp = new QueryProcessor(query);
ResultSet rsSet = qp.getQueryResultSet();
return rsSet;
}
public List<LinkedHashMap<String, Object>> getResult(String query) throws Exception {
return this.getValues(this.getQueryResult(query));
}
public ArrayList getHeaders() {
ArrayList<String> dataHeader = new ArrayList<>();
for (LinkedHashMap<String, Object> aRsList : this.rsList) {
for (Map.Entry<String, Object> dh : aRsList.entrySet()) {
if (!dataHeader.contains(dh.getKey()))
dataHeader.add(dh.getKey());
}
}
return dataHeader;
}
private List<LinkedHashMap<String, Object>> getValues(ResultSet rs) throws SQLException {
ResultSetMetaData rmd = rs.getMetaData();
int columns = rmd.getColumnCount();
while (rs.next()) {
LinkedHashMap<String, Object> row = new LinkedHashMap<>(columns);
for (int i = 1; i <= columns; ++i) {
row.put(rmd.getColumnName(i), rs.getObject(i));
}
this.rsList.add(row);
}
return this.rsList;
}
public int getRowCount() {
return this.rsList.size();
}
}

It depends on how you use this class.
With your doPost method in controller it is thread safe for multiple users because you create new SqlDataset object every time.
It means it will be used only by thread which processes a single request.
Your Controller code is re-entrant and thread safe.
BTW in case if you plan to use your SqlDataset as singleton (e.g. Spring bean or such) - it is not thread safe. It has instance variables used in process - it means SqlDataset methods are not re-entrant.
just think about them...
do you really need private QueryProcessor qp; while you create new instance every time in the getQueryResult() method?
do you really need private ArrayList<String> dataHeader = new ArrayList<>();
while you just return it from getHeaders() - why do not just create new ArrayList before for loops inside the method.
...and so on...
If you make everything passed to methods as parameters and return everything created inside methods it will be fully thread safe.
Singletons may have only immutable instance variables (logically almost constants) to keep some settings or properties applicable to any threads which use it.

The SqlDataset.java itself is not thread safe as you have instance variables in it.
However if you only use it in some of your request controller's methods then there will be no problems. This is because a Servlet is not thread safe but the Servlets methods are that.

ArrayLists With Zero and Two Outcomes for Junit Tests

I need help with this piece of code. I need to write a test just like testfindOverviewByStatus() example in the code below but I need to write it for 2 pieces of data and 0 pieces. testfindOverviewByStatus() is a test for if there is only one piece of data in the list. I tried writing the two different tests I need at the end of the code but was unsuccessful in doing so. Can anyone help me?
package com.primatics.greensight.dashboard.controller;
import static org.junit.Assert.assertEquals;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.test.context.junit4.SpringRunner;
import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
import com.primatics.greensight.dashboard.UnitTest;
import com.primatics.greensight.dashboard.model.Overview;
import com.primatics.greensight.dashboard.repositories.ScenarioDashboardRepository;
import com.primatics.greensight.dashboard.repositories.impl.ScenarioDashboardRepositoryImpl;
import com.primatics.greensight.dashboard.services.ScenarioDashboardService;
import com.primatics.greensight.dashboard.services.impl.ScenarioDashboardServiceImpl;
#RunWith(SpringRunner.class)
#SpringBootTest(classes = ScenarioDashboardServiceImpl.class)
#AutoConfigureMockMvc
#EnableAutoConfiguration
#Category(UnitTest.class)
public class ScenarioDashboardServiceImplUnitTest {
#Configuration
static class AccountServiceTestContextConfiguration {
#Bean
public ScenarioDashboardRepositoryImpl scenarioDashboardService() {
return new ScenarioDashboardRepositoryImpl();
}
#Bean
public ScenarioDashboardRepository scenarioDashboardtRepository() {
return Mockito.mock(ScenarioDashboardRepositoryImpl.class);
}
}
#Mock
private ScenarioDashboardService scenarioDashboardService;
#Before
public void doSetUp() {
List<Overview> scenarioListTest = new ArrayList<Overview>();
String name = "first_scenario-test";
ISO8601DateFormat df = new ISO8601DateFormat();
Date estimationDate = null;
try {
estimationDate = df.parse("2017-01-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
Date creationDate = null;
try {
creationDate = df.parse("2017-02-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
Double balance = 131750000.0;
Double individualReviewImpairment = 1000.00;
Map<String, Double> baseline = new HashMap<String, Double>();
baseline.put("complete", 1000.0);
Map<String, Double> macroAdjustment = new HashMap<String, Double>();
macroAdjustment.put("complete", 2000.0);
Map<String, Double> qualitativeAdjustment = new HashMap<String, Double>();
qualitativeAdjustment.put("complete", 3000.0);
Date positionDate = null;
try {
positionDate = df.parse("2017-01-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
Date lossHistoryDate = null;
try {
lossHistoryDate = df.parse("2017-01-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
String status = "active";
Map<String, Integer> period = new HashMap<String, Integer>();
period.put("Q1", 2017);
boolean publish = true;
Overview ac = new Overview(4, name, estimationDate, creationDate, balance, individualReviewImpairment, baseline,
macroAdjustment, qualitativeAdjustment, positionDate, lossHistoryDate, status, period, publish);
scenarioListTest.add(ac);
Mockito.when(scenarioDashboardService.getOverviewByStatus("active")).thenReturn(scenarioListTest);
}
#Before
public void getTwoOverviews() {
List<Overview> scenarioListTest2 = new ArrayList<Overview>();
String name = "first_scenario-test";
ISO8601DateFormat df = new ISO8601DateFormat();
Date estimationDate = null;
try {
estimationDate = df.parse("2017-01-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
Date creationDate = null;
try {
creationDate = df.parse("2017-02-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
Double balance = 131750000.0;
Double individualReviewImpairment = 1000.00;
Map<String, Double> baseline = new HashMap<String, Double>();
baseline.put("complete", 1000.0);
Map<String, Double> macroAdjustment = new HashMap<String, Double>();
macroAdjustment.put("complete", 2000.0);
Map<String, Double> qualitativeAdjustment = new HashMap<String, Double>();
qualitativeAdjustment.put("complete", 3000.0);
Date positionDate = null;
try {
positionDate = df.parse("2017-01-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
Date lossHistoryDate = null;
try {
lossHistoryDate = df.parse("2017-01-28T22:25:51Z");
} catch (ParseException e) {
e.printStackTrace();
}
String status = "active";
Map<String, Integer> period = new HashMap<String, Integer>();
period.put("Q1", 2017);
boolean publish = true;
Overview ac = new Overview(4, name, estimationDate, creationDate, balance, individualReviewImpairment, baseline,
macroAdjustment, qualitativeAdjustment, positionDate, lossHistoryDate, status, period, publish);
scenarioListTest2.add(ac);
Mockito.when(scenarioDashboardService.getOverviewByStatus("active")).thenReturn(scenarioListTest2);
}
#Test
public void testfindOverviewByStatus() {
List<Overview> scenarioListTest = scenarioDashboardService.getOverviewByStatus("active");
assertEquals(1, scenarioListTest.size());
String scenarioName = scenarioListTest.get(0).getScenarioName();
assertEquals("first_scenario-test", scenarioName);
}
#Test
public void testfindOverviewByStatusTwo() {
List<Overview> scenarioList = scenarioDashboardService.getOverviewByStatus("active");
assertEquals(2, scenarioList.size());
String scenarioName = scenarioList.get(0).getScenarioName();
String scenarioName2 = scenarioList.get(1).getScenarioName();
assertEquals("first_scenario-test", scenarioName);
assertEquals("second_scenario-test", scenarioName2);
}
#Test
public void testfindOverviewByStatusNull() {
List<Overview> scenarioListNull = scenarioDashboardService.getOverviewByStatus("active");
assertEquals(0, scenarioListNull.size());
}
}

The problem is with the layout of your test. You have multiple #Before methods which all mock scenarioDashboardService.getOverviewByStatus("active")
The simplest way to get your tests working would be to remove the #Before annotations and instead invoke them methods in the #Test method.
So for example:
public void getTwoOverviews() {
//Contents of method
}
#Test
public void testfindOverviewByStatusTwo() {
//Setup
getTwoOverviews();
List<Overview> scenarioList = scenarioDashboardService.getOverviewByStatus("active");
assertEquals(2, scenarioList.size());
String scenarioName = scenarioList.get(0).getScenarioName();
String scenarioName2 = scenarioList.get(1).getScenarioName();
assertEquals("first_scenario-test", scenarioName);
assertEquals("second_scenario-test", scenarioName2);
}
As I mentioned this would be the simplest way but in the long run there most likely is a better way to layout your test class.

In situations where Mockito is already a dependency you might find it easier to use a custom argument matcher for this:
private class isListOfSize extends ArgumentMatcher<List<Overview>> {
int size;
isListOfSize(int size) {
this.size = size;
}
#Override
public boolean matches(Object list) {
return ((List) list).size() == size;
}
}
Usage:
assertThat(scenarioList, argThat(new isListOfSize(2)))

Extended Html Unit Driver with screenshot capabilities

I am working on code (java) that will open up a selenium headless browser with HTML Unit webdriver and then take a screenshot. Unfortunately, HTML Unit does not support screenshots on its own, so I had to download an extended version:
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.internal.Base64Encoder;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements TakesScreenshot {
private static Map<String, byte[]> imagesCache = Collections.synchronizedMap(new HashMap<String, byte[]>());
private static Map<String, String> cssjsCache = Collections.synchronizedMap(new HashMap<String, String>());
// http://stackoverflow.com/questions/4652777/java-regex-to-get-the-urls-from-css
private final static Pattern cssUrlPattern = Pattern.compile("background(-image)?[\\s]*:[^url]*url[\\s]*\\([\\s]*([^\\)]*)[\\s]*\\)[\\s]*");// ?<url>
public ScreenCaptureHtmlUnitDriver() {
super();
}
public ScreenCaptureHtmlUnitDriver(boolean enableJavascript) {
super(enableJavascript);
}
public ScreenCaptureHtmlUnitDriver(Capabilities capabilities) {
super(capabilities);
}
public ScreenCaptureHtmlUnitDriver(BrowserVersion version) {
super(version);
DesiredCapabilities var = ((DesiredCapabilities) getCapabilities());
var.setCapability(CapabilityType.TAKES_SCREENSHOT, true);
}
//#Override
#SuppressWarnings("unchecked")
public <X> X getScreenshotAs(OutputType<X> target) throws WebDriverException {
byte[] archive = new byte[0];
try {
archive = downloadCssAndImages(getWebClient(), (HtmlPage) getCurrentWindow().getEnclosedPage());
} catch (Exception e) {
}
if(target.equals(OutputType.BASE64)){
return target.convertFromBase64Png(new Base64Encoder().encode(archive));
}
if(target.equals(OutputType.BYTES)){
return (X) archive;
}
return (X) archive;
}
// http://stackoverflow.com/questions/2244272/how-can-i-tell-htmlunits-webclient-to-download-images-and-css
protected byte[] downloadCssAndImages(WebClient webClient, HtmlPage page) throws Exception {
WebWindow currentWindow = webClient.getCurrentWindow();
Map<String, String> urlMapping = new HashMap<String, String>();
Map<String, byte[]> files = new HashMap<String, byte[]>();
WebWindow window = null;
try {
window = webClient.getWebWindowByName(page.getUrl().toString()+"_screenshot");
webClient.getPage(window, new WebRequest(page.getUrl()));
} catch (Exception e) {
window = webClient.openWindow(page.getUrl(), page.getUrl().toString()+"_screenshot");
}
String xPathExpression = "//*[name() = 'img' or name() = 'link' and (#type = 'text/css' or #type = 'image/x-icon') or #type = 'text/javascript']";
List<?> resultList = page.getByXPath(xPathExpression);
Iterator<?> i = resultList.iterator();
while (i.hasNext()) {
try {
HtmlElement el = (HtmlElement) i.next();
String resourceSourcePath = el.getAttribute("src").equals("") ? el.getAttribute("href") : el
.getAttribute("src");
if (resourceSourcePath == null || resourceSourcePath.equals(""))
continue;
URL resourceRemoteLink = page.getFullyQualifiedUrl(resourceSourcePath);
String resourceLocalPath = mapLocalUrl(page, resourceRemoteLink, resourceSourcePath, urlMapping);
urlMapping.put(resourceSourcePath, resourceLocalPath);
if (!resourceRemoteLink.toString().endsWith(".css")) {
byte[] image = downloadImage(webClient, window, resourceRemoteLink);
files.put(resourceLocalPath, image);
} else {
String css = downloadCss(webClient, window, resourceRemoteLink);
for (String cssImagePath : getLinksFromCss(css)) {
URL cssImagelink = page.getFullyQualifiedUrl(cssImagePath.replace("\"", "").replace("\'", "")
.replace(" ", ""));
String cssImageLocalPath = mapLocalUrl(page, cssImagelink, cssImagePath, urlMapping);
files.put(cssImageLocalPath, downloadImage(webClient, window, cssImagelink));
}
files.put(resourceLocalPath, replaceRemoteUrlsWithLocal(css, urlMapping)
.replace("resources/", "./").getBytes());
}
} catch (Exception e) {
}
}
String pagesrc = replaceRemoteUrlsWithLocal(page.getWebResponse().getContentAsString(), urlMapping);
files.put("page.html", pagesrc.getBytes());
webClient.setCurrentWindow(currentWindow);
return createZip(files);
}
String downloadCss(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (cssjsCache.get(resourceUrl.toString()) == null) {
cssjsCache.put(resourceUrl.toString(), webClient.getPage(window, new WebRequest(resourceUrl))
.getWebResponse().getContentAsString());
}
return cssjsCache.get(resourceUrl.toString());
}
byte[] downloadImage(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (imagesCache.get(resourceUrl.toString()) == null) {
imagesCache.put(
resourceUrl.toString(),
IOUtils.toByteArray(webClient.getPage(window, new WebRequest(resourceUrl)).getWebResponse()
.getContentAsStream()));
}
return imagesCache.get(resourceUrl.toString());
}
public static byte[] createZip(Map<String, byte[]> files) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipOutputStream zipfile = new ZipOutputStream(bos);
Iterator<String> i = files.keySet().iterator();
String fileName = null;
ZipEntry zipentry = null;
while (i.hasNext()) {
fileName = i.next();
zipentry = new ZipEntry(fileName);
zipfile.putNextEntry(zipentry);
zipfile.write(files.get(fileName));
}
zipfile.close();
return bos.toByteArray();
}
List<String> getLinksFromCss(String css) {
List<String> result = new LinkedList<String>();
Matcher m = cssUrlPattern.matcher(css);
while (m.find()) { // find next match
result.add( m.group(2));
}
return result;
}
String replaceRemoteUrlsWithLocal(String source, Map<String, String> replacement) {
for (String object : replacement.keySet()) {
// background:url(http://org.com/images/image.gif)
source = source.replace(object, replacement.get(object));
}
return source;
}
String mapLocalUrl(HtmlPage page, URL link, String path, Map<String, String> replacementToAdd) throws Exception {
String resultingFileName = "resources/" + FilenameUtils.getName(link.getFile());
replacementToAdd.put(path, resultingFileName);
return resultingFileName;
}
}
Anyway, I found this code, but I am unsure how to use this code to actually take a screenshot.
I would like the screenshot to be a .jpg and to be able to be stored in a certain folder.
In the long haul, I am also going to need to run the screenshot code repeatedly.
Any help would be appreciated.