Develop Reference

Many objects to one Object using jolt

Hi working in Jolt transformation tool and getting know some in-depth details
Input
{
"characteristic": [
{
"name": "usageCharacteristicName",
"value": "availableBalance",
"#type": "usageCharacteristic",
"arrayIndex": "2"
},
{
"name": "usageCharacteristicValue",
"value": "2999.25",
"#type": "usageCharacteristic",
"arrayIndex": "2"
},
{
"name": "usageCharacteristicName",
"value": "Name",
"#type": "usageCharacteristic",
"arrayIndex": "0"
},
{
"name": "usageCharacteristicValue",
"value": "Jack",
"#type": "usageCharacteristic",
"arrayIndex": "0"
},
{
"name": "usageCharacteristicName",
"value": "Likes",
"#type": "usageCharacteristic",
"arrayIndex": "1"
},
{
"name": "usageCharacteristicValue",
"value": "Code",
"#type": "usageCharacteristic",
"arrayIndex": "1"
}
]
}
Jolt Spec :
[
{
"operation": "shift",
"spec": {
"characteristic": {
"*": {
"arrayIndex": {
"2": {
"#(2,value)": "data.queryBalance.accountBalance"
},
"0": {
"#(2,value)": "data.queryBalance.Name"
},
"1": {
"#(2,value)": "data.queryBalance.Likes"
},
}
}
}
}
}
]
Output :
{
"data" : {
"queryBalance" : {
"accountBalance" : [ "availableBalance", "2999.25" ],
"Name" : [ "Name", "Jack" ],
"Likes" : [ "Likes", "Code" ]
}
}
}
the Output i was getting is not good enough for me, want to do like String like bellow Expected.
Expected Output :
"availableBalance" : "2999.25",
"Name" : "Jack",
"Likes" : "Code"
how do i get like a String ?

I think this solve the problem
spec
[
{
//group keys and valeus for arrayIndex
"operation": "shift",
"spec": {
"characteristic": {
"*": {
"value": "newArray[#(1,arrayIndex)].#(1,name)"
}
}
}
},
{
//create a new object use a key and valeus matchs previous
"operation": "shift",
"spec": {
"newArray": {
"*": {
"usageCharacteristicValue": "#(1,usageCharacteristicName)"
}
}
}
}
]
output
{
"Name" : "Jack",
"Likes" : "Code",
"availableBalance" : "2999.25"
}

You need a modify transformation spec as well such as
[
{
// reduce to simple array "value" : [ "availableBalance", "2999.25" ] nested within an object
"operation": "shift",
"spec": {
"*": {
"*": {
"value": "&"
}
}
}
},
{
// combine components of the array so as to reform as a string
"operation": "modify-overwrite-beta",
"spec": {
"*": "=join(' : ',#(1,&))"
}
}
]
Edit: You can alternatively try the following for the new case :
[
{
// dissipate each array into different object with respect to arrayIndex values
"operation": "shift",
"spec": {
"*": {
"*": {
"value": "#(1,arrayIndex).&"
}
}
}
},
{
// combine components of each array colon-separatedly
"operation": "modify-overwrite-beta",
"spec": {
"*": {
"*": "=join(' : ',#(1,&))"
}
}
},
{
// convert to a unique array
"operation": "shift",
"spec": {
"*": {
"*": "&"
}
}
},
{
// combine components of each array comma-separatedly
"operation": "modify-overwrite-beta",
"spec": {
"*": "=join(' , ',#(1,value))"
}
}
]

I know how to do it by another library Josson. You may consider to use it.
https://github.com/octomix/josson
Deserialization
Josson josson = Josson.fromJsonString(inputJSON);
Transform to an array of string
JsonNode node = josson.getNode(
"characteristic" +
".group(arrayIndex)" +
".concat(elements[name='usageCharacteristicName'].value" +
" ,' : '" +
" ,elements[name='usageCharacteristicValue'].value)");
System.out.println(node.toPrettyString());
Output
[ "availableBalance : 2999.25", "Name : Jack", "Likes : Code" ]
Transform to a single string
String str = josson.getString(
"characteristic" +
".group(arrayIndex)" +
".concat(elements[name='usageCharacteristicName'].value" +
" ,' : '" +
" ,elements[name='usageCharacteristicValue'].value)" +
".join(', ')");
System.out.println(str);
Output
availableBalance : 2999.25, Name : Jack, Likes : Code
Transform to a map
JsonNode node = josson.getNode(
"characteristic" +
".group(arrayIndex)" +
".map(elements[name='usageCharacteristicName'].value::elements[name='usageCharacteristicValue'].value)" +
".mergeObjects()");
System.out.println(node.toPrettyString());
Output
{
"availableBalance" : "2999.25",
"Name" : "Jack",
"Likes" : "Code"
}
Transform to a map with sorted "arrayIndex" as key
JsonNode node = josson.getNode(
"characteristic" +
".group(arrayIndex)" +
".sort(arrayIndex)" +
".map(arrayIndex.prepend('arrayIndex')::" +
" concat(elements[name='usageCharacteristicName'].value" +
" ,' : '" +
" ,elements[name='usageCharacteristicValue'].value))" +
".mergeObjects()");
System.out.println(node.toPrettyString());
Output
{
"arrayIndex0" : "Name : Jack",
"arrayIndex1" : "Likes : Code",
"arrayIndex2" : "availableBalance : 2999.25"
}

How does Elasticsearch get the doc_count value of the filter under aggs?

I encountered some problems when using elasticsearch's aggregation query.This is the DSL statement I used
{
"size": 0,
"aggs": {
"terms_vlid_year":{
"terms": {
"field": "valid_year",
"size": 5,
"order": {
"_key": "asc"
}
}
},
"filterss":{
"filter": {
"range": {
"valid_year": {
"gte": 6
}
}
}
}
}
}
This is the aggregate result
"aggregations" : {
"terms_vlid_year" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 13918850,
"buckets" : [
{
"key" : 1,
"doc_count" : 1632006
},
{
"key" : 2,
"doc_count" : 3588976
},
{
"key" : 3,
"doc_count" : 3710637
},
{
"key" : 4,
"doc_count" : 2948910
},
{
"key" : 5,
"doc_count" : 2672394
}
]
},
"filters" : {
"doc_count" : 13918850
}
}
Now I want to use java to get the doc_count of filters.Below is my java code, But the doc_count value of the filter object cannot be obtained through the filterResult.getDocCount(). How to solve this problem?
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("terms_vlid_year");
FilterAggregationBuilder filter = AggregationBuilders.filter("filters", QueryBuilders.rangeQuery("valid_year").gte(6));
termsAggregationBuilder.field("valid_year");
termsAggregationBuilder.size(5);
termsAggregationBuilder.order(BucketOrder.key(true));
sourceBuilder.aggregation(termsAggregationBuilder.subAggregation(filter));
searchRequest.source(sourceBuilder);
ReturnValue returnValue = new ReturnValue();
LinkedList<Object> list = new LinkedList<>();
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Terms terms = searchResponse.getAggregations().get("terms_vlid_year");
List<? extends Terms.Bucket> buckets = terms.getBuckets();
Filter filterResult = searchResponse.getAggregations().get("filters");
long docCount1 = filterResult.getDocCount();
System.out.println("docCount1 = " + docCount1);
for (Terms.Bucket bucket : buckets) {
String keyAsString = bucket.getKeyAsString();
long docCount = bucket.getDocCount();
returnValue.setFieldName(keyAsString);
returnValue.setDocValue(docCount);
System.out.println(returnValue.toString());
}
}

Now the problem has been solved.FilterAggregation cannot use subAggregation nested in termsAggregation, they belong to the same level.It should be this way to get the desired result:
sourceBuilder.aggregation(termsAggregationBuilder);
FilterAggregationBuilder filter = AggregationBuilders.filter("filters", QueryBuilders.rangeQuery("valid_year").gte(6));
sourceBuilder.aggregation(filter);

How to parse JSON records which have JSON array elements in it?

I am getting the error:
JSONObject["Transaction_details"] not found.
Why is it not able to find Transaction_details?
Code:
for(Iterator it = mapper.readValues(new JsonFactory().createJsonParser(in),JSONObject.class);it.hasNext();)
{
JSONObject js = (JSONObject) it.next();
JSONArray recs = (JSONArray) js.get("Transaction_details");
for (int i = 1; i < recs.length(); ++i) {
JSONObject rec = recs.getJSONObject(i);
int id = rec.getInt("merchantid");
String loc = rec.getString("transaction_amount");
}
Snapshot of a part of my schema:
{
"name" : "cica",
"type" : "long"
}, {
"name" : "pica",
"type" : "long"
}, {
"name" : "issuingcountry",
"type" : "string"
}, {
"name" : "Transaction_details",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "Transaction_details",
"doc" : "Transaction event details",
"fields" : [ {
"name" : "transactionid",
"type" : "long"
}, {
"name" : "transactiondate",
"type" : "string"
}, {
"name" : "Productcode",
"type" : "string"
}
]
}

Something like the below snippet should work. But, I have not compiled or executed it.
Basically, you would have to check if you are encountering a "Transaction_details" name value. If yes, then you need to retrieve its 'type' array and work with its elements.
Hope this helps!
for(Iterator it = mapper.readValues(new JsonFactory().createJsonParser(in),JSONObject.class);it.hasNext();)
{
JSONObject js = (JSONObject) it.next();
String nameValue = (String) js.get("name");
if ("Transaction_details".equals(nameValue)) {
JSONArray recs = (JSONArray) js.get("type");
for (int i = 1; i < recs.length(); ++i) {
JSONObject rec = recs.getJSONObject(i);
...
}
}
}

modifying Highcharts tool tip with for a bar chart dynamic data

I m using highcharts.js for my project What I'm trying to do is format my tool tip. I have written a java method which is returning me the output in the json format as
syList:
[{
"value1": "Some Area",
"value2": "1",
"agrAcronym": "AB"
}, {
"value1": "Some AREA",
"value2": "2",
"agrAcronym": "BA"
}, {
"value1": "Some Area",
"value2": "3",
"agrAcronym": "NA"
}, {
"value1": "Some Area",
"value2": "4",
"agrAcronym": "MW"
}, {
"value1": "Some Area",
"value2": "5",
"agrAcronym": "PW"
}, {
"value1": "Some Area",
"value2": "6",
"agrAcronym": "NP"
}, {
"value1": "Some Area",
"value2": "7",
"agrAcronym": "SP"
}, {
"value1": "Some Area",
"value2": "8",
"agrAcronym": "MS"
}, {
"value1": "Some Area",
"value2": "9",
"agrAcronym": "SA"
}],
my chart script is
var options2 = {
chart : {
renderTo : 'outgoingContainer',
type : 'column',
marginTop: 50,
},
title : {
text : 'Something Area',
style: {
color : '#2B1B17',
fontWeight: 'bold'
}
},
xAxis : {
title : {
text : null
},
labels: {
y: 30,
}
},
yAxis : {
title : {
text : 'something',
align : 'middle'
},
stackLabels : {
enabled : true,
style : {
fontWeight : 'bold',
color : (Highcharts.theme && Highcharts.theme.textColor)
|| 'charcoal'
},
formatter : function() {
return getCurrencyFormat(parseInt(this.total));
}
}
},
tooltip : {
formatter : function() {
return '' + this.x + ': ' + getCurrencyFormat(parseInt(this.y));
}
},
plotOptions : {
series: {
borderWidth: 1,
borderColor: 'black'
},
column : {
stacking : 'normal',
dataLabels : {
enabled : false,
color : (Highcharts.theme && Highcharts.theme.dataLabelsColor)
|| 'black',
style : {
fontWeight : 'bold',
}
}
}
},
legend : {
enabled : false
},
credits : {
enabled : false
},
series : [ {
color : '#00FFFF',
} ]
};
$.getJSON(
function(data) {
options2.xAxis.categories = [];
options2.series[0].data = [];
if (data["syList"] != null)
{
$.each(data["syList"], function( index, value ) {
options2.xAxis.categories[index] = data["syList"][index] ["value1"];
options2.series[0].data[index] = parseFloat(data["syList"][index]["value2"]);
});
$("#outgoingContainer").show();
} else {
options2.yAxis.title.text = "";
$("#outgoingContainer").hide();
}
new Highcharts.Chart(options2);
});

You need to set useHtml = true on your tooltip and this will allow you to format your tooltip any way that you want. Make sure that the data for your series is set to syList and set the headerFormat, pointFormat, and footerFormat on your tooltip like this. Note the {point.agrAcronym}.
tooltip: {
shared: true,
useHTML: true,
headerFormat: '<table>',
pointFormat: '<tr><td style="color: {series.color};">{series.name}: </td>' +
'<td>{point.agrAcronym}</td></tr>',
footerFormat: '</table>'
},
series: [{
data: syList
}]

Incremental Mongodb Update

I'm trying to update my db incrementally by parsing a csv and counting how many times each person has been to a country or state, if they're within the US.
public static void main(String[] args) throws UnknownHostException {
Scanner dir = new Scanner(System.in);
String dir_loc;
System.out.println("Enter File Directory:");
dir_loc=dir.nextLine();
Scanner dbloc = new Scanner(System.in);
String db_loc;
System.out.println("Enter Database location:");
db_loc=dbloc.nextLine();
Scanner port = new Scanner(System.in);
int port_loc;
System.out.println("Enter Port:");
port_loc=port.nextInt();
dir.close();
dbloc.close();
port.close();
System.out.println("uploading");
String directoryName = dir_loc;
File directory = new File(directoryName);
File[] fileList = directory.listFiles();
DB dB = (new MongoClient(db_loc, port_loc)).getDB("visits");
DBCollection dBCollection = dB.getCollection("Channel");
BasicDBObject doc = null;
for(File file: fileList) {
String dataFileName = file.toString();
BufferedReader bReader = null;
try {
bReader = new BufferedReader(new FileReader(dataFileName));
} catch (FileNotFoundException e2) {
e2.printStackTrace();
System.exit(-1);
}
String line = null;
int count = 0;
int statecount = 0;
try {
while((line = bReader.readLine()) !=null) {
String dataValue[] = line.split("\t");
String user_id = (dataValue[2]);
String country = (dataValue[1]);
boolean user_idExists = dBCollection.equals(user_id);
if(user_idExists){
if(dataValue[1].equals("US")) {
String state = (dataValue[5]);
System.out.println(state);
boolean stateExists = dBCollection.equals(state);
if(stateExists){
statecount ++;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("state", state)
.append("count", statecount);
}
else{
statecount = 1;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("state", state)
.append("count", statecount);
}
}
else{
boolean countryExists = dBCollection.equals(country);
if(countryExists){
count ++;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("count", count);
}
else{
count = 1;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("count", count);
}
}
}
else {
count = 1;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("count", count);
}
}
} catch (IOException e1) {
e1.printStackTrace();
System.exit(-1);
}
try {
bReader.close();
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
}
dBCollection.insert(doc);
System.out.println("upload complete");
}
}
The output I'm getting looks like this:
{ "_id" : { "$oid" : "53861247d121c16b38b3421d"} , "user_id" : "5406" , "country" : "US" , "state" : "NY" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b3421e"} , "user_id" : "5406" , "country" : "US" , "state" : "NY" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b3421f"} , "user_id" : "5406" , "country" : "US" , "state" : "NY" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34220"} , "user_id" : "5406" , "country" : "JA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34220"} , "user_id" : "5406" , "country" : "JA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34222"} , "user_id" : "5406" , "country" : "US" , "state" : "TX" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34223"} , "user_id" : "5406" , "country" : "US" , "state" : "TX" , "count" : 1}
but it should look like this:
{ "_id" : { "$oid" : "53861247d121c16b38b3421d"} , "user_id" : "5406" , "country" : "VN" , "state" : "NY" , "count" : 3}
{ "_id" : { "$oid" : "53861247d121c16b38b34220"} , "user_id" : "5406" , "country" : "JA", "count" : 2}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA", "count" : 3}
{ "_id" : { "$oid" : "53861247d121c16b38b34222"} , "user_id" : "5406" , "country" : "US" , "state" : "TX" , "count" : 2}
I'm not sure why I can't get state/country counts to increment?