Incremental Mongodb Update - java

I'm trying to update my db incrementally by parsing a csv and counting how many times each person has been to a country or state, if they're within the US.
public static void main(String[] args) throws UnknownHostException {
Scanner dir = new Scanner(System.in);
String dir_loc;
System.out.println("Enter File Directory:");
dir_loc=dir.nextLine();
Scanner dbloc = new Scanner(System.in);
String db_loc;
System.out.println("Enter Database location:");
db_loc=dbloc.nextLine();
Scanner port = new Scanner(System.in);
int port_loc;
System.out.println("Enter Port:");
port_loc=port.nextInt();
dir.close();
dbloc.close();
port.close();
System.out.println("uploading");
String directoryName = dir_loc;
File directory = new File(directoryName);
File[] fileList = directory.listFiles();
DB dB = (new MongoClient(db_loc, port_loc)).getDB("visits");
DBCollection dBCollection = dB.getCollection("Channel");
BasicDBObject doc = null;
for(File file: fileList) {
String dataFileName = file.toString();
BufferedReader bReader = null;
try {
bReader = new BufferedReader(new FileReader(dataFileName));
} catch (FileNotFoundException e2) {
e2.printStackTrace();
System.exit(-1);
}
String line = null;
int count = 0;
int statecount = 0;
try {
while((line = bReader.readLine()) !=null) {
String dataValue[] = line.split("\t");
String user_id = (dataValue[2]);
String country = (dataValue[1]);
boolean user_idExists = dBCollection.equals(user_id);
if(user_idExists){
if(dataValue[1].equals("US")) {
String state = (dataValue[5]);
System.out.println(state);
boolean stateExists = dBCollection.equals(state);
if(stateExists){
statecount ++;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("state", state)
.append("count", statecount);
}
else{
statecount = 1;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("state", state)
.append("count", statecount);
}
}
else{
boolean countryExists = dBCollection.equals(country);
if(countryExists){
count ++;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("count", count);
}
else{
count = 1;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("count", count);
}
}
}
else {
count = 1;
doc = new BasicDBObject("user_id", user_id).append("country", country).append("count", count);
}
}
} catch (IOException e1) {
e1.printStackTrace();
System.exit(-1);
}
try {
bReader.close();
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
}
dBCollection.insert(doc);
System.out.println("upload complete");
}
}
The output I'm getting looks like this:
{ "_id" : { "$oid" : "53861247d121c16b38b3421d"} , "user_id" : "5406" , "country" : "US" , "state" : "NY" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b3421e"} , "user_id" : "5406" , "country" : "US" , "state" : "NY" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b3421f"} , "user_id" : "5406" , "country" : "US" , "state" : "NY" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34220"} , "user_id" : "5406" , "country" : "JA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34220"} , "user_id" : "5406" , "country" : "JA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34222"} , "user_id" : "5406" , "country" : "US" , "state" : "TX" , "count" : 1}
{ "_id" : { "$oid" : "53861247d121c16b38b34223"} , "user_id" : "5406" , "country" : "US" , "state" : "TX" , "count" : 1}
but it should look like this:
{ "_id" : { "$oid" : "53861247d121c16b38b3421d"} , "user_id" : "5406" , "country" : "VN" , "state" : "NY" , "count" : 3}
{ "_id" : { "$oid" : "53861247d121c16b38b34220"} , "user_id" : "5406" , "country" : "JA", "count" : 2}
{ "_id" : { "$oid" : "53861247d121c16b38b34221"} , "user_id" : "5406" , "country" : "SA", "count" : 3}
{ "_id" : { "$oid" : "53861247d121c16b38b34222"} , "user_id" : "5406" , "country" : "US" , "state" : "TX" , "count" : 2}
I'm not sure why I can't get state/country counts to increment?

Related

Problem displaying an expandable list android studio java

I have made a page allowing to see tools arranged in classes and subclasses (3rd floor).
The tools database is on a server, and I receive the data as JSON.
Everything was working fine until I added a number of tools (and therefore classes)
My tools appear several times without reason in classes that are not the right ones.
I checked the JSON, no problem.
Here are screenshots of the problem and the code that allows me to retrieve the data and send it to my adapter.
Thanks for your help.
Random display problem image 1
Random display problem image 2
Random display problem image 3
ArrayList<String> famille1 = new ArrayList<>();
ArrayList<String> typeliste = new ArrayList<>();
HashMap<String, HashMap<String, HashMap<String, ArrayList<MatosItem>>>> ma = new HashMap<>();
ArrayList<String> materiauliste = new ArrayList<>();
HashMap<String, HashMap<String, HashMap<String, ArrayList<MatosItem>>>> materielsItemList = new HashMap<>();
Handler handler = new Handler(Looper.getMainLooper());
handler.post(new Runnable() {
#Override
public void run() {
FetchData fetchData = new FetchData("http://" + adresseIp.getAdresseIP() + "/jsonmateriels.php");
if (fetchData.startFetch()) {
if (fetchData.onComplete()) {
String json = fetchData.getResult();
try {
HashMap<String, HashMap<String, ArrayList<MatosItem>>> typeItemList = new HashMap<>();
JSONObject jsonObject = new JSONObject(json);
JSONArray jsonArrayOutils = jsonObject.getJSONArray("Outils");
for (int j = 0; j < jsonArrayOutils.length(); j++) {
HashMap<String, HashMap<String, ArrayList<MatosItem>>> typeItemList1 = new HashMap<>();
JSONObject fam = jsonArrayOutils.getJSONObject(j);
String famille = fam.names().getString(0);//permet d'avoir les choses contenu dans la classe soudeurs
JSONArray jsonArrayFamille = fam.getJSONArray(famille);//permet de récupérer le suivant
famille1.add(famille);
for (int i = 0; i < jsonArrayFamille.length(); i++) {
HashMap<String, ArrayList<MatosItem>> matItemList = new HashMap<>();
JSONObject typ = jsonArrayFamille.getJSONObject(i);
String type = typ.names().getString(0);
JSONArray jsonArrayType = typ.getJSONArray(type);
typeliste.add(type);
for (int k = 0; k < jsonArrayType.length(); k++) {
JSONObject mat = jsonArrayType.getJSONObject(k);
ArrayList<MatosItem> outils = new ArrayList<>();
String materiau = mat.names().getString(0);
JSONArray jsonArraylistout = mat.getJSONArray(materiau);
materiauliste.add(materiau);
for (int l = 0; l < jsonArraylistout.length(); l++) {
JSONObject out = jsonArraylistout.getJSONObject(l);
String description = out.getString("description");
String id = out.getString("id");
String ref = out.getString("ref");
String code_fourn = out.getString("code_fourn");
Log.e("boucle",description+id);
outils.add(new MatosItem(description, id, false, ref, code_fourn));
}
matItemList.put(materiau, outils);
}
typeItemList1.put(type, matItemList);
}
materielsItemList.put(famille, typeItemList1);
tabLayout.addTab(tabLayout.newTab().setText(famille));
}
ArrayList<String> list = new ArrayList<>();
for (String i : Objects.requireNonNull(materielsItemList.get(famille1.get(0))).keySet()) {
list.add(i);
Log.e(TAG, "run: "+i );
}
typeItemList = materielsItemList.get(famille1.get(0));
Log.e(TAG, "run: "+typeItemList );
matosListView.setAdapter(new MatosItemAdapter(context, typeItemList,list ));
tabLayout.addOnTabSelectedListener(new TabLayout.OnTabSelectedListener() {
#Override
public void onTabSelected(TabLayout.Tab tab) {
// get the current selected tab's position and replace the fragment accordingly
Fragment fragment = null;
if (tab.isSelected()) {
ArrayList<String> list = new ArrayList<>();
for (String i : Objects.requireNonNull(materielsItemList.get(famille1.get(tab.getPosition()))).keySet()) {
list.add(i);
}
HashMap<String, HashMap<String, ArrayList<MatosItem>>> typeItemList1 = new HashMap<>();
typeItemList1 = materielsItemList.get(famille1.get(tab.getPosition()));
matosListView.setAdapter(new MatosItemAdapter(context, typeItemList1, list));
}
FragmentManager fm = getSupportFragmentManager();
FragmentTransaction ft = fm.beginTransaction();
ft.setTransition(FragmentTransaction.TRANSIT_FRAGMENT_OPEN);
ft.commit();
}
#Override
public void onTabUnselected(TabLayout.Tab tab) {
}
#Override
public void onTabReselected(TabLayout.Tab tab) {
}
});
progressBar.setVisibility(GONE);
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
});
I checked the JSON
I tried to use a thread
{
"Outils" : [{
"Soudeurs" : [
{
"Fils" : [
{
"Fil" : [
{
"description" : "Fil MIG G3SI1 diam 1.2",
"id" : "26",
"ref" : "bobine de 15 kg",
"code_fourn" : "x"
},
{
"description" : "Fourre OK1413 diam 1.2",
"id" : "45",
"ref" : "bobine 16 kg avec gaz",
"code_fourn" : "x"
},
{
"description" : "MIG inox 316L SI diam 1.2",
"id" : "46",
"ref" : "bobine 15 kg",
"code_fourn" : "x"
}]
}]
},
{
"Materiel de soudure" : [
{
"Gaine guide" : [
{
"description" : "fil 25 x 45 4M diam 1.2",
"id" : "27",
"ref" : "x",
"code_fourn" : "x"
}
]},
{
"Batterie" : [
{
"description" : "Warrior AIR ESAB",
"id" : "47",
"ref" : "x",
"code_fourn" : "x"
}
]},
{
"Cagoule" : [
{
"description" : "Couvre nuque noire ESAB",
"id" : "48",
"ref" : "x",
"code_fourn" : "x"
}
]},
{
"Protection" : [
{
"description" : "Ecran de protection bleu",
"id" : "49",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Visiere transparente pour ecran HG930B",
"id" : "50",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Visiere teinte pour ecran HG930B",
"id" : "51",
"ref" : "x",
"code_fourn" : "x"
}
]},
{
"Pince" : [
{
"description" : "Stubby original 400 A",
"id" : "52",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Masse bronze GCB 450 A",
"id" : "53",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Masse bronze GCB 600 A",
"id" : "54",
"ref" : "x",
"code_fourn" : "x"
}]
}]
},
{
"Electrodes" : [
{
"OK4627" : [
{
"description" : "Diam 2.5 x 350 rutile",
"id" : "25",
"ref" : "x",
"code_fourn" : "x"
}]
},
{
"OK4800" : [
{
"description" : "Diam 2.5 x 350",
"id" : "32",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Diam 3.2 x 350",
"id" : "33",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Diam 4 x 350",
"id" : "34",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Diam 5 x 450",
"id" : "35",
"ref" : "x",
"code_fourn" : "x"
}]
},
{
"Inox 316 L 17" : [
{
"description" : "Diam 2.5 x 300",
"id" : "36",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Diam 3.2 x 350",
"id" : "37",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "Diam 4 x 350",
"id" : "38",
"ref" : "x",
"code_fourn" : "x"
}]
},
{
"Gougeage" : [
{
"description" : "6 x 305",
"id" : "39",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "8 x 305",
"id" : "40",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "10 x 305",
"id" : "41",
"ref" : "x",
"code_fourn" : "x"
}]
},
{
"Nue" : [
{
"description" : "9.5 x 914 slice",
"id" : "42",
"ref" : "x",
"code_fourn" : "x"
}]
}]
}]},
{
"Ajusteurs" : [
{
"Accessoires" : [
{
"Nettoyant" : [
{
"description" : "26.5 kg alfa marine",
"id" : "44",
"ref" : "x",
"code_fourn" : "x"
}]
}]
},
{
"Hydraulique" : [
{
"Huile" : [
{
"description" : "Enerpac HF95Y bidon 5L",
"id" : "43",
"ref" : "x",
"code_fourn" : "x"
}]
}]
}]
},
{
"Machinistes" : [
{
"Accessoires" : [
{
"Lame ebavureur" : [
{
"description" : "E100S (longue) boite de 10",
"id" : "30",
"ref" : "x",
"code_fourn" : "x"
},
{
"description" : "B10S (courte) boite de 10",
"id" : "31",
"ref" : "x",
"code_fourn" : "x"
}]
}]
}]
}]
}
I would like to have a correct display of the tools and their class. I don't have to have expandables list

How does Elasticsearch get the doc_count value of the filter under aggs?

I encountered some problems when using elasticsearch's aggregation query.This is the DSL statement I used
{
"size": 0,
"aggs": {
"terms_vlid_year":{
"terms": {
"field": "valid_year",
"size": 5,
"order": {
"_key": "asc"
}
}
},
"filterss":{
"filter": {
"range": {
"valid_year": {
"gte": 6
}
}
}
}
}
}
This is the aggregate result
"aggregations" : {
"terms_vlid_year" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 13918850,
"buckets" : [
{
"key" : 1,
"doc_count" : 1632006
},
{
"key" : 2,
"doc_count" : 3588976
},
{
"key" : 3,
"doc_count" : 3710637
},
{
"key" : 4,
"doc_count" : 2948910
},
{
"key" : 5,
"doc_count" : 2672394
}
]
},
"filters" : {
"doc_count" : 13918850
}
}
Now I want to use java to get the doc_count of filters.Below is my java code, But the doc_count value of the filter object cannot be obtained through the filterResult.getDocCount(). How to solve this problem?
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("terms_vlid_year");
FilterAggregationBuilder filter = AggregationBuilders.filter("filters", QueryBuilders.rangeQuery("valid_year").gte(6));
termsAggregationBuilder.field("valid_year");
termsAggregationBuilder.size(5);
termsAggregationBuilder.order(BucketOrder.key(true));
sourceBuilder.aggregation(termsAggregationBuilder.subAggregation(filter));
searchRequest.source(sourceBuilder);
ReturnValue returnValue = new ReturnValue();
LinkedList<Object> list = new LinkedList<>();
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Terms terms = searchResponse.getAggregations().get("terms_vlid_year");
List<? extends Terms.Bucket> buckets = terms.getBuckets();
Filter filterResult = searchResponse.getAggregations().get("filters");
long docCount1 = filterResult.getDocCount();
System.out.println("docCount1 = " + docCount1);
for (Terms.Bucket bucket : buckets) {
String keyAsString = bucket.getKeyAsString();
long docCount = bucket.getDocCount();
returnValue.setFieldName(keyAsString);
returnValue.setDocValue(docCount);
System.out.println(returnValue.toString());
}
}

Now the problem has been solved.FilterAggregation cannot use subAggregation nested in termsAggregation, they belong to the same level.It should be this way to get the desired result:
sourceBuilder.aggregation(termsAggregationBuilder);
FilterAggregationBuilder filter = AggregationBuilders.filter("filters", QueryBuilders.rangeQuery("valid_year").gte(6));
sourceBuilder.aggregation(filter);

ElasticSearch - create query in Java

I have this query I made using sense, I'm breaking my head how to transform it into Java.
I can manage the aggs part, the real pain is the "constant_score"
GET /xxxx/yyyy/_search
{
"size": 0,
"query" : {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{ "term" : {"userId" : 275}},
{ "range" :{"logDate" : { "gte" : "2016-04-30", "lte" : "now/d" }}}
]
}
}
}
},
"aggs" : {
"datebucket" : {
"date_histogram" : {
"field" : "logDate",
"interval": "day",
"format": "yyyy-MM-dd",
"min_doc_count": 0
},
"aggs": {
"info": {
"filters": {
"filters" : [
{"term": { "logAction": "sleep" }},
{"term": { "logAction": "stop" }}
]
}
}
}
}
}
}
I had a more simple query I managed to do it like this
SearchResponse res = client.prepareSearch("xxxx").setTypes("yyyy")
.setSize(0)
.setQuery(QueryBuilders.termQuery("userId", 95))
.addAggregation(
AggregationBuilders.dateHistogram("date_histogram")
.field("logDate")
.interval(DateHistogramInterval.DAY)
.format("yyyy-MM-dd")
.minDocCount(0)
).execute().get();

Well, I found the solution. I hope this will help someone
String query = "{\"constant_score\" : "
+ "{ \"filter\" : "
+ "{\"bool\" : "
+ "{\"must\" : "
+ "[{ \"term\" : {\"userId\" : " + userID + "}}, "
+ "{ \"range\" :{\"logDate\" : { \"gte\" : \"" + startdate + "\", \"lte\" : \"" + enddate + "\" }}}]"
+ "}"
+ "}"
+ "}"
+ "}";
SearchResponse res = client.prepareSearch(xxxx).setTypes(yyyy)
.setQuery(query).addAggregation(
AggregationBuilders.dateHistogram("date_histogram")
.field("logDate")
.interval(DateHistogramInterval.DAY)
.format("dd-MM-yyyy")
.minDocCount(0)
.subAggregation(AggregationBuilders.filters("info")
.filter(QueryBuilders.termQuery("logAction", "click"))
.filter(QueryBuilders.termQuery("logAction", "view")))
).setSize(0).execute().get();

acess nested object of json in java

This is my JSON file
{
"content": [
{
"a":{
"b" : "abcd",
"c" : "bcd"
}
"ab" : "123",
"abc":{
"id" : "12345",
"name" : "abcde"
}
"cd": "afsf"
},
{
"a":{
"b" : "abcd",
"c" : "bcd"
}
"ab" : "123",
"abc":{
"id" : "12345",
"name" : "abcde"
}
"cd": "afsf"
}
]
}
I want the "id" of object "abc" in Java?
I created a object of content and the code is below
JSONArray content = (JSONArray) jsonObject.get("content");
for (int i = 0; i < content.length(); i++) {
JSONObject inner = (JSONObject) content.getJSONObject(i);
String abc = inner.getString("Loan_Application__r");
}
How to i access now the "id" and "name" in java?
pls ignore if any syntax mistake is their...

Take the below code as a hint to approach your problem:-
public static void main(String[] args){
String json="{ \"content\": [ {\"a\":{ \"b\" : \"abcd\", \"c\" : \"bcd\"},\"ab\" : \"123\",\"abc\":{\"id\" : \"12345\", \"name\" : \"abcde\"},\"cd\": \"afsf\"},{\"a\":{\"b\" : \"abcd\", \"c\" : \"bcd\"},\"ab\" : \"123\",\"abc\":{\"id\" : \"12346\",\"name\" : \"abcde\"},\"cd\": \"afsf\"}]}";
JSONObject jsonObject = new JSONObject(json);
JSONArray jsonArray = jsonObject.getJSONArray("content");
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject objects = jsonArray.getJSONObject(i);
String[] elementNames = JSONObject.getNames(objects);
for (String elementName : elementNames)
{
if(elementName.equalsIgnoreCase("abc")){
JSONObject value = objects.getJSONObject(elementName);
String[] elementList = JSONObject.getNames(value);
for(String j:elementList){
if(j.equalsIgnoreCase("id")){
System.out.println(value.getString("id"));
}
}
}
}
}
}
Output:-
12345
12346

How to parse JSON records which have JSON array elements in it?

I am getting the error:
JSONObject["Transaction_details"] not found.
Why is it not able to find Transaction_details?
Code:
for(Iterator it = mapper.readValues(new JsonFactory().createJsonParser(in),JSONObject.class);it.hasNext();)
{
JSONObject js = (JSONObject) it.next();
JSONArray recs = (JSONArray) js.get("Transaction_details");
for (int i = 1; i < recs.length(); ++i) {
JSONObject rec = recs.getJSONObject(i);
int id = rec.getInt("merchantid");
String loc = rec.getString("transaction_amount");
}
Snapshot of a part of my schema:
{
"name" : "cica",
"type" : "long"
}, {
"name" : "pica",
"type" : "long"
}, {
"name" : "issuingcountry",
"type" : "string"
}, {
"name" : "Transaction_details",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "Transaction_details",
"doc" : "Transaction event details",
"fields" : [ {
"name" : "transactionid",
"type" : "long"
}, {
"name" : "transactiondate",
"type" : "string"
}, {
"name" : "Productcode",
"type" : "string"
}
]
}

Something like the below snippet should work. But, I have not compiled or executed it.
Basically, you would have to check if you are encountering a "Transaction_details" name value. If yes, then you need to retrieve its 'type' array and work with its elements.
Hope this helps!
for(Iterator it = mapper.readValues(new JsonFactory().createJsonParser(in),JSONObject.class);it.hasNext();)
{
JSONObject js = (JSONObject) it.next();
String nameValue = (String) js.get("name");
if ("Transaction_details".equals(nameValue)) {
JSONArray recs = (JSONArray) js.get("type");
for (int i = 1; i < recs.length(); ++i) {
JSONObject rec = recs.getJSONObject(i);
...
}
}
}

Categories

Resources