Develop Reference

Is it possible to create a map in runtime mode, which will be filled in passing through all documents and returned at the end ES

For ex: I have 2 documents with this body:
{
"id": "doc_one",
"name": "test_name",
"date_creation": "some_date_cr_1",
"date_updation": "some_date_up_1"
}
And the second doc:
{
"id": "doc_two",
"name": "test_name",
"date_creation": "some_date_cr_2",
"date_updation": "some_date_up_2"
}
What I want to do: to create two runtime field or Map('data_creation',count_of_doc_where_field_not_null_AND_the_condition_is_met).
For ex: I've got the 1st doc, there is date_creation IS NOT NULL and the condition startDate<=date_creation<=endDate is met, so, I create some field count = 0 and when I've got this case I do count++. When I will get all the docs I will set finally count value from map as result: Map('data_creation',final_count) and the same for another field but in the same map.
I tried to use script, but there is return Map for each doc, for ex:
{
"_index": "my_index_001",
"_type": "_doc",
"_id": "1",
"_score": 1.0,
"fields": {
"my_doubled_field": [
{
"NEW": 2
}
]
}
},
{
"_index": "my_index_001",
"_type": "_doc",
"_id": "2",
"_score": 1.0,
"fields": {
"my_doubled_field": [
{
"NEW": 2
}
]
}
}

I have index below 3 documents to index where one document dont have date_creation field:
POST sample/_doc
{
"id": "doc_two",
"name": "test_name",
"date_updation": "some_date_up_2"
}
POST sample/_doc
{
"id": "doc_one",
"name": "test_name",
"date_creation": "some_date_cr_1",
"date_updation": "some_date_up_1"
}
POST sample/_doc
{
"id": "doc_two",
"name": "test_name",
"date_creation": "some_date_cr_2",
"date_updation": "some_date_up_2"
}
Now you can use filter aggregation from elasticsearch as shown below:
{
"size": 0,
"aggs": {
"date_creation": {
"filter": {
"range": {
"date_creation": {
"gte": "2020-01-09T10:20:10"
}
}
}
},
"date_updation": {
"filter": {
"range": {
"date_updation": {
"gte": "2020-01-09T10:20:10"
}
}
}
}
}
}
Response:
"hits" : {
"total" : {
"value" : 3,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"date_updation" : {
"meta" : { },
"doc_count" : 3
},
"date_creation" : {
"meta" : { },
"doc_count" : 2
}
}
You can see date_updation field is available in 3 doc so it is showing count as 3 and date_creation field is available in 2 doc so it is showing count as 2.

Mongo upsert do not return modified object _id on update

Mongodb 4.2.15
I'm tring to use mongo Updates with Aggregation Pipeline
My request is very big but here it's core structure
db.runCommand({
"update": "collectionName",
"updates": [
{
"q": { ... },
"u": [
{
"$project": { ... },
"$set": { ... },
"$set": { ... },
"$project": { ... }
}
],
"multi": false,
"upsert": true
}
]
});
After the first execute I receive a result with newly created object's _id
{
"n" : 1,
"nModified" : 0,
"upserted" : [
{
"index" : 0,
"_id" : ObjectId("619997f11501d6eb40c6f64a")
}
],
"opTime" : {
"ts" : Timestamp(1637455857, 61),
"t" : NumberLong(5)
},
"electionId" : ObjectId("7fffffff0000000000000005"),
"ok" : 1.0,
"$clusterTime" : {
"clusterTime" : Timestamp(1637455857, 61),
"signature" : {
"hash" : { "$binary" : "hA0tf5DXMqTNmnVXdMVnpnAKCU0=", "$type" : "00" },
"keyId" : NumberLong(7018546168816730116)
}
},
"operationTime" : Timestamp(1637455857, 61)
}
After the second execution of the same request there is no modified object's _id
{
"n" : 1,
"nModified" : 1,
"opTime" : {
"ts" : Timestamp(1637456057, 19),
"t" : NumberLong(5)
},
"electionId" : ObjectId("7fffffff0000000000000005"),
"ok" : 1.0,
"$clusterTime" : {
"clusterTime" : Timestamp(1637456057, 19),
"signature" : {
"hash" : { "$binary" : "U2yCP6nXUrjBN9ZiLanyl0rgxww=", "$type" : "00" },
"keyId" : NumberLong(7018546168816730116)
}
},
"operationTime" : Timestamp(1637456057, 19)
}
The thing is that my filter conditions do not contain _id of the object but I need to return it with response. I see no useful request configurations. Any suggestions is it possible to get _id at response on update case?

must_not is not giving expected result in Elasticsearch for empty field

This is my sample es index document:
"hits" : [
{
"_index" : "project_note",
"_type" : "project_note",
"_id" : "19",
"_score" : 1.0,
"_source" : {
"createTime" : "2021-10-04T13:43:55.330",
"createTimeInMs" : 1633333435330,
"createdBy" : "test",
"editTime" : "2021-10-04T13:43:55.330",
"editTimeInMs" : 1633333435330,
"editedBy" : "test",
"versionId" : 1,
"id" : "19",
"organizationId" : "28",
"accessLevel" : "PUBLIC",
"status" : "ACTIVE",
"projectId" : "95",
"userId" : 129,
"noteType" : "SYSTEM_GENERATED",
"projectDemographicLogId" : "1"
},
{
"_index" : "project_note",
"_type" : "project_note",
"_id" : "19",
"_score" : 1.0,
"_source" : {
"createTime" : "2021-10-04T13:43:55.330",
"createTimeInMs" : 1633333435330,
"createdBy" : "test",
"editTime" : "2021-10-04T13:43:55.330",
"editTimeInMs" : 1633333435330,
"editedBy" : "test",
"versionId" : 1,
"id" : "19",
"organizationId" : "28",
"accessLevel" : "PUBLIC",
"status" : "ACTIVE",
"projectId" : "95",
"userId" : 129
}
]
In the first doc, it has noteType but in the second, I don't have that field stored in db.
I want to exclude the documents where noteType==null or noteType is absent.
But, I am getting only the docs which have noteType="SYSTEM_GENERATED"
My approach:
{
"query":
{
"bool" : {
"must" : [
{
"term" : {
"projectId" : {
"value" : "95",
"boost" : 1.0
}
}
},
{
"range" : {
"createTimeInMs" : {
"from" : null,
"to" : 1633594455000,
"include_lower" : true,
"include_upper" : true,
"boost" : 1.0
}
}
}
],
"must_not" : [
{
"term" : {
"noteType" : {
"value" : "SYSTEM_GENERATED",
"boost" : 1.0
}
}
}
],
"adjust_pure_negative" : true,
"boost" : 1.0
}
}
}
Equivalent java code:
BoolQueryBuilder queryBuilder= QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.termQuery("projectId", requestInfo.getProjectId()));
queryBuilder.must(rangeQuery("createTimeInMs").lte(requestInfo.getCreateTimeInMs()));
if(!requestInfo.isIncludeLog()) {
queryBuilder.mustNot(QueryBuilders.termQuery("noteType", Defs.SYSTEM_NOTE_TYPE));
}

If only the must_not part of the query is used (excluding the must part)
{
"query": {
"bool": {
"must_not": [
{
"term": {
"noteType.keyword": {
"value": "SYSTEM_GENERATED",
"boost": 1.0
}
}
}
],
"adjust_pure_negative": true,
"boost": 1.0
}
}
}
The search result is similar to what you expect to get
"hits": [
{
"_index": "69477995",
"_type": "_doc",
"_id": "2",
"_score": 0.0,
"_source": {
"createTime": "2021-09-26T15:54:08.373",
"createTimeInMs": 1632650048373,
"createdBy": "test",
"editTime": "2021-09-26T15:54:08.373",
"editTimeInMs": 1632650048373,
"editedBy": "test",
"versionId": 1,
"id": "18",
"note": "note-1, simple note ",
"organizationId": "28",
"accessLevel": "PUBLIC",
"status": "ACTIVE",
"taskId": "5",
"userId": 129
}
}
]

QueryString to search String with colon

i am trying to achieve below condition
orgId = "z2store" and type = "web" and dateTime = "12:17:08"
below query i have written
GET /sample/_search
{
"bool" : {
"must" : [
{
"term" : {
"orgId" : {
"value" : "z2store",
"boost" : 1.0
}
}
},
{
"term" : {
"type" : {
"value" : "web",
"boost" : 1.0
}
}
},
{
"query_string" : {
"query" : "12:17:08",
"default_field" : "dateTime",
"fields" : [ ],
"type" : "best_fields",
"default_operator" : "or",
"max_determinized_states" : 10000,
"enable_position_increments" : true,
"fuzziness" : "AUTO",
"fuzzy_prefix_length" : 0,
"fuzzy_max_expansions" : 50,
"phrase_slop" : 0,
"escape" : false,
"auto_generate_synonyms_phrase_query" : true,
"fuzzy_transpositions" : true,
"boost" : 1.0
}
}
],
"adjust_pure_negative" : true,
"boost" : 1.0
}
}
below is my java code
BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(QueryBuilders.termQuery("orgId", orgId))
.must(QueryBuilders.termQuery("type", "web"));
QueryStringQueryBuilder builder = new QueryStringQueryBuilder("12:17:08");
builder.defaultField("dateTime").queryString();
boolQuery.must(builder);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(builder)
.from((batchNumber - 1) * batchSize).size(batchSize)
.sort("#timestamp", SortOrder.DESC);
Above query is not working. Any help will be appreciated. I am using elasticSearch 7.4.

You can create your dateTime field with type as date and giving format as hour_minute_second(which takes format as HH:mm:ss) . You can read more about different date formats here https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html.
Below is the mapping of dateTime field:
{
"mappings": {
"properties": {
"dateTime": {
"type" : "date",
"format" : "hour_minute_second"
}
}
}
}
Now when you search data with below search query :
{
"query" : {
"bool" : {
"must" : [
{
"term" : {
"orgId" : {
"value" : "z2store",
"boost" : 1.0
}
}
},
{
"term" : {
"type" : {
"value" : "web",
"boost" : 1.0
}
}
},
{
"term" :{
"dateTime":"12:17:08"
}
}
],
"adjust_pure_negative" : true,
"boost" : 1.0
}
}
}
You will get your required result :
"hits": [
{
"_index": "datetimeindexf",
"_type": "_doc",
"_id": "1",
"_score": 1.5753641,
"_source": {
"dateTime": "12:17:08",
"orgId": "z2store",
"type": "web"
}
}
]

Elasticsearch composite group by queries across the documents

We have an elastic search document which has a dimension called city. Each document will have only one value for city field. I have a scenario where I need to query the person based on the city or cities.
Documents in Elasticsearch
{
person_id: "1",
property_value : 25000,
city: "Bangalore"
}
{
person_id: "2",
property_value : 100000,
city: "Bangalore"
}
{
person_id: "1",
property_value : 15000,
city: "Delhi"
}
Note: The aggregation should be performed on property_value and group by on person_id.
For eg.,
If I query for Bangalore it should return document with person_id 1 and 2.
If I query for both Delhi and Bangalore it should return this
{
person_id: "1",
property_value : 40000,
city: ["Bangalore", "Delhi"]
}

Looking at your data, I've come up with a sample mapping, request query and the response.
Mapping:
PUT my_index_city
{
"mappings": {
"properties": {
"person_id":{
"type": "keyword"
},
"city":{
"type":"text",
"fields":{
"keyword":{
"type": "keyword"
}
}
},
"property_value":{
"type": "long"
}
}
}
}
Sample Request:
Note that I've made use of simple query string to filter the documents having Bangalore and Delhi.
For aggregation I've made use of Terms Aggregation on person_id and Sum Aggregation on the property_value field.
POST my_index_city/_search
{
"size": 0,
"query": {
"query_string": {
"default_field": "city",
"query": "Bangalore Delhi"
}
},
"aggs": {
"my_person": {
"terms": {
"field": "person_id",
"size": 10,
"min_doc_count": 2
},
"aggs": {
"sum_property_value": {
"sum": {
"field": "property_value"
}
}
}
}
}
}
Sample Response:
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 3,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"my_person" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "1",
"doc_count" : 2,
"sum_property_value" : {
"value" : 40000.0
}
}
]
}
}
}
Note: This query would only work if the person_id has multiple documents but each document having unique/different city value.
What I mean to say is, if the person_id has multiple documents with same city, the aggregation would not give right answer.
Updated Answer:
There is no direct way to achieve what you are looking for unless you modify the mapping. What I've done is, made use of nested datatype and ingested all the documents for person_id as a single document.
Mapping:
PUT my_sample_city_index
{
"mappings": {
"properties": {
"person_id":{
"type": "keyword"
},
"property_details":{
"type":"nested", <------ Note this
"properties": {
"city":{
"type": "text",
"fields":{
"keyword":{
"type":"keyword"
}
}
},
"property_value":{
"type": "long"
}
}
}
}
}
}
Sample Documents:
POST my_sample_city_index/_doc/1
{
"person_id": "1",
"property_details":[
{
"property_value" : 25000,
"city": "Bangalore"
},
{
"property_value" : 15000,
"city": "Delhi"
}
]
}
POST my_sample_city_index/_doc/2
{
"person_id": "2",
"property_details":[
{
"property_value" : 100000,
"city": "Bangalore"
}
]
}
Aggregation Query:
POST my_sample_city_index/_search
{
"size": 0,
"query": {
"nested": {
"path": "property_details",
"query": {
"query_string": {
"default_field": "property_details.city",
"query": "bangalore delhi"
}
}
}
},
"aggs": {
"persons": {
"terms": {
"field": "person_id",
"size": 10
},
"aggs": {
"property_sum": {
"nested": { <------ Note this
"path": "property_details"
},
"aggs": {
"total_sum": {
"sum": {
"field": "property_details.property_value"
}
}
}
}
}
}
}
}
Note that I've applied initially a term query on person_id post which I've applied Nested Aggregation, further on which I've applied metric sum aggregation query.
This should also work correctly if a person has multiple properties in the same city.
Response:
{
"took" : 31,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"persons" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "1",
"doc_count" : 1,
"property_sum" : {
"doc_count" : 2,
"total_sum" : {
"value" : 40000.0
}
}
},
{
"key" : "2",
"doc_count" : 1,
"property_sum" : {
"doc_count" : 1,
"total_sum" : {
"value" : 100000.0
}
}
}
]
}
}
}
Let me know if this helps!