I am querying ElasticSearch and sorting the documents locally in Bash with jq, as sorting in ES is too slow for me.
The original purpose is to create a CSV file.
But I find the sorting does not work properly, it gives wrong order.
As I am launching cURL requests, I thought the wrong order is due to content is chunked so I save some results into a local test.json file and tried again, but it still does not work.
test.json:
{
"took": 680,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"max_score": 1.0,
"hits": [
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113584925",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113584925"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111121254059",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111121254059"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111116879444",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111116879444"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111116879484",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111116879484"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114472530",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114472530"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113372966",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113372966"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113046053",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113046053"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113034864",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113034864"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111116770197",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111116770197"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111123578950",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111123578950"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114472544",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114472544"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111123578971",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111123578971"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124882870",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124882870"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124178732",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124178732"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113649568",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113649568"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113034877",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113034877"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114049560",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114049560"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113034894",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113034894"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111116879498",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111116879498"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111115634604",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111115634604"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117550495",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117550495"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117964387",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117964387"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111120647956",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111120647956"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124178784",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124178784"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111123579003",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111123579003"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114049597",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114049597"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113922927",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113922927"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113649622",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113649622"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113922948",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113922948"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124178797",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124178797"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114775147",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114775147"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111115634973",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111115634973"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111115634987",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111115634987"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124882959",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124882959"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114049664",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114049664"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117964419",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117964419"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117069687",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117069687"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117453564",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117453564"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113046124",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113046124"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113586875",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113586875"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111115735339",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111115735339"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111119990227",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111119990227"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111120648051",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111120648051"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117453835",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117453835"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113899778",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113899778"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111120648071",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111120648071"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111117964461",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111117964461"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114049666",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114049666"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114049678",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114049678"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113923053",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113923053"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111119990248",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111119990248"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111115735378",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111115735378"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111115735384",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111115735384"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111116770587",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111116770587"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124883047",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124883047"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111114775219",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111114775219"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113373129",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113373129"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113046216",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113046216"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111124883526",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111124883526"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113046278",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113046278"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111113587608",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111113587608"
]
}
},
{
"_index": "my-index",
"_type": "_doc",
"_id": "111111116770696",
"_score": 1.0,
"fields": {
"field2": [
"FOO"
],
"field1": [
"111111116770696"
]
}
}
]
}
}
Command that I use:
jq '.hits.hits[].fields | [.field1[0] + "," + .field2[0]] | sort | .[0]' -r test.json
The result:
111111113584925,FOO
111111121254059,FOO
111111116879444,FOO
111111116879484,FOO
111111114472530,FOO
111111113372966,FOO
111111113046053,FOO
111111113034864,FOO
111111116770197,FOO
111111123578950,FOO
111111114472544,FOO
111111123578971,FOO
111111124882870,FOO
111111124178732,FOO
111111113649568,FOO
111111113034877,FOO
111111114049560,FOO
111111113034894,FOO
111111116879498,FOO
111111115634604,FOO
111111117550495,FOO
111111117964387,FOO
111111120647956,FOO
111111124178784,FOO
111111123579003,FOO
111111114049597,FOO
111111113922927,FOO
111111113649622,FOO
111111113922948,FOO
111111124178797,FOO
111111114775147,FOO
111111115634973,FOO
111111115634987,FOO
111111124882959,FOO
111111114049664,FOO
111111117964419,FOO
111111117069687,FOO
111111117453564,FOO
111111113046124,FOO
111111113586875,FOO
111111115735339,FOO
111111119990227,FOO
111111120648051,FOO
111111117453835,FOO
111111113899778,FOO
111111120648071,FOO
111111117964461,FOO
111111114049666,FOO
111111114049678,FOO
111111113923053,FOO
111111119990248,FOO
111111115735378,FOO
111111115735384,FOO
111111116770587,FOO
111111124883047,FOO
111111114775219,FOO
111111113373129,FOO
111111113046216,FOO
111111124883526,FOO
111111113046278,FOO
111111113587608,FOO
111111116770696,FOO
Why?
Should I rely on jq sorting? Am I using it correctly? I mean I want to do string comparison by alphabetical order, and field1 all have unique values, so it will never be a tie and start to compare values of field2(it also could have various values but I only want to sort by field1)
Should I use Bash sort -k 1 instead? Which is faster when it comes to 100K rows?