I need to use elastic agent’s custom Logs integration which is essentially filebeat to push my json data into elastic search, but I’m having trouble decoding my json fields into separate fields extracted from the message field.

i use elastic stack with Elasticsearch.

Integration settings:

- decode_json_fields:
    fields: ["message"]
    target: ""
    process_array: true
    max_depth: 8
    overwrite_keys: true
- drop_fields:
    fields: ["message"]

  multiline.pattern: '^['
  multiline.negate: true
  multiline.match: after
  multiline.max_lines: 5000
  multiline.timeout: 10

source json file:

{
    "report": {
        "mtr": {
            "src": "something.com",
            "dst": "something.else",
            "tos": 0,
            "tests": 10,
            "psize": "64",
            "bitpattern": "0x00"
        },
        "hubs": [
            {
                "count": 1,
                "host": "192.18.2.19",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 0.184,
                "Avg": 0.218,
                "Best": 0.173,
                "Wrst": 0.247,
                "StDev": 0.023
            },
            {
                "count": 2,
                "host": "199.4.48.1",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 0.38,
                "Avg": 0.372,
                "Best": 0.278,
                "Wrst": 0.449,
                "StDev": 0.046
            },
            {
                "count": 3,
                "host": "38.2.24.9",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 0.819,
                "Avg": 1.053,
                "Best": 0.819,
                "Wrst": 1.923,
                "StDev": 0.348
            },
            {
                "count": 4,
                "host": "14.24.3.11",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 0.869,
                "Avg": 1.04,
                "Best": 0.869,
                "Wrst": 1.138,
                "StDev": 0.087
            },
            {
                "count": 5,
                "host": "54.4.41.113",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 1.677,
                "Avg": 2.014,
                "Best": 1.669,
                "Wrst": 3.472,
                "StDev": 0.555
            },
            {
                "count": 6,
                "host": "14.5.4.86",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 69.771,
                "Avg": 79.623,
                "Best": 69.656,
                "Wrst": 104.19,
                "StDev": 15.887
            },
            {
                "count": 7,
                "host": "10.17.48.206",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 67.818,
                "Avg": 67.888,
                "Best": 67.702,
                "Wrst": 68.107,
                "StDev": 0.111
            },
            {
                "count": 8,
                "host": "149.4.81.106",
                "Loss%": 0.0,
                "Snt": 10,
                "Last": 69.569,
                "Avg": 69.587,
                "Best": 69.514,
                "Wrst": 69.745,
                "StDev": 0.075
            },
            {
                "count": 9,
                "host": "???",
                "Loss%": 100.0,
                "Snt": 10,
                "Last": 0.0,
                "Avg": 0.0,
                "Best": 0.0,
                "Wrst": 0.0,
                "StDev": 0.0
            }
        ]
    }
}

with the settings nothings happans but the multiline alone works so all of the file shown in message field and not every line in the file is a log file in elastic.

i expect to parse every JSON object at the message field to be its own quarriable field.