Hello all
Sadly, I’m stuck at Elasticsearch 1.6.2, which means I’m stuck at Grafana 4.0.1. I cannot simply upgrade Elasticsearch, I’m locked in (believe me I wish I could upgrade).
I’m trying to scale a CPU query (against metricbeats indices) by adding this in the script:
_value * 100
If the script field is empty, the queries succeed - I get beautiful graphs.
But if I simply insert “_value * 100” I get “unknown elatic error response” on a red-exclamation-mark-triangle - and clicking on it reveals the full error like this.
I’ve tried researching this for a couple days with creative Googling but am unable to find an easy solution and my Elastic-Fu skills are sub-par. Anyone have a clue what I can be looking at?
“SearchPhaseExecutionException[Failed to execute phase [query], all shards failed; shardFailures {[dSRQK5ZmR-mTQlO51bLTPw][metricbeat-2019.09.18][0]: RemoteTransportException[[77596958-30db-4cb4-bf11-09e114a44012][inet[/192.168.1.51:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[metricbeat-2019.09.18][0]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Failed to parse source [{“size”:0,“query”:{“filtered”:{“query”:{“query_string”:{“analyze_wildcard”:true,“query”:“beat.hostname = name-of-host”}},“filter”:{“bool”:{“must”:[{“range”:{”@timestamp":{“gte”:“1568838538865”,“lte”:“1568840338866”}}}]}}}},“aggs”:{“2”:{“date_histogram”:{“interval”:“10s”,“field”:"@timestamp",“min_doc_count”:1,“extended_bounds”:{“min”:“1568838538865”,“max”:“1568840338866”}},“aggs”:{“3”:{“max”:{“field”:“system.cpu.idle.pct”,“script”:{“inline”:"_value * 100"}}}}}}}]]]; nested: SearchParseException[[metricbeat-2019.09.18][0]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Unexpected token START_OBJECT in [3].]]; }{[dSRQK5ZmR-mTQlO51bLTPw][metricbeat-2019.09.18][1]: RemoteTransportException[[77596958-30db-4cb4-bf11-09e114a44012][inet[/192.168.1.51:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[metricbeat-2019.09.18][1]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Failed to parse source [{“size”:0,“query”:{“filtered”:{“query”:{“query_string”:{“analyze_wildcard”:true,“query”:“beat.hostname = name-of-host”}},“filter”:{“bool”:{“must”:[{“range”:{"@timestamp":{“gte”:“1568838538865”,“lte”:“1568840338866”}}}]}}}},“aggs”:{“2”:{“date_histogram”:{“interval”:“10s”,“field”:"@timestamp",“min_doc_count”:1,“extended_bounds”:{“min”:“1568838538865”,“max”:“1568840338866”}},“aggs”:{“3”:{“max”:{“field”:“system.cpu.idle.pct”,“script”:{“inline”:"_value * 100"}}}}}}}]]]; nested: SearchParseException[[metricbeat-2019.09.18][1]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Unexpected token START_OBJECT in [3].]]; }{[1g1i_pV3TZSgZftYioWuWg][metricbeat-2019.09.18][2]: RemoteTransportException[[e38e8f57-86f7-467a-a4ef-849d57c973ae][inet[/192.168.1.52:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[metricbeat-2019.09.18][2]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Failed to parse source [{“size”:0,“query”:{“filtered”:{“query”:{“query_string”:{“analyze_wildcard”:true,“query”:“beat.hostname = name-of-host”}},“filter”:{“bool”:{“must”:[{“range”:{"@timestamp":{“gte”:“1568838538865”,“lte”:“1568840338866”}}}]}}}},“aggs”:{“2”:{“date_histogram”:{“interval”:“10s”,“field”:"@timestamp",“min_doc_count”:1,“extended_bounds”:{“min”:“1568838538865”,“max”:“1568840338866”}},“aggs”:{“3”:{“max”:{“field”:“system.cpu.idle.pct”,“script”:{“inline”:"_value * 100"}}}}}}}]]]; nested: SearchParseException[[metricbeat-2019.09.18][2]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Unexpected token START_OBJECT in [3].]]; }{[0tzSv2ppSmmG6RMbzZlUdQ][metricbeat-2019.09.18][3]: RemoteTransportException[[39e75611-f913-4be5-969e-b6ad41fd5437][inet[/192.168.1.53:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[metricbeat-2019.09.18][3]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Failed to parse source [{“size”:0,“query”:{“filtered”:{“query”:{“query_string”:{“analyze_wildcard”:true,“query”:“beat.hostname = name-of-host”}},“filter”:{“bool”:{“must”:[{“range”:{"@timestamp":{“gte”:“1568838538865”,“lte”:“1568840338866”}}}]}}}},“aggs”:{“2”:{“date_histogram”:{“interval”:“10s”,“field”:"@timestamp",“min_doc_count”:1,“extended_bounds”:{“min”:“1568838538865”,“max”:“1568840338866”}},“aggs”:{“3”:{“max”:{“field”:“system.cpu.idle.pct”,“script”:{“inline”:"_value * 100"}}}}}}}]]]; nested: SearchParseException[[metricbeat-2019.09.18][3]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Unexpected token START_OBJECT in [3].]]; }{[1g1i_pV3TZSgZftYioWuWg][metricbeat-2019.09.18][4]: RemoteTransportException[[e38e8f57-86f7-467a-a4ef-849d57c973ae][inet[/192.168.1.52:9300]][indices:data/read/search[phase/query]]]; nested: SearchParseException[[metricbeat-2019.09.18][4]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Failed to parse source [{“size”:0,“query”:{“filtered”:{“query”:{“query_string”:{“analyze_wildcard”:true,“query”:“beat.hostname = name-of-host”}},“filter”:{“bool”:{“must”:[{“range”:{"@timestamp":{“gte”:“1568838538865”,“lte”:“1568840338866”}}}]}}}},“aggs”:{“2”:{“date_histogram”:{“interval”:“10s”,“field”:"@timestamp",“min_doc_count”:1,“extended_bounds”:{“min”:“1568838538865”,“max”:“1568840338866”}},“aggs”:{“3”:{“max”:{“field”:“system.cpu.idle.pct”,“script”:{“inline”:"_value * 100"}}}}}}}]]]; nested: SearchParseException[[metricbeat-2019.09.18][4]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838538865 TO 1568840338866]))],from[-1],size[0]: Parse Failure [Unexpected token START_OBJECT in [3].]]; }]
And we’ll see this in the Elasticsearch logs
[2019-09-18 16:58:38,907][DEBUG][action.search.type ] [dda7f85c-6641-4b98-b573-fbdf7121c025] All shards failed for phase: [query]
org.elasticsearch.transport.RemoteTransportException: [39e75611-f913-4be5-969e-b6ad41fd5437][inet[/192.168.1.53:9300]][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.search.SearchParseException: [metricbeat-2019.09.18][3]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838518856 TO 1568840318856]))],from[-1],size[0]: Parse Failure [Failed to parse source [{“size”:0,“query”:{“filtered”:{“query”:{“query_string”:{“analyze_wildcard”:true,“query”:“beat.hostname = name-of-host”}},“filter”:{“bool”:{“must”:[{“range”:{"@timestamp":{“gte”:“1568838518856”,“lte”:“1568840318856”}}}]}}}},“aggs”:{“2”:{“date_histogram”:{“interval”:“10s”,“field”:"@timestamp",“min_doc_count”:1,“extended_bounds”:{“min”:“1568838518856”,“max”:“1568840318856”}},“aggs”:{“3”:{“max”:{“field”:“system.cpu.idle.pct”,“script”:{“inline”:"_value * 100"}}}}}}}]]
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:747)
at org.elasticsearch.search.SearchService.createContext(SearchService.java:572)
at org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:544)
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:306)
at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:776)
at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:767)
at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:279)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.elasticsearch.search.SearchParseException: [metricbeat-2019.09.18][3]: query[filtered(_all:beat.hostname _all:name-of-host)->BooleanFilter(+cache(@timestamp:[1568838518856 TO 1568840318856]))],from[-1],size[0]: Parse Failure [Unexpected token START_OBJECT in [3].]
at org.elasticsearch.search.aggregations.metrics.NumericValuesSourceMetricsAggregatorParser.parse(NumericValuesSourceMetricsAggregatorParser.java:61)
at org.elasticsearch.search.aggregations.AggregatorParsers.parseAggregators(AggregatorParsers.java:148)
at org.elasticsearch.search.aggregations.AggregatorParsers.parseAggregators(AggregatorParsers.java:138)
at org.elasticsearch.search.aggregations.AggregatorParsers.parseAggregators(AggregatorParsers.java:78)
at org.elasticsearch.search.aggregations.AggregationParseElement.parse(AggregationParseElement.java:60)
at org.elasticsearch.search.SearchService.parseSource(SearchService.java:731)
… 10 more
