Introducing query_failed as part of nodes, indices and shards stats

CHANGELOG.md

@@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Optimize gRPC transport thread management for improved throughput ([#19278](https://github.com/opensearch-project/OpenSearch/pull/19278))
 - Implement GRPC Boolean query and inject registry for all internal query converters ([#19391](https://github.com/opensearch-project/OpenSearch/pull/19391))
 - Implement GRPC Script query ([#19455](https://github.com/opensearch-project/OpenSearch/pull/19455))
+- [Search Stats] Add search & star-tree search query failure count metrics ([#19210](https://github.com/opensearch-project/OpenSearch/issues/19210))
 
 ### Changed
 - Refactor `if-else` chains to use `Java 17 pattern matching switch expressions`(([#18965](https://github.com/opensearch-project/OpenSearch/pull/18965))

rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml

@@ -1,13 +1,113 @@
 "Help":
   - skip:
-      version: " - 3.1.99"
+      version: " - 3.2.99"
+      reason: search query failure stats is added in 3.3.0
+      features: node_selector
+  - do:
+      cat.shards:
+        help: true
+      node_selector:
+        version: "3.3.0 - "
+
+  - match:
+      $body: |
+        /^   index                            .+   \n
+             shard                            .+   \n
+             prirep                           .+   \n
+             state                            .+   \n
+             docs                             .+   \n
+             store                            .+   \n
+             ip                               .+   \n
+             id                               .+   \n
+             node                             .+   \n
+             sync_id                          .+   \n
+             unassigned.reason                .+   \n
+             unassigned.at                    .+   \n
+             unassigned.for                   .+   \n
+             unassigned.details               .+   \n
+             recoverysource.type              .+   \n
+             completion.size                  .+   \n
+             fielddata.memory_size            .+   \n
+             fielddata.evictions              .+   \n
+             query_cache.memory_size          .+   \n
+             query_cache.evictions            .+   \n
+             flush.total                      .+   \n
+             flush.total_time                 .+   \n
+             get.current                      .+   \n
+             get.time                         .+   \n
+             get.total                        .+   \n
+             get.exists_time                  .+   \n
+             get.exists_total                 .+   \n
+             get.missing_time                 .+   \n
+             get.missing_total                .+   \n
+             indexing.delete_current          .+   \n
+             indexing.delete_time             .+   \n
+             indexing.delete_total            .+   \n
+             indexing.index_current           .+   \n
+             indexing.index_time              .+   \n
+             indexing.index_total             .+   \n
+             indexing.index_failed            .+   \n
+             merges.current                   .+   \n
+             merges.current_docs              .+   \n
+             merges.current_size              .+   \n
+             merges.total                     .+   \n
+             merges.total_docs                .+   \n
+             merges.total_size                .+   \n
+             merges.total_time                .+   \n
+             refresh.total                    .+   \n
+             refresh.time                     .+   \n
+             refresh.external_total           .+   \n
+             refresh.external_time            .+   \n
+             refresh.listeners                .+   \n
+             search.fetch_current             .+   \n
+             search.fetch_time                .+   \n
+             search.fetch_total               .+   \n
+             search.open_contexts             .+   \n
+             search.query_current             .+   \n
+             search.query_time                .+   \n
+             search.query_total               .+   \n
+             search.query_failed              .+   \n
+             search.concurrent_query_current  .+   \n
+             search.concurrent_query_time     .+   \n
+             search.concurrent_query_total    .+   \n
+             search.concurrent_avg_slice_count  .+   \n
+             search.startree_query_current    .+   \n
+             search.startree_query_time       .+   \n
+             search.startree_query_total      .+   \n
+             search.startree_query_failed     .+   \n
+             search.scroll_current            .+   \n
+             search.scroll_time               .+   \n
+             search.scroll_total              .+   \n
+             search.point_in_time_current     .+   \n
+             search.point_in_time_time        .+   \n
+             search.point_in_time_total       .+   \n
+             search.search_idle_reactivate_count_total  .+   \n
+             segments.count                   .+   \n
+             segments.memory                  .+   \n
+             segments.index_writer_memory     .+   \n
+             segments.version_map_memory      .+   \n
+             segments.fixed_bitset_memory     .+   \n
+             seq_no.max                       .+   \n
+             seq_no.local_checkpoint          .+   \n
+             seq_no.global_checkpoint         .+   \n
+             warmer.current                   .+   \n
+             warmer.total                     .+   \n
+             warmer.total_time                .+   \n
+             path.data                        .+   \n
+             path.state                       .+   \n
+             docs.deleted                     .+   \n
+        $/
+---
+"Help from 3.2.0 to 3.2.99":
+  - skip:
+      version: " - 3.1.99, 3.3.0 - "
       reason:  star-tree search stats is only added in 3.2.0
       features: node_selector
   - do:
       cat.shards:
         help: true
       node_selector:
-        version: "3.2.0 - "
+        version: "3.2.0 - 3.2.99"
 
   - match:
       $body: |

server/src/main/java/org/opensearch/index/search/stats/SearchStats.java

@@ -142,6 +142,7 @@ public static class Stats implements Writeable, ToXContentFragment {
         private long queryCount;
         private long queryTimeInMillis;
         private long queryCurrent;
+        private long queryFailedCount;
 
         private long concurrentQueryCount;
         private long concurrentQueryTimeInMillis;
@@ -169,6 +170,7 @@ public static class Stats implements Writeable, ToXContentFragment {
         private long starTreeQueryCount;
         private long starTreeQueryTimeInMillis;
         private long starTreeQueryCurrent;
+        private long starTreeQueryFailed;
 
         @Nullable
         private RequestStatsLongHolder requestStatsLongHolder;
@@ -191,6 +193,7 @@ private Stats(Builder builder) {
             this.queryCount = builder.queryCount;
             this.queryTimeInMillis = builder.queryTimeInMillis;
             this.queryCurrent = builder.queryCurrent;
+            this.queryFailedCount = builder.queryFailedCount;
 
             this.concurrentQueryCount = builder.concurrentQueryCount;
             this.concurrentQueryTimeInMillis = builder.concurrentQueryTimeInMillis;
@@ -218,6 +221,7 @@ private Stats(Builder builder) {
             this.starTreeQueryCount = builder.starTreeQueryCount;
             this.starTreeQueryTimeInMillis = builder.starTreeQueryTimeInMillis;
             this.starTreeQueryCurrent = builder.starTreeQueryCurrent;
+            this.starTreeQueryFailed = builder.starTreeQueryFailed;
         }
 
         /**
@@ -319,12 +323,18 @@ private Stats(StreamInput in) throws IOException {
                 starTreeQueryTimeInMillis = in.readVLong();
                 starTreeQueryCurrent = in.readVLong();
             }
+
+            if (in.getVersion().onOrAfter(Version.V_3_3_0)) {
+                queryFailedCount = in.readVLong();
+                starTreeQueryFailed = in.readVLong();
+            }
         }
 
         public void add(Stats stats) {
             queryCount += stats.queryCount;
             queryTimeInMillis += stats.queryTimeInMillis;
             queryCurrent += stats.queryCurrent;
+            queryFailedCount += stats.queryFailedCount;
 
             concurrentQueryCount += stats.concurrentQueryCount;
             concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis;
@@ -352,11 +362,13 @@ public void add(Stats stats) {
             starTreeQueryCount += stats.starTreeQueryCount;
             starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
             starTreeQueryCurrent += stats.starTreeQueryCurrent;
+            starTreeQueryFailed += stats.starTreeQueryFailed;
         }
 
         public void addForClosingShard(Stats stats) {
             queryCount += stats.queryCount;
             queryTimeInMillis += stats.queryTimeInMillis;
+            queryFailedCount += stats.queryFailedCount;
 
             concurrentQueryCount += stats.concurrentQueryCount;
             concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis;
@@ -381,6 +393,7 @@ public void addForClosingShard(Stats stats) {
 
             starTreeQueryCount += stats.starTreeQueryCount;
             starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
+            starTreeQueryFailed += stats.starTreeQueryFailed;
         }
 
         public long getQueryCount() {
@@ -399,6 +412,10 @@ public long getQueryCurrent() {
             return queryCurrent;
         }
 
+        public long getQueryFailedCount() {
+            return queryFailedCount;
+        }
+
         public long getConcurrentQueryCount() {
             return concurrentQueryCount;
         }
@@ -507,6 +524,10 @@ public long getStarTreeQueryCurrent() {
             return starTreeQueryCurrent;
         }
 
+        public long getStarTreeQueryFailed() {
+            return starTreeQueryFailed;
+        }
+
         public static Stats readStats(StreamInput in) throws IOException {
             return new Stats(in);
         }
@@ -562,13 +583,19 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeVLong(starTreeQueryTimeInMillis);
                 out.writeVLong(starTreeQueryCurrent);
             }
+
+            if (out.getVersion().onOrAfter(Version.V_3_3_0)) {
+                out.writeVLong(queryFailedCount);
+                out.writeVLong(starTreeQueryFailed);
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(Fields.QUERY_TOTAL, queryCount);
             builder.humanReadableField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, getQueryTime());
             builder.field(Fields.QUERY_CURRENT, queryCurrent);
+            builder.field(Fields.QUERY_FAILED_TOTAL, queryFailedCount);
 
             builder.field(Fields.CONCURRENT_QUERY_TOTAL, concurrentQueryCount);
             builder.humanReadableField(Fields.CONCURRENT_QUERY_TIME_IN_MILLIS, Fields.CONCURRENT_QUERY_TIME, getConcurrentQueryTime());
@@ -578,6 +605,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.STARTREE_QUERY_TOTAL, starTreeQueryCount);
             builder.humanReadableField(Fields.STARTREE_QUERY_TIME_IN_MILLIS, Fields.STARTREE_QUERY_TIME, getStarTreeQueryTime());
             builder.field(Fields.STARTREE_QUERY_CURRENT, getStarTreeQueryCurrent());
+            builder.field(Fields.STARTREE_QUERY_FAILED, getStarTreeQueryFailed());
 
             builder.field(Fields.FETCH_TOTAL, fetchCount);
             builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime());
@@ -633,6 +661,7 @@ public static class Builder {
             private long queryCount = 0;
             private long queryTimeInMillis = 0;
             private long queryCurrent = 0;
+            private long queryFailedCount = 0;
             private long concurrentQueryCount = 0;
             private long concurrentQueryTimeInMillis = 0;
             private long concurrentQueryCurrent = 0;
@@ -653,6 +682,7 @@ public static class Builder {
             private long starTreeQueryCount = 0;
             private long starTreeQueryTimeInMillis = 0;
             private long starTreeQueryCurrent = 0;
+            private long starTreeQueryFailed = 0;
             @Nullable
             private RequestStatsLongHolder requestStatsLongHolder = null;
 
@@ -673,6 +703,11 @@ public Builder queryCurrent(long current) {
                 return this;
             }
 
+            public Builder queryFailed(long count) {
+                this.queryFailedCount = count;
+                return this;
+            }
+
             public Builder concurrentQueryCount(long count) {
                 this.concurrentQueryCount = count;
                 return this;
@@ -773,6 +808,11 @@ public Builder starTreeQueryCurrent(long current) {
                 return this;
             }
 
+            public Builder starTreeQueryFailed(long count) {
+                this.starTreeQueryFailed = count;
+                return this;
+            }
+
             /**
              * Creates a {@link Stats} object from the builder's current state.
              * @return A new Stats instance.
@@ -916,6 +956,7 @@ static final class Fields {
         static final String QUERY_TIME = "query_time";
         static final String QUERY_TIME_IN_MILLIS = "query_time_in_millis";
         static final String QUERY_CURRENT = "query_current";
+        static final String QUERY_FAILED_TOTAL = "query_failed";
         static final String CONCURRENT_QUERY_TOTAL = "concurrent_query_total";
         static final String CONCURRENT_QUERY_TIME = "concurrent_query_time";
         static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis";
@@ -925,6 +966,7 @@ static final class Fields {
         static final String STARTREE_QUERY_TIME = "startree_query_time";
         static final String STARTREE_QUERY_TIME_IN_MILLIS = "startree_query_time_in_millis";
         static final String STARTREE_QUERY_CURRENT = "startree_query_current";
+        static final String STARTREE_QUERY_FAILED = "startree_query_failed";
         static final String FETCH_TOTAL = "fetch_total";
         static final String FETCH_TIME = "fetch_time";
         static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis";

server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java

@@ -104,6 +104,7 @@ public void onPreQueryPhase(SearchContext searchContext) {
     @Override
     public void onFailedQueryPhase(SearchContext searchContext) {
         computeStats(searchContext, statsHolder -> {
+            statsHolder.queryFailed.inc();
             if (searchContext.hasOnlySuggest()) {
                 statsHolder.suggestCurrent.dec();
                 assert statsHolder.suggestCurrent.count() >= 0;
@@ -115,6 +116,7 @@ public void onFailedQueryPhase(SearchContext searchContext) {
                     assert statsHolder.concurrentQueryCurrent.count() >= 0;
                 }
                 if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) {
+                    statsHolder.starTreeQueryFailed.inc();
                     statsHolder.starTreeCurrent.dec();
                     assert statsHolder.starTreeCurrent.count() >= 0;
                 }
@@ -237,6 +239,7 @@ public void onSearchIdleReactivation() {
      */
     static final class StatsHolder {
         final MeanMetric queryMetric = new MeanMetric();
+        final CounterMetric queryFailed = new CounterMetric();
         final MeanMetric concurrentQueryMetric = new MeanMetric();
         final CounterMetric queryConcurrencyMetric = new CounterMetric();
         final MeanMetric fetchMetric = new MeanMetric();
@@ -259,11 +262,13 @@ static final class StatsHolder {
         final CounterMetric searchIdleMetric = new CounterMetric();
         final MeanMetric starTreeQueryMetric = new MeanMetric();
         final CounterMetric starTreeCurrent = new CounterMetric();
+        final CounterMetric starTreeQueryFailed = new CounterMetric();
 
         SearchStats.Stats stats() {
             return new SearchStats.Stats.Builder().queryCount(queryMetric.count())
                 .queryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()))
                 .queryCurrent(queryCurrent.count())
+                .queryFailed(queryFailed.count())
                 .concurrentQueryCount(concurrentQueryMetric.count())
                 .concurrentQueryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(concurrentQueryMetric.sum()))
                 .concurrentQueryCurrent(concurrentQueryCurrent.count())
@@ -284,6 +289,7 @@ SearchStats.Stats stats() {
                 .starTreeQueryCount(starTreeQueryMetric.count())
                 .starTreeQueryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(starTreeQueryMetric.sum()))
                 .starTreeQueryCurrent(starTreeCurrent.count())
+                .starTreeQueryFailed(starTreeQueryFailed.count())
                 .build();
         }
     }