Skip to content

Add support for query profiler with concurrent aggregation#9248

Merged
reta merged 9 commits intoopensearch-project:mainfrom
ticheng-aws:cs-query-profiler
Sep 5, 2023
Merged

Add support for query profiler with concurrent aggregation#9248
reta merged 9 commits intoopensearch-project:mainfrom
ticheng-aws:cs-query-profiler

Conversation

@ticheng-aws
Copy link
Copy Markdown
Contributor

@ticheng-aws ticheng-aws commented Aug 11, 2023 •
edited
Loading

Description

At present, the query profiler is the total timing obtained by summing all segments. However, this does not accurately represent the detailed timing of the query, including queue time and gap time, during concurrent execution. This PR will address the problem.

A sample query response for a concurrent search with 2 segment slices:

{
    "took": 61,
    "timed_out": false,
    "num_reduce_phases": 2,
    "_shards": {...},
    "hits": {...},
    "profile":
    {
        "shards":
        [
            {
                "id": "[FFjuubYPTGSYwNLsL2ccbA][test][0]",
                "inbound_network_time_in_millis": 6,
                "outbound_network_time_in_millis": 1,
                "searches":
                [
                    {
                        "query":
                        [
                            {
                                "type": "TermQuery",
                                "description": "field1:one",
                                "time_in_nanos": 3944209,
                                "max_slice_time_in_nanos": 3944209,
                                "min_slice_time_in_nanos": 3892625,
                                "avg_slice_time_in_nanos": 3918417,
                                "breakdown":
                                {
                                    "max_match": 0,
                                    "set_min_competitive_score_count": 0,
                                    "match_count": 0,
                                    "avg_score_count": 4,
                                    "shallow_advance_count": 0,
                                    "next_doc": 50625,
                                    "min_build_scorer": 1327791,
                                    "score_count": 8,
                                    "compute_max_score_count": 0,
                                    "advance": 96583,
                                    "min_set_min_competitive_score": 0,
                                    "min_advance": 4042,
                                    "score": 96751,
                                    "avg_set_min_competitive_score_count": 0,
                                    "min_match_count": 0,
                                    "avg_score": 65438,
                                    "max_next_doc_count": 7,
                                    "max_compute_max_score_count": 0,
                                    "avg_shallow_advance": 0,
                                    "max_shallow_advance_count": 0,
                                    "set_min_competitive_score": 0,
                                    "min_build_scorer_count": 2,
                                    "next_doc_count": 8,
                                    "min_match": 0,
                                    "avg_next_doc": 26250,
                                    "compute_max_score": 0,
                                    "min_set_min_competitive_score_count": 0,
                                    "max_build_scorer": 1722750,
                                    "avg_match_count": 0,
                                    "avg_advance": 50125,
                                    "build_scorer_count": 6,
                                    "avg_build_scorer_count": 3,
                                    "min_next_doc_count": 1,
                                    "min_shallow_advance_count": 0,
                                    "max_score_count": 7,
                                    "avg_match": 0,
                                    "avg_compute_max_score": 0,
                                    "max_advance": 96208,
                                    "avg_shallow_advance_count": 0,
                                    "avg_set_min_competitive_score": 0,
                                    "avg_compute_max_score_count": 0,
                                    "avg_build_scorer": 1525270,
                                    "max_set_min_competitive_score_count": 0,
                                    "advance_count": 3,
                                    "max_build_scorer_count": 4,
                                    "shallow_advance": 0,
                                    "min_compute_max_score": 0,
                                    "max_match_count": 0,
                                    "create_weight_count": 1,
                                    "build_scorer": 1830250,
                                    "max_set_min_competitive_score": 0,
                                    "max_compute_max_score": 0,
                                    "min_shallow_advance": 0,
                                    "match": 0,
                                    "max_shallow_advance": 0,
                                    "avg_advance_count": 1,
                                    "min_next_doc": 1875,
                                    "max_advance_count": 2,
                                    "min_score": 34209,
                                    "max_next_doc": 50625,
                                    "create_weight": 472375,
                                    "avg_next_doc_count": 4,
                                    "max_score": 96667,
                                    "min_compute_max_score_count": 0,
                                    "min_score_count": 1,
                                    "min_advance_count": 1
                                }
                            }
                        ],
                        "rewrite_time": 175791,
                        "collector": [...]
                    }
                ],
                "aggregations": []
            },
            {...},
            {...}
        ]
    }
}

Related Issues

Resolves #8330 #7354

Check List

  • New functionality includes testing.
    • All tests pass
  • New functionality has been documented.
    • New functionality has javadoc added
  • Commits are signed per the DCO using --signoff
  • Commit changes are listed out in CHANGELOG.md file (See: Changelog)

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check here.

ticheng-aws added a commit to ticheng-aws/OpenSearch that referenced this pull request Aug 11, 2023
@ticheng-aws
Add support for query profiler with concurrent aggregation (opensearc…
ea74aef 
…h-project#9248)

Signed-off-by: Ticheng Lin <ticheng@amazon.com>
@opensearch-trigger-bot
Copy link
Copy Markdown
Contributor

opensearch-trigger-bot bot commented Aug 11, 2023

Compatibility status:



> Task :checkCompatibility
Incompatible components: [https://github.com/opensearch-project/geospatial.git, https://github.com/opensearch-project/security.git, https://github.com/opensearch-project/notifications.git, https://github.com/opensearch-project/neural-search.git, https://github.com/opensearch-project/index-management.git, https://github.com/opensearch-project/security-analytics.git, https://github.com/opensearch-project/sql.git, https://github.com/opensearch-project/job-scheduler.git, https://github.com/opensearch-project/k-nn.git, https://github.com/opensearch-project/observability.git, https://github.com/opensearch-project/alerting.git, https://github.com/opensearch-project/cross-cluster-replication.git, https://github.com/opensearch-project/anomaly-detection.git, https://github.com/opensearch-project/asynchronous-search.git, https://github.com/opensearch-project/ml-commons.git, https://github.com/opensearch-project/performance-analyzer.git, https://github.com/opensearch-project/common-utils.git, https://github.com/opensearch-project/reporting.git]
Compatible components: [https://github.com/opensearch-project/opensearch-oci-object-storage.git, https://github.com/opensearch-project/performance-analyzer-rca.git]

BUILD SUCCESSFUL in 30m 47s

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Aug 11, 2023

Gradle Check (Jenkins) Run Completed with:

@opensearch-trigger-bot
Copy link
Copy Markdown
Contributor

opensearch-trigger-bot bot commented Aug 11, 2023

Compatibility status:



> Task :checkCompatibility
Incompatible components: [https://github.com/opensearch-project/geospatial.git, https://github.com/opensearch-project/security.git, https://github.com/opensearch-project/notifications.git, https://github.com/opensearch-project/neural-search.git, https://github.com/opensearch-project/index-management.git, https://github.com/opensearch-project/sql.git, https://github.com/opensearch-project/security-analytics.git, https://github.com/opensearch-project/job-scheduler.git, https://github.com/opensearch-project/observability.git, https://github.com/opensearch-project/k-nn.git, https://github.com/opensearch-project/cross-cluster-replication.git, https://github.com/opensearch-project/alerting.git, https://github.com/opensearch-project/ml-commons.git, https://github.com/opensearch-project/anomaly-detection.git, https://github.com/opensearch-project/asynchronous-search.git, https://github.com/opensearch-project/common-utils.git, https://github.com/opensearch-project/performance-analyzer.git, https://github.com/opensearch-project/reporting.git]
Compatible components: [https://github.com/opensearch-project/opensearch-oci-object-storage.git, https://github.com/opensearch-project/performance-analyzer-rca.git]

BUILD SUCCESSFUL in 21m 42s

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Aug 11, 2023

Gradle Check (Jenkins) Run Completed with:

ticheng-aws added a commit to ticheng-aws/OpenSearch that referenced this pull request Aug 11, 2023
@ticheng-aws
Add support for query profiler with concurrent aggregation (opensearc…
d347dc0 
…h-project#9248)

Signed-off-by: Ticheng Lin <ticheng@amazon.com>
@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Compatibility status:

Checks if related components are compatible with change 2cbf8ed

Incompatible components

Skipped components

Compatible components

Compatible components: [https://github.com/opensearch-project/geospatial.git, https://github.com/opensearch-project/security.git, https://github.com/opensearch-project/notifications.git, https://github.com/opensearch-project/neural-search.git, https://github.com/opensearch-project/index-management.git, https://github.com/opensearch-project/sql.git, https://github.com/opensearch-project/security-analytics.git, https://github.com/opensearch-project/observability.git, https://github.com/opensearch-project/job-scheduler.git, https://github.com/opensearch-project/opensearch-oci-object-storage.git, https://github.com/opensearch-project/k-nn.git, https://github.com/opensearch-project/cross-cluster-replication.git, https://github.com/opensearch-project/alerting.git, https://github.com/opensearch-project/anomaly-detection.git, https://github.com/opensearch-project/asynchronous-search.git, https://github.com/opensearch-project/ml-commons.git, https://github.com/opensearch-project/performance-analyzer.git, https://github.com/opensearch-project/performance-analyzer-rca.git, https://github.com/opensearch-project/common-utils.git, https://github.com/opensearch-project/reporting.git]

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Gradle Check (Jenkins) Run Completed with:

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Gradle Check (Jenkins) Run Completed with:

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Compatibility status:

Checks if related components are compatible with change 48d59a0

Incompatible components

Skipped components

Compatible components

Compatible components: [https://github.com/opensearch-project/geospatial.git, https://github.com/opensearch-project/security.git, https://github.com/opensearch-project/notifications.git, https://github.com/opensearch-project/neural-search.git, https://github.com/opensearch-project/index-management.git, https://github.com/opensearch-project/security-analytics.git, https://github.com/opensearch-project/sql.git, https://github.com/opensearch-project/job-scheduler.git, https://github.com/opensearch-project/observability.git, https://github.com/opensearch-project/opensearch-oci-object-storage.git, https://github.com/opensearch-project/k-nn.git, https://github.com/opensearch-project/cross-cluster-replication.git, https://github.com/opensearch-project/alerting.git, https://github.com/opensearch-project/anomaly-detection.git, https://github.com/opensearch-project/performance-analyzer.git, https://github.com/opensearch-project/asynchronous-search.git, https://github.com/opensearch-project/ml-commons.git, https://github.com/opensearch-project/performance-analyzer-rca.git, https://github.com/opensearch-project/common-utils.git, https://github.com/opensearch-project/reporting.git]

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Compatibility status:

Checks if related components are compatible with change 2b84d5a

Incompatible components

Skipped components

Compatible components

Compatible components: [https://github.com/opensearch-project/geospatial.git, https://github.com/opensearch-project/security.git, https://github.com/opensearch-project/notifications.git, https://github.com/opensearch-project/neural-search.git, https://github.com/opensearch-project/index-management.git, https://github.com/opensearch-project/sql.git, https://github.com/opensearch-project/security-analytics.git, https://github.com/opensearch-project/job-scheduler.git, https://github.com/opensearch-project/observability.git, https://github.com/opensearch-project/k-nn.git, https://github.com/opensearch-project/opensearch-oci-object-storage.git, https://github.com/opensearch-project/cross-cluster-replication.git, https://github.com/opensearch-project/alerting.git, https://github.com/opensearch-project/anomaly-detection.git, https://github.com/opensearch-project/performance-analyzer.git, https://github.com/opensearch-project/asynchronous-search.git, https://github.com/opensearch-project/performance-analyzer-rca.git, https://github.com/opensearch-project/ml-commons.git, https://github.com/opensearch-project/common-utils.git, https://github.com/opensearch-project/reporting.git]

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Gradle Check (Jenkins) Run Completed with:

@sohami
Copy link
Copy Markdown
Contributor

sohami commented Sep 1, 2023

Gradle Check (Jenkins) Run Completed with:

@ticheng-aws There is a test failure with recent changes, can you please take a look

Related:

[org.opensearch.search.profile.query.QueryProfilerIT.testProfileMatchesRegular {p0={"search.concurrent_segment_search.enabled":"true"} p1=true}](https://build.ci.opensearch.org/job/gradle-check/24161/testReport/junit/org.opensearch.search.profile.query/QueryProfilerIT/testProfileMatchesRegular__p0___search_concurrent_segment_search_enabled___true___p1_true_/)

Unrelated:

[org.opensearch.indices.replication.SegmentReplicationAllocationIT.testSingleIndexShardAllocation](https://build.ci.opensearch.org/job/gradle-check/24161/testReport/junit/org.opensearch.indices.replication/SegmentReplicationAllocationIT/testSingleIndexShardAllocation/)
[org.opensearch.remotestore.RemoteStoreRestoreIT.testRateLimitedRemoteDownloads](https://build.ci.opensearch.org/job/gradle-check/24161/testReport/junit/org.opensearch.remotestore/RemoteStoreRestoreIT/testRateLimitedRemoteDownloads/)
[org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT.testDeleteOperations](https://build.ci.opensearch.org/job/gradle-check/24161/testReport/junit/org.opensearch.remotestore/SegmentReplicationUsingRemoteStoreIT/testDeleteOperations/)
[org.opensearch.search.SearchWeightedRoutingIT.testShardRoutingWithNetworkDisruption_FailOpenEnabled](https://build.ci.opensearch.org/job/gradle-check/24161/testReport/junit/org.opensearch.search/SearchWeightedRoutingIT/testShardRoutingWithNetworkDisruption_FailOpenEnabled/)

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 1, 2023

Gradle Check (Jenkins) Run Completed with:

@sohami
Copy link
Copy Markdown
Contributor

sohami commented Sep 1, 2023

Gradle Check (Jenkins) Run Completed with:

This is known flaky test failure: #9688

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 2, 2023

Gradle Check (Jenkins) Run Completed with:

ticheng-aws and others added 9 commits September 5, 2023 09:42
@ticheng-aws
Add support for query profiler with concurrent aggregation (opensearc…
e6ad355 
…h-project#9248)

Signed-off-by: Ticheng Lin <ticheng@amazon.com>
@ticheng-aws
Refactor and work on the PR comments
a6cee80 
Signed-off-by: Ticheng Lin <ticheng@amazon.com>
@sohami @ticheng-aws
Update collectorToLeaves mapping for children breakdowns post profile…
adc9b51 
… metric collection and before creating the results

Signed-off-by: Sorabh Hamirwasia <sohami.apache@gmail.com>
@sohami @ticheng-aws
Refactor logic to compute the slice level breakdown stats and query l…
0868b20 
…evel breakdown stats for concurrent search case

Signed-off-by: Sorabh Hamirwasia <sohami.apache@gmail.com>
@ticheng-aws
Fix QueryProfilePhaseTests and QueryProfileTests, and parameterize Qu…
b5af44e 
…eryProfilerIT with concurrent search enabled

Signed-off-by: Ticheng Lin <ticheng@amazon.com>
@sohami @ticheng-aws
Handle the case when there are no leaf context to compute the profile…
c9fcca0 
… stats to return default stats

for all breakdown type along with min/max/avg values. Replace queryStart and queryEnd time with queryNodeTime

Signed-off-by: Sorabh Hamirwasia <sohami.apache@gmail.com>
@sohami @ticheng-aws
Add UTs for ConcurrentQueryProfileBreakdown
34b048e 
Signed-off-by: Sorabh Hamirwasia <sohami.apache@gmail.com>
@ticheng-aws
Add concurrent search stats test into the QueryProfilerIT
ae1af62 
Signed-off-by: Ticheng Lin <ticheng@amazon.com>
@sohami @ticheng-aws
Address review comments
6644e02 
Signed-off-by: Sorabh Hamirwasia <sohami.apache@gmail.com>
@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 5, 2023

Compatibility status:

Checks if related components are compatible with change 6644e02

Incompatible components

Incompatible components: [https://github.com/opensearch-project/anomaly-detection.git, https://github.com/opensearch-project/sql.git, https://github.com/opensearch-project/neural-search.git]

Skipped components

Compatible components

Compatible components: [https://github.com/opensearch-project/security.git, https://github.com/opensearch-project/alerting.git, https://github.com/opensearch-project/index-management.git, https://github.com/opensearch-project/asynchronous-search.git, https://github.com/opensearch-project/job-scheduler.git, https://github.com/opensearch-project/observability.git, https://github.com/opensearch-project/common-utils.git, https://github.com/opensearch-project/k-nn.git, https://github.com/opensearch-project/reporting.git, https://github.com/opensearch-project/cross-cluster-replication.git, https://github.com/opensearch-project/geospatial.git, https://github.com/opensearch-project/notifications.git, https://github.com/opensearch-project/ml-commons.git, https://github.com/opensearch-project/performance-analyzer.git, https://github.com/opensearch-project/performance-analyzer-rca.git, https://github.com/opensearch-project/security-analytics.git, https://github.com/opensearch-project/opensearch-oci-object-storage.git]

@github-actions
Copy link
Copy Markdown
Contributor

github-actions bot commented Sep 5, 2023

Gradle Check (Jenkins) Run Completed with:

@opensearch-trigger-bot opensearch-trigger-bot bot mentioned this pull request Sep 5, 2023
Closed
@reta reta mentioned this pull request Sep 6, 2023
Closed
@ticheng-aws ticheng-aws mentioned this pull request Oct 23, 2023
Merged
7 tasks
@sohami sohami mentioned this pull request Oct 24, 2023
Merged
8 tasks
@ticheng-aws ticheng-aws mentioned this pull request Oct 27, 2023
Closed
4 tasks
@ticheng-aws ticheng-aws mentioned this pull request Dec 26, 2023
Merged
1 task
@BrewTestBot BrewTestBot mentioned this pull request Feb 21, 2024
Merged
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@reta reta reta approved these changes

@sohami sohami sohami approved these changes

@anasalkouz anasalkouz Awaiting requested review from anasalkouz anasalkouz is a code owner

@andrross andrross Awaiting requested review from andrross andrross is a code owner

@Bukhtawar Bukhtawar Awaiting requested review from Bukhtawar Bukhtawar is a code owner

@CEHENKLE CEHENKLE Awaiting requested review from CEHENKLE CEHENKLE is a code owner

@dblock dblock Awaiting requested review from dblock

@gbbafna gbbafna Awaiting requested review from gbbafna gbbafna is a code owner

@setiah setiah Awaiting requested review from setiah

@kartg kartg Awaiting requested review from kartg

@kotwanikunal kotwanikunal Awaiting requested review from kotwanikunal kotwanikunal is a code owner

@mch2 mch2 Awaiting requested review from mch2 mch2 is a code owner

@nknize nknize Awaiting requested review from nknize

@owaiskazi19 owaiskazi19 Awaiting requested review from owaiskazi19 owaiskazi19 is a code owner

@Rishikesh1159 Rishikesh1159 Awaiting requested review from Rishikesh1159 Rishikesh1159 is a code owner

@ryanbogan ryanbogan Awaiting requested review from ryanbogan

@saratvemulapalli saratvemulapalli Awaiting requested review from saratvemulapalli saratvemulapalli is a code owner

@shwetathareja shwetathareja Awaiting requested review from shwetathareja shwetathareja is a code owner

@dreamer-89 dreamer-89 Awaiting requested review from dreamer-89

@tlfeng tlfeng Awaiting requested review from tlfeng

@VachaShah VachaShah Awaiting requested review from VachaShah

@dbwiddis dbwiddis Awaiting requested review from dbwiddis dbwiddis is a code owner

@sachinpkale sachinpkale Awaiting requested review from sachinpkale sachinpkale is a code owner

@kkmr kkmr Awaiting requested review from kkmr

@msfroh msfroh Awaiting requested review from msfroh msfroh is a code owner

@peternied peternied Awaiting requested review from peternied

Assignees

No one assigned

Labels

backport 2.x Backport to 2.x branch Indexing & Search v2.10.0

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

[Concurrent Segment Search] Add support for query profiler with concurrent aggregation

4 participants

@ticheng-aws @sohami @reta @kkmr