diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java index 3ac7db6c8..2feb19a92 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java @@ -128,6 +128,14 @@ public void collectMetrics(long startTime) { .append(OSMetrics.THREAD_BLOCKED_EVENT) .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) .append(threadState.blockedCount); + value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) + .append(OSMetrics.THREAD_WAITED_TIME) + .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) + .append(threadState.avgWaitedTime); + value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) + .append(OSMetrics.THREAD_WAITED_EVENT) + .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor) + .append(threadState.waitedCount); } if (diskIOMetricsGenerator.hasDiskIOMetrics(threadId)) { diff --git a/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java b/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java index 4ec278ccd..891b043f3 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java +++ b/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java @@ -82,9 +82,12 @@ public static class ThreadState { public Thread.State state; public long blockedCount; public long blockedTime; + public long waitedCount; + public long waitedTime; public double heapAllocRate; public double avgBlockedTime; + public double avgWaitedTime; ThreadState() { javaTid = -1; @@ -93,7 +96,10 @@ public static class ThreadState { heapAllocRate = 0; blockedCount = 0; blockedTime = 0; + waitedCount = 0; + waitedTime = 0; avgBlockedTime = 0; + avgWaitedTime = 0; threadName = ""; tState = ""; } @@ -118,6 +124,10 @@ public String toString() { .append(avgBlockedTime) .append(":") .append(blockedCount) + .append(" wTime: ") + .append(avgWaitedTime) + .append(":") + .append(waitedCount) .toString(); } } @@ -254,7 +264,15 @@ private static void parseThreadInfo(final ThreadInfo info) { t.state = state; t.blockedCount = info.getBlockedCount(); t.blockedTime = info.getBlockedTime(); - ThreadHistory.add(t.nativeTid, (state == Thread.State.BLOCKED) ? samplingInterval : 0); + t.waitedCount = info.getWaitedCount(); + t.waitedTime = info.getWaitedTime(); + ThreadHistory.addBlocked( + t.nativeTid, (state == Thread.State.BLOCKED) ? samplingInterval : 0); + ThreadHistory.addWaited( + t.nativeTid, + (state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING) + ? samplingInterval + : 0); long curRunTime = System.currentTimeMillis(); ThreadState oldt = oldNativeTidMap.get(t.nativeTid); @@ -267,12 +285,24 @@ private static void parseThreadInfo(final ThreadInfo info) { * (t.blockedTime - oldt.blockedTime) / (t.blockedCount - oldt.blockedCount); } else { - CircularLongArray arr = ThreadHistory.tidHistoryMap.get(t.nativeTid); + CircularLongArray arr = ThreadHistory.blockedTidHistoryMap.get(t.nativeTid); // NOTE: this is an upper bound if (arr != null) { t.avgBlockedTime = 1.0 * arr.getAvgValue() / samplingInterval; } } + if (t.waitedTime != -1 && t.waitedCount > oldt.waitedCount) { + t.avgWaitedTime = + 1.0e-3 + * (t.waitedTime - oldt.waitedTime) + / (t.waitedCount - oldt.waitedCount); + } else { + CircularLongArray arr = ThreadHistory.waitedTidHistoryMap.get(t.nativeTid); + // NOTE: this is an upper bound + if (arr != null) { + t.avgWaitedTime = 1.0 * arr.getAvgValue() / samplingInterval; + } + } } jTidNameMap.put(id, name); } @@ -352,10 +382,25 @@ private static void createMap(InputStream in) throws Exception { // currently stores thread states to track locking periods static class ThreadHistory { - public static Map tidHistoryMap = new HashMap<>(); + public static Map blockedTidHistoryMap = new HashMap<>(); + public static Map waitedTidHistoryMap = new HashMap<>(); private static final int HISTORY_SIZE = 60; // 60 * samplingInterval - public static void add(long tid, long value) { + public static void addBlocked(long tid, long value) { + add(tid, value, blockedTidHistoryMap); + } + + public static void addWaited(long tid, long value) { + add(tid, value, waitedTidHistoryMap); + } + + public static void cleanup() { + long curTime = System.currentTimeMillis(); + cleanUp(curTime, blockedTidHistoryMap); + cleanUp(curTime, waitedTidHistoryMap); + } + + private static void add(long tid, long value, Map tidHistoryMap) { CircularLongArray arr = tidHistoryMap.get(tid); if (arr == null) { arr = new CircularLongArray(HISTORY_SIZE); @@ -366,8 +411,7 @@ public static void add(long tid, long value) { } } - public static void cleanup() { - long curTime = System.currentTimeMillis(); + private static void cleanUp(long curTime, Map tidHistoryMap) { for (Iterator> it = tidHistoryMap.entrySet().iterator(); it.hasNext(); ) { diff --git a/src/main/java/org/opensearch/performanceanalyzer/metrics/AllMetrics.java b/src/main/java/org/opensearch/performanceanalyzer/metrics/AllMetrics.java index 2760af0f5..c3247050e 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/metrics/AllMetrics.java +++ b/src/main/java/org/opensearch/performanceanalyzer/metrics/AllMetrics.java @@ -962,7 +962,9 @@ public enum OSMetrics { IO_WRITE_SYSCALL_RATE(Constants.WRITE_SYSCALL_RATE_VALUE), IO_TOTAL_SYSCALL_RATE(Constants.TOTAL_SYSCALL_RATE_VALUE), THREAD_BLOCKED_TIME(Constants.BLOCKED_TIME_VALUE), - THREAD_BLOCKED_EVENT(Constants.BLOCKED_COUNT_VALUE); + THREAD_BLOCKED_EVENT(Constants.BLOCKED_COUNT_VALUE), + THREAD_WAITED_TIME(Constants.WAITED_TIME_VALUE), + THREAD_WAITED_EVENT(Constants.WAITED_COUNT_VALUE); private final String value; @@ -992,6 +994,8 @@ public static class Constants { public static final String TOTAL_SYSCALL_RATE_VALUE = "IO_TotalSyscallRate"; public static final String BLOCKED_TIME_VALUE = "Thread_Blocked_Time"; public static final String BLOCKED_COUNT_VALUE = "Thread_Blocked_Event"; + public static final String WAITED_TIME_VALUE = "Thread_Waited_Time"; + public static final String WAITED_COUNT_VALUE = "Thread_Waited_Event"; } } diff --git a/src/main/java/org/opensearch/performanceanalyzer/model/MetricsModel.java b/src/main/java/org/opensearch/performanceanalyzer/model/MetricsModel.java index 840b1d5f7..f5ca213bb 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/model/MetricsModel.java +++ b/src/main/java/org/opensearch/performanceanalyzer/model/MetricsModel.java @@ -133,7 +133,13 @@ public class MetricsModel { allMetricsInitializer.put( OSMetrics.THREAD_BLOCKED_EVENT.toString(), new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values())); - + allMetricsInitializer.put( + OSMetrics.THREAD_WAITED_TIME.toString(), + new MetricAttributes( + MetricUnits.SEC_PER_EVENT.toString(), AggregatedOSDimension.values())); + allMetricsInitializer.put( + OSMetrics.THREAD_WAITED_EVENT.toString(), + new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values())); // Latency Metric allMetricsInitializer.put( CommonMetric.LATENCY.toString(), diff --git a/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Event.java b/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Event.java new file mode 100644 index 000000000..0aca05983 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Event.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.opensearch.performanceanalyzer.rca.framework.api.metrics; + + +import org.opensearch.performanceanalyzer.metrics.AllMetrics; +import org.opensearch.performanceanalyzer.rca.framework.api.Metric; + +public class Thread_Waited_Event extends Metric { + public Thread_Waited_Event(long evaluationIntervalSeconds) { + super(AllMetrics.OSMetrics.THREAD_WAITED_EVENT.name(), evaluationIntervalSeconds); + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Time.java b/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Time.java new file mode 100644 index 000000000..ef7b3ee7c --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Time.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.opensearch.performanceanalyzer.rca.framework.api.metrics; + + +import org.opensearch.performanceanalyzer.metrics.AllMetrics; +import org.opensearch.performanceanalyzer.rca.framework.api.Metric; + +public class Thread_Waited_Time extends Metric { + public Thread_Waited_Time(long evaluationIntervalSeconds) { + super(AllMetrics.OSMetrics.THREAD_WAITED_TIME.name(), evaluationIntervalSeconds); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java b/src/test/java/org/opensearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java index 37a800704..34e2b0f13 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java @@ -234,6 +234,8 @@ private HashMap getMetricsMap() { this.put(AllMetrics.OSMetrics.Constants.TOTAL_SYSCALL_RATE_VALUE, 2342343223d); this.put(AllMetrics.OSMetrics.Constants.BLOCKED_TIME_VALUE, 23432.32432d); this.put(AllMetrics.OSMetrics.Constants.BLOCKED_COUNT_VALUE, 23432.32432d); + this.put(AllMetrics.OSMetrics.Constants.WAITED_TIME_VALUE, 23432.32432d); + this.put(AllMetrics.OSMetrics.Constants.WAITED_COUNT_VALUE, 23432.32432d); } }; }