Adds Thread_Waited_Time and Thread_Waited_Event metrics

src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java

@@ -128,6 +128,14 @@ public void collectMetrics(long startTime) {
                         .append(OSMetrics.THREAD_BLOCKED_EVENT)
                         .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor)
                         .append(threadState.blockedCount);
+                value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
+                        .append(OSMetrics.THREAD_WAITED_TIME)
+                        .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor)
+                        .append(threadState.avgWaitedTime);
+                value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
+                        .append(OSMetrics.THREAD_WAITED_EVENT)
+                        .append(PerformanceAnalyzerMetrics.sKeyValueDelimitor)
+                        .append(threadState.waitedCount);
             }
 
             if (diskIOMetricsGenerator.hasDiskIOMetrics(threadId)) {

src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java

@@ -82,9 +82,12 @@ public static class ThreadState {
         public Thread.State state;
         public long blockedCount;
         public long blockedTime;
+        public long waitedCount;
+        public long waitedTime;
 
         public double heapAllocRate;
         public double avgBlockedTime;
+        public double avgWaitedTime;
 
         ThreadState() {
             javaTid = -1;
@@ -93,7 +96,10 @@ public static class ThreadState {
             heapAllocRate = 0;
             blockedCount = 0;
             blockedTime = 0;
+            waitedCount = 0;
+            waitedTime = 0;
             avgBlockedTime = 0;
+            avgWaitedTime = 0;
             threadName = "";
             tState = "";
         }
@@ -118,6 +124,10 @@ public String toString() {
                     .append(avgBlockedTime)
                     .append(":")
                     .append(blockedCount)
+                    .append(" wTime: ")
+                    .append(avgWaitedTime)
+                    .append(":")
+                    .append(waitedCount)
                     .toString();
         }
     }
@@ -254,7 +264,15 @@ private static void parseThreadInfo(final ThreadInfo info) {
         t.state = state;
         t.blockedCount = info.getBlockedCount();
         t.blockedTime = info.getBlockedTime();
-        ThreadHistory.add(t.nativeTid, (state == Thread.State.BLOCKED) ? samplingInterval : 0);
+        t.waitedCount = info.getWaitedCount();
+        t.waitedTime = info.getWaitedTime();
+        ThreadHistory.addBlocked(
+                t.nativeTid, (state == Thread.State.BLOCKED) ? samplingInterval : 0);
+        ThreadHistory.addWaited(
+                t.nativeTid,
+                (state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING)
+                        ? samplingInterval
+                        : 0);
 
         long curRunTime = System.currentTimeMillis();
         ThreadState oldt = oldNativeTidMap.get(t.nativeTid);
@@ -267,12 +285,24 @@ private static void parseThreadInfo(final ThreadInfo info) {
                                 * (t.blockedTime - oldt.blockedTime)
                                 / (t.blockedCount - oldt.blockedCount);
             } else {
-                CircularLongArray arr = ThreadHistory.tidHistoryMap.get(t.nativeTid);
+                CircularLongArray arr = ThreadHistory.blockedTidHistoryMap.get(t.nativeTid);
                 // NOTE: this is an upper bound
                 if (arr != null) {
                     t.avgBlockedTime = 1.0 * arr.getAvgValue() / samplingInterval;
                 }
             }
+            if (t.waitedTime != -1 && t.waitedCount > oldt.waitedCount) {
+                t.avgWaitedTime =
+                        1.0e-3
+                                * (t.waitedTime - oldt.waitedTime)
+                                / (t.waitedCount - oldt.waitedCount);
+            } else {
+                CircularLongArray arr = ThreadHistory.waitedTidHistoryMap.get(t.nativeTid);
+                // NOTE: this is an upper bound
+                if (arr != null) {
+                    t.avgWaitedTime = 1.0 * arr.getAvgValue() / samplingInterval;
+                }
+            }
         }
         jTidNameMap.put(id, name);
     }
@@ -352,10 +382,25 @@ private static void createMap(InputStream in) throws Exception {
 
     // currently stores thread states to track locking periods
     static class ThreadHistory {
-        public static Map<Long, CircularLongArray> tidHistoryMap = new HashMap<>();
+        public static Map<Long, CircularLongArray> blockedTidHistoryMap = new HashMap<>();
+        public static Map<Long, CircularLongArray> waitedTidHistoryMap = new HashMap<>();
         private static final int HISTORY_SIZE = 60; // 60 * samplingInterval
 
-        public static void add(long tid, long value) {
+        public static void addBlocked(long tid, long value) {
+            add(tid, value, blockedTidHistoryMap);
+        }
+
+        public static void addWaited(long tid, long value) {
+            add(tid, value, waitedTidHistoryMap);
+        }
+
+        public static void cleanup() {
+            long curTime = System.currentTimeMillis();
+            cleanUp(curTime, blockedTidHistoryMap);
+            cleanUp(curTime, waitedTidHistoryMap);
+        }
+
+        private static void add(long tid, long value, Map<Long, CircularLongArray> tidHistoryMap) {
             CircularLongArray arr = tidHistoryMap.get(tid);
             if (arr == null) {
                 arr = new CircularLongArray(HISTORY_SIZE);
@@ -366,8 +411,7 @@ public static void add(long tid, long value) {
             }
         }
 
-        public static void cleanup() {
-            long curTime = System.currentTimeMillis();
+        private static void cleanUp(long curTime, Map<Long, CircularLongArray> tidHistoryMap) {
             for (Iterator<Map.Entry<Long, CircularLongArray>> it =
                             tidHistoryMap.entrySet().iterator();
                     it.hasNext(); ) {

src/main/java/org/opensearch/performanceanalyzer/metrics/AllMetrics.java

@@ -962,7 +962,9 @@ public enum OSMetrics {
         IO_WRITE_SYSCALL_RATE(Constants.WRITE_SYSCALL_RATE_VALUE),
         IO_TOTAL_SYSCALL_RATE(Constants.TOTAL_SYSCALL_RATE_VALUE),
         THREAD_BLOCKED_TIME(Constants.BLOCKED_TIME_VALUE),
-        THREAD_BLOCKED_EVENT(Constants.BLOCKED_COUNT_VALUE);
+        THREAD_BLOCKED_EVENT(Constants.BLOCKED_COUNT_VALUE),
+        THREAD_WAITED_TIME(Constants.WAITED_TIME_VALUE),
+        THREAD_WAITED_EVENT(Constants.WAITED_COUNT_VALUE);
 
         private final String value;
 
@@ -992,6 +994,8 @@ public static class Constants {
             public static final String TOTAL_SYSCALL_RATE_VALUE = "IO_TotalSyscallRate";
             public static final String BLOCKED_TIME_VALUE = "Thread_Blocked_Time";
             public static final String BLOCKED_COUNT_VALUE = "Thread_Blocked_Event";
+            public static final String WAITED_TIME_VALUE = "Thread_Waited_Time";
+            public static final String WAITED_COUNT_VALUE = "Thread_Waited_Event";
         }
     }
 

src/main/java/org/opensearch/performanceanalyzer/model/MetricsModel.java

@@ -133,7 +133,13 @@ public class MetricsModel {
         allMetricsInitializer.put(
                 OSMetrics.THREAD_BLOCKED_EVENT.toString(),
                 new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values()));
-
+        allMetricsInitializer.put(
+                OSMetrics.THREAD_WAITED_TIME.toString(),
+                new MetricAttributes(
+                        MetricUnits.SEC_PER_EVENT.toString(), AggregatedOSDimension.values()));
+        allMetricsInitializer.put(
+                OSMetrics.THREAD_WAITED_EVENT.toString(),
+                new MetricAttributes(MetricUnits.COUNT.toString(), AggregatedOSDimension.values()));
         // Latency Metric
         allMetricsInitializer.put(
                 CommonMetric.LATENCY.toString(),

src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Event.java

@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package org.opensearch.performanceanalyzer.rca.framework.api.metrics;
+
+
+import org.opensearch.performanceanalyzer.metrics.AllMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.api.Metric;
+
+public class Thread_Waited_Event extends Metric {
+    public Thread_Waited_Event(long evaluationIntervalSeconds) {
+        super(AllMetrics.OSMetrics.THREAD_WAITED_EVENT.name(), evaluationIntervalSeconds);
+    }
+}

src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/metrics/Thread_Waited_Time.java

@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package org.opensearch.performanceanalyzer.rca.framework.api.metrics;
+
+
+import org.opensearch.performanceanalyzer.metrics.AllMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.api.Metric;
+
+public class Thread_Waited_Time extends Metric {
+    public Thread_Waited_Time(long evaluationIntervalSeconds) {
+        super(AllMetrics.OSMetrics.THREAD_WAITED_TIME.name(), evaluationIntervalSeconds);
+    }
+}

src/test/java/org/opensearch/performanceanalyzer/reader/OSMetricsSnapshotTests.java

@@ -234,6 +234,8 @@ private HashMap<String, Double> getMetricsMap() {
                 this.put(AllMetrics.OSMetrics.Constants.TOTAL_SYSCALL_RATE_VALUE, 2342343223d);
                 this.put(AllMetrics.OSMetrics.Constants.BLOCKED_TIME_VALUE, 23432.32432d);
                 this.put(AllMetrics.OSMetrics.Constants.BLOCKED_COUNT_VALUE, 23432.32432d);
+                this.put(AllMetrics.OSMetrics.Constants.WAITED_TIME_VALUE, 23432.32432d);
+                this.put(AllMetrics.OSMetrics.Constants.WAITED_COUNT_VALUE, 23432.32432d);
             }
         };
     }