Skip to content

Commit bbced21

Browse files
ShephalimittalShephali Mittal
Shephalimittal
authored and
Shephali Mittal
committed
[Backport 2.x] Added Integration tests support for Telemetry span Validations (opensearch-project#8444)
Signed-off-by: Shephali Mittal <shephalm@amazon.com>
1 parent ab8e306 commit bbced21

File tree

16 files changed

+773
-51
lines changed

16 files changed

+773
-51
lines changed

‎plugins/telemetry-otel/build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import org.opensearch.gradle.Architecture
1111
import org.opensearch.gradle.OS
1212
import org.opensearch.gradle.info.BuildParams
1313

14+
apply plugin: 'opensearch.internal-cluster-test'
15+
1416
opensearchplugin {
1517
description 'Opentelemetry based telemetry implementation.'
1618
classname 'org.opensearch.telemetry.OTelTelemetryPlugin'
@@ -29,6 +31,7 @@ dependencies {
2931
api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha"
3032
api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}-alpha"
3133
api "io.opentelemetry:opentelemetry-api-logs:${versions.opentelemetry}-alpha"
34+
testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${versions.opentelemetry}"
3235
}
3336

3437

‎plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/InMemorySingletonSpanExporter.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.telemetry.tracing;
10+
11+
import io.opentelemetry.sdk.common.CompletableResultCode;
12+
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
13+
import io.opentelemetry.sdk.trace.data.SpanData;
14+
import io.opentelemetry.sdk.trace.export.SpanExporter;
15+
import org.opensearch.test.telemetry.tracing.MockSpanData;
16+
17+
import java.util.Collection;
18+
import java.util.List;
19+
import java.util.stream.Collectors;
20+
21+
public class InMemorySingletonSpanExporter implements SpanExporter {
22+
23+
private static final InMemorySingletonSpanExporter INSTANCE = new InMemorySingletonSpanExporter(InMemorySpanExporter.create());
24+
25+
private static InMemorySpanExporter delegate;
26+
27+
public static InMemorySingletonSpanExporter create() {
28+
return INSTANCE;
29+
}
30+
31+
private InMemorySingletonSpanExporter(InMemorySpanExporter delegate) {
32+
InMemorySingletonSpanExporter.delegate = delegate;
33+
}
34+
35+
@Override
36+
public CompletableResultCode export(Collection<SpanData> spans) {
37+
return delegate.export(spans);
38+
}
39+
40+
@Override
41+
public CompletableResultCode flush() {
42+
return delegate.flush();
43+
}
44+
45+
@Override
46+
public CompletableResultCode shutdown() {
47+
return delegate.shutdown();
48+
}
49+
50+
public List<MockSpanData> getFinishedSpanItems() {
51+
return convertSpanDataListToMockSpanDataList(delegate.getFinishedSpanItems());
52+
}
53+
54+
private List<MockSpanData> convertSpanDataListToMockSpanDataList(List<SpanData> spanDataList) {
55+
List<MockSpanData> mockSpanDataList = spanDataList.stream()
56+
.map(
57+
spanData -> new MockSpanData(
58+
spanData.getSpanId(),
59+
spanData.getParentSpanId(),
60+
spanData.getTraceId(),
61+
spanData.getStartEpochNanos(),
62+
spanData.getEndEpochNanos(),
63+
spanData.hasEnded(),
64+
spanData.getName()
65+
)
66+
)
67+
.collect(Collectors.toList());
68+
return mockSpanDataList;
69+
}
70+
}

‎plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/IntegrationTestOTelTelemetryPlugin.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.telemetry.tracing;
10+
11+
import io.opentelemetry.api.GlobalOpenTelemetry;
12+
import org.opensearch.common.settings.Settings;
13+
import org.opensearch.telemetry.OTelTelemetryPlugin;
14+
import org.opensearch.telemetry.Telemetry;
15+
import org.opensearch.telemetry.TelemetrySettings;
16+
17+
import java.util.Optional;
18+
19+
/**
20+
* Telemetry plugin used for Integration tests.
21+
*/
22+
public class IntegrationTestOTelTelemetryPlugin extends OTelTelemetryPlugin {
23+
/**
24+
* Creates IntegrationTestOTelTelemetryPlugin
25+
* @param settings cluster settings
26+
*/
27+
public IntegrationTestOTelTelemetryPlugin(Settings settings) {
28+
super(settings);
29+
}
30+
31+
/**
32+
* This method overrides getTelemetry() method in OTel plugin class, so we create only one instance of global OpenTelemetry
33+
* resetForTest() will set OpenTelemetry to null again.
34+
* @param settings cluster settings
35+
*/
36+
public Optional<Telemetry> getTelemetry(TelemetrySettings settings) {
37+
GlobalOpenTelemetry.resetForTest();
38+
return super.getTelemetry(settings);
39+
}
40+
}

‎plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.telemetry.tracing;
10+
11+
import org.opensearch.client.Client;
12+
import org.opensearch.common.settings.Settings;
13+
import org.opensearch.common.unit.TimeValue;
14+
import org.opensearch.plugins.Plugin;
15+
import org.opensearch.telemetry.OTelTelemetrySettings;
16+
import org.opensearch.telemetry.TelemetrySettings;
17+
import org.opensearch.test.OpenSearchIntegTestCase;
18+
19+
import java.util.Arrays;
20+
import java.util.Collection;
21+
22+
import static org.opensearch.index.query.QueryBuilders.queryStringQuery;
23+
24+
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, supportsDedicatedMasters = false, minNumDataNodes = 2)
25+
public class TelemetryTracerDisabledSanityIT extends OpenSearchIntegTestCase {
26+
27+
@Override
28+
protected Settings nodeSettings(int nodeOrdinal) {
29+
return Settings.builder()
30+
.put(super.nodeSettings(nodeOrdinal))
31+
.put(
32+
OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(),
33+
"org.opensearch.telemetry.tracing.InMemorySingletonSpanExporter"
34+
)
35+
.put(OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING.getKey(), TimeValue.timeValueSeconds(1))
36+
.build();
37+
}
38+
39+
@Override
40+
protected Collection<Class<? extends Plugin>> nodePlugins() {
41+
return Arrays.asList(IntegrationTestOTelTelemetryPlugin.class);
42+
}
43+
44+
@Override
45+
protected boolean addMockTelemetryPlugin() {
46+
return false;
47+
}
48+
49+
public void testSanityCheckWhenTracingDisabled() throws Exception {
50+
Client client = client();
51+
// DISABLE TRACING
52+
client.admin()
53+
.cluster()
54+
.prepareUpdateSettings()
55+
.setTransientSettings(Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), false))
56+
.get();
57+
58+
// Create Index and ingest data
59+
String indexName = "test-index-11";
60+
Settings basicSettings = Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 1).build();
61+
createIndex(indexName, basicSettings);
62+
indexRandom(true, client.prepareIndex(indexName).setId("1").setSource("field1", "t`"));
63+
64+
ensureGreen();
65+
refresh();
66+
67+
// Make the search call;
68+
client.prepareSearch().setQuery(queryStringQuery("fox")).get();
69+
70+
// Sleep for about 3s to wait for traces are published (the delay is 1s)
71+
Thread.sleep(3000);
72+
73+
InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.create();
74+
assertTrue(exporter.getFinishedSpanItems().isEmpty());
75+
}
76+
77+
}

‎plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.telemetry.tracing;
10+
11+
import org.opensearch.common.settings.Settings;
12+
import org.opensearch.common.unit.TimeValue;
13+
import org.opensearch.telemetry.TelemetrySettings;
14+
import org.opensearch.test.OpenSearchIntegTestCase;
15+
import org.opensearch.telemetry.OTelTelemetrySettings;
16+
import org.opensearch.plugins.Plugin;
17+
import org.opensearch.client.Client;
18+
import org.opensearch.test.telemetry.tracing.TelemetryValidators;
19+
import org.opensearch.test.telemetry.tracing.validators.AllSpansAreEndedProperly;
20+
import org.opensearch.test.telemetry.tracing.validators.AllSpansHaveUniqueId;
21+
import org.opensearch.test.telemetry.tracing.validators.NumberOfTraceIDsEqualToRequests;
22+
import org.opensearch.test.telemetry.tracing.validators.TotalRootSpansEqualToRequests;
23+
24+
import java.util.Arrays;
25+
import java.util.Collection;
26+
27+
import static org.opensearch.index.query.QueryBuilders.queryStringQuery;
28+
29+
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, minNumDataNodes = 2)
30+
public class TelemetryTracerEnabledSanityIT extends OpenSearchIntegTestCase {
31+
32+
@Override
33+
protected Settings nodeSettings(int nodeOrdinal) {
34+
return Settings.builder()
35+
.put(super.nodeSettings(nodeOrdinal))
36+
.put(
37+
OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(),
38+
"org.opensearch.telemetry.tracing.InMemorySingletonSpanExporter"
39+
)
40+
.put(OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING.getKey(), TimeValue.timeValueSeconds(1))
41+
.build();
42+
}
43+
44+
@Override
45+
protected Collection<Class<? extends Plugin>> nodePlugins() {
46+
return Arrays.asList(IntegrationTestOTelTelemetryPlugin.class);
47+
}
48+
49+
@Override
50+
protected boolean addMockTelemetryPlugin() {
51+
return false;
52+
}
53+
54+
public void testSanityChecksWhenTracingEnabled() throws Exception {
55+
Client client = client();
56+
// ENABLE TRACING
57+
client.admin()
58+
.cluster()
59+
.prepareUpdateSettings()
60+
.setTransientSettings(Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true))
61+
.get();
62+
63+
// Create Index and ingest data
64+
String indexName = "test-index-11";
65+
Settings basicSettings = Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0).build();
66+
createIndex(indexName, basicSettings);
67+
indexRandom(true, client.prepareIndex(indexName).setId("1").setSource("field1", "the fox jumps in the well"));
68+
indexRandom(true, client.prepareIndex(indexName).setId("1").setSource("field2", "another fox did the same."));
69+
70+
ensureGreen();
71+
refresh();
72+
73+
// Make the search calls;
74+
client.prepareSearch().setQuery(queryStringQuery("fox")).get();
75+
client.prepareSearch().setQuery(queryStringQuery("jumps")).get();
76+
77+
// Sleep for about 3s to wait for traces are published, delay is (the delay is 1s).
78+
Thread.sleep(3000);
79+
80+
TelemetryValidators validators = new TelemetryValidators(
81+
Arrays.asList(
82+
new AllSpansAreEndedProperly(),
83+
new AllSpansHaveUniqueId(),
84+
new NumberOfTraceIDsEqualToRequests(),
85+
new TotalRootSpansEqualToRequests()
86+
)
87+
);
88+
89+
InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.create();
90+
if (!exporter.getFinishedSpanItems().isEmpty()) {
91+
validators.validate(exporter.getFinishedSpanItems(), 2);
92+
}
93+
}
94+
95+
}

‎test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,12 +157,13 @@
157157
import org.opensearch.search.SearchBootstrapSettings;
158158
import org.opensearch.search.SearchHit;
159159
import org.opensearch.search.SearchService;
160+
import org.opensearch.telemetry.TelemetrySettings;
160161
import org.opensearch.test.client.RandomizingClient;
161162
import org.opensearch.test.disruption.NetworkDisruption;
162163
import org.opensearch.test.disruption.ServiceDisruptionScheme;
163164
import org.opensearch.test.store.MockFSIndexStore;
164-
import org.opensearch.test.transport.MockTransportService;
165165
import org.opensearch.test.telemetry.MockTelemetryPlugin;
166+
import org.opensearch.test.transport.MockTransportService;
166167
import org.opensearch.transport.TransportInterceptor;
167168
import org.opensearch.transport.TransportRequest;
168169
import org.opensearch.transport.TransportRequestHandler;
@@ -788,6 +789,7 @@ protected Settings featureFlagSettings() {
788789
for (Setting builtInFlag : FeatureFlagSettings.BUILT_IN_FEATURE_FLAGS) {
789790
featureSettings.put(builtInFlag.getKey(), builtInFlag.getDefaultRaw(Settings.EMPTY));
790791
}
792+
// Enabling Telemetry setting by default
791793
featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true);
792794
return featureSettings.build();
793795
}
@@ -1959,6 +1961,10 @@ protected Settings nodeSettings(int nodeOrdinal) {
19591961
// when tests are run with concurrent segment search enabled
19601962
builder.put(SearchBootstrapSettings.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_KEY, 2);
19611963
}
1964+
// Enable tracer only when Telemetry Setting is enabled
1965+
if (featureFlagSettings().getAsBoolean(FeatureFlags.TELEMETRY_SETTING.getKey(), false)) {
1966+
builder.put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true);
1967+
}
19621968
return builder.build();
19631969
}
19641970

@@ -2114,6 +2120,11 @@ protected boolean addMockGeoShapeFieldMapper() {
21142120
return true;
21152121
}
21162122

2123+
/** Returns {@code true} if this test cluster should have tracing enabled with MockTelemetryPlugin */
2124+
protected boolean addMockTelemetryPlugin() {
2125+
return true;
2126+
}
2127+
21172128
/**
21182129
* Returns a function that allows to wrap / filter all clients that are exposed by the test cluster. This is useful
21192130
* for debugging or request / response pre and post processing. It also allows to intercept all calls done by the test
@@ -2158,8 +2169,9 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
21582169
if (addMockGeoShapeFieldMapper()) {
21592170
mocks.add(TestGeoShapeFieldMapperPlugin.class);
21602171
}
2161-
mocks.add(MockTelemetryPlugin.class);
2162-
2172+
if (addMockTelemetryPlugin()) {
2173+
mocks.add(MockTelemetryPlugin.class);
2174+
}
21632175
return Collections.unmodifiableList(mocks);
21642176
}
21652177

0 commit comments

Comments
 (0)