[BUG] org.opensearch.action.admin.indices.create.RemoteCloneIndexIT.testCreateCloneIndex is flaky #11179
Description
Describe the bug
The test case org.opensearch.action.admin.indices.create.RemoteCloneIndexIT.testCreateCloneIndex is flaky
org.opensearch.action.admin.indices.create.RemoteCloneIndexIT.testCreateCloneIndex
java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1089)
at org.opensearch.test.InternalTestCluster.assertRequestsFinished(InternalTestCluster.java:2787)
at org.opensearch.test.InternalTestCluster.assertAfterTest(InternalTestCluster.java:2762)
at org.opensearch.test.OpenSearchIntegTestCase.afterInternal(OpenSearchIntegTestCase.java:626)
at org.opensearch.test.OpenSearchIntegTestCase.cleanUpCluster(OpenSearchIntegTestCase.java:2409)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:833)
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Suppressed: java.lang.AssertionError: All incoming requests on node [node_s2] should have finished. Expected 0 but got 237; pending tasks [[]]
at org.opensearch.test.InternalTestCluster.lambda$assertRequestsFinished$44(InternalTestCluster.java:2796)
at org.opensearch.test.OpenSearchTestCase.assertBusy(OpenSearchTestCase.java:1077)
... 42 more
Kas 13, 2023 12:08:38 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
WARNING: Uncaught exception in thread: Thread[opensearch[node_s2][generic][T#3],5,TGRP-RemoteCloneIndexIT]
java.lang.AssertionError: inconsistent generation
at __randomizedtesting.SeedInfo.seed([12ED910E5B8FEFC]:0)
at org.opensearch.index.translog.transfer.TranslogCheckpointTransferSnapshot$Builder.build(TranslogCheckpointTransferSnapshot.java:180)
at org.opensearch.index.translog.RemoteFsTranslog.upload(RemoteFsTranslog.java:338)
at org.opensearch.index.translog.RemoteFsTranslog.prepareAndUpload(RemoteFsTranslog.java:310)
at org.opensearch.index.translog.RemoteFsTranslog.sync(RemoteFsTranslog.java:365)
at org.opensearch.index.translog.InternalTranslogManager.syncTranslog(InternalTranslogManager.java:196)
at org.opensearch.index.engine.InternalEngine.syncTranslog(InternalEngine.java:610)
at org.opensearch.index.shard.IndexShard.postActivatePrimaryMode(IndexShard.java:3449)
at org.opensearch.index.shard.IndexShard.activateWithPrimaryContext(IndexShard.java:3440)
at org.opensearch.indices.recovery.RecoveryTarget.handoffPrimaryContext(RecoveryTarget.java:303)
at org.opensearch.indices.recovery.PeerRecoveryTargetService$HandoffPrimaryContextRequestHandler.messageReceived(PeerRecoveryTargetService.java:418)
at org.opensearch.indices.recovery.PeerRecoveryTargetService$HandoffPrimaryContextRequestHandler.messageReceived(PeerRecoveryTargetService.java:412)
at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:106)
at org.opensearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:471)
at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:908)
at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
To Reproduce
./gradlew ':server:internalClusterTest' --tests "org.opensearch.action.admin.indices.create.RemoteCloneIndexIT.testCreateCloneIndex" -Dtests.seed=12ED910E5B8FEFC
Expected behavior
The test must always pass
Plugins
Standard
Screenshots
N/A
Host/Environment (please complete the following information):
- CI
Additional context