[BUG] ./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' fails

[andrross]

Running the following test seems to fail consistently on main:

./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest'

Failures:

REPRODUCE WITH: ./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' --tests "org.opensearch.upgrades.FullClusterRestartIT.testRecoveryWithTranslogRetentionDisabled" -Dtests.seed=4BBBB3F7DB8DEDF8 -Dtests.security.manager=true -Dtests.jvm.argline="-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m" -Dtests.locale=nb -Dtests.timezone=Asia/Tomsk -Druntime.java=17

org.opensearch.upgrades.FullClusterRestartIT > testRecoveryWithTranslogRetentionDisabled FAILED
    org.opensearch.client.WarningFailureException: method [PUT], host [http://[::1]:43897], URI [/testrecoverywithtranslogretentiondisabled], status line [HTTP/1.1 200 OK]
    {"acknowledged":true,"shards_acknowledged":true,"index":"testrecoverywithtranslogretentiondisabled"}
        at __randomizedtesting.SeedInfo.seed([4BBBB3F7DB8DEDF8:36AD1B03BAABB8F4]:0)
        at app//org.opensearch.client.RestClient.convertResponse(RestClient.java:346)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:320)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:295)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:976)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:956)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:952)
        at app//org.opensearch.upgrades.FullClusterRestartIT.testRecoveryWithTranslogRetentionDisabled(FullClusterRestartIT.java:1352)

REPRODUCE WITH: ./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' --tests "org.opensearch.upgrades.FullClusterRestartIT.testEnableSoftDeletesOnRestore" -Dtests.seed=4BBBB3F7DB8DEDF8 -Dtests.security.manager=true -Dtests.jvm.argline="-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m" -Dtests.locale=nb -Dtests.timezone=Asia/Tomsk -Druntime.java=17

org.opensearch.upgrades.FullClusterRestartIT > testEnableSoftDeletesOnRestore FAILED
    org.opensearch.client.WarningFailureException: method [PUT], host [http://127.0.0.1:33817], URI [/testenablesoftdeletesonrestore], status line [HTTP/1.1 200 OK]
    {"acknowledged":true,"shards_acknowledged":true,"index":"testenablesoftdeletesonrestore"}
        at __randomizedtesting.SeedInfo.seed([4BBBB3F7DB8DEDF8:7FE4775E071E53C3]:0)
        at app//org.opensearch.client.RestClient.convertResponse(RestClient.java:346)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:320)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:295)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:976)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:956)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:952)
        at app//org.opensearch.upgrades.FullClusterRestartIT.testEnableSoftDeletesOnRestore(FullClusterRestartIT.java:1551)

REPRODUCE WITH: ./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' --tests "org.opensearch.upgrades.FullClusterRestartIT.testOperationBasedRecovery" -Dtests.seed=4BBBB3F7DB8DEDF8 -Dtests.security.manager=true -Dtests.jvm.argline="-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m" -Dtests.locale=nb -Dtests.timezone=Asia/Tomsk -Druntime.java=17

org.opensearch.upgrades.FullClusterRestartIT > testOperationBasedRecovery FAILED
    org.opensearch.client.WarningFailureException: method [PUT], host [http://127.0.0.1:36513], URI [/testoperationbasedrecovery], status line [HTTP/1.1 200 OK]
    {"acknowledged":true,"shards_acknowledged":true,"index":"testoperationbasedrecovery"}
        at __randomizedtesting.SeedInfo.seed([4BBBB3F7DB8DEDF8:5657BFD2DD0B178F]:0)
        at app//org.opensearch.client.RestClient.convertResponse(RestClient.java:346)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:320)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:295)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:976)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:956)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:952)
        at app//org.opensearch.upgrades.FullClusterRestartIT.testOperationBasedRecovery(FullClusterRestartIT.java:1289)

REPRODUCE WITH: ./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' --tests "org.opensearch.upgrades.FullClusterRestartIT.testSnapshotRestore" -Dtests.seed=4BBBB3F7DB8DEDF8 -Dtests.security.manager=true -Dtests.jvm.argline="-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m" -Dtests.locale=nb -Dtests.timezone=Asia/Tomsk -Druntime.java=17

org.opensearch.upgrades.FullClusterRestartIT > testSnapshotRestore FAILED
    org.opensearch.client.WarningFailureException: method [PUT], host [http://[::1]:43897], URI [/testsnapshotrestore], status line [HTTP/1.1 200 OK]
    {"acknowledged":true,"shards_acknowledged":true,"index":"testsnapshotrestore"}
        at __randomizedtesting.SeedInfo.seed([4BBBB3F7DB8DEDF8:74DE590E1130EA87]:0)
        at app//org.opensearch.client.RestClient.convertResponse(RestClient.java:346)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:320)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:295)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:976)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:956)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:952)
        at app//org.opensearch.upgrades.FullClusterRestartIT.testSnapshotRestore(FullClusterRestartIT.java:810)

[andrross]

The tests pass using this 1.x commit:

./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' -Dbwc.refspec.1.x=13bb6cb46a1e2069c084775a3946a48772254132

but fail using this commit:

./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' -Dbwc.refspec.1.x=6004f051402ba61f9d5225a9c799fd4bbd20f679

looks like the backport PR #2315 might be the cause. @VachaShah care to take a look?

[VachaShah]

Sure I will take a look

[VachaShah]

@andrross Looks like this failure is on main, the original PR #1660 for backport #2315 has been merged since a few months to main.

[andrross]

@VachaShah The ':qa:full-cluster-restart:v1.3.0#oldClusterTest' test task checks out the latest commit on the 1.x branch to run some backward compatibility tests. That test fails when running on main. The following command tells the test to use an older commit from 1.x (the commit before your backport commit), and then the tests pass:

./gradlew ':qa:full-cluster-restart:v1.3.0#oldClusterTest' -Dbwc.refspec.1.x=13bb6cb46a1e2069c084775a3946a48772254132

If you point the test at the commit of your backport PR then it fails again. That's why I suspect the backport PR to 1.x is causing the backward compatibility test failure on main.

[VachaShah]

Ah got it! That makes sense. I jumped to the comment too fast 🤦🏼‍♀️ . I will take a look at why this is failing bwc. Should I revert the commit till then?

[andrross]

It looks like there's a problem with an unexpected warning. If it is a quick fix then we can just go ahead with the fix. If you can't figure it out quickly then a revert would be better.

[VachaShah]

Sounds good, on it.

[dreamer-89]

@VachaShah : I started seeing testRecoverReplica more frequently in main gradle check failures. Is this also covered as part of fix here ?

REPRODUCE WITH: ./gradlew ':qa:translog-policy:v1.3.0#Step1OldClusterTest' --tests "org.opensearch.upgrades.TranslogPolicyIT.testRecoverReplica" -Dtests.seed=A7F39AB3B5FF3CF9 -Dtests.security.manager=true -Dtests.jvm.argline="-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m" -Dtests.locale=mt -Dtests.timezone=Chile/Continental -Druntime.java=17

org.opensearch.upgrades.TranslogPolicyIT > testRecoverReplica FAILED
    org.opensearch.client.WarningFailureException: method [PUT], host [http://127.0.0.1:33619], URI [/testrecoverreplica], status line [HTTP/1.1 200 OK]
    {"acknowledged":true,"shards_acknowledged":true,"index":"testrecoverreplica"}
        at __randomizedtesting.SeedInfo.seed([A7F39AB3B5FF3CF9:5D6A55174026FDF4]:0)
        at app//org.opensearch.client.RestClient.convertResponse(RestClient.java:346)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:320)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:295)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:976)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:956)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:952)
        at app//org.opensearch.upgrades.TranslogPolicyIT.testRecoverReplica(TranslogPolicyIT.java:129)

Sample failure: #2335 (comment)

[VachaShah]

@VachaShah : I started seeing testRecoverReplica more frequently in main gradle check failures. Is this also covered as part of fix here ?

REPRODUCE WITH: ./gradlew ':qa:translog-policy:v1.3.0#Step1OldClusterTest' --tests "org.opensearch.upgrades.TranslogPolicyIT.testRecoverReplica" -Dtests.seed=A7F39AB3B5FF3CF9 -Dtests.security.manager=true -Dtests.jvm.argline="-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m" -Dtests.locale=mt -Dtests.timezone=Chile/Continental -Druntime.java=17

org.opensearch.upgrades.TranslogPolicyIT > testRecoverReplica FAILED
    org.opensearch.client.WarningFailureException: method [PUT], host [http://127.0.0.1:33619], URI [/testrecoverreplica], status line [HTTP/1.1 200 OK]
    {"acknowledged":true,"shards_acknowledged":true,"index":"testrecoverreplica"}
        at __randomizedtesting.SeedInfo.seed([A7F39AB3B5FF3CF9:5D6A55174026FDF4]:0)
        at app//org.opensearch.client.RestClient.convertResponse(RestClient.java:346)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:320)
        at app//org.opensearch.client.RestClient.performRequest(RestClient.java:295)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:976)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:956)
        at app//org.opensearch.test.rest.OpenSearchRestTestCase.createIndex(OpenSearchRestTestCase.java:952)
        at app//org.opensearch.upgrades.TranslogPolicyIT.testRecoverReplica(TranslogPolicyIT.java:129)

Sample failure: #2335 (comment)

Yes it will fix the soft deletes warning that are causing this error.

[dblock]

Closed via #2339