Compatibility issue with /_mget: RHLC 2.x connected to OpenSearch Cluster 1.x (opensearch-project#4812)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
(cherry picked from commit d85f8cd)
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

Author: reta

.ci/bwcVersions

@@ -43,6 +43,7 @@ BWC_VERSION:
   - "1.3.4"
   - "1.3.5"
   - "1.3.6"
+  - "1.3.7"
   - "2.0.0"
   - "2.0.1"
   - "2.0.2"

qa/mixed-cluster/build.gradle

@@ -38,6 +38,10 @@ apply plugin: 'opensearch.standalone-test'
 apply from : "$rootDir/gradle/bwc-test.gradle"
 apply plugin: 'opensearch.rest-resources'
 
+dependencies {
+  testImplementation project(":client:rest-high-level")
+}
+
 restResources {
   restTests {
     includeCore '*'

qa/mixed-cluster/src/test/java/org/opensearch/backwards/SearchingIT.java

@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.backwards;
+
+import org.apache.http.HttpHost;
+import org.opensearch.action.get.MultiGetRequest;
+import org.opensearch.action.get.MultiGetResponse;
+import org.opensearch.client.Request;
+import org.opensearch.client.RequestOptions;
+import org.opensearch.client.Response;
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.test.rest.OpenSearchRestTestCase;
+import org.opensearch.test.rest.yaml.ObjectPath;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class SearchingIT extends OpenSearchRestTestCase {
+    public void testMultiGet() throws Exception {
+        final Set<HttpHost> nodes = buildNodes();
+
+        final MultiGetRequest multiGetRequest = new MultiGetRequest();
+        multiGetRequest.add("index", "id1");
+
+        try (RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(nodes.toArray(HttpHost[]::new)))) {
+            MultiGetResponse response = client.mget(multiGetRequest, RequestOptions.DEFAULT);
+            assertEquals(1, response.getResponses().length);
+    
+            assertTrue(response.getResponses()[0].isFailed());
+            assertNotNull(response.getResponses()[0].getFailure());
+            assertEquals(response.getResponses()[0].getFailure().getId(), "id1");
+            assertEquals(response.getResponses()[0].getFailure().getIndex(), "index");
+            assertThat(response.getResponses()[0].getFailure().getMessage(), containsString("no such index [index]"));
+       }
+    }
+
+    private Set<HttpHost> buildNodes() throws IOException, URISyntaxException {
+        Response response = client().performRequest(new Request("GET", "_nodes"));
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
+        final Set<HttpHost> nodes = new HashSet<>();
+        for (String id : nodesAsMap.keySet()) {
+            nodes.add(HttpHost.create((String) objectPath.evaluate("nodes." + id + ".http.publish_address")));
+        }
+
+        return nodes;
+    }
+}

server/src/main/java/org/opensearch/Version.java

@@ -90,6 +90,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_1_3_4 = new Version(1030499, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_1_3_5 = new Version(1030599, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_1_3_6 = new Version(1030699, org.apache.lucene.util.Version.LUCENE_8_10_1);
+    public static final Version V_1_3_7 = new Version(1030799, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_2_0_1 = new Version(2000199, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_2_0_2 = new Version(2000299, org.apache.lucene.util.Version.LUCENE_9_1_0);

server/src/main/java/org/opensearch/action/get/MultiGetResponse.java

@@ -63,6 +63,10 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
     private static final ParseField ID = new ParseField("_id");
     private static final ParseField ERROR = new ParseField("error");
     private static final ParseField DOCS = new ParseField("docs");
+    // In mixed clusters, the 1.x cluster could still return the '_type' in the response payload, it has to
+    // be handled gracefully
+    @Deprecated(forRemoval = true)
+    private static final ParseField TYPE = new ParseField("_type");
 
     /**
      * Represents a failure.
@@ -212,6 +216,7 @@ private static MultiGetItemResponse parseItem(XContentParser parser) throws IOEx
                     currentFieldName = parser.currentName();
                     if (INDEX.match(currentFieldName, parser.getDeprecationHandler()) == false
                         && ID.match(currentFieldName, parser.getDeprecationHandler()) == false
+                        && TYPE.match(currentFieldName, parser.getDeprecationHandler()) == false
                         && ERROR.match(currentFieldName, parser.getDeprecationHandler()) == false) {
                         getResult = GetResult.fromXContentEmbedded(parser, index, id);
                     }