Bump Apache HttpClient5 to 5.6 and Apache HttpCore5 to 5.4

CHANGELOG.md

@@ -48,6 +48,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.http-client:google-http-client-appengine` from 2.0.2 to 2.0.3 ([#20332](https://github.com/opensearch-project/OpenSearch/pull/20332))
 - Bump `jackson` from 2.18.1 to 2.20.1 ([#20343](https://github.com/opensearch-project/OpenSearch/pull/20343))
 - Bump `opensearch-protobufs` from 0.24.0 to 1.0.0 and update transport-grpc module compatibility ([#20335](https://github.com/opensearch-project/OpenSearch/pull/20335))
+- Bump Apache HttpClient5 to 5.6 ([#20358](https://github.com/opensearch-project/OpenSearch/pull/20358))
+- Bump Apache HttpCore5 to 5.4 ([#20358](https://github.com/opensearch-project/OpenSearch/pull/20358))
 
 ### Removed
 

client/rest/build.gradle

@@ -149,7 +149,16 @@ thirdPartyAudit {
       'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
       'io.micrometer.core.instrument.search.Search',
       'reactor.blockhound.BlockHound$Builder',
-      'reactor.blockhound.integration.BlockHoundIntegration'
+      'reactor.blockhound.integration.BlockHoundIntegration',
+      //optional compression
+      'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
+      'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
+      'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
+      'com.aayushatharva.brotli4j.encoder.EncoderJNI$Operation',
+      'com.aayushatharva.brotli4j.encoder.EncoderJNI$Wrapper',
+      'com.github.luben.zstd.ZstdDecompressCtx',
+      'com.github.luben.zstd.ZstdDirectBufferCompressingStream',
+      'org.apache.commons.compress.compressors.CompressorStreamFactory'
    )
    ignoreViolations(
       'reactor.core.publisher.CallSiteSupplierFactory$SharedSecretsCallSiteSupplierFactory',

client/rest/licenses/httpclient5-5.6.jar.sha1

@@ -0,0 +1 @@
+f502ee00ba82d44a6a29bda06a18f5b959808e09

client/rest/licenses/httpcore5-5.4.jar.sha1

@@ -0,0 +1 @@
+e40011ec0dae056466399f8e414ede4772001621

client/rest/licenses/httpcore5-h2-5.4.jar.sha1

@@ -0,0 +1 @@
+83cdd62ef3140664f46be59c2c2727141d1c5a32

client/rest/licenses/httpcore5-reactive-5.4.jar.sha1

@@ -0,0 +1 @@
+7ebbb91027261db6f0e80fe03c928b63b116c247

client/rest/src/main/java/org/opensearch/client/RestClient.java

@@ -50,7 +50,6 @@
 import org.apache.hc.client5.http.classic.methods.HttpTrace;
 import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;
 import org.apache.hc.client5.http.config.RequestConfig;
-import org.apache.hc.client5.http.entity.GzipDecompressingEntity;
 import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
 import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
 import org.apache.hc.client5.http.impl.auth.BasicScheme;
@@ -484,12 +483,6 @@ private ResponseOrResponseException convertResponse(InternalRequest request, Nod
         RequestLogger.logResponse(logger, request.httpRequest, node.getHost(), httpResponse);
         int statusCode = httpResponse.getCode();
 
-        Optional.ofNullable(httpResponse.getEntity())
-            .map(HttpEntity::getContentEncoding)
-            .filter("gzip"::equalsIgnoreCase)
-            .map(gzipHeaderValue -> new GzipDecompressingEntity(httpResponse.getEntity()))
-            .ifPresent(httpResponse::setEntity);
-
         Response response = new Response(new RequestLine(request.httpRequest), node.getHost(), httpResponse);
         if (isSuccessfulResponse(statusCode) || request.ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) {
             onResponse(node);

client/rest/src/test/java/org/opensearch/client/RestClientSingleHostIntegTests.java

@@ -378,7 +378,7 @@ public void testRequestResetAndAbort() throws Exception {
      */
     public void testHeaders() throws Exception {
         for (String method : getHttpMethods()) {
-            final Set<String> standardHeaders = new HashSet<>(Arrays.asList("Connection", "Host", "User-agent", "Date"));
+            final Set<String> standardHeaders = new HashSet<>(Arrays.asList("Connection", "Host", "User-agent", "Date", "Accept-encoding"));
             if (method.equals("HEAD") == false) {
                 standardHeaders.add("Content-length");
             }

client/sniffer/build.gradle

@@ -113,5 +113,14 @@ thirdPartyAudit.ignoreMissingClasses(
   'org.apache.logging.log4j.util.StackLocatorUtil',
   //commons-logging provided dependencies
   'javax.servlet.ServletContextEvent',
-  'javax.servlet.ServletContextListener'
+  'javax.servlet.ServletContextListener',
+  //optional compression
+  'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
+  'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
+  'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
+  'com.aayushatharva.brotli4j.encoder.EncoderJNI$Operation',
+  'com.aayushatharva.brotli4j.encoder.EncoderJNI$Wrapper',
+  'com.github.luben.zstd.ZstdDecompressCtx',
+  'com.github.luben.zstd.ZstdDirectBufferCompressingStream',
+  'org.apache.commons.compress.compressors.CompressorStreamFactory'
 )

client/sniffer/licenses/httpclient5-5.6.jar.sha1

@@ -0,0 +1 @@
+f502ee00ba82d44a6a29bda06a18f5b959808e09

client/sniffer/licenses/httpcore5-5.4.jar.sha1

@@ -0,0 +1 @@
+e40011ec0dae056466399f8e414ede4772001621

gradle/libs.versions.toml

@@ -46,8 +46,8 @@ reactor_netty     = "1.3.1"
 reactor           = "3.8.1"
 
 # client dependencies
-httpclient5       = "5.4.4"
-httpcore5         = "5.3.4"
+httpclient5       = "5.6"
+httpcore5         = "5.4"
 httpclient        = "4.5.14"
 httpcore          = "4.4.16"
 httpasyncclient   = "4.1.5"

qa/smoke-test-http/src/test/java/org/opensearch/http/HttpCompressionIT.java

@@ -34,6 +34,7 @@
 import org.apache.hc.client5.http.entity.GzipDecompressingEntity;
 import org.apache.hc.core5.http.HttpHeaders;
 import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.opensearch.client.Request;
 import org.opensearch.client.RequestOptions;
@@ -59,39 +60,41 @@ public class HttpCompressionIT extends OpenSearchRestTestCase {
         }
         """;
 
-    public void testCompressesResponseIfRequested() throws IOException, ParseException {
+    public void testUncompressesResponseIfRequested() throws IOException, ParseException {
+        // See please https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Encoding
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
+                .addHeader(HttpHeaders.ACCEPT_ENCODING, "identity")
+                .build();
+
         Request request = new Request("POST", "/company/_doc/2");
+        request.setOptions(requestOptions);
         request.setJsonEntity(SAMPLE_DOCUMENT);
+
         Response response = client().performRequest(request);
         assertEquals(201, response.getStatusLine().getStatusCode());
         assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING));
         assertThat(response.getEntity(), is(not(instanceOf(GzipDecompressingEntity.class))));
 
         request = new Request("GET", "/company/_doc/2");
-        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
-            .addHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING)
-            .build();
-
-        request.setOptions(requestOptions);
         response = client().performRequest(request);
         assertEquals(200, response.getStatusLine().getStatusCode());
         assertEquals(GZIP_ENCODING, response.getHeader(HttpHeaders.CONTENT_ENCODING));
-        assertThat(response.getEntity(), instanceOf(GzipDecompressingEntity.class));
+        assertThat(response.getEntity(), instanceOf(ByteArrayEntity.class));
 
         String body = EntityUtils.toString(response.getEntity());
         assertThat(body, containsString(SAMPLE_DOCUMENT));
     }
 
-    public void testUncompressedResponseByDefault() throws IOException {
+    public void testCompressedResponseByDefault() throws IOException {
         Response response = client().performRequest(new Request("GET", "/"));
         assertEquals(200, response.getStatusLine().getStatusCode());
-        assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING));
+        assertEquals(GZIP_ENCODING, response.getHeader(HttpHeaders.CONTENT_ENCODING));
 
         Request request = new Request("POST", "/company/_doc/1");
         request.setJsonEntity(SAMPLE_DOCUMENT);
         response = client().performRequest(request);
         assertEquals(201, response.getStatusLine().getStatusCode());
-        assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING));
+        assertEquals(GZIP_ENCODING, response.getHeader(HttpHeaders.CONTENT_ENCODING));
         assertThat(response.getEntity(), is(not(instanceOf(GzipDecompressingEntity.class))));
     }