Bump to 3.7.0-SNAPSHOT, fix security plugin compatibility, use admin cert auth for health checks

.github/workflows/build.yml

@@ -20,7 +20,6 @@ jobs:
           - 25
         os:
           - windows-latest
-          - macos-latest
     # Job name
     name: Java ${{ matrix.java }} On ${{ matrix.os }}
     runs-on: ${{ matrix.os }}

.gitignore

@@ -13,3 +13,6 @@ build/
 __pycache__/
 Pipfile
 Pipfile.lock
+bin/main/
+bin/test/
+.kotlin/

build.gradle

@@ -35,7 +35,7 @@ import org.opensearch.gradle.test.RestIntegTestTask
 buildscript {
     ext {
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "3.6.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.7.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         // e.g. 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT
         version_tokens = opensearch_version.tokenize('-')
@@ -274,6 +274,8 @@ class CrossClusterWaitForHttpResource {
     private URL url;
     private String username;
     private String password;
+    private String clientCertPath;
+    private String clientKeyPath;
     Set<Integer> validResponseCodes = Collections.singleton(200);
 
     CrossClusterWaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
@@ -312,7 +314,54 @@ class CrossClusterWaitForHttpResource {
         this.password = password;
     }
 
+    void setClientCert(String certPath, String keyPath) {
+        this.clientCertPath = certPath;
+        this.clientKeyPath = keyPath;
+    }
+
     void checkResource() throws IOException {
+        if (clientCertPath != null && clientKeyPath != null) {
+            // Use java.net.http.HttpClient which properly handles client certs
+            def certFactory = java.security.cert.CertificateFactory.getInstance("X.509")
+            def cert = new File(clientCertPath).withInputStream { certFactory.generateCertificate(it) }
+            def rootCa = new File(new File(clientCertPath).parent, "root-ca.pem").withInputStream { certFactory.generateCertificate(it) }
+            def keyLines = new File(clientKeyPath).readLines().findAll { !it.startsWith("-----") }.join("")
+            def keyBytes = Base64.getDecoder().decode(keyLines)
+            def keySpec = new java.security.spec.PKCS8EncodedKeySpec(keyBytes)
+            def key = java.security.KeyFactory.getInstance("RSA").generatePrivate(keySpec)
+
+            def ks = java.security.KeyStore.getInstance("JKS")
+            ks.load(null, null)
+            ks.setKeyEntry("admin", key, "".toCharArray(), [cert, rootCa] as java.security.cert.Certificate[])
+            def kmf = javax.net.ssl.KeyManagerFactory.getInstance(javax.net.ssl.KeyManagerFactory.getDefaultAlgorithm())
+            kmf.init(ks, "".toCharArray())
+
+            TrustManager[] trustAllCerts = [ new X509TrustManager() {
+                X509Certificate[] getAcceptedIssuers() { return null }
+                void checkClientTrusted(X509Certificate[] certs, String authType) {}
+                void checkServerTrusted(X509Certificate[] certs, String authType) {}
+            }] as TrustManager[]
+
+            def sc = javax.net.ssl.SSLContext.getInstance("TLS")
+            sc.init(kmf.getKeyManagers(), trustAllCerts, new java.security.SecureRandom())
+
+            def sslParams = sc.getDefaultSSLParameters()
+            sslParams.setEndpointIdentificationAlgorithm("")
+            def httpClient = java.net.http.HttpClient.newBuilder()
+                .sslContext(sc)
+                .sslParameters(sslParams)
+                .build()
+            def request = java.net.http.HttpRequest.newBuilder()
+                .uri(new URI(this.@url.toString().replace(">=", "%3E%3D")))
+                .header("Authorization", "Basic ")
+                .GET()
+                .build()
+            def response = httpClient.send(request, java.net.http.HttpResponse.BodyHandlers.ofString())
+            if (validResponseCodes.contains(response.statusCode())) {
+                return
+            }
+            throw new IOException(response.statusCode() + " " + response.body())
+        }
         final HttpURLConnection connection = buildConnection()
         connection.connect();
         final Integer response = connection.getResponseCode();
@@ -340,7 +389,25 @@ class CrossClusterWaitForHttpResource {
                 }
             ] as TrustManager[];
             SSLContext sc = SSLContext.getInstance("SSL");
-            sc.init(null, trustAllCerts, new java.security.SecureRandom());
+            javax.net.ssl.KeyManager[] keyManagers = null
+            if (clientCertPath != null && clientKeyPath != null) {
+                def certFactory = java.security.cert.CertificateFactory.getInstance("X.509")
+                def cert = new File(clientCertPath).withInputStream { certFactory.generateCertificate(it) }
+                def rootCa = new File(new File(clientCertPath).parent, "root-ca.pem").withInputStream { certFactory.generateCertificate(it) }
+                def keyLines = new File(clientKeyPath).readLines()
+                    .findAll { !it.startsWith("-----") }
+                    .join("")
+                def keyBytes = Base64.getDecoder().decode(keyLines)
+                def keySpec = new java.security.spec.PKCS8EncodedKeySpec(keyBytes)
+                def key = java.security.KeyFactory.getInstance("RSA").generatePrivate(keySpec)
+                def ks = java.security.KeyStore.getInstance("JKS")
+                ks.load(null, null)
+                ks.setKeyEntry("admin", key, "".toCharArray(), [cert, rootCa] as java.security.cert.Certificate[])
+                def kmf = javax.net.ssl.KeyManagerFactory.getInstance(javax.net.ssl.KeyManagerFactory.getDefaultAlgorithm())
+                kmf.init(ks, "".toCharArray())
+                keyManagers = kmf.getKeyManagers()
+            }
+            sc.init(keyManagers, trustAllCerts, new java.security.SecureRandom());
             connection.setSSLSocketFactory(sc.getSocketFactory());
             // Create all-trusting host name verifier
             HostnameVerifier allHostsValid = new HostnameVerifier() {
@@ -441,8 +508,15 @@ def configureCluster(OpenSearchCluster cluster, Boolean securityEnabled) {
             protocol = "https"
         }
         CrossClusterWaitForHttpResource wait = new CrossClusterWaitForHttpResource(protocol, cluster.getFirstNode().getHttpSocketURI(), cluster.nodes.size())
-        wait.setUsername("admin")
-        wait.setPassword("admin")
+        if (securityEnabled) {
+            wait.setClientCert(
+                "$projectDir/src/test/resources/security/plugin/kirk.pem",
+                "$projectDir/src/test/resources/security/plugin/kirk-key.pem"
+            )
+        } else {
+            wait.setUsername("admin")
+            wait.setPassword("admin")
+        }
         return wait.wait(500)
     }
 
@@ -576,9 +650,10 @@ def addSecurityConfig(OpenSearchNode node, Boolean securityEnabled) {
         node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
         node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
         node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+        node.setting("plugins.security.ssl.http.clientauth_mode", "OPTIONAL")
         node.setting("plugins.security.allow_unsafe_democertificates", "true")
         node.setting("plugins.security.allow_default_init_securityindex", "true")
-        node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
+        node.setting("plugins.security.authcz.admin_dn", "[\"CN=kirk,OU=client,O=client,L=test,C=de\"]")
         node.setting("plugins.security.audit.type", "internal_opensearch")
         node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
         node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")

src/main/kotlin/org/opensearch/replication/action/stats/AutoFollowStatsAction.kt

@@ -27,6 +27,9 @@ import java.io.IOException
 
 class AutoFollowStatsAction : ActionType<AutoFollowStatsResponses>(NAME, reader) {
     companion object {
+        // TODO: Rename to "cluster:admin/plugins/replication/autofollow/stats" in OpenSearch 4.0
+        // This is a cluster-level action but uses the indices: prefix for backward compatibility.
+        // See https://github.com/opensearch-project/security/pull/6038 for plugin-defined default roles.
         const val NAME = "indices:admin/plugins/replication/autofollow/stats"
         val INSTANCE = AutoFollowStatsAction()
         val reader = Writeable.Reader { inp -> AutoFollowStatsResponses(inp) }

src/main/kotlin/org/opensearch/replication/action/stats/FollowerStatsAction.kt

@@ -16,6 +16,9 @@ import org.opensearch.core.common.io.stream.Writeable
 
 class FollowerStatsAction : ActionType<FollowerStatsResponse>(NAME, reader) {
     companion object {
+        // TODO: Rename to "cluster:admin/plugins/replication/follower/stats" in OpenSearch 4.0
+        // This is a cluster-level action but uses the indices: prefix for backward compatibility.
+        // See https://github.com/opensearch-project/security/pull/6038 for plugin-defined default roles.
         const val NAME = "indices:admin/plugins/replication/follower/stats"
         val INSTANCE = FollowerStatsAction()
         val reader = Writeable.Reader { inp -> FollowerStatsResponse(inp) }

src/main/kotlin/org/opensearch/replication/action/stats/LeaderStatsAction.kt

@@ -16,6 +16,9 @@ import org.opensearch.core.common.io.stream.Writeable
 
 class LeaderStatsAction : ActionType<LeaderStatsResponse>(NAME, reader) {
     companion object {
+        // TODO: Rename to "cluster:admin/plugins/replication/index/stats" in OpenSearch 4.0
+        // This is a cluster-level action but uses the indices: prefix for backward compatibility.
+        // See https://github.com/opensearch-project/security/pull/6038 for plugin-defined default roles.
         const val NAME = "indices:admin/plugins/replication/index/stats"
         val INSTANCE = LeaderStatsAction()
         val reader = Writeable.Reader { inp -> LeaderStatsResponse(inp) }

src/main/kotlin/org/opensearch/replication/action/stats/TransportAutoFollowStatsAction.kt

@@ -23,6 +23,9 @@ import org.opensearch.cluster.service.ClusterService
 import org.opensearch.common.inject.Inject
 import org.opensearch.replication.task.autofollow.AutoFollowTask
 import org.opensearch.threadpool.ThreadPool
+import org.opensearch.action.support.TransportIndicesResolvingAction
+import org.opensearch.cluster.metadata.OptionallyResolvedIndices
+import org.opensearch.cluster.metadata.ResolvedIndices
 import org.opensearch.transport.TransportService
 
 class TransportAutoFollowStatsAction @Inject constructor(transportService: TransportService,
@@ -31,7 +34,10 @@ class TransportAutoFollowStatsAction @Inject constructor(transportService: Trans
                                                             ) :
         TransportTasksAction<AutoFollowTask, AutoFollowStatsRequest, AutoFollowStatsResponses, AutoFollowStatsResponse>(AutoFollowStatsAction.NAME,
               clusterService, transportService,  actionFilters,
-                ::AutoFollowStatsRequest,  ::AutoFollowStatsResponses, ::AutoFollowStatsResponse, ThreadPool.Names.MANAGEMENT), CoroutineScope by GlobalScope {
+                ::AutoFollowStatsRequest,  ::AutoFollowStatsResponses, ::AutoFollowStatsResponse, ThreadPool.Names.MANAGEMENT), CoroutineScope by GlobalScope,
+        TransportIndicesResolvingAction<AutoFollowStatsRequest> {
+
+    override fun resolveIndices(request: AutoFollowStatsRequest): OptionallyResolvedIndices = ResolvedIndices.of(listOf<String>())
 
     companion object {
         private val log = LogManager.getLogger(TransportAutoFollowStatsAction::class.java)

src/main/kotlin/org/opensearch/replication/action/stats/TransportFollowerStatsAction.kt

@@ -25,6 +25,9 @@ import org.opensearch.replication.metadata.state.ReplicationStateMetadata
 import org.opensearch.replication.seqno.RemoteClusterStats
 import org.opensearch.replication.task.shard.FollowerClusterStats
 import org.opensearch.threadpool.ThreadPool
+import org.opensearch.action.support.TransportIndicesResolvingAction
+import org.opensearch.cluster.metadata.OptionallyResolvedIndices
+import org.opensearch.cluster.metadata.ResolvedIndices
 import org.opensearch.transport.TransportService
 
 class TransportFollowerStatsAction @Inject constructor(transportService: TransportService,
@@ -36,7 +39,10 @@ class TransportFollowerStatsAction @Inject constructor(transportService: Transpo
                                                        private val followerStats: FollowerClusterStats) :
         TransportNodesAction<FollowerStatsRequest, FollowerStatsResponse, NodeStatsRequest, FollowerNodeStatsResponse>(FollowerStatsAction.NAME,
              threadPool, clusterService, transportService,  actionFilters, ::FollowerStatsRequest,  ::NodeStatsRequest, ThreadPool.Names.MANAGEMENT,
-                FollowerNodeStatsResponse::class.java), CoroutineScope by GlobalScope {
+                FollowerNodeStatsResponse::class.java), CoroutineScope by GlobalScope,
+        TransportIndicesResolvingAction<FollowerStatsRequest> {
+
+    override fun resolveIndices(request: FollowerStatsRequest): OptionallyResolvedIndices = ResolvedIndices.of(listOf<String>())
 
     companion object {
         private val log = LogManager.getLogger(TransportFollowerStatsAction::class.java)

src/main/kotlin/org/opensearch/replication/action/stats/TransportLeaderStatsAction.kt

@@ -26,6 +26,9 @@ import org.opensearch.indices.IndicesService
 import org.opensearch.replication.seqno.RemoteClusterRetentionLeaseHelper.Companion.RETENTION_LEASE_PREFIX
 import org.opensearch.replication.seqno.RemoteClusterStats
 import org.opensearch.threadpool.ThreadPool
+import org.opensearch.action.support.TransportIndicesResolvingAction
+import org.opensearch.cluster.metadata.OptionallyResolvedIndices
+import org.opensearch.cluster.metadata.ResolvedIndices
 import org.opensearch.transport.TransportService
 import java.util.concurrent.TimeUnit
 
@@ -38,7 +41,10 @@ class TransportLeaderStatsAction @Inject constructor(transportService: Transport
                                                      private val client: NodeClient) :
         TransportNodesAction<LeaderStatsRequest, LeaderStatsResponse, NodeStatsRequest, LeaderNodeStatsResponse>(LeaderStatsAction.NAME,
              threadPool, clusterService, transportService,  actionFilters, ::LeaderStatsRequest,  ::NodeStatsRequest, ThreadPool.Names.MANAGEMENT,
-                LeaderNodeStatsResponse::class.java), CoroutineScope by GlobalScope {
+                LeaderNodeStatsResponse::class.java), CoroutineScope by GlobalScope,
+        TransportIndicesResolvingAction<LeaderStatsRequest> {
+
+    override fun resolveIndices(request: LeaderStatsRequest): OptionallyResolvedIndices = ResolvedIndices.of(listOf<String>())
 
     companion object {
         private val log = LogManager.getLogger(TransportLeaderStatsAction::class.java)

src/main/kotlin/org/opensearch/replication/metadata/TransportUpdateMetadataAction.kt

@@ -49,10 +49,12 @@ import org.opensearch.tasks.Task
 import org.opensearch.threadpool.ThreadPool
 import org.opensearch.transport.TransportService
 import java.util.*
+import org.opensearch.action.support.TransportIndicesResolvingAction
 
 /*
  This action allows the replication plugin to update the index metadata(mapping, setting & aliases) on the follower index
  when there is a metadata write block(added by the plugin).
+
  */
 class TransportUpdateMetadataAction @Inject constructor(
     transportService: TransportService, actionFilters: ActionFilters, threadPool: ThreadPool,
@@ -62,7 +64,12 @@ class TransportUpdateMetadataAction @Inject constructor(
     val indexAliasService: MetadataIndexAliasesService,
     val indexStateService: MetadataIndexStateService
 ) : TransportClusterManagerNodeAction<UpdateMetadataRequest, AcknowledgedResponse>(UpdateMetadataAction.NAME,
-    transportService, clusterService, threadPool, actionFilters, ::UpdateMetadataRequest, indexNameExpressionResolver) {
+    transportService, clusterService, threadPool, actionFilters, ::UpdateMetadataRequest, indexNameExpressionResolver),
+    TransportIndicesResolvingAction<UpdateMetadataRequest> {
+
+    override fun resolveIndices(request: UpdateMetadataRequest): OptionallyResolvedIndices {
+        return ResolvedIndices.of(request.indexName)
+    }
 
     companion object {
         private val log = LogManager.getLogger(TransportUpdateMetadataAction::class.java)

src/test/resources/security/plugin/README.md

@@ -0,0 +1,62 @@
+# Test Certificates
+
+Self-signed certificates used for integration testing with the security plugin.
+
+## Certificate Details
+
+| File | Description | Subject |
+|---|---|---|
+| `root-ca.pem` | Root CA certificate | `C=de, L=test, O=node, OU=node, CN=root` |
+| `esnode.pem` | Node certificate (transport + HTTP) | `C=de, L=test, O=node, OU=node, CN=node-0.example.com` |
+| `esnode-key.pem` | Node private key (PKCS8, RSA 2048) | — |
+| `kirk.pem` | Admin client certificate | `C=de, L=test, O=client, OU=client, CN=kirk` |
+| `kirk-key.pem` | Admin client private key (PKCS8, RSA 2048) | — |
+
+The `esnode.pem` certificate includes the following Subject Alternative Names:
+- `DNS:node-0.example.com`
+- `DNS:localhost`
+- `IP:127.0.0.1`
+- `IP:::1` (IPv6 localhost)
+- `RID:1.2.3.4.5.5`
+
+The admin DN configured in `opensearch.yml` must match the kirk certificate subject:
+```yaml
+plugins.security.authcz.admin_dn: ["CN=kirk,OU=client,O=client,L=test,C=de"]
+```
+
+## How to Regenerate
+
+```bash
+cd src/test/resources/security/plugin
+
+# Generate Root CA
+openssl genrsa -out root-ca-key.pem 2048
+openssl req -new -x509 -sha256 -key root-ca-key.pem \
+  -subj "/C=de/L=test/O=node/OU=node/CN=root" \
+  -out root-ca.pem -days 3650
+
+# Generate node certificate with SANs (including IPv6 localhost)
+openssl genrsa -out esnode-key.pem 2048
+openssl req -new -key esnode-key.pem \
+  -subj "/C=de/L=test/O=node/OU=node/CN=node-0.example.com" \
+  -out esnode.csr
+echo "subjectAltName=DNS:node-0.example.com,DNS:localhost,IP:127.0.0.1,IP:::1,RID:1.2.3.4.5.5" > esnode-ext.cnf
+openssl x509 -req -in esnode.csr -CA root-ca.pem -CAkey root-ca-key.pem \
+  -CAcreateserial -out esnode.pem -days 3650 -sha256 -extfile esnode-ext.cnf
+
+# Generate kirk admin certificate
+openssl genrsa -out kirk-key.pem 2048
+openssl req -new -key kirk-key.pem \
+  -subj "/C=de/L=test/O=client/OU=client/CN=kirk" \
+  -out kirk.csr
+openssl x509 -req -in kirk.csr -CA root-ca.pem -CAkey root-ca-key.pem \
+  -CAcreateserial -out kirk.pem -days 3650 -sha256
+
+# Cleanup temporary files
+rm -f *.csr *.srl root-ca-key.pem esnode-ext.cnf
+```
+
+## References
+
+- [OpenSearch self-signed certificate documentation](https://opensearch.org/docs/latest/security/configuration/generate-certificates/)
+- [Security plugin DEVELOPER_GUIDE.md](https://github.com/opensearch-project/security/blob/main/DEVELOPER_GUIDE.md)