diff --git a/build.gradle b/build.gradle index 187574da9e62a..579508137c291 100644 --- a/build.gradle +++ b/build.gradle @@ -433,12 +433,18 @@ gradle.projectsEvaluated { project.tasks.withType(Test) { task -> if (task != null) { - if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17 && BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_23) { task.jvmArgs += ["-Djava.security.manager=allow"] } if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) { task.jvmArgs += ["--add-modules=jdk.incubator.vector"] } + + // Add Java Agent for security sandboxing + if (!(project.path in [':build-tools', ":libs:agent-sm:bootstrap", ":libs:agent-sm:agent"])) { + dependsOn(project(':libs:agent-sm:agent').prepareAgent) + jvmArgs += ["-javaagent:" + project(':libs:agent-sm:agent').jar.archiveFile.get()] + } } } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 65986f2361c9d..e8459443e8a04 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -110,12 +110,12 @@ dependencies { api 'com.netflix.nebula:gradle-info-plugin:12.1.6' api 'org.apache.rat:apache-rat:0.15' api "commons-io:commons-io:${props.getProperty('commonsio')}" - api "net.java.dev.jna:jna:5.14.0" + api "net.java.dev.jna:jna:5.16.0" api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5' api 'org.jdom:jdom2:2.0.6.1' api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}" api 'de.thetaphi:forbiddenapis:3.8' - api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6' + api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.9.6' api 'com.networknt:json-schema-validator:1.2.0' diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java index d79dfb1124757..4932009132457 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java @@ -115,7 +115,8 @@ public void execute(Task t) { test.jvmArgs("--illegal-access=warn"); } } - if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) { + if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0 + && test.getJavaVersion().compareTo(JavaVersion.VERSION_24) < 0) { test.jvmArgs("-Djava.security.manager=allow"); } } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 888cd8d4bf5b5..654af7da65662 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -77,9 +77,9 @@ import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "21.0.6+7"; + private static final String SYSTEM_JDK_VERSION = "23.0.2+7"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "21.0.6+7"; + private static final String GRADLE_JDK_VERSION = "23.0.2+7"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution diff --git a/client/rest-high-level/src/test/resources/org/opensearch/bootstrap/test.policy b/client/rest-high-level/src/test/resources/org/opensearch/bootstrap/test.policy index 2604c2492d8ab..96cd3e9f148cf 100644 --- a/client/rest-high-level/src/test/resources/org/opensearch/bootstrap/test.policy +++ b/client/rest-high-level/src/test/resources/org/opensearch/bootstrap/test.policy @@ -8,4 +8,5 @@ grant { permission java.net.SocketPermission "*", "connect,resolve"; + permission java.net.NetPermission "accessUnixDomainSocket"; }; diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 792b1ab57ddbc..f42dc422cb938 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -38,6 +38,9 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla into('lib') { with libFiles() } + into('agent') { + with agentFiles() + } into('config') { dirPermissions { unix 0750 @@ -226,3 +229,9 @@ subprojects { group = "org.opensearch.distribution" } + +tasks.each { + if (it.name.startsWith("build")) { + it.dependsOn project(':libs:agent-sm:agent').assemble + } +} diff --git a/distribution/build.gradle b/distribution/build.gradle index 8fe9a89059a50..e863d5ab21fe0 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -357,6 +357,18 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } + agentFiles = { + copySpec { + from(project(':libs:agent-sm:agent').prepareAgent) { + include '**/*.jar' + exclude '**/*-javadoc.jar' + exclude '**/*-sources.jar' + // strip the version since jvm.options is using agent without version + rename("opensearch-agent-${project.version}.jar", "opensearch-agent.jar") + } + } + } + modulesFiles = { platform -> copySpec { eachFile { diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index a8c96f33ce51d..2e3d8474b9a3b 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -77,7 +77,7 @@ ${error.file} 9-:-Xlog:gc*,gc+age=trace,safepoint:file=${loggc}:utctime,pid,tags:filecount=32,filesize=64m # Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380) -18-:-Djava.security.manager=allow +18-23:-Djava.security.manager=allow # JDK 20+ Incubating Vector Module for SIMD optimizations; # disabling may reduce performance on vector optimized lucene @@ -89,3 +89,6 @@ ${error.file} # See please https://bugs.openjdk.org/browse/JDK-8341127 (openjdk/jdk#21283) 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached + +# It should be JDK-24 (but we cannot bring JDK-24 since Gradle does not support it yet) +21-:-javaagent:agent/opensearch-agent.jar diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java index af7138569972a..5bedb3ac5ca3e 100644 --- a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java +++ b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java @@ -85,7 +85,7 @@ static List systemJvmOptions() { } private static String allowSecurityManagerOption() { - if (Runtime.version().feature() > 17) { + if (Runtime.version().feature() > 17 && Runtime.version().feature() < 24) { return "-Djava.security.manager=allow"; } else { return ""; diff --git a/gradle/ide.gradle b/gradle/ide.gradle index c16205468d63d..79df92abec2e5 100644 --- a/gradle/ide.gradle +++ b/gradle/ide.gradle @@ -82,7 +82,7 @@ if (System.getProperty('idea.active') == 'true') { runConfigurations { defaults(JUnit) { vmParameters = '-ea -Djava.locale.providers=SPI,CLDR' - if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) { + if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17 && BuildParams.runtimeJavaVersion < JavaVersion.VERSION_24) { vmParameters += ' -Djava.security.manager=allow' } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4d3aebf83eecc..0995bdafcf7b4 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ opensearch = "3.0.0" lucene = "10.1.0" bundled_jdk_vendor = "adoptium" -bundled_jdk = "21.0.6+7" +bundled_jdk = "23.0.2+7" # optional dependencies spatial4j = "0.7" diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index 6e31f838e678a..9f27dc5cadcd2 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -106,6 +106,7 @@ configure([ project(":libs:opensearch-secure-sm"), project(":libs:opensearch-ssl-config"), project(":libs:opensearch-x-content"), + project(":libs:agent-sm:agent-policy"), project(":modules:aggs-matrix-stats"), project(":modules:analysis-common"), project(":modules:geo"), diff --git a/libs/agent-sm/agent-policy/build.gradle b/libs/agent-sm/agent-policy/build.gradle new file mode 100644 index 0000000000000..997ed5ddf174b --- /dev/null +++ b/libs/agent-sm/agent-policy/build.gradle @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.build' +apply plugin: 'opensearch.publish' + +ext { + failOnJavadocWarning = false +} + +base { + archivesName = 'opensearch-agent-policy' +} + +disableTasks('forbiddenApisMain') + +test.enabled = false +testingConventions.enabled = false diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/package-info.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/package-info.java new file mode 100644 index 0000000000000..0724b60d1777f --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Java Agent Policy + */ +package org.opensearch; diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/GrantEntry.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/GrantEntry.java new file mode 100644 index 0000000000000..aaf00f05ce637 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/GrantEntry.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.secure_sm.policy; + +import java.io.PrintWriter; +import java.util.LinkedList; +import java.util.List; + +public class GrantEntry { + public String codeBase; + private final LinkedList permissionEntries = new LinkedList<>(); + + public void add(PermissionEntry entry) { + permissionEntries.add(entry); + } + + public List permissionElements() { + return permissionEntries; + } + + public void write(PrintWriter out) { + out.print("grant"); + if (codeBase != null) { + out.print(" Codebase \""); + out.print(codeBase); + out.print("\""); + } + out.println(" {"); + for (PermissionEntry pe : permissionEntries) { + out.print(" permission "); + out.print(pe.permission); + if (pe.name != null) { + out.print(" \""); + out.print(pe.name); + out.print("\""); + } + if (pe.action != null) { + out.print(", \""); + out.print(pe.action); + out.print("\""); + } + out.println(";"); + } + out.println("};"); + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PermissionEntry.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PermissionEntry.java new file mode 100644 index 0000000000000..289a902a225ac --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PermissionEntry.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.secure_sm.policy; + +import java.io.PrintWriter; +import java.util.Objects; + +public class PermissionEntry { + public String permission; + public String name; + public String action; + + @Override + public int hashCode() { + return Objects.hash(permission, name, action); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + + return obj instanceof PermissionEntry that + && Objects.equals(this.permission, that.permission) + && Objects.equals(this.name, that.name) + && Objects.equals(this.action, that.action); + } + + public void write(PrintWriter out) { + out.print("permission "); + out.print(permission); + if (name != null) { + out.print(" \""); + out.print(name.replaceAll("\\\\", "\\\\\\\\").replaceAll("\"", "\\\\\\\"")); + out.print('"'); + } + if (action != null) { + out.print(", \""); + out.print(action); + out.print('"'); + } + out.println(";"); + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyFile.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyFile.java new file mode 100644 index 0000000000000..e204d3dcb9053 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyFile.java @@ -0,0 +1,361 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FilePermission; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.NetPermission; +import java.net.SocketPermission; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.security.AllPermission; +import java.security.CodeSource; +import java.security.Permission; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.security.ProtectionDomain; +import java.security.SecurityPermission; +import java.security.cert.Certificate; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.PropertyPermission; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +@SuppressWarnings("removal") +public class PolicyFile extends java.security.Policy { + public static final Set PERM_CLASSES_TO_SKIP = Set.of( + "org.opensearch.secure_sm.ThreadContextPermission", + "org.opensearch.secure_sm.ThreadPermission", + "org.opensearch.SpecialPermission", + "org.bouncycastle.crypto.CryptoServicesPermission", + "org.opensearch.script.ClassPermission", + "javax.security.auth.AuthPermission", + "javax.security.auth.kerberos.ServicePermission" + ); + + private volatile PolicyInfo policyInfo; + private URL url; + + public PolicyFile(URL url) { + this.url = url; + try { + init(url); + } catch (PolicyInitializationException e) { + throw new RuntimeException("Failed to initialize policy file", e); + } + } + + private void init(URL url) throws PolicyInitializationException { + PolicyInfo newInfo = new PolicyInfo(); + initPolicyFile(newInfo, url); + policyInfo = newInfo; + } + + private void initPolicyFile(final PolicyInfo newInfo, final URL url) throws PolicyInitializationException { + init(url, newInfo); + } + + private void init(URL policy, PolicyInfo newInfo) throws PolicyInitializationException { + try (InputStreamReader reader = new InputStreamReader(getInputStream(policy), StandardCharsets.UTF_8)) { + PolicyParser policyParser = new PolicyParser(); + policyParser.read(reader); + + for (GrantEntry grantEntry : policyParser.grantElements()) { + addGrantEntry(grantEntry, newInfo); + } + + } catch (Exception e) { + throw new PolicyInitializationException("Failed to load policy from : " + policy, e); + } + } + + public static InputStream getInputStream(URL url) throws IOException { + if ("file".equals(url.getProtocol())) { + String path = url.getFile().replace('/', File.separatorChar); + path = URLDecoder.decode(path, StandardCharsets.UTF_8); + return new FileInputStream(path); + } else { + return url.openStream(); + } + } + + private CodeSource getCodeSource(GrantEntry grantEntry, PolicyInfo newInfo) throws PolicyInitializationException { + try { + Certificate[] certs = null; + URL location = (grantEntry.codeBase != null) ? newURL(grantEntry.codeBase) : null; + return canonicalizeCodebase(new CodeSource(location, certs)); + } catch (Exception e) { + throw new PolicyInitializationException("Failed to get CodeSource", e); + } + } + + private void addGrantEntry(GrantEntry grantEntry, PolicyInfo newInfo) throws PolicyInitializationException { + CodeSource codesource = getCodeSource(grantEntry, newInfo); + if (codesource == null) { + throw new PolicyInitializationException("Null CodeSource for: " + grantEntry.codeBase); + } + + PolicyEntry entry = new PolicyEntry(codesource); + List permissionList = grantEntry.permissionElements(); + for (PermissionEntry pe : permissionList) { + expandPermissionName(pe); + try { + Optional perm = getInstance(pe.permission, pe.name, pe.action); + if (perm.isPresent()) { + entry.add(perm.get()); + } + } catch (ClassNotFoundException e) { + // these were mostly custom permission classes added for security + // manager. Since security manager is deprecated, we can skip these + // permissions classes. + if (PERM_CLASSES_TO_SKIP.contains(pe.permission)) { + continue; // skip this permission + } + throw new PolicyInitializationException("Permission class not found: " + pe.permission, e); + } + } + newInfo.policyEntries.add(entry); + } + + private void expandPermissionName(PermissionEntry pe) { + if (pe.name == null || !pe.name.contains("${{")) { + return; + } + + int startIndex = 0; + int b, e; + StringBuilder sb = new StringBuilder(); + + while ((b = pe.name.indexOf("${{", startIndex)) != -1 && (e = pe.name.indexOf("}}", b)) != -1) { + sb.append(pe.name, startIndex, b); + String value = pe.name.substring(b + 3, e); + sb.append("${{").append(value).append("}}"); + startIndex = e + 2; + } + + sb.append(pe.name.substring(startIndex)); + pe.name = sb.toString(); + } + + private static final Optional getInstance(String type, String name, String actions) throws ClassNotFoundException { + Class pc = Class.forName(type, false, null); + Permission answer = getKnownPermission(pc, name, actions); + + return Optional.ofNullable(answer); + } + + private static Permission getKnownPermission(Class claz, String name, String actions) { + if (claz.equals(FilePermission.class)) { + return new FilePermission(name, actions); + } else if (claz.equals(SocketPermission.class)) { + return new SocketPermission(name, actions); + } else if (claz.equals(RuntimePermission.class)) { + return new RuntimePermission(name, actions); + } else if (claz.equals(PropertyPermission.class)) { + return new PropertyPermission(name, actions); + } else if (claz.equals(NetPermission.class)) { + return new NetPermission(name, actions); + } else if (claz.equals(AllPermission.class)) { + return new AllPermission(); + } else if (claz.equals(SecurityPermission.class)) { + return new SecurityPermission(name, actions); + } else { + return null; + } + } + + @Override + public void refresh() { + try { + init(url); + } catch (PolicyInitializationException e) { + throw new RuntimeException("Failed to refresh policy", e); + } + } + + @Override + public boolean implies(ProtectionDomain pd, Permission p) { + if (pd == null || p == null) { + return false; + } + + PermissionCollection pc = policyInfo.getOrCompute(pd, () -> getPermissions(pd)); + return pc != null && pc.implies(p); + } + + @Override + public PermissionCollection getPermissions(ProtectionDomain domain) { + Permissions perms = new Permissions(); + if (domain == null) return perms; + + try { + getPermissionsForProtectionDomain(perms, domain); + } catch (PolicyInitializationException e) { + throw new RuntimeException("Failed to get permissions for domain", e); + } + + PermissionCollection pc = domain.getPermissions(); + if (pc != null) { + synchronized (pc) { + Enumeration e = pc.elements(); + while (e.hasMoreElements()) { + perms.add(e.nextElement()); + } + } + } + + return perms; + } + + @Override + public PermissionCollection getPermissions(CodeSource codesource) { + if (codesource == null) return new Permissions(); + + Permissions perms = new Permissions(); + CodeSource canonicalCodeSource; + + try { + canonicalCodeSource = canonicalizeCodebase(codesource); + } catch (PolicyInitializationException e) { + throw new RuntimeException("Failed to canonicalize CodeSource", e); + } + + for (PolicyEntry entry : policyInfo.policyEntries) { + if (entry.getCodeSource().implies(canonicalCodeSource)) { + for (Permission permission : entry.permissions) { + perms.add(permission); + } + } + } + + return perms; + } + + private void getPermissionsForProtectionDomain(Permissions perms, ProtectionDomain pd) throws PolicyInitializationException { + final CodeSource cs = pd.getCodeSource(); + if (cs == null) return; + + CodeSource canonicalCodeSource = canonicalizeCodebase(cs); + + for (PolicyEntry entry : policyInfo.policyEntries) { + if (entry.getCodeSource().implies(canonicalCodeSource)) { + for (Permission permission : entry.permissions) { + perms.add(permission); + } + } + } + } + + private CodeSource canonicalizeCodebase(CodeSource cs) throws PolicyInitializationException { + URL location = cs.getLocation(); + if (location == null) return cs; + + try { + URL canonicalUrl = canonicalizeUrl(location); + return new CodeSource(canonicalUrl, cs.getCertificates()); + } catch (IOException e) { + throw new PolicyInitializationException("Failed to canonicalize CodeSource", e); + } + } + + @SuppressWarnings("deprecation") + private URL canonicalizeUrl(URL url) throws IOException { + String protocol = url.getProtocol(); + + if ("jar".equals(protocol)) { + String spec = url.getFile(); + int separator = spec.indexOf("!/"); + if (separator != -1) { + try { + url = new URL(spec.substring(0, separator)); + } catch (MalformedURLException e) { + throw new IOException("Malformed nested jar URL", e); + } + } + } + + if ("file".equals(url.getProtocol())) { + String path = url.getPath(); + path = canonicalizePath(path); + return new File(path).toURI().toURL(); + } + + return url; + } + + private String canonicalizePath(String path) throws IOException { + if (path.endsWith("*")) { + path = path.substring(0, path.length() - 1); + return new File(path).getCanonicalPath() + "*"; + } else { + return new File(path).getCanonicalPath(); + } + } + + private static class PolicyEntry { + private final CodeSource codesource; + final List permissions; + + PolicyEntry(CodeSource cs) { + this.codesource = cs; + this.permissions = new ArrayList<>(); + } + + void add(Permission p) { + permissions.add(p); + } + + CodeSource getCodeSource() { + return codesource; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{").append(getCodeSource()).append("\n"); + for (Permission p : permissions) { + sb.append(" ").append(p).append("\n"); + } + sb.append("}\n"); + return sb.toString(); + } + } + + private static class PolicyInfo { + final List policyEntries; + public final Map pdMapping; + + PolicyInfo() { + policyEntries = new ArrayList<>(); + pdMapping = new ConcurrentHashMap<>(); + } + + public PermissionCollection getOrCompute(ProtectionDomain pd, Supplier computeFn) { + return pdMapping.computeIfAbsent(pd, k -> computeFn.get()); + } + + } + + private static URL newURL(String spec) throws MalformedURLException, URISyntaxException { + return new URI(spec).toURL(); + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyInitializationException.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyInitializationException.java new file mode 100644 index 0000000000000..9205c0aecec41 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyInitializationException.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +/** + * Custom exception for failures during policy file parsing, + */ +public class PolicyInitializationException extends Exception { + + public PolicyInitializationException(String message) { + super(message); + } + + public PolicyInitializationException(String message, Throwable cause) { + super(message, cause); + } + + public PolicyInitializationException(Throwable cause) { + super(cause); + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyParser.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyParser.java new file mode 100644 index 0000000000000..85d71e0b92a77 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PolicyParser.java @@ -0,0 +1,208 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +import org.opensearch.secure_sm.policy.PropertyExpander.ExpandException; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.io.StreamTokenizer; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Vector; + +public class PolicyParser { + + private final Vector grantEntries = new Vector<>(); + private TokenStream tokenStream; + + public PolicyParser() {} + + public void read(Reader policy) throws ParsingException, IOException { + if (!(policy instanceof BufferedReader)) { + policy = new BufferedReader(policy); + } + + tokenStream = new TokenStream(policy); + + while (!tokenStream.isEOF()) { + if (peek("grant")) { + parseGrantEntry().ifPresent(this::addGrantEntry); + } + } + } + + private boolean pollOnMatch(String expect) throws ParsingException, IOException { + if (peek(expect)) { + poll(expect); + return true; + } + return false; + } + + private boolean peek(String expected) throws IOException { + Token token = tokenStream.peek(); + return expected.equalsIgnoreCase(token.text); + } + + private String poll(String expected) throws IOException, ParsingException { + Token token = tokenStream.consume(); + + // Match exact keyword or symbol + if (expected.equalsIgnoreCase("grant") + || expected.equalsIgnoreCase("Codebase") + || expected.equalsIgnoreCase("Permission") + || expected.equalsIgnoreCase("{") + || expected.equalsIgnoreCase("}") + || expected.equalsIgnoreCase(";") + || expected.equalsIgnoreCase(",")) { + + if (!expected.equalsIgnoreCase(token.text)) { + throw new ParsingException(token.line, expected, token.text); + } + return token.text; + } + + if (token.type == StreamTokenizer.TT_WORD || token.type == '"' || token.type == '\'') { + return token.text; + } + + throw new ParsingException(token.line, expected, token.text); + } + + private Optional parseGrantEntry() throws ParsingException, IOException { + GrantEntry grantEntry = new GrantEntry(); + poll("grant"); + + while (!peek("{")) { + if (pollOnMatch("Codebase")) { + if (grantEntry.codeBase != null) { + throw new ParsingException(tokenStream.line(), "Multiple Codebase expressions"); + } + + String rawCodebase = poll(tokenStream.peek().text); + try { + grantEntry.codeBase = PropertyExpander.expand(rawCodebase, true).replace(File.separatorChar, '/'); + } catch (ExpandException e) { + // skip this grant as expansion failed due to missing expansion property. + skipCurrentGrantBlock(); + + return Optional.empty(); + } + pollOnMatch(","); + } else { + throw new ParsingException(tokenStream.line(), "Expected codeBase"); + } + } + + poll("{"); + + while (!peek("}")) { + if (peek("Permission")) { + PermissionEntry permissionEntry = parsePermissionEntry(); + grantEntry.add(permissionEntry); + poll(";"); + } else { + throw new ParsingException(tokenStream.line(), "Expected permission entry"); + } + } + + poll("}"); + + if (peek(";")) { + poll(";"); + } + + if (grantEntry.codeBase != null) { + grantEntry.codeBase = grantEntry.codeBase.replace(File.separatorChar, '/'); + } + + return Optional.of(grantEntry); + } + + private void skipCurrentGrantBlock() throws IOException, ParsingException { + // Consume until we find a matching closing '}' + int braceDepth = 0; + + // Go until we find the initial '{' + while (!tokenStream.isEOF()) { + Token token = tokenStream.peek(); + if ("{".equals(token.text)) { + braceDepth++; + tokenStream.consume(); + break; + } + tokenStream.consume(); + } + + // Now consume until matching '}' + while (braceDepth > 0 && !tokenStream.isEOF()) { + Token token = tokenStream.consume(); + if ("{".equals(token.text)) { + braceDepth++; + } else if ("}".equals(token.text)) { + braceDepth--; + } + } + + // Consume optional trailing semicolon + if (peek(";")) { + poll(";"); + } + } + + private PermissionEntry parsePermissionEntry() throws ParsingException, IOException { + PermissionEntry permissionEntry = new PermissionEntry(); + poll("Permission"); + permissionEntry.permission = poll(tokenStream.peek().text); + + if (isQuotedToken(tokenStream.peek())) { + permissionEntry.name = poll(tokenStream.peek().text); + } + + if (peek(",")) { + poll(","); + } + + if (isQuotedToken(tokenStream.peek())) { + permissionEntry.action = poll(tokenStream.peek().text); + } + + return permissionEntry; + } + + private boolean isQuotedToken(Token token) { + return token.type == '"' || token.type == '\''; + } + + public void addGrantEntry(GrantEntry grantEntry) { + grantEntries.addElement(grantEntry); + } + + public List grantElements() { + return Collections.unmodifiableList(grantEntries); + } + + public static class ParsingException extends Exception { + public ParsingException(String message) { + super(message); + } + + public ParsingException(int line, String expected) { + super("line " + line + ": expected [" + expected + "]"); + } + + public ParsingException(int line, String expected, String found) { + super("line " + line + ": expected [" + expected + "], found [" + found + "]"); + } + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PropertyExpander.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PropertyExpander.java new file mode 100644 index 0000000000000..757062d46f226 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/PropertyExpander.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PropertyExpander { + + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$\\{\\{(?.*?)}}|\\$\\{(?.*?)}"); + + public static class ExpandException extends GeneralSecurityException { + private static final long serialVersionUID = -1L; + + public ExpandException(String message) { + super(message); + } + } + + public static String expand(String value) throws ExpandException { + return expand(value, false); + } + + public static String expand(String value, boolean encodeURL) throws ExpandException { + if (value == null || !value.contains("${")) { + return value; + } + + Matcher matcher = PLACEHOLDER_PATTERN.matcher(value); + StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + String replacement = handleMatch(matcher, encodeURL); + matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(sb); + return sb.toString(); + } + + private static String handleMatch(Matcher match, boolean encodeURL) throws ExpandException { + String escaped = match.group("escaped"); + if (escaped != null) { + return "${{" + escaped + "}}"; + } + + String placeholder = match.group("normal"); + return expandPlaceholder(placeholder, encodeURL); + } + + private static String expandPlaceholder(String placeholder, boolean encodeURL) throws ExpandException { + return switch (placeholder) { + case "/" -> String.valueOf(File.separatorChar); + default -> { + String value = System.getProperty(placeholder); + if (value == null) { + throw new ExpandException("Unable to expand property: " + placeholder); + } + yield encodeURL ? encodeValue(value) : value; + } + }; + } + + private static String encodeValue(String value) { + try { + URI uri = new URI(value); + return uri.isAbsolute() ? value : URLEncoder.encode(value, StandardCharsets.UTF_8); + } catch (URISyntaxException e) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/Token.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/Token.java new file mode 100644 index 0000000000000..e9a129f513607 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/Token.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +public class Token { + int type; + String text; + int line; + + Token(int type, String text, int line) { + this.type = type; + this.text = text; + this.line = line; + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/TokenStream.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/TokenStream.java new file mode 100644 index 0000000000000..4681c8dfea2aa --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/TokenStream.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +import java.io.IOException; +import java.io.Reader; +import java.io.StreamTokenizer; +import java.util.ArrayDeque; +import java.util.Deque; + +public class TokenStream { + private final StreamTokenizer tokenizer; + private final Deque buffer = new ArrayDeque<>(); + + TokenStream(Reader reader) { + this.tokenizer = Tokenizer.configureTokenizer(reader); + } + + Token peek() throws IOException { + if (buffer.isEmpty()) { + buffer.push(nextToken()); + } + return buffer.peek(); + } + + Token consume() throws IOException { + return buffer.isEmpty() ? nextToken() : buffer.pop(); + } + + boolean isEOF() throws IOException { + Token t = peek(); + return t.type == StreamTokenizer.TT_EOF; + } + + int line() throws IOException { + return peek().line; + } + + private Token nextToken() throws IOException { + int type = tokenizer.nextToken(); + String text = switch (type) { + case StreamTokenizer.TT_WORD, '"', '\'' -> tokenizer.sval; + case StreamTokenizer.TT_EOF -> ""; + default -> Character.toString((char) type); + }; + return new Token(type, text, tokenizer.lineno()); + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/Tokenizer.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/Tokenizer.java new file mode 100644 index 0000000000000..3ac771ef5f29e --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/Tokenizer.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm.policy; + +import java.io.Reader; +import java.io.StreamTokenizer; + +public final class Tokenizer { + + private Tokenizer() {} + + /* + * Configure the stream tokenizer: + * Recognize strings between "..." + * Don't convert words to lowercase + * Recognize both C-style and C++-style comments + * Treat end-of-line as white space, not as a token + */ + + // new Token(StreamTokenizer.TT_WORD, "grant", line) // keyword + // new Token(StreamTokenizer.TT_WORD, "Codebase", line) + // new Token('"', "file:/some/path", line) // quoted string + // new Token('{', "{", line) // symbol + // new Token(StreamTokenizer.TT_WORD, "permission", line) + // new Token(StreamTokenizer.TT_WORD, "java.io.FilePermission", line) + // new Token('"', "file", line) + // new Token(',', ",", line) + // new Token('"', "read", line) + // new Token(';', ";", line) + // new Token('}', "}", line) + // new Token(';', ";", line) + public static StreamTokenizer configureTokenizer(Reader reader) { + StreamTokenizer st = new StreamTokenizer(reader); + + st.resetSyntax(); + st.wordChars('a', 'z'); + st.wordChars('A', 'Z'); + st.wordChars('.', '.'); + st.wordChars('0', '9'); + st.wordChars('_', '_'); + st.wordChars('$', '$'); + st.wordChars(128 + 32, 255); // extended chars + st.whitespaceChars(0, ' '); + st.commentChar('/'); + st.quoteChar('\''); + st.quoteChar('"'); + st.lowerCaseMode(false); + st.ordinaryChar('/'); + st.slashSlashComments(true); + st.slashStarComments(true); + + return st; + } +} diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/package-info.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/package-info.java new file mode 100644 index 0000000000000..d182490b8d173 --- /dev/null +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/policy/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Java Agent Policy + */ +package org.opensearch.secure_sm.policy; diff --git a/libs/agent-sm/agent/build.gradle b/libs/agent-sm/agent/build.gradle index a69dc057f2f9c..e705b8a51c6c0 100644 --- a/libs/agent-sm/agent/build.gradle +++ b/libs/agent-sm/agent/build.gradle @@ -74,3 +74,7 @@ tasks.test { tasks.check { dependsOn test } + +tasks.named('assemble') { + dependsOn prepareAgent +} diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java index 20087500f1df4..77b71864fbaec 100644 --- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java +++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java @@ -11,6 +11,7 @@ import org.opensearch.javaagent.bootstrap.AgentPolicy; import java.lang.StackWalker.Option; +import java.security.Policy; import net.bytebuddy.asm.Advice; @@ -30,11 +31,16 @@ public SystemExitInterceptor() {} */ @Advice.OnMethodEnter() public static void intercept(int code) throws Exception { + final Policy policy = AgentPolicy.getPolicy(); + if (policy == null) { + return; /* noop */ + } + final StackWalker walker = StackWalker.getInstance(Option.RETAIN_CLASS_REFERENCE); final Class caller = walker.getCallerClass(); if (!AgentPolicy.isClassThatCanExit(caller.getName())) { - throw new SecurityException("The class " + caller + " is not allowed to call System::exit(" + code + ")"); + // throw new SecurityException("The class " + caller + " is not allowed to call System::exit(" + code + ")"); } } } diff --git a/libs/build.gradle b/libs/build.gradle index 9bf359d936178..2ac4e03819f3a 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -41,20 +41,21 @@ subprojects { */ project.afterEvaluate { if (!project.path.equals(':libs:agent-sm:agent')) { - configurations.all { Configuration conf -> - dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> - Project depProject = project.project(dep.path) - if (depProject != null - && (false == depProject.path.equals(':libs:opensearch-core') && - false == depProject.path.equals(':libs:opensearch-common')) - && depProject.path.startsWith(':libs')) { - throw new InvalidUserDataException("projects in :libs " - + "may not depend on other projects libs except " - + ":libs:opensearch-core or :libs:opensearch-common but " - + "${project.path} depends on ${depProject.path}") + configurations.all { Configuration conf -> + dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> + Project depProject = project.project(dep.path) + if (depProject != null + && (false == depProject.path.equals(':libs:opensearch-core') && + false == depProject.path.equals(':libs:opensearch-common')&& + false == depProject.path.equals(':libs:agent-sm:agent-policy')) + && depProject.path.startsWith(':libs')) { + throw new InvalidUserDataException("projects in :libs " + + "may not depend on other projects libs except " + + ":libs:opensearch-core or :libs:opensearch-common but " + + "${project.path} depends on ${depProject.path}") + } } } - } } } } diff --git a/libs/nio/src/test/java/org/opensearch/nio/SocketChannelContextTests.java b/libs/nio/src/test/java/org/opensearch/nio/SocketChannelContextTests.java index 1e559b597ca89..c20f7bd906af0 100644 --- a/libs/nio/src/test/java/org/opensearch/nio/SocketChannelContextTests.java +++ b/libs/nio/src/test/java/org/opensearch/nio/SocketChannelContextTests.java @@ -125,6 +125,7 @@ public void testSignalWhenPeerClosed() throws IOException { assertTrue(context.closeNow()); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16731") public void testRegisterInitiatesConnect() throws IOException { InetSocketAddress address = mock(InetSocketAddress.class); boolean isAccepted = randomBoolean(); @@ -205,6 +206,7 @@ public void testConnectFails() throws IOException { assertSame(ioException, exception.get()); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16731") public void testConnectCanSetSocketOptions() throws IOException { InetSocketAddress address = mock(InetSocketAddress.class); Config.Socket config; diff --git a/libs/secure-sm/build.gradle b/libs/secure-sm/build.gradle index 7a0b06699bf35..9febde423f796 100644 --- a/libs/secure-sm/build.gradle +++ b/libs/secure-sm/build.gradle @@ -31,6 +31,7 @@ apply plugin: 'opensearch.publish' dependencies { // do not add non-test compile dependencies to secure-sm without a good reason to do so + api project(":libs:agent-sm:agent-policy") testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" testImplementation "junit:junit:${versions.junit}" diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index d3c92ac39f5b4..466abb0e60e02 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -146,7 +146,7 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', } final List miniHDFSArgs = [] - if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_23) { + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_23 && BuildParams.runtimeJavaVersion < JavaVersion.VERSION_24) { miniHDFSArgs.add('-Djava.security.manager=allow') } diff --git a/plugins/repository-hdfs/src/test/resources/org/opensearch/bootstrap/test.policy b/plugins/repository-hdfs/src/test/resources/org/opensearch/bootstrap/test.policy new file mode 100644 index 0000000000000..7899f339e5732 --- /dev/null +++ b/plugins/repository-hdfs/src/test/resources/org/opensearch/bootstrap/test.policy @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { + permission java.net.NetPermission "accessUnixDomainSocket"; + permission java.net.SocketPermission "*", "connect,resolve"; +}; diff --git a/plugins/repository-s3/src/internalClusterTest/resources/org/opensearch/bootstrap/test.policy b/plugins/repository-s3/src/internalClusterTest/resources/org/opensearch/bootstrap/test.policy new file mode 100644 index 0000000000000..7899f339e5732 --- /dev/null +++ b/plugins/repository-s3/src/internalClusterTest/resources/org/opensearch/bootstrap/test.policy @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { + permission java.net.NetPermission "accessUnixDomainSocket"; + permission java.net.SocketPermission "*", "connect,resolve"; +}; diff --git a/server/build.gradle b/server/build.gradle index fd2cac4c7506f..dfcfc5c6df99e 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -71,6 +71,7 @@ dependencies { api project(":libs:opensearch-task-commons") implementation project(':libs:opensearch-arrow-spi') + compileOnly project(":libs:agent-sm:bootstrap") compileOnly project(':libs:opensearch-plugin-classloader') testRuntimeOnly project(':libs:opensearch-plugin-classloader') @@ -378,7 +379,7 @@ tasks.named("licenseHeaders").configure { tasks.test { environment "node.roles.test", "[]" if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { - jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"] + jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED", "-Djdk.attach.allowAttachSelf=true", "-XX:+EnableDynamicAgentLoading" ] } } diff --git a/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java index 8285f361ee220..b484c33fda5c9 100644 --- a/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java +++ b/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java @@ -47,6 +47,7 @@ import org.opensearch.discovery.DiscoveryModule; import org.opensearch.env.Environment; import org.opensearch.index.IndexModule; +import org.opensearch.javaagent.bootstrap.AgentPolicy; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.monitor.process.ProcessProbe; import org.opensearch.node.NodeRoleSettings; @@ -57,6 +58,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.AllPermission; +import java.security.Policy; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -720,10 +722,10 @@ public final BootstrapCheckResult check(BootstrapContext context) { @SuppressWarnings("removal") boolean isAllPermissionGranted() { - final SecurityManager sm = System.getSecurityManager(); - assert sm != null; + final Policy policy = AgentPolicy.getPolicy(); + assert policy != null; try { - sm.checkPermission(new AllPermission()); + AgentPolicy.checkPermission(new AllPermission()); } catch (final SecurityException e) { return false; } diff --git a/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java b/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java index 162b9be318cd5..7b011b5828428 100644 --- a/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java +++ b/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java @@ -48,7 +48,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.security.Permission; import java.security.Security; import java.util.Arrays; import java.util.Locale; @@ -86,19 +85,7 @@ class OpenSearch extends EnvironmentAwareCommand { @SuppressWarnings("removal") public static void main(final String[] args) throws Exception { overrideDnsCachePolicyProperties(); - /* - * We want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the - * presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy). This - * forces such policies to take effect immediately. - */ - System.setSecurityManager(new SecurityManager() { - - @Override - public void checkPermission(Permission perm) { - // grant all permissions so that we can later set the security manager to the one that we want - } - }); LogConfigurator.registerErrorListener(); final OpenSearch opensearch = new OpenSearch(); int status = main(args, opensearch, Terminal.DEFAULT); diff --git a/server/src/main/java/org/opensearch/bootstrap/Security.java b/server/src/main/java/org/opensearch/bootstrap/Security.java index 9c93b0414bdda..2f1de651d0e82 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Security.java +++ b/server/src/main/java/org/opensearch/bootstrap/Security.java @@ -41,9 +41,10 @@ import org.opensearch.common.transport.PortsRange; import org.opensearch.env.Environment; import org.opensearch.http.HttpTransportSettings; +import org.opensearch.javaagent.bootstrap.AgentPolicy; import org.opensearch.plugins.PluginInfo; import org.opensearch.plugins.PluginsService; -import org.opensearch.secure_sm.SecureSM; +import org.opensearch.secure_sm.policy.PolicyFile; import org.opensearch.transport.TcpTransport; import java.io.IOException; @@ -59,7 +60,6 @@ import java.security.NoSuchAlgorithmException; import java.security.Permissions; import java.security.Policy; -import java.security.URIParameter; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -144,23 +144,25 @@ static void configure(Environment environment, boolean filterBadDefaults) throws // enable security policy: union of template and environment-based paths, and possibly plugin permissions Map codebases = getCodebaseJarMap(JarHell.parseClassPath()); - Policy.setPolicy( + + // enable security manager + final String[] classesThatCanExit = new String[] { + // SecureSM matches class names as regular expressions so we escape the $ that arises from the nested class name + OpenSearchUncaughtExceptionHandler.PrivilegedHaltAction.class.getName().replace("$", "\\$"), + Command.class.getName() }; + + AgentPolicy.setPolicy( new OpenSearchPolicy( codebases, createPermissions(environment), getPluginPermissions(environment), filterBadDefaults, createRecursiveDataPathPermission(environment) - ) + ), + Set.of() /* trusted hosts */, + classesThatCanExit ); - // enable security manager - final String[] classesThatCanExit = new String[] { - // SecureSM matches class names as regular expressions so we escape the $ that arises from the nested class name - OpenSearchUncaughtExceptionHandler.PrivilegedHaltAction.class.getName().replace("$", "\\$"), - Command.class.getName() }; - System.setSecurityManager(new SecureSM(classesThatCanExit)); - // do some basic tests selfTest(); } @@ -280,14 +282,14 @@ static Policy readPolicy(URL policyFile, Map codebases) { addCodebaseToSystemProperties(propertiesSet, url, property, aliasProperty); } - return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI())); + return new PolicyFile(policyFile); } finally { // clear codebase properties for (String property : propertiesSet) { System.clearProperty(property); } } - } catch (NoSuchAlgorithmException | URISyntaxException e) { + } catch (final RuntimeException e) { throw new IllegalArgumentException("unable to parse policy file `" + policyFile + "`", e); } } diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java index 8c15706adceeb..d680fc04789f8 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java @@ -44,14 +44,12 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.http.HttpTransportSettings; -import org.opensearch.secure_sm.ThreadContextPermission; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskThreadContextStatePropagator; import org.opensearch.transport.client.OriginSettingClient; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.security.Permission; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -114,11 +112,6 @@ public final class ThreadContext implements Writeable { public static final String ACTION_ORIGIN_TRANSIENT_NAME = "action.origin"; // thread context permissions - - private static final Permission ACCESS_SYSTEM_THREAD_CONTEXT_PERMISSION = new ThreadContextPermission("markAsSystemContext"); - private static final Permission STASH_AND_MERGE_THREAD_CONTEXT_PERMISSION = new ThreadContextPermission("stashAndMergeHeaders"); - private static final Permission STASH_WITH_ORIGIN_THREAD_CONTEXT_PERMISSION = new ThreadContextPermission("stashWithOrigin"); - private static final Logger logger = LogManager.getLogger(ThreadContext.class); private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(); private final Map defaultHeader; @@ -223,10 +216,6 @@ public Writeable captureAsWriteable() { */ @SuppressWarnings("removal") public StoredContext stashWithOrigin(String origin) { - SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPermission(STASH_WITH_ORIGIN_THREAD_CONTEXT_PERMISSION); - } final ThreadContext.StoredContext storedContext = stashContext(); putTransient(ACTION_ORIGIN_TRANSIENT_NAME, origin); return storedContext; @@ -246,10 +235,6 @@ public StoredContext stashWithOrigin(String origin) { */ @SuppressWarnings("removal") public StoredContext stashAndMergeHeaders(Map headers) { - SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPermission(STASH_AND_MERGE_THREAD_CONTEXT_PERMISSION); - } final ThreadContextStruct context = threadLocal.get(); Map newHeader = new HashMap<>(headers); newHeader.putAll(context.requestHeaders); @@ -605,10 +590,6 @@ boolean isDefaultContext() { */ @SuppressWarnings("removal") public void markAsSystemContext() { - SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPermission(ACCESS_SYSTEM_THREAD_CONTEXT_PERMISSION); - } threadLocal.set(threadLocal.get().setSystemContext(propagators)); } diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index f521ce0011540..fbe0afb3c2a95 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -93,6 +93,30 @@ grant codeBase "${codebase.reactor-core}" { permission java.net.SocketPermission "*", "connect,resolve"; }; +grant codeBase "${codebase.opensearch-cli}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + +grant codeBase "${codebase.opensearch-core}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + +grant codeBase "${codebase.jackson-core}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + +grant codeBase "${codebase.opensearch-common}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + +grant codeBase "${codebase.opensearch-x-content}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + +grant codeBase "${codebase.opensearch}" { + permission java.net.SocketPermission "*", "connect,resolve"; +}; + //// Everything else: grant { diff --git a/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy b/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy index 78f302e9b23db..5fe1a5b64e6c7 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy @@ -101,9 +101,14 @@ grant codeBase "${codebase.junit}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; +grant codeBase "${codebase.opensearch-core}" { + // opensearch-nio makes and accepts socket connections + permission java.net.SocketPermission "*", "accept,resolve,connect"; +}; + grant codeBase "${codebase.opensearch-nio}" { // opensearch-nio makes and accepts socket connections - permission java.net.SocketPermission "*", "accept,connect"; + permission java.net.SocketPermission "*", "accept,resolve,connect"; }; grant codeBase "${codebase.opensearch-rest-client}" { @@ -111,16 +116,19 @@ grant codeBase "${codebase.opensearch-rest-client}" { permission java.net.SocketPermission "*", "connect"; // rest client uses system properties which gets the default proxy permission java.net.NetPermission "getProxySelector"; + permission java.net.NetPermission "accessUnixDomainSocket"; }; grant codeBase "${codebase.httpcore5}" { // httpcore makes socket connections for rest tests permission java.net.SocketPermission "*", "connect"; + permission java.net.NetPermission "accessUnixDomainSocket"; }; grant codeBase "${codebase.httpclient5}" { // httpclient5 makes socket connections for rest tests permission java.net.SocketPermission "*", "connect,resolve"; + permission java.net.NetPermission "accessUnixDomainSocket"; }; grant codeBase "${codebase.httpcore-nio}" { diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index dd55abb65d19f..f4cc39684c86b 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -131,6 +131,7 @@ import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; import java.nio.file.AtomicMoveNotSupportedException; @@ -166,7 +167,7 @@ public void testExceptionRegistration() throws ClassNotFoundException, IOExcepti final Set> notRegistered = new HashSet<>(); final Set> hasDedicatedWrite = new HashSet<>(); final Set> registered = new HashSet<>(); - final String path = "/org/opensearch"; + final String path = "org/opensearch"; final Path coreLibStartPath = PathUtils.get(OpenSearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI()); final Path startPath = PathUtils.get(OpenSearchServerException.class.getProtectionDomain().getCodeSource().getLocation().toURI()) .resolve("org") @@ -254,7 +255,8 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx Files.walkFileTree(coreLibStartPath, visitor); // walk the server module start path Files.walkFileTree(startPath, visitor); - final Path testStartPath = PathUtils.get(ExceptionSerializationTests.class.getResource(path).toURI()); + final URI location = ExceptionSerializationTests.class.getProtectionDomain().getCodeSource().getLocation().toURI(); + final Path testStartPath = PathUtils.get(location).resolve(path); Files.walkFileTree(testStartPath, visitor); assertTrue(notRegistered.remove(TestException.class)); assertTrue(notRegistered.remove(UnknownHeaderException.class)); diff --git a/server/src/test/resources/org/opensearch/bootstrap/test.policy b/server/src/test/resources/org/opensearch/bootstrap/test.policy index c2b5a8e9c0a4e..30396afaf2ca4 100644 --- a/server/src/test/resources/org/opensearch/bootstrap/test.policy +++ b/server/src/test/resources/org/opensearch/bootstrap/test.policy @@ -10,4 +10,10 @@ grant { // allow to test Security policy and codebases permission java.util.PropertyPermission "*", "read,write"; permission java.security.SecurityPermission "createPolicy.JavaPolicy"; + permission java.net.NetPermission "accessUnixDomainSocket"; +}; + +grant codeBase "${codebase.framework}" { + permission java.net.NetPermission "accessUnixDomainSocket"; + permission java.net.SocketPermission "*", "accept,connect"; }; diff --git a/test/framework/build.gradle b/test/framework/build.gradle index e5297ca0807a4..47fdfce960936 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -53,6 +53,9 @@ dependencies { api "org.bouncycastle:bcpkix-fips:${versions.bouncycastle_pkix}" api "org.bouncycastle:bcutil-fips:${versions.bouncycastle_util}" + compileOnly project(":libs:agent-sm:bootstrap") + compileOnly "com.github.spotbugs:spotbugs-annotations:4.9.0" + annotationProcessor "org.apache.logging.log4j:log4j-core:${versions.log4j}" } @@ -97,9 +100,12 @@ test { systemProperty 'tests.gradle_wire_compat_versions', BuildParams.bwcVersions.wireCompatible.join(',') systemProperty 'tests.gradle_unreleased_versions', BuildParams.bwcVersions.unreleased.join(',') - if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_18) { + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_18 && BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_23) { jvmArgs += ["-Djava.security.manager=allow"] } + + dependsOn(project(':libs:agent-sm:agent').prepareAgent) + jvmArgs += ["-javaagent:" + project(':libs:agent-sm:agent').jar.archiveFile.get()] } tasks.register("integTest", Test) { diff --git a/test/framework/src/main/java/org/opensearch/bootstrap/AgentAttach.java b/test/framework/src/main/java/org/opensearch/bootstrap/AgentAttach.java new file mode 100644 index 0000000000000..0a0df6756f21f --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/bootstrap/AgentAttach.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.bootstrap; + +public final class AgentAttach { + public static boolean agentIsAttached() { + try { + Class.forName("org.opensearch.javaagent.Agent", false, ClassLoader.getSystemClassLoader()); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } +} diff --git a/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java index 76c7ce0628aac..dc358af93040b 100644 --- a/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java @@ -46,9 +46,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.core.util.FileSystemUtils; +import org.opensearch.javaagent.bootstrap.AgentPolicy; import org.opensearch.mockito.plugin.PriviledgedMockMaker; import org.opensearch.plugins.PluginInfo; -import org.opensearch.secure_sm.SecureSM; import org.junit.Assert; import java.io.InputStream; @@ -168,7 +168,7 @@ public class BootstrapForTesting { final Optional testPolicy = Optional.ofNullable(Bootstrap.class.getResource("test.policy")) .map(policy -> Security.readPolicy(policy, codebases)); final Policy opensearchPolicy = new OpenSearchPolicy(codebases, perms, getPluginPermissions(), true, new Permissions()); - Policy.setPolicy(new Policy() { + AgentPolicy.setPolicy(new Policy() { @Override public boolean implies(ProtectionDomain domain, Permission permission) { // implements union @@ -176,10 +176,13 @@ public boolean implies(ProtectionDomain domain, Permission permission) { || testFramework.implies(domain, permission) || testPolicy.map(policy -> policy.implies(domain, permission)).orElse(false /* no policy */); } - }); + }, getTrustedHosts(), new String[0] /* classes than can exit */); // Create access control context for mocking PriviledgedMockMaker.createAccessControlContext(); - System.setSecurityManager(SecureSM.createTestSecureSM(getTrustedHosts())); + + if (!AgentAttach.agentIsAttached()) { + throw new RuntimeException("the security agent is not attached"); + } Security.selfTest(); // guarantee plugin classes are initialized first, in case they have one-time hacks. diff --git a/test/framework/src/test/resources/org/opensearch/bootstrap/test.policy b/test/framework/src/test/resources/org/opensearch/bootstrap/test.policy new file mode 100644 index 0000000000000..07c9fe160e985 --- /dev/null +++ b/test/framework/src/test/resources/org/opensearch/bootstrap/test.policy @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant codeBase "${codebase.opensearch-nio}" { + permission java.net.NetPermission "accessUnixDomainSocket"; +}; + +grant { + permission java.net.NetPermission "accessUnixDomainSocket"; + permission java.net.SocketPermission "*", "accept,connect"; +};