From 76f413bc8dd87e36d6bfccb4d4b0e7683ccb9161 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 20 Nov 2024 15:12:10 -0500 Subject: [PATCH 01/10] Move system index protection to the core Signed-off-by: Craig Perkins --- gradle/missing-javadoc.gradle | 1 + modules/system-index-protection/build.gradle | 26 + .../mapper/size/SystemIndexPluginIT.java | 57 ++ .../size/SystemIndexProtectionTests.java | 57 ++ .../index/filter/ClusterInfoHolder.java | 95 ++ .../index/filter/IndexResolverReplacer.java | 829 ++++++++++++++++++ .../index/filter/SnapshotRestoreHelper.java | 108 +++ .../index/filter/SystemIndexFilter.java | 77 ++ .../index/filter/WildcardMatcher.java | 559 ++++++++++++ .../SystemIndexProtectionPlugin.java | 147 ++++ .../SystemIndexProtectionYamlTestSuiteIT.java | 51 ++ .../test/system_index_protection/10_basic.yml | 22 + 12 files changed, 2029 insertions(+) create mode 100644 modules/system-index-protection/build.gradle create mode 100644 modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java create mode 100644 modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java create mode 100644 modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java create mode 100644 modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java create mode 100644 modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java create mode 100644 modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java create mode 100644 modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java create mode 100644 modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java create mode 100644 modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java create mode 100644 modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index 751da941d25dd..38d2346713273 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -123,6 +123,7 @@ configure([ project(":modules:rank-eval"), project(":modules:reindex"), project(":modules:repository-url"), + project(":modules:system-index-protection"), project(":modules:systemd"), project(":modules:transport-netty4"), project(":plugins:analysis-icu"), diff --git a/modules/system-index-protection/build.gradle b/modules/system-index-protection/build.gradle new file mode 100644 index 0000000000000..d3b5ae0c67d63 --- /dev/null +++ b/modules/system-index-protection/build.gradle @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.internal-cluster-test' + +opensearchplugin { + description 'The System Index Protection Plugin provides native protection to system indices' + classname 'org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin' +} + +restResources { + restApi { + includeCore '_common', 'indices', 'index', 'get' + } +} +// no unit tests +test.enabled = false diff --git a/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java b/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java new file mode 100644 index 0000000000000..f7f991146a030 --- /dev/null +++ b/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.mapper.size; + +import org.opensearch.OpenSearchSecurityException; +import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.tasks.TaskResultsService.TASK_INDEX; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class SystemIndexPluginIT extends OpenSearchIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(SystemIndexProtectionPlugin.class); + } + + public void testBasic() throws Exception { + assertAcked(prepareCreate(TASK_INDEX)); + client().prepareDelete().setIndex(TASK_INDEX); + assertThrows(OpenSearchSecurityException.class, () -> { admin().indices().prepareDelete(TASK_INDEX).get(); }); + } +} diff --git a/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java b/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java new file mode 100644 index 0000000000000..59283a0fd8871 --- /dev/null +++ b/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.mapper.size; + +import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.util.Collection; + +import static org.opensearch.tasks.TaskResultsService.TASK_INDEX; + +public class SystemIndexProtectionTests extends OpenSearchSingleNodeTestCase { + @Override + protected Collection> getPlugins() { + return pluginList(SystemIndexProtectionPlugin.class); + } + + public void testBasic() throws Exception { + createIndex(TASK_INDEX); + DeleteIndexRequestBuilder deleteIndexRequestBuilder = client().admin().indices().prepareDelete(TASK_INDEX); + assertThrows(OpenSearchSecurityException.class, deleteIndexRequestBuilder::get); + } + +} diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java new file mode 100644 index 0000000000000..9b9d5a10a3185 --- /dev/null +++ b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java @@ -0,0 +1,95 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.filter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; + +public class ClusterInfoHolder implements ClusterStateListener { + + protected final Logger log = LogManager.getLogger(this.getClass()); + private volatile DiscoveryNodes nodes = null; + private volatile Boolean isLocalNodeElectedClusterManager = null; + private volatile boolean initialized; + private final String clusterName; + + public ClusterInfoHolder(String clusterName) { + this.clusterName = clusterName; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (nodes == null || event.nodesChanged()) { + nodes = event.state().nodes(); + if (log.isDebugEnabled()) { + log.debug("Cluster Info Holder now initialized for 'nodes'"); + } + initialized = true; + } + + isLocalNodeElectedClusterManager = event.localNodeClusterManager() ? Boolean.TRUE : Boolean.FALSE; + } + + public Boolean isLocalNodeElectedClusterManager() { + return isLocalNodeElectedClusterManager; + } + + public boolean isInitialized() { + return initialized; + } + + public Version getMinNodeVersion() { + if (nodes == null) { + if (log.isDebugEnabled()) { + log.debug("Cluster Info Holder not initialized yet for 'nodes'"); + } + return null; + } + + return nodes.getMinNodeVersion(); + } + + public Boolean hasNode(DiscoveryNode node) { + if (nodes == null) { + if (log.isDebugEnabled()) { + log.debug("Cluster Info Holder not initialized yet for 'nodes'"); + } + return null; + } + + return nodes.nodeExists(node) ? Boolean.TRUE : Boolean.FALSE; + } + + public String getClusterName() { + return this.clusterName; + } +} diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java new file mode 100644 index 0000000000000..2dc80f0251fa6 --- /dev/null +++ b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java @@ -0,0 +1,829 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.filter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.IndicesRequest; +import org.opensearch.action.IndicesRequest.Replaceable; +import org.opensearch.action.OriginalIndices; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; +import org.opensearch.action.admin.indices.shrink.ResizeRequest; +import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkShardRequest; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.fieldcaps.FieldCapabilitiesIndexRequest; +import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetRequest.Item; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.main.MainRequest; +import org.opensearch.action.search.ClearScrollRequest; +import org.opensearch.action.search.MultiSearchRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.nodes.BaseNodesRequest; +import org.opensearch.action.support.replication.ReplicationRequest; +import org.opensearch.action.support.single.shard.SingleShardRequest; +import org.opensearch.action.termvectors.MultiTermVectorsRequest; +import org.opensearch.action.termvectors.TermVectorsRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexAbstraction; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.util.IndexUtils; +import org.opensearch.core.index.Index; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.reindex.ReindexRequest; +import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; +import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.transport.RemoteClusterService; +import org.opensearch.transport.TransportRequest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; +import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; + +import static org.opensearch.cluster.metadata.IndexAbstraction.Type.ALIAS; + +public class IndexResolverReplacer { + + private static final Set NULL_SET = new HashSet<>(Collections.singleton(null)); + private final Logger log = LogManager.getLogger(this.getClass()); + private final IndexNameExpressionResolver resolver; + private final Supplier clusterStateSupplier; + private final ClusterInfoHolder clusterInfoHolder; + private volatile boolean respectRequestIndicesOptions = false; + + public IndexResolverReplacer( + IndexNameExpressionResolver resolver, + Supplier clusterStateSupplier, + ClusterInfoHolder clusterInfoHolder + ) { + this.resolver = resolver; + this.clusterStateSupplier = clusterStateSupplier; + this.clusterInfoHolder = clusterInfoHolder; + } + + private static boolean isAllWithNoRemote(final String... requestedPatterns) { + + final List patterns = requestedPatterns == null ? null : Arrays.asList(requestedPatterns); + + if (IndexNameExpressionResolver.isAllIndices(patterns)) { + return true; + } + + if (patterns.size() == 1 && patterns.contains("*")) { + return true; + } + + if (new HashSet(patterns).equals(NULL_SET)) { + return true; + } + + return false; + } + + private static boolean isLocalAll(String... requestedPatterns) { + return isLocalAll(requestedPatterns == null ? null : Arrays.asList(requestedPatterns)); + } + + private static boolean isLocalAll(Collection patterns) { + if (IndexNameExpressionResolver.isAllIndices(patterns)) { + return true; + } + + if (patterns.contains("_all")) { + return true; + } + + if (new HashSet(patterns).equals(NULL_SET)) { + return true; + } + + return false; + } + + private class ResolvedIndicesProvider implements IndicesProvider { + private final Set aliases; + private final Set allIndices; + private final Set originalRequested; + private final Set remoteIndices; + // set of previously resolved index requests to avoid resolving + // the same index more than once while processing bulk requests + private final Set alreadyResolved; + private final String name; + + private final class AlreadyResolvedKey { + + private final IndicesOptions indicesOptions; + + private final boolean enableCrossClusterResolution; + + private final String[] original; + + private AlreadyResolvedKey(final IndicesOptions indicesOptions, final boolean enableCrossClusterResolution) { + this(indicesOptions, enableCrossClusterResolution, null); + } + + private AlreadyResolvedKey( + final IndicesOptions indicesOptions, + final boolean enableCrossClusterResolution, + final String[] original + ) { + this.indicesOptions = indicesOptions; + this.enableCrossClusterResolution = enableCrossClusterResolution; + this.original = original; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AlreadyResolvedKey that = (AlreadyResolvedKey) o; + return enableCrossClusterResolution == that.enableCrossClusterResolution + && Objects.equals(indicesOptions, that.indicesOptions) + && Arrays.equals(original, that.original); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, enableCrossClusterResolution); + result = 31 * result + Arrays.hashCode(original); + return result; + } + } + + ResolvedIndicesProvider(Object request) { + aliases = new HashSet<>(); + allIndices = new HashSet<>(); + originalRequested = new HashSet<>(); + remoteIndices = new HashSet<>(); + alreadyResolved = new HashSet<>(); + name = request.getClass().getSimpleName(); + } + + private void resolveIndexPatterns( + final String name, + final IndicesOptions indicesOptions, + final boolean enableCrossClusterResolution, + final String[] original + ) { + final boolean isTraceEnabled = log.isTraceEnabled(); + if (isTraceEnabled) { + log.trace("resolve requestedPatterns: " + Arrays.toString(original)); + } + + if (isAllWithNoRemote(original)) { + if (isTraceEnabled) { + log.trace(Arrays.toString(original) + " is an ALL pattern without any remote indices"); + } + resolveToLocalAll(); + return; + } + + Set remoteIndices; + final List localRequestedPatterns = new ArrayList<>(Arrays.asList(original)); + + final RemoteClusterService remoteClusterService = SystemIndexProtectionPlugin.GuiceHolder.getRemoteClusterService(); + + if (remoteClusterService != null && remoteClusterService.isCrossClusterSearchEnabled() && enableCrossClusterResolution) { + remoteIndices = new HashSet<>(); + final Map remoteClusterIndices = SystemIndexProtectionPlugin.GuiceHolder.getRemoteClusterService() + .groupIndices(indicesOptions, original, idx -> resolver.hasIndexAbstraction(idx, clusterStateSupplier.get())); + final Set remoteClusters = remoteClusterIndices.keySet() + .stream() + .filter(k -> !RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY.equals(k)) + .collect(Collectors.toSet()); + for (String remoteCluster : remoteClusters) { + for (String remoteIndex : remoteClusterIndices.get(remoteCluster).indices()) { + remoteIndices.add(RemoteClusterService.buildRemoteIndexName(remoteCluster, remoteIndex)); + } + } + + final Iterator iterator = localRequestedPatterns.iterator(); + while (iterator.hasNext()) { + final String[] split = iterator.next().split(String.valueOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR), 2); + final WildcardMatcher matcher = WildcardMatcher.from(split[0]); + if (split.length > 1 && matcher.matchAny(remoteClusters)) { + iterator.remove(); + } + } + + if (isTraceEnabled) { + log.trace( + "CCS is enabled, we found this local patterns " + + localRequestedPatterns + + " and this remote patterns: " + + remoteIndices + ); + } + + } else { + remoteIndices = Collections.emptySet(); + } + + final Collection matchingAliases; + Collection matchingAllIndices; + Collection matchingDataStreams = null; + + if (isLocalAll(original)) { + if (isTraceEnabled) { + log.trace(Arrays.toString(original) + " is an LOCAL ALL pattern"); + } + matchingAliases = Resolved.All_SET; + matchingAllIndices = Resolved.All_SET; + + } else if (!remoteIndices.isEmpty() && localRequestedPatterns.isEmpty()) { + if (isTraceEnabled) { + log.trace(Arrays.toString(original) + " is an LOCAL EMPTY request"); + } + matchingAllIndices = Collections.emptySet(); + matchingAliases = Collections.emptySet(); + } + + else { + final ClusterState state = clusterStateSupplier.get(); + final Set dateResolvedLocalRequestedPatterns = localRequestedPatterns.stream() + .map(resolver::resolveDateMathExpression) + .collect(Collectors.toSet()); + final WildcardMatcher dateResolvedMatcher = WildcardMatcher.from(dateResolvedLocalRequestedPatterns); + // fill matchingAliases + final Map lookup = state.metadata().getIndicesLookup(); + matchingAliases = lookup.entrySet() + .stream() + .filter(e -> e.getValue().getType() == ALIAS) + .map(Map.Entry::getKey) + .filter(dateResolvedMatcher) + .collect(Collectors.toSet()); + + final boolean isDebugEnabled = log.isDebugEnabled(); + try { + matchingAllIndices = Arrays.asList( + resolver.concreteIndexNames(state, indicesOptions, localRequestedPatterns.toArray(new String[0])) + ); + matchingDataStreams = resolver.dataStreamNames(state, indicesOptions, localRequestedPatterns.toArray(new String[0])); + + if (isDebugEnabled) { + log.debug( + "Resolved pattern {} to indices: {} and data-streams: {}", + localRequestedPatterns, + matchingAllIndices, + matchingDataStreams + ); + } + } catch (IndexNotFoundException e1) { + if (isDebugEnabled) { + log.debug("No such indices for pattern {}, use raw value", localRequestedPatterns); + } + + matchingAllIndices = dateResolvedLocalRequestedPatterns; + } + } + + if (matchingDataStreams == null || matchingDataStreams.size() == 0) { + matchingDataStreams = Arrays.asList(NOOP); + } + + if (isTraceEnabled) { + log.trace( + "Resolved patterns {} for {} ({}) to [aliases {}, allIndices {}, dataStreams {}, originalRequested{}, remote indices {}]", + original, + name, + this.name, + matchingAliases, + matchingAllIndices, + matchingDataStreams, + Arrays.toString(original), + remoteIndices + ); + } + + resolveTo(matchingAliases, matchingAllIndices, matchingDataStreams, original, remoteIndices); + } + + private void resolveToLocalAll() { + aliases.add(Resolved.ANY); + allIndices.add(Resolved.ANY); + originalRequested.add(Resolved.ANY); + } + + private void resolveTo( + Collection matchingAliases, + Collection matchingAllIndices, + Collection matchingDataStreams, + String[] original, + Collection remoteIndices + ) { + aliases.addAll(matchingAliases); + allIndices.addAll(matchingAllIndices); + allIndices.addAll(matchingDataStreams); + originalRequested.addAll(Arrays.stream(original).collect(Collectors.toList())); + this.remoteIndices.addAll(remoteIndices); + } + + @Override + public String[] provide(String[] original, Object localRequest, boolean supportsReplace) { + final IndicesOptions indicesOptions = indicesOptionsFrom(localRequest); + final boolean enableCrossClusterResolution = localRequest instanceof FieldCapabilitiesRequest + || localRequest instanceof SearchRequest + || localRequest instanceof ResolveIndexAction.Request; + // skip the whole thing if we have seen this exact resolveIndexPatterns request + final AlreadyResolvedKey alreadyResolvedKey; + if (original != null) { + alreadyResolvedKey = new AlreadyResolvedKey(indicesOptions, enableCrossClusterResolution, original); + } else { + alreadyResolvedKey = new AlreadyResolvedKey(indicesOptions, enableCrossClusterResolution); + } + if (alreadyResolved.add(alreadyResolvedKey)) { + resolveIndexPatterns(localRequest.getClass().getSimpleName(), indicesOptions, enableCrossClusterResolution, original); + } + return IndicesProvider.NOOP; + } + + Resolved resolved(IndicesOptions indicesOptions) { + final Resolved resolved = alreadyResolved.isEmpty() + ? Resolved._LOCAL_ALL + : new Resolved(aliases, allIndices, originalRequested, remoteIndices, indicesOptions); + + if (log.isTraceEnabled()) { + log.trace("Finally resolved for {}: {}", name, resolved); + } + + return resolved; + } + } + + // dnfof + public boolean replace(final TransportRequest request, boolean retainMode, String... replacements) { + return getOrReplaceAllIndices(request, new IndicesProvider() { + + @Override + public String[] provide(String[] original, Object request, boolean supportsReplace) { + if (supportsReplace) { + if (retainMode && !isAllWithNoRemote(original)) { + final Resolved resolved = resolveRequest(request); + final List retained = WildcardMatcher.from(resolved.getAllIndices()) + .getMatchAny(replacements, Collectors.toList()); + retained.addAll(resolved.getRemoteIndices()); + return retained.toArray(new String[0]); + } + return replacements; + } else { + return NOOP; + } + } + }, false); + } + + public boolean replace(final TransportRequest request, boolean retainMode, Collection replacements) { + return replace(request, retainMode, replacements.toArray(new String[replacements.size()])); + } + + public Resolved resolveRequest(final Object request) { + if (log.isDebugEnabled()) { + log.debug("Resolve aliases, indices and types from {}", request.getClass().getSimpleName()); + } + + final ResolvedIndicesProvider resolvedIndicesProvider = new ResolvedIndicesProvider(request); + + getOrReplaceAllIndices(request, resolvedIndicesProvider, false); + + return resolvedIndicesProvider.resolved(indicesOptionsFrom(request)); + } + + public final static class Resolved { + private static final String ANY = "*"; + private static final Set All_SET = Set.of(ANY); + private static final Set types = All_SET; + public static final Resolved _LOCAL_ALL = new Resolved(All_SET, All_SET, All_SET, Set.of(), SearchRequest.DEFAULT_INDICES_OPTIONS); + + private static final IndicesOptions EXACT_INDEX_OPTIONS = new IndicesOptions( + EnumSet.of(IndicesOptions.Option.FORBID_ALIASES_TO_MULTIPLE_INDICES), + EnumSet.noneOf(IndicesOptions.WildcardStates.class) + ); + + private final Set aliases; + private final Set allIndices; + private final Set originalRequested; + private final Set remoteIndices; + private final boolean isLocalAll; + private final IndicesOptions indicesOptions; + + public Resolved( + final Set aliases, + final Set allIndices, + final Set originalRequested, + final Set remoteIndices, + IndicesOptions indicesOptions + ) { + this.aliases = aliases; + this.allIndices = allIndices; + this.originalRequested = originalRequested; + this.remoteIndices = remoteIndices; + this.isLocalAll = IndexResolverReplacer.isLocalAll(originalRequested.toArray(new String[0])) + || (aliases.contains("*") && allIndices.contains("*")); + this.indicesOptions = indicesOptions; + } + + public boolean isLocalAll() { + return isLocalAll; + } + + public Set getAliases() { + return aliases; + } + + public Set getAllIndices() { + return allIndices; + } + + public Set getAllIndicesResolved(ClusterService clusterService, IndexNameExpressionResolver resolver) { + return getAllIndicesResolved(clusterService::state, resolver); + } + + public Set getAllIndicesResolved(Supplier clusterStateSupplier, IndexNameExpressionResolver resolver) { + if (isLocalAll) { + return new HashSet<>(Arrays.asList(resolver.concreteIndexNames(clusterStateSupplier.get(), indicesOptions, "*"))); + } else { + return allIndices; + } + } + + public boolean isAllIndicesEmpty() { + return allIndices.isEmpty(); + } + + public Set getTypes() { + return types; + } + + public Set getRemoteIndices() { + return remoteIndices; + } + + @Override + public String toString() { + return "Resolved [aliases=" + + aliases + + ", allIndices=" + + allIndices + + ", types=" + + types + + ", originalRequested=" + + originalRequested + + ", remoteIndices=" + + remoteIndices + + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((aliases == null) ? 0 : aliases.hashCode()); + result = prime * result + ((allIndices == null) ? 0 : allIndices.hashCode()); + result = prime * result + ((originalRequested == null) ? 0 : originalRequested.hashCode()); + result = prime * result + ((remoteIndices == null) ? 0 : remoteIndices.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + Resolved other = (Resolved) obj; + if (aliases == null) { + if (other.aliases != null) return false; + } else if (!aliases.equals(other.aliases)) return false; + if (allIndices == null) { + if (other.allIndices != null) return false; + } else if (!allIndices.equals(other.allIndices)) return false; + if (originalRequested == null) { + if (other.originalRequested != null) return false; + } else if (!originalRequested.equals(other.originalRequested)) return false; + if (remoteIndices == null) { + if (other.remoteIndices != null) return false; + } else if (!remoteIndices.equals(other.remoteIndices)) return false; + return true; + } + + public static Resolved ofIndex(String index) { + Set indexSet = Set.of(index); + return new Resolved(Set.of(), indexSet, indexSet, Set.of(), EXACT_INDEX_OPTIONS); + } + } + + private List renamedIndices(final RestoreSnapshotRequest request, final List filteredIndices) { + try { + final List renamedIndices = new ArrayList<>(); + for (final String index : filteredIndices) { + String renamedIndex = index; + if (request.renameReplacement() != null && request.renamePattern() != null) { + renamedIndex = index.replaceAll(request.renamePattern(), request.renameReplacement()); + } + renamedIndices.add(renamedIndex); + } + return renamedIndices; + } catch (PatternSyntaxException e) { + log.error("Unable to parse the regular expression denoted in 'rename_pattern'. Please correct the pattern an try again."); + throw e; + } + } + + // -- + + @FunctionalInterface + public interface IndicesProvider { + public static final String[] NOOP = new String[0]; + + String[] provide(String[] original, Object request, boolean supportsReplace); + } + + private boolean checkIndices(Object request, String[] indices, boolean needsToBeSizeOne, boolean allowEmpty) { + + if (indices == IndicesProvider.NOOP) { + return false; + } + + final boolean isTraceEnabled = log.isTraceEnabled(); + if (!allowEmpty && (indices == null || indices.length == 0)) { + if (isTraceEnabled && request != null) { + log.trace("Null or empty indices for " + request.getClass().getName()); + } + return false; + } + + if (!allowEmpty && needsToBeSizeOne && indices.length != 1) { + if (isTraceEnabled && request != null) { + log.trace("To much indices for " + request.getClass().getName()); + } + return false; + } + + for (int i = 0; i < indices.length; i++) { + final String index = indices[i]; + if (index == null || index.isEmpty()) { + // not allowed + if (isTraceEnabled && request != null) { + log.trace("At least one null or empty index for " + request.getClass().getName()); + } + return false; + } + } + + return true; + } + + /** + * new + * @param request + * @param allowEmptyIndices + * @return + */ + @SuppressWarnings("rawtypes") + private boolean getOrReplaceAllIndices(final Object request, final IndicesProvider provider, boolean allowEmptyIndices) { + final boolean isDebugEnabled = log.isDebugEnabled(); + final boolean isTraceEnabled = log.isTraceEnabled(); + if (isTraceEnabled) { + log.trace("getOrReplaceAllIndices() for " + request.getClass()); + } + + boolean result = true; + + if (request instanceof BulkRequest) { + + for (DocWriteRequest ar : ((BulkRequest) request).requests()) { + result = getOrReplaceAllIndices(ar, provider, false) && result; + } + + } else if (request instanceof MultiGetRequest) { + + for (ListIterator it = ((MultiGetRequest) request).getItems().listIterator(); it.hasNext();) { + Item item = it.next(); + result = getOrReplaceAllIndices(item, provider, false) && result; + /*if(item.index() == null || item.indices() == null || item.indices().length == 0) { + it.remove(); + }*/ + } + + } else if (request instanceof MultiSearchRequest) { + + for (ListIterator it = ((MultiSearchRequest) request).requests().listIterator(); it.hasNext();) { + SearchRequest ar = it.next(); + result = getOrReplaceAllIndices(ar, provider, false) && result; + /*if(ar.indices() == null || ar.indices().length == 0) { + it.remove(); + }*/ + } + + } else if (request instanceof MultiTermVectorsRequest) { + + for (ActionRequest ar : (Iterable) () -> ((MultiTermVectorsRequest) request).iterator()) { + result = getOrReplaceAllIndices(ar, provider, false) && result; + } + + } else if (request instanceof PutMappingRequest) { + PutMappingRequest pmr = (PutMappingRequest) request; + Index concreteIndex = pmr.getConcreteIndex(); + if (concreteIndex != null && (pmr.indices() == null || pmr.indices().length == 0)) { + String[] newIndices = provider.provide(new String[] { concreteIndex.getName() }, request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + + ((PutMappingRequest) request).indices(newIndices); + ((PutMappingRequest) request).setConcreteIndex(null); + } else { + String[] newIndices = provider.provide(((PutMappingRequest) request).indices(), request, true); + if (checkIndices(request, newIndices, false, allowEmptyIndices) == false) { + return false; + } + ((PutMappingRequest) request).indices(newIndices); + } + } else if (request instanceof RestoreSnapshotRequest) { + + if (clusterInfoHolder.isLocalNodeElectedClusterManager() == Boolean.FALSE) { + return true; + } + + final RestoreSnapshotRequest restoreRequest = (RestoreSnapshotRequest) request; + final SnapshotInfo snapshotInfo = SnapshotRestoreHelper.getSnapshotInfo(restoreRequest); + + if (snapshotInfo == null) { + log.warn( + "snapshot repository '" + restoreRequest.repository() + "', snapshot '" + restoreRequest.snapshot() + "' not found" + ); + provider.provide(new String[] { "*" }, request, false); + } else { + final List requestedResolvedIndices = IndexUtils.filterIndices( + snapshotInfo.indices(), + restoreRequest.indices(), + restoreRequest.indicesOptions() + ); + final List renamedTargetIndices = renamedIndices(restoreRequest, requestedResolvedIndices); + // final Set indices = new HashSet<>(requestedResolvedIndices); + // indices.addAll(renamedTargetIndices); + if (isDebugEnabled) { + log.debug("snapshot: {} contains this indices: {}", snapshotInfo.snapshotId().getName(), renamedTargetIndices); + } + provider.provide(renamedTargetIndices.toArray(new String[0]), request, false); + } + + } else if (request instanceof IndicesAliasesRequest) { + for (AliasActions ar : ((IndicesAliasesRequest) request).getAliasActions()) { + result = getOrReplaceAllIndices(ar, provider, false) && result; + } + } else if (request instanceof DeleteRequest) { + String[] newIndices = provider.provide(((DeleteRequest) request).indices(), request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + ((DeleteRequest) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof UpdateRequest) { + String[] newIndices = provider.provide(((UpdateRequest) request).indices(), request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + ((UpdateRequest) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof SingleShardRequest) { + final SingleShardRequest singleShardRequest = (SingleShardRequest) request; + final String index = singleShardRequest.index(); + String[] indices = provider.provide(index == null ? null : new String[] { index }, request, true); + if (!checkIndices(request, indices, true, allowEmptyIndices)) { + return false; + } + singleShardRequest.index(indices.length != 1 ? null : indices[0]); + } else if (request instanceof FieldCapabilitiesIndexRequest) { + // FieldCapabilitiesIndexRequest does not support replacing the indexes. + // However, the indexes are always determined by FieldCapabilitiesRequest which will be reduced below + // (implements Replaceable). So IF an index arrives here, we can be sure that we have + // at least privileges for indices:data/read/field_caps + FieldCapabilitiesIndexRequest fieldCapabilitiesRequest = (FieldCapabilitiesIndexRequest) request; + + String index = fieldCapabilitiesRequest.index(); + + String[] newIndices = provider.provide(new String[] { index }, request, true); + if (!checkIndices(request, newIndices, true, allowEmptyIndices)) { + return false; + } + } else if (request instanceof IndexRequest) { + String[] newIndices = provider.provide(((IndexRequest) request).indices(), request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + ((IndexRequest) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof Replaceable) { + String[] newIndices = provider.provide(((Replaceable) request).indices(), request, true); + if (checkIndices(request, newIndices, false, allowEmptyIndices) == false) { + return false; + } + ((Replaceable) request).indices(newIndices); + } else if (request instanceof BulkShardRequest) { + provider.provide(((ReplicationRequest) request).indices(), request, false); + // replace not supported? + } else if (request instanceof ReplicationRequest) { + String[] newIndices = provider.provide(((ReplicationRequest) request).indices(), request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + ((ReplicationRequest) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof MultiGetRequest.Item) { + String[] newIndices = provider.provide(((MultiGetRequest.Item) request).indices(), request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + ((MultiGetRequest.Item) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof CreateIndexRequest) { + String[] newIndices = provider.provide(((CreateIndexRequest) request).indices(), request, true); + if (checkIndices(request, newIndices, true, allowEmptyIndices) == false) { + return false; + } + ((CreateIndexRequest) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof ResizeRequest) { + // clone or shrink operations + provider.provide(((ResizeRequest) request).indices(), request, true); + provider.provide(((ResizeRequest) request).getTargetIndexRequest().indices(), request, true); + } else if (request instanceof CreateDataStreamAction.Request) { + provider.provide(((CreateDataStreamAction.Request) request).indices(), request, false); + } else if (request instanceof ReindexRequest) { + result = getOrReplaceAllIndices(((ReindexRequest) request).getDestination(), provider, false) && result; + result = getOrReplaceAllIndices(((ReindexRequest) request).getSearchRequest(), provider, false) && result; + } else if (request instanceof BaseNodesRequest) { + // do nothing + } else if (request instanceof MainRequest) { + // do nothing + } else if (request instanceof ClearScrollRequest) { + // do nothing + } else if (request instanceof SearchScrollRequest) { + // do nothing + } else if (request instanceof PutComponentTemplateAction.Request) { + // do nothing + } else { + if (isDebugEnabled) { + log.debug(request.getClass() + " not supported (It is likely not a indices related request)"); + } + result = false; + } + + return result; + } + + private IndicesOptions indicesOptionsFrom(Object localRequest) { + if (IndicesRequest.class.isInstance(localRequest)) { + return ((IndicesRequest) localRequest).indicesOptions(); + } else if (RestoreSnapshotRequest.class.isInstance(localRequest)) { + return ((RestoreSnapshotRequest) localRequest).indicesOptions(); + } else { + return IndicesOptions.fromOptions(false, true, true, false, true); + } + } +} diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java new file mode 100644 index 0000000000000..2005736fc91bb --- /dev/null +++ b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java @@ -0,0 +1,108 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.filter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.SpecialPermission; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.util.IndexUtils; +import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.snapshots.SnapshotId; +import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.threadpool.ThreadPool; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.List; +import java.util.Objects; + +public class SnapshotRestoreHelper { + + protected static final Logger log = LogManager.getLogger(SnapshotRestoreHelper.class); + + public static List resolveOriginalIndices(RestoreSnapshotRequest restoreRequest) { + final SnapshotInfo snapshotInfo = getSnapshotInfo(restoreRequest); + + if (snapshotInfo == null) { + log.warn("snapshot repository '{}', snapshot '{}' not found", restoreRequest.repository(), restoreRequest.snapshot()); + return null; + } else { + return IndexUtils.filterIndices(snapshotInfo.indices(), restoreRequest.indices(), restoreRequest.indicesOptions()); + } + + } + + public static SnapshotInfo getSnapshotInfo(RestoreSnapshotRequest restoreRequest) { + final RepositoriesService repositoriesService = Objects.requireNonNull( + SystemIndexProtectionPlugin.GuiceHolder.getRepositoriesService(), + "RepositoriesService not initialized" + ); + final Repository repository = repositoriesService.repository(restoreRequest.repository()); + final String threadName = Thread.currentThread().getName(); + SnapshotInfo snapshotInfo = null; + + try { + setCurrentThreadName("[" + ThreadPool.Names.GENERIC + "]"); + for (SnapshotId snapshotId : PlainActionFuture.get(repository::getRepositoryData).getSnapshotIds()) { + if (snapshotId.getName().equals(restoreRequest.snapshot())) { + + if (log.isDebugEnabled()) { + log.debug("snapshot found: {} (UUID: {})", snapshotId.getName(), snapshotId.getUUID()); + } + + snapshotInfo = repository.getSnapshotInfo(snapshotId); + break; + } + } + } finally { + setCurrentThreadName(threadName); + } + return snapshotInfo; + } + + @SuppressWarnings("removal") + private static void setCurrentThreadName(final String name) { + final SecurityManager sm = System.getSecurityManager(); + + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Object run() { + Thread.currentThread().setName(name); + return null; + } + }); + } + +} diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java new file mode 100644 index 0000000000000..733977320da7b --- /dev/null +++ b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.filter; + +import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.support.ActionFilter; +import org.opensearch.action.support.ActionFilterChain; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.indices.SystemIndexRegistry; +import org.opensearch.tasks.Task; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class SystemIndexFilter implements ActionFilter { + + private final IndexResolverReplacer indexResolverReplacer; + private final WildcardMatcher deniedActionsMatcher; + + public SystemIndexFilter(final IndexResolverReplacer indexResolverReplacer) { + this.indexResolverReplacer = indexResolverReplacer; + + final List deniedActionPatternsList = deniedActionPatterns(); + + deniedActionsMatcher = WildcardMatcher.from(deniedActionPatternsList); + } + + private static List deniedActionPatterns() { + final List systemIndexDeniedActionPatternsList = new ArrayList<>(); + systemIndexDeniedActionPatternsList.add("indices:data/write*"); + systemIndexDeniedActionPatternsList.add("indices:admin/delete*"); + systemIndexDeniedActionPatternsList.add("indices:admin/mapping/delete*"); + systemIndexDeniedActionPatternsList.add("indices:admin/mapping/put*"); + systemIndexDeniedActionPatternsList.add("indices:admin/freeze*"); + systemIndexDeniedActionPatternsList.add("indices:admin/settings/update*"); + systemIndexDeniedActionPatternsList.add("indices:admin/aliases"); + systemIndexDeniedActionPatternsList.add("indices:admin/close*"); + systemIndexDeniedActionPatternsList.add("cluster:admin/snapshot/restore*"); + return systemIndexDeniedActionPatternsList; + } + + @Override + public int order() { + return 0; + } + + @Override + public void apply( + Task task, + String action, + Request request, + ActionListener listener, + ActionFilterChain chain + ) { + if (deniedActionsMatcher.test(action)) { + final IndexResolverReplacer.Resolved resolved = indexResolverReplacer.resolveRequest(request); + final Set allIndices = resolved.getAllIndices(); + Set matchingSystemIndices = SystemIndexRegistry.matchesSystemIndexPattern(allIndices); + if (!matchingSystemIndices.isEmpty()) { + String err = String.format("Cannot perform %s on matching system indices %s", action, matchingSystemIndices); + listener.onFailure(new OpenSearchSecurityException(err, RestStatus.FORBIDDEN)); + return; + } + } + chain.proceed(task, action, request, listener); + } +} diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java new file mode 100644 index 0000000000000..2a75fec123411 --- /dev/null +++ b/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java @@ -0,0 +1,559 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.filter; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public abstract class WildcardMatcher implements Predicate { + + private static void checkArgument(boolean expression) { + if (!expression) { + throw new IllegalArgumentException(); + } + } + + public static final WildcardMatcher ANY = new WildcardMatcher() { + + @Override + public boolean matchAny(Stream candidates) { + return true; + } + + @Override + public boolean matchAny(Collection candidates) { + return true; + } + + @Override + public boolean matchAny(String... candidates) { + return true; + } + + @Override + public boolean matchAll(Stream candidates) { + return true; + } + + @Override + public boolean matchAll(Collection candidates) { + return true; + } + + @Override + public boolean matchAll(String[] candidates) { + return true; + } + + @Override + public > T getMatchAny(Stream candidates, Collector collector) { + return candidates.collect(collector); + } + + @Override + public boolean test(String candidate) { + return true; + } + + @Override + public String toString() { + return "*"; + } + }; + + public static final WildcardMatcher NONE = new WildcardMatcher() { + + @Override + public boolean matchAny(Stream candidates) { + return false; + } + + @Override + public boolean matchAny(Collection candidates) { + return false; + } + + @Override + public boolean matchAny(String... candidates) { + return false; + } + + @Override + public boolean matchAll(Stream candidates) { + return false; + } + + @Override + public boolean matchAll(Collection candidates) { + return false; + } + + @Override + public boolean matchAll(String[] candidates) { + return false; + } + + @Override + public > T getMatchAny(Stream candidates, Collector collector) { + return Stream.empty().collect(collector); + } + + @Override + public > T getMatchAny(Collection candidate, Collector collector) { + return Stream.empty().collect(collector); + } + + @Override + public > T getMatchAny(String[] candidate, Collector collector) { + return Stream.empty().collect(collector); + } + + @Override + public boolean test(String candidate) { + return false; + } + + @Override + public String toString() { + return ""; + } + }; + + public static WildcardMatcher from(String pattern, boolean caseSensitive) { + if (pattern == null) { + return NONE; + } else if (pattern.equals("*")) { + return ANY; + } else if (pattern.startsWith("/") && pattern.endsWith("/")) { + return new RegexMatcher(pattern, caseSensitive); + } else if (pattern.indexOf('?') >= 0 || pattern.indexOf('*') >= 0) { + return caseSensitive ? new SimpleMatcher(pattern) : new CasefoldingMatcher(pattern, SimpleMatcher::new); + } else { + return caseSensitive ? new Exact(pattern) : new CasefoldingMatcher(pattern, Exact::new); + } + } + + public static WildcardMatcher from(String pattern) { + return from(pattern, true); + } + + // This may in future use more optimized techniques to combine multiple WildcardMatchers in a single automaton + public static WildcardMatcher from(Stream stream, boolean caseSensitive) { + Collection matchers = stream.map(t -> { + if (t == null) { + return NONE; + } else if (t instanceof String) { + return WildcardMatcher.from(((String) t), caseSensitive); + } else if (t instanceof WildcardMatcher) { + return ((WildcardMatcher) t); + } + throw new UnsupportedOperationException("WildcardMatcher can't be constructed from " + t.getClass().getSimpleName()); + }).collect(Collectors.toSet()); + + if (matchers.isEmpty()) { + return NONE; + } else if (matchers.size() == 1) { + return matchers.stream().findFirst().get(); + } + return new MatcherCombiner(matchers); + } + + public static WildcardMatcher from(Collection collection, boolean caseSensitive) { + if (collection == null || collection.isEmpty()) { + return NONE; + } else if (collection.size() == 1) { + T t = collection.stream().findFirst().get(); + if (t instanceof String) { + return from(((String) t), caseSensitive); + } else if (t instanceof WildcardMatcher) { + return ((WildcardMatcher) t); + } + throw new UnsupportedOperationException("WildcardMatcher can't be constructed from " + t.getClass().getSimpleName()); + } + return from(collection.stream(), caseSensitive); + } + + public static WildcardMatcher from(String[] patterns, boolean caseSensitive) { + if (patterns == null || patterns.length == 0) { + return NONE; + } else if (patterns.length == 1) { + return from(patterns[0], caseSensitive); + } + return from(Arrays.stream(patterns), caseSensitive); + } + + public static WildcardMatcher from(Stream patterns) { + return from(patterns, true); + } + + public static WildcardMatcher from(Collection patterns) { + return from(patterns, true); + } + + public static WildcardMatcher from(String... patterns) { + return from(patterns, true); + } + + public WildcardMatcher concat(Stream matchers) { + return new WildcardMatcher.MatcherCombiner(Stream.concat(matchers, Stream.of(this)).collect(Collectors.toSet())); + } + + public WildcardMatcher concat(Collection matchers) { + if (matchers.isEmpty()) { + return this; + } + return concat(matchers.stream()); + } + + public WildcardMatcher concat(WildcardMatcher... matchers) { + if (matchers.length == 0) { + return this; + } + return concat(Arrays.stream(matchers)); + } + + public boolean matchAny(Stream candidates) { + return candidates.anyMatch(this); + } + + public boolean matchAny(Collection candidates) { + return matchAny(candidates.stream()); + } + + public boolean matchAny(String... candidates) { + return matchAny(Arrays.stream(candidates)); + } + + public boolean matchAll(Stream candidates) { + return candidates.allMatch(this); + } + + public boolean matchAll(Collection candidates) { + return matchAll(candidates.stream()); + } + + public boolean matchAll(String[] candidates) { + return matchAll(Arrays.stream(candidates)); + } + + public > T getMatchAny(Stream candidates, Collector collector) { + return candidates.filter(this).collect(collector); + } + + public > T getMatchAny(Collection candidate, Collector collector) { + return getMatchAny(candidate.stream(), collector); + } + + public > T getMatchAny(final String[] candidate, Collector collector) { + return getMatchAny(Arrays.stream(candidate), collector); + } + + public Optional findFirst(final String candidate) { + return Optional.ofNullable(test(candidate) ? this : null); + } + + public Iterable iterateMatching(Iterable candidates) { + return iterateMatching(candidates, Function.identity()); + } + + public Iterable iterateMatching(Iterable candidates, Function toStringFunction) { + return new Iterable() { + + @Override + public Iterator iterator() { + Iterator delegate = candidates.iterator(); + + return new Iterator() { + private E next; + + @Override + public boolean hasNext() { + if (next == null) { + init(); + } + + return next != null; + } + + @Override + public E next() { + if (next == null) { + init(); + } + + E result = next; + next = null; + return result; + } + + private void init() { + while (delegate.hasNext()) { + E candidate = delegate.next(); + + if (test(toStringFunction.apply(candidate))) { + next = candidate; + break; + } + } + } + }; + } + }; + } + + public static List matchers(Collection patterns) { + return patterns.stream().map(p -> WildcardMatcher.from(p, true)).collect(Collectors.toList()); + } + + public static List getAllMatchingPatterns(final Collection matchers, final String candidate) { + return matchers.stream().filter(p -> p.test(candidate)).map(Objects::toString).collect(Collectors.toList()); + } + + public static List getAllMatchingPatterns(final Collection pattern, final Collection candidates) { + return pattern.stream().filter(p -> p.matchAny(candidates)).map(Objects::toString).collect(Collectors.toList()); + } + + public static boolean isExact(String pattern) { + return pattern == null || !(pattern.contains("*") || pattern.contains("?") || (pattern.startsWith("/") && pattern.endsWith("/"))); + } + + // + // --- Implementation specializations --- + // + // Casefolding matcher - sits on top of case-sensitive matcher + // and proxies toLower() of input string to the wrapped matcher + private static final class CasefoldingMatcher extends WildcardMatcher { + + private final WildcardMatcher inner; + + public CasefoldingMatcher(String pattern, Function simpleWildcardMatcher) { + this.inner = simpleWildcardMatcher.apply(pattern.toLowerCase()); + } + + @Override + public boolean test(String candidate) { + return inner.test(candidate.toLowerCase()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CasefoldingMatcher that = (CasefoldingMatcher) o; + return inner.equals(that.inner); + } + + @Override + public int hashCode() { + return inner.hashCode(); + } + + @Override + public String toString() { + return inner.toString(); + } + } + + public static final class Exact extends WildcardMatcher { + + private final String pattern; + + private Exact(String pattern) { + this.pattern = pattern; + } + + @Override + public boolean test(String candidate) { + return pattern.equals(candidate); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Exact that = (Exact) o; + return pattern.equals(that.pattern); + } + + @Override + public int hashCode() { + return pattern.hashCode(); + } + + @Override + public String toString() { + return pattern; + } + } + + // RegexMatcher uses JDK Pattern to test for matching, + // assumes "//" strings as input pattern + private static final class RegexMatcher extends WildcardMatcher { + + private final Pattern pattern; + + private RegexMatcher(String pattern, boolean caseSensitive) { + checkArgument(pattern.length() > 1 && pattern.startsWith("/") && pattern.endsWith("/")); + final String stripSlashesPattern = pattern.substring(1, pattern.length() - 1); + this.pattern = caseSensitive + ? Pattern.compile(stripSlashesPattern) + : Pattern.compile(stripSlashesPattern, Pattern.CASE_INSENSITIVE); + } + + @Override + public boolean test(String candidate) { + return pattern.matcher(candidate).matches(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RegexMatcher that = (RegexMatcher) o; + return pattern.pattern().equals(that.pattern.pattern()); + } + + @Override + public int hashCode() { + return pattern.pattern().hashCode(); + } + + @Override + public String toString() { + return "/" + pattern.pattern() + "/"; + } + } + + // Simple implementation of WildcardMatcher matcher with * and ? without + // using exlicit stack or recursion (as long as we don't need sub-matches it does work) + // allows us to save on resources and heap allocations unless Regex is required + private static final class SimpleMatcher extends WildcardMatcher { + + private final String pattern; + + SimpleMatcher(String pattern) { + this.pattern = pattern; + } + + @Override + public boolean test(String candidate) { + int i = 0; + int j = 0; + int n = candidate.length(); + int m = pattern.length(); + int text_backup = -1; + int wild_backup = -1; + while (i < n) { + if (j < m && pattern.charAt(j) == '*') { + text_backup = i; + wild_backup = ++j; + } else if (j < m && (pattern.charAt(j) == '?' || pattern.charAt(j) == candidate.charAt(i))) { + i++; + j++; + } else { + if (wild_backup == -1) return false; + i = ++text_backup; + j = wild_backup; + } + } + while (j < m && pattern.charAt(j) == '*') + j++; + return j >= m; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SimpleMatcher that = (SimpleMatcher) o; + return pattern.equals(that.pattern); + } + + @Override + public int hashCode() { + return pattern.hashCode(); + } + + @Override + public String toString() { + return pattern; + } + } + + // MatcherCombiner is a combination of a set of matchers + // matches if any of the set do + // Empty MultiMatcher always returns false + private static final class MatcherCombiner extends WildcardMatcher { + + private final Collection wildcardMatchers; + private final int hashCode; + + MatcherCombiner(Collection wildcardMatchers) { + checkArgument(wildcardMatchers.size() > 1); + this.wildcardMatchers = wildcardMatchers; + hashCode = wildcardMatchers.hashCode(); + } + + @Override + public boolean test(String candidate) { + return wildcardMatchers.stream().anyMatch(m -> m.test(candidate)); + } + + @Override + public Optional findFirst(final String candidate) { + return wildcardMatchers.stream().filter(m -> m.test(candidate)).findFirst(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MatcherCombiner that = (MatcherCombiner) o; + return wildcardMatchers.equals(that.wildcardMatchers); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public String toString() { + return wildcardMatchers.toString(); + } + } +} diff --git a/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java b/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java new file mode 100644 index 0000000000000..cb22c6d78138d --- /dev/null +++ b/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.plugin.systemindex; + +import org.opensearch.action.support.ActionFilter; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.index.filter.ClusterInfoHolder; +import org.opensearch.index.filter.IndexResolverReplacer; +import org.opensearch.index.filter.SystemIndexFilter; +import org.opensearch.plugins.ActionPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.RemoteClusterService; +import org.opensearch.transport.TransportService; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +public class SystemIndexProtectionPlugin extends Plugin implements ActionPlugin { + + private volatile SystemIndexFilter sif; + private volatile IndexResolverReplacer irr; + + @Override + public Collection createComponents( + Client localClient, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + final ClusterInfoHolder cih = new ClusterInfoHolder(clusterService.getClusterName().value()); + + final IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(threadPool.getThreadContext()); + irr = new IndexResolverReplacer(resolver, clusterService::state, cih); + sif = new SystemIndexFilter(irr); + return Collections.emptySet(); + } + + @Override + public List getActionFilters() { + List filters = new ArrayList<>(1); + filters.add(Objects.requireNonNull(sif)); + return filters; + } + + @Override + public Collection> getGuiceServiceClasses() { + final List> services = new ArrayList<>(1); + services.add(GuiceHolder.class); + return services; + } + + public static class GuiceHolder implements LifecycleComponent { + + private static RepositoriesService repositoriesService; + private static RemoteClusterService remoteClusterService; + + @Inject + public GuiceHolder(final RepositoriesService repositoriesService, final TransportService remoteClusterService) { + GuiceHolder.repositoriesService = repositoriesService; + GuiceHolder.remoteClusterService = remoteClusterService.getRemoteClusterService(); + } + + public static RepositoriesService getRepositoriesService() { + return repositoriesService; + } + + public static RemoteClusterService getRemoteClusterService() { + return remoteClusterService; + } + + @Override + public void close() {} + + @Override + public Lifecycle.State lifecycleState() { + return null; + } + + @Override + public void addLifecycleListener(LifecycleListener listener) {} + + @Override + public void removeLifecycleListener(LifecycleListener listener) {} + + @Override + public void start() {} + + @Override + public void stop() {} + + } +} diff --git a/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java b/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java new file mode 100644 index 0000000000000..119f1e2d495b7 --- /dev/null +++ b/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.mapper.size; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class SystemIndexProtectionYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + + public SystemIndexProtectionYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return createParameters(); + } +} diff --git a/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml b/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml new file mode 100644 index 0000000000000..ce35b54391557 --- /dev/null +++ b/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml @@ -0,0 +1,22 @@ +# Integration tests for Mapper Size components +# + +--- +"System Index Protection": + - skip: + features: "allowed_warnings" + + - do: + indices.create: + index: .tasks + + - do: + catch: forbidden + allowed_warnings: + - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default" + indices.delete: + index: .tasks + + - match: { status: 403 } + - match: { error.root_cause.0.type: "security_exception" } + - match: { error.root_cause.0.reason: "Cannot perform indices:admin/delete on matching system indices [.tasks]" } From 981e75659129182c3e58ef2b3ae3a8f6211bb5ff Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 20 Nov 2024 15:18:33 -0500 Subject: [PATCH 02/10] Add new sandbox module that moves system index protection to the core Signed-off-by: Craig Perkins --- .../modules}/system-index-protection/build.gradle | 0 .../java/org/opensearch/index/system}/SystemIndexPluginIT.java | 2 +- .../opensearch/index/system}/SystemIndexProtectionTests.java | 2 +- .../java/org/opensearch/index/filter/ClusterInfoHolder.java | 0 .../java/org/opensearch/index/filter/IndexResolverReplacer.java | 0 .../java/org/opensearch/index/filter/SnapshotRestoreHelper.java | 0 .../java/org/opensearch/index/filter/SystemIndexFilter.java | 0 .../main/java/org/opensearch/index/filter/WildcardMatcher.java | 0 .../plugin/systemindex/SystemIndexProtectionPlugin.java | 0 .../index/system}/SystemIndexProtectionYamlTestSuiteIT.java | 2 +- .../rest-api-spec/test/system_index_protection/10_basic.yml | 0 11 files changed, 3 insertions(+), 3 deletions(-) rename {modules => sandbox/modules}/system-index-protection/build.gradle (100%) rename {modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size => sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system}/SystemIndexPluginIT.java (97%) rename {modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size => sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system}/SystemIndexProtectionTests.java (97%) rename {modules => sandbox/modules}/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java (100%) rename {modules => sandbox/modules}/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java (100%) rename {modules => sandbox/modules}/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java (100%) rename {modules => sandbox/modules}/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java (100%) rename {modules => sandbox/modules}/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java (100%) rename {modules => sandbox/modules}/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java (100%) rename {modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size => sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system}/SystemIndexProtectionYamlTestSuiteIT.java (97%) rename {modules => sandbox/modules}/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml (100%) diff --git a/modules/system-index-protection/build.gradle b/sandbox/modules/system-index-protection/build.gradle similarity index 100% rename from modules/system-index-protection/build.gradle rename to sandbox/modules/system-index-protection/build.gradle diff --git a/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java similarity index 97% rename from modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java rename to sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java index f7f991146a030..0c73ed89a7ae1 100644 --- a/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexPluginIT.java +++ b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.index.mapper.size; +package org.opensearch.index.system; import org.opensearch.OpenSearchSecurityException; import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; diff --git a/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java similarity index 97% rename from modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java rename to sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java index 59283a0fd8871..8bb4b9d4fb9a3 100644 --- a/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionTests.java +++ b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.mapper.size; +package org.opensearch.index.system; import org.opensearch.OpenSearchSecurityException; import org.opensearch.action.admin.indices.delete.DeleteIndexRequestBuilder; diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java similarity index 100% rename from modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java rename to sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/ClusterInfoHolder.java diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java similarity index 100% rename from modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java rename to sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/IndexResolverReplacer.java diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java similarity index 100% rename from modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java rename to sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SnapshotRestoreHelper.java diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java similarity index 100% rename from modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java rename to sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java diff --git a/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java similarity index 100% rename from modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java rename to sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java diff --git a/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java similarity index 100% rename from modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java rename to sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java diff --git a/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java b/sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java similarity index 97% rename from modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java rename to sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java index 119f1e2d495b7..c62ceb5bfa9e3 100644 --- a/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/mapper/size/SystemIndexProtectionYamlTestSuiteIT.java +++ b/sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.mapper.size; +package org.opensearch.index.system; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; diff --git a/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml b/sandbox/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml similarity index 100% rename from modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml rename to sandbox/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml From 92de73b9e2a8594503263bbea81fbb78327dbe53 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 20 Nov 2024 15:19:11 -0500 Subject: [PATCH 03/10] Update missing-javadoc Signed-off-by: Craig Perkins --- gradle/missing-javadoc.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index 38d2346713273..f95e822465dee 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -123,7 +123,6 @@ configure([ project(":modules:rank-eval"), project(":modules:reindex"), project(":modules:repository-url"), - project(":modules:system-index-protection"), project(":modules:systemd"), project(":modules:transport-netty4"), project(":plugins:analysis-icu"), @@ -153,6 +152,7 @@ configure([ project(":plugins:crypto-kms"), project(":qa:die-with-dignity"), project(":qa:wildfly"), + project(":sandbox:modules:system-index-protection"), project(":test:external-modules:test-delayed-aggs"), project(":test:fixtures:azure-fixture"), project(":test:fixtures:gcs-fixture"), From dc757242ca5137566632471d226a8e55db667daf Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 20 Nov 2024 15:20:42 -0500 Subject: [PATCH 04/10] Update license headers Signed-off-by: Craig Perkins --- .../index/system/SystemIndexPluginIT.java | 23 ------------------ .../system/SystemIndexProtectionTests.java | 24 ------------------- .../SystemIndexProtectionYamlTestSuiteIT.java | 24 ------------------- 3 files changed, 71 deletions(-) diff --git a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java index 0c73ed89a7ae1..481fc292370fb 100644 --- a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java +++ b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java @@ -6,29 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.system; import org.opensearch.OpenSearchSecurityException; diff --git a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java index 8bb4b9d4fb9a3..deb1ba3a367d3 100644 --- a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java +++ b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.system; import org.opensearch.OpenSearchSecurityException; diff --git a/sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java b/sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java index c62ceb5bfa9e3..45661ebc819a5 100644 --- a/sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java +++ b/sandbox/modules/system-index-protection/src/yamlRestTest/java/org/opensearch/index/system/SystemIndexProtectionYamlTestSuiteIT.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.system; import com.carrotsearch.randomizedtesting.annotations.Name; From fb2cbb0a7d07ae7ae9eca2c27bcb1215df11b339 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 20 Nov 2024 15:51:39 -0500 Subject: [PATCH 05/10] Fix precommit failures Signed-off-by: Craig Perkins --- .../java/org/opensearch/index/filter/SystemIndexFilter.java | 3 ++- .../java/org/opensearch/index/filter/WildcardMatcher.java | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java index 733977320da7b..4d896e32ef854 100644 --- a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Set; public class SystemIndexFilter implements ActionFilter { @@ -67,7 +68,7 @@ public void app final Set allIndices = resolved.getAllIndices(); Set matchingSystemIndices = SystemIndexRegistry.matchesSystemIndexPattern(allIndices); if (!matchingSystemIndices.isEmpty()) { - String err = String.format("Cannot perform %s on matching system indices %s", action, matchingSystemIndices); + String err = String.format(Locale.ROOT, "Cannot perform %s on matching system indices %s", action, matchingSystemIndices); listener.onFailure(new OpenSearchSecurityException(err, RestStatus.FORBIDDEN)); return; } diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java index 2a75fec123411..207aa8219861e 100644 --- a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/WildcardMatcher.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.function.Function; @@ -361,12 +362,12 @@ private static final class CasefoldingMatcher extends WildcardMatcher { private final WildcardMatcher inner; public CasefoldingMatcher(String pattern, Function simpleWildcardMatcher) { - this.inner = simpleWildcardMatcher.apply(pattern.toLowerCase()); + this.inner = simpleWildcardMatcher.apply(pattern.toLowerCase(Locale.ROOT)); } @Override public boolean test(String candidate) { - return inner.test(candidate.toLowerCase()); + return inner.test(candidate.toLowerCase(Locale.ROOT)); } @Override From d48afcfbb5594385759d1df0b7f09489674a5368 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 21 Nov 2024 15:14:54 -0500 Subject: [PATCH 06/10] Allow system to perform actions on system indices Signed-off-by: Craig Perkins --- .../org/opensearch/index/filter/SystemIndexFilter.java | 9 ++++++++- .../plugin/systemindex/SystemIndexProtectionPlugin.java | 2 +- .../java/org/opensearch/client/OriginSettingClient.java | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java index 4d896e32ef854..9061244ab6634 100644 --- a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/index/filter/SystemIndexFilter.java @@ -17,6 +17,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.indices.SystemIndexRegistry; import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.List; @@ -27,9 +28,11 @@ public class SystemIndexFilter implements ActionFilter { private final IndexResolverReplacer indexResolverReplacer; private final WildcardMatcher deniedActionsMatcher; + private final ThreadPool threadPool; - public SystemIndexFilter(final IndexResolverReplacer indexResolverReplacer) { + public SystemIndexFilter(final IndexResolverReplacer indexResolverReplacer, final ThreadPool threadPool) { this.indexResolverReplacer = indexResolverReplacer; + this.threadPool = threadPool; final List deniedActionPatternsList = deniedActionPatterns(); @@ -63,6 +66,10 @@ public void app ActionListener listener, ActionFilterChain chain ) { + if (threadPool.getThreadContext().isSystemContext()) { + chain.proceed(task, action, request, listener); + return; + } if (deniedActionsMatcher.test(action)) { final IndexResolverReplacer.Resolved resolved = indexResolverReplacer.resolveRequest(request); final Set allIndices = resolved.getAllIndices(); diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java index cb22c6d78138d..6082fc598f4bb 100644 --- a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java @@ -86,7 +86,7 @@ public Collection createComponents( final IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(threadPool.getThreadContext()); irr = new IndexResolverReplacer(resolver, clusterService::state, cih); - sif = new SystemIndexFilter(irr); + sif = new SystemIndexFilter(irr, localClient.threadPool()); return Collections.emptySet(); } diff --git a/server/src/main/java/org/opensearch/client/OriginSettingClient.java b/server/src/main/java/org/opensearch/client/OriginSettingClient.java index 27d87227df7bc..78a5a03890407 100644 --- a/server/src/main/java/org/opensearch/client/OriginSettingClient.java +++ b/server/src/main/java/org/opensearch/client/OriginSettingClient.java @@ -71,6 +71,8 @@ protected void () -> in().threadPool().getThreadContext().stashWithOrigin(origin) ) ) { + ThreadContext threadContext = in().threadPool().getThreadContext(); + ThreadContextAccess.doPrivilegedVoid(threadContext::markAsSystemContext); super.doExecute(action, request, new ContextPreservingActionListener<>(supplier, listener)); } } From 0b11785284e555b111af28c78f8357ceb902196d Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 21 Nov 2024 15:57:27 -0500 Subject: [PATCH 07/10] Add setting to enable and disable Signed-off-by: Craig Perkins --- .../system-index-protection/build.gradle | 6 ++++ .../index/system/SystemIndexPluginIT.java | 9 ++++++ .../system/SystemIndexProtectionTests.java | 9 ++++++ .../SystemIndexProtectionPlugin.java | 31 ++++++++++++++++++- .../test/system_index_protection/10_basic.yml | 2 +- 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/sandbox/modules/system-index-protection/build.gradle b/sandbox/modules/system-index-protection/build.gradle index d3b5ae0c67d63..8bb42fa1d98ad 100644 --- a/sandbox/modules/system-index-protection/build.gradle +++ b/sandbox/modules/system-index-protection/build.gradle @@ -1,3 +1,5 @@ +import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE + /* * SPDX-License-Identifier: Apache-2.0 * @@ -24,3 +26,7 @@ restResources { } // no unit tests test.enabled = false + +testClusters.yamlRestTest { + setting 'modules.system_index_protection.system_indices.enabled', 'true' +} diff --git a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java index 481fc292370fb..b5da08ac2f7b1 100644 --- a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java +++ b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexPluginIT.java @@ -9,6 +9,7 @@ package org.opensearch.index.system; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.common.settings.Settings; import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -26,6 +27,14 @@ protected Collection> nodePlugins() { return Arrays.asList(SystemIndexProtectionPlugin.class); } + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(SystemIndexProtectionPlugin.SYSTEM_INDEX_PROTECTION_ENABLED_KEY, true) + .build(); + } + public void testBasic() throws Exception { assertAcked(prepareCreate(TASK_INDEX)); client().prepareDelete().setIndex(TASK_INDEX); diff --git a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java index deb1ba3a367d3..1fd480e60983a 100644 --- a/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java +++ b/sandbox/modules/system-index-protection/src/internalClusterTest/java/org/opensearch/index/system/SystemIndexProtectionTests.java @@ -10,6 +10,7 @@ import org.opensearch.OpenSearchSecurityException; import org.opensearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.opensearch.common.settings.Settings; import org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -24,6 +25,14 @@ protected Collection> getPlugins() { return pluginList(SystemIndexProtectionPlugin.class); } + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put(SystemIndexProtectionPlugin.SYSTEM_INDEX_PROTECTION_ENABLED_KEY, true) + .build(); + } + public void testBasic() throws Exception { createIndex(TASK_INDEX); DeleteIndexRequestBuilder deleteIndexRequestBuilder = client().admin().indices().prepareDelete(TASK_INDEX); diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java index 6082fc598f4bb..eebd1fa34c4b2 100644 --- a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java @@ -40,6 +40,8 @@ import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.common.lifecycle.LifecycleListener; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; @@ -56,6 +58,7 @@ import org.opensearch.transport.TransportService; import org.opensearch.watcher.ResourceWatcherService; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -65,8 +68,15 @@ public class SystemIndexProtectionPlugin extends Plugin implements ActionPlugin { + public static final String SYSTEM_INDEX_PROTECTION_ENABLED_KEY = "modules.system_index_protection.system_indices.enabled"; + private volatile SystemIndexFilter sif; private volatile IndexResolverReplacer irr; + private volatile Settings settings; + + public SystemIndexProtectionPlugin(final Settings settings, final Path configPath) { + this.settings = settings; + } @Override public Collection createComponents( @@ -93,7 +103,10 @@ public Collection createComponents( @Override public List getActionFilters() { List filters = new ArrayList<>(1); - filters.add(Objects.requireNonNull(sif)); + boolean isEnabled = settings.getAsBoolean(SYSTEM_INDEX_PROTECTION_ENABLED_KEY, false); + if (isEnabled) { + filters.add(Objects.requireNonNull(sif)); + } return filters; } @@ -144,4 +157,20 @@ public void start() {} public void stop() {} } + + @Override + public List> getSettings() { + List> settings = new ArrayList>(); + + settings.add( + Setting.boolSetting( + SYSTEM_INDEX_PROTECTION_ENABLED_KEY, + false, + Setting.Property.NodeScope, + Setting.Property.Filtered, + Setting.Property.Final + ) + ); + return settings; + } } diff --git a/sandbox/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml b/sandbox/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml index ce35b54391557..f3abb05415ec1 100644 --- a/sandbox/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml +++ b/sandbox/modules/system-index-protection/src/yamlRestTest/resources/rest-api-spec/test/system_index_protection/10_basic.yml @@ -1,4 +1,4 @@ -# Integration tests for Mapper Size components +# Integration tests for System Index Protection components # --- From 83896cd06e35b512eecb56d30d029070e2cad8f1 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 21 Nov 2024 16:35:19 -0500 Subject: [PATCH 08/10] Create SystemIndexSearcherWrapper Signed-off-by: Craig Perkins --- .../systemindex/EmptyFilterLeafReader.java | 97 +++++++++++++++++++ .../SystemIndexProtectionPlugin.java | 11 +++ .../SystemIndexSearcherWrapper.java | 62 ++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/EmptyFilterLeafReader.java create mode 100644 sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexSearcherWrapper.java diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/EmptyFilterLeafReader.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/EmptyFilterLeafReader.java new file mode 100644 index 0000000000000..5efbd137d3c24 --- /dev/null +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/EmptyFilterLeafReader.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.plugin.systemindex; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.Bits; + +import java.io.IOException; + +//copied from org.apache.lucene.index.AllDeletedFilterReader +//https://github.com/apache/lucene-solr/blob/1d85cd783863f75cea133fb9c452302214165a4d/lucene/test-framework/src/java/org/apache/lucene/index/AllDeletedFilterReader.java + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class EmptyFilterLeafReader extends FilterLeafReader { + + final Bits liveDocs; + + public EmptyFilterLeafReader(LeafReader in) { + super(in); + liveDocs = new Bits.MatchNoBits(in.maxDoc()); + assert maxDoc() == 0 || hasDeletions(); + } + + @Override + public Bits getLiveDocs() { + return liveDocs; + } + + @Override + public int numDocs() { + return 0; + } + + @Override + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return null; + } + + private static class EmptySubReaderWrapper extends FilterDirectoryReader.SubReaderWrapper { + + @Override + public LeafReader wrap(final LeafReader reader) { + return new EmptyFilterLeafReader(reader); + } + + } + + static class EmptyDirectoryReader extends FilterDirectoryReader { + + public EmptyDirectoryReader(final DirectoryReader in) throws IOException { + super(in, new EmptySubReaderWrapper()); + } + + @Override + protected DirectoryReader doWrapDirectoryReader(final DirectoryReader in) throws IOException { + return new EmptyDirectoryReader(in); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } + } +} diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java index eebd1fa34c4b2..9526abce56597 100644 --- a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexProtectionPlugin.java @@ -46,6 +46,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; +import org.opensearch.index.IndexModule; import org.opensearch.index.filter.ClusterInfoHolder; import org.opensearch.index.filter.IndexResolverReplacer; import org.opensearch.index.filter.SystemIndexFilter; @@ -110,6 +111,16 @@ public List getActionFilters() { return filters; } + @Override + public void onIndexModule(IndexModule indexModule) { + // called for every index! + boolean isEnabled = settings.getAsBoolean(SYSTEM_INDEX_PROTECTION_ENABLED_KEY, false); + + if (isEnabled) { + indexModule.setReaderWrapper(indexService -> new SystemIndexSearcherWrapper(indexService, settings)); + } + } + @Override public Collection> getGuiceServiceClasses() { final List> services = new ArrayList<>(1); diff --git a/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexSearcherWrapper.java b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexSearcherWrapper.java new file mode 100644 index 0000000000000..05d621c1b6922 --- /dev/null +++ b/sandbox/modules/system-index-protection/src/main/java/org/opensearch/plugin/systemindex/SystemIndexSearcherWrapper.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.plugin.systemindex; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.DirectoryReader; +import org.opensearch.common.CheckedFunction; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.index.Index; +import org.opensearch.index.IndexService; +import org.opensearch.indices.SystemIndexRegistry; + +import java.io.IOException; +import java.util.Set; + +import static org.opensearch.plugin.systemindex.SystemIndexProtectionPlugin.SYSTEM_INDEX_PROTECTION_ENABLED_KEY; + +public class SystemIndexSearcherWrapper implements CheckedFunction { + + protected final Logger log = LogManager.getLogger(this.getClass()); + protected final ThreadContext threadContext; + protected final Index index; + private final Boolean systemIndexEnabled; + + // constructor is called per index, so avoid costly operations here + public SystemIndexSearcherWrapper(final IndexService indexService, final Settings settings) { + index = indexService.index(); + threadContext = indexService.getThreadPool().getThreadContext(); + + this.systemIndexEnabled = settings.getAsBoolean(SYSTEM_INDEX_PROTECTION_ENABLED_KEY, false); + } + + @Override + public final DirectoryReader apply(DirectoryReader reader) throws IOException { + + if (systemIndexEnabled && isBlockedSystemIndexRequest() && !threadContext.isSystemContext()) { + log.warn("search action for {} is not allowed", index.getName()); + return new EmptyFilterLeafReader.EmptyDirectoryReader(reader); + } + + return wrap(reader); + } + + protected DirectoryReader wrap(final DirectoryReader reader) { + return reader; + } + + protected final boolean isBlockedSystemIndexRequest() { + return !SystemIndexRegistry.matchesSystemIndexPattern(Set.of(index.getName())).isEmpty(); + } +} From 0ef49bfa4ca28f512d2ec6212ffedc435e139886 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 22 Nov 2024 09:12:16 -0500 Subject: [PATCH 09/10] Move markAsSystemContext Signed-off-by: Craig Perkins --- .../main/java/org/opensearch/client/OriginSettingClient.java | 2 -- .../org/opensearch/common/util/concurrent/ThreadContext.java | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/client/OriginSettingClient.java b/server/src/main/java/org/opensearch/client/OriginSettingClient.java index 78a5a03890407..27d87227df7bc 100644 --- a/server/src/main/java/org/opensearch/client/OriginSettingClient.java +++ b/server/src/main/java/org/opensearch/client/OriginSettingClient.java @@ -71,8 +71,6 @@ protected void () -> in().threadPool().getThreadContext().stashWithOrigin(origin) ) ) { - ThreadContext threadContext = in().threadPool().getThreadContext(); - ThreadContextAccess.doPrivilegedVoid(threadContext::markAsSystemContext); super.doExecute(action, request, new ContextPreservingActionListener<>(supplier, listener)); } } diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java index 75a7ef94978d4..4e43326cdbf27 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java @@ -227,6 +227,7 @@ public StoredContext stashWithOrigin(String origin) { sm.checkPermission(STASH_WITH_ORIGIN_THREAD_CONTEXT_PERMISSION); } final ThreadContext.StoredContext storedContext = stashContext(); + ThreadContextAccess.doPrivilegedVoid(this::markAsSystemContext); putTransient(ACTION_ORIGIN_TRANSIENT_NAME, origin); return storedContext; } From 8a2a8a47e55cbe5006f50e78e0b246eefa3b8923 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 25 Nov 2024 10:01:21 -0500 Subject: [PATCH 10/10] Add WildcardMatcherTests Signed-off-by: Craig Perkins --- .../system-index-protection/build.gradle | 2 - .../index/filter/WildcardMatcherTests.java | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 sandbox/modules/system-index-protection/src/test/java/org/opensearch/index/filter/WildcardMatcherTests.java diff --git a/sandbox/modules/system-index-protection/build.gradle b/sandbox/modules/system-index-protection/build.gradle index 8bb42fa1d98ad..556e390028bbc 100644 --- a/sandbox/modules/system-index-protection/build.gradle +++ b/sandbox/modules/system-index-protection/build.gradle @@ -24,8 +24,6 @@ restResources { includeCore '_common', 'indices', 'index', 'get' } } -// no unit tests -test.enabled = false testClusters.yamlRestTest { setting 'modules.system_index_protection.system_indices.enabled', 'true' diff --git a/sandbox/modules/system-index-protection/src/test/java/org/opensearch/index/filter/WildcardMatcherTests.java b/sandbox/modules/system-index-protection/src/test/java/org/opensearch/index/filter/WildcardMatcherTests.java new file mode 100644 index 0000000000000..9e0681499a49e --- /dev/null +++ b/sandbox/modules/system-index-protection/src/test/java/org/opensearch/index/filter/WildcardMatcherTests.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.filter; + +import org.opensearch.test.OpenSearchTestCase; + +public class WildcardMatcherTests extends OpenSearchTestCase { + + static private WildcardMatcher wc(String pattern) { + return WildcardMatcher.from(pattern); + } + + static private WildcardMatcher iwc(String pattern) { + return WildcardMatcher.from(pattern, false); + } + + public void testWildcardMatcherClasses() { + assertFalse(wc("a*?").test("a")); + assertTrue(wc("a*?").test("aa")); + assertTrue(wc("a*?").test("ab")); + assertTrue(wc("a*?").test("abb")); + assertTrue(wc("*my*index").test("myindex")); + assertFalse(wc("*my*index").test("myindex1")); + assertTrue(wc("*my*index?").test("myindex1")); + assertTrue(wc("*my*index").test("this_is_my_great_index")); + assertFalse(wc("*my*index").test("MYindex")); + assertFalse(wc("?kibana").test("kibana")); + assertTrue(wc("?kibana").test(".kibana")); + assertFalse(wc("?kibana").test("kibana.")); + assertTrue(wc("?kibana?").test("?kibana.")); + assertTrue(wc("/(\\d{3}-?\\d{2}-?\\d{4})/").test("123-45-6789")); + assertFalse(wc("(\\d{3}-?\\d{2}-?\\d{4})").test("123-45-6789")); + assertTrue(wc("/\\S+/").test("abc")); + assertTrue(wc("abc").test("abc")); + assertFalse(wc("ABC").test("abc")); + assertFalse(wc(null).test("abc")); + assertTrue(WildcardMatcher.from(null, "abc").test("abc")); + } + + public void testWildcardMatcherClassesCaseInsensitive() { + assertTrue(iwc("AbC").test("abc")); + assertTrue(iwc("abc").test("aBC")); + assertTrue(iwc("A*b").test("ab")); + assertTrue(iwc("A*b").test("aab")); + assertTrue(iwc("A*b").test("abB")); + assertFalse(iwc("abc").test("AB")); + assertTrue(iwc("/^\\w+$/").test("AbCd")); + } + + public void testWildcardMatchers() { + assertTrue(!WildcardMatcher.from("a*?").test("a")); + assertTrue(WildcardMatcher.from("a*?").test("aa")); + assertTrue(WildcardMatcher.from("a*?").test("ab")); + // assertTrue(WildcardMatcher.pattern("a*?").test( "abb")); + assertTrue(WildcardMatcher.from("*my*index").test("myindex")); + assertTrue(!WildcardMatcher.from("*my*index").test("myindex1")); + assertTrue(WildcardMatcher.from("*my*index?").test("myindex1")); + assertTrue(WildcardMatcher.from("*my*index").test("this_is_my_great_index")); + assertTrue(!WildcardMatcher.from("*my*index").test("MYindex")); + assertTrue(!WildcardMatcher.from("?kibana").test("kibana")); + assertTrue(WildcardMatcher.from("?kibana").test(".kibana")); + assertTrue(!WildcardMatcher.from("?kibana").test("kibana.")); + assertTrue(WildcardMatcher.from("?kibana?").test("?kibana.")); + assertTrue(WildcardMatcher.from("/(\\d{3}-?\\d{2}-?\\d{4})/").test("123-45-6789")); + assertTrue(!WildcardMatcher.from("(\\d{3}-?\\d{2}-?\\d{4})").test("123-45-6789")); + assertTrue(WildcardMatcher.from("/\\S*/").test("abc")); + assertTrue(WildcardMatcher.from("abc").test("abc")); + assertTrue(!WildcardMatcher.from("ABC").test("abc")); + } +}