Fix/collision of indexpatterns

CHANGELOG.md

@@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Move pull-based ingestion classes from experimental to publicAPI ([#20704](https://github.com/opensearch-project/OpenSearch/pull/20704))
 
 ### Fixed
+- Relax index template pattern overlap check to use minimum-string heuristic, allowing distinguishable multi-wildcard patterns at the same priority ([#20702](https://github.com/opensearch-project/OpenSearch/pull/20702))
 - Fix `AutoForceMergeMetrics` silently dropping tags due to unreassigned `addTag()` return value ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
 - Fix flaky test failures in ShardsLimitAllocationDeciderIT ([#20375](https://github.com/opensearch-project/OpenSearch/pull/20375))
 - Prevent criteria update for context aware indices ([#20250](https://github.com/opensearch-project/OpenSearch/pull/20250))

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml

@@ -102,6 +102,51 @@
               number_of_shards:   1
               number_of_replicas: 0
 
+---
+"Put distinguishable multi-wildcard templates at the same priority":
+  - skip:
+      features: allowed_warnings
+      version: " - 3.5.99"
+      reason: "template overlap check was relaxed in 3.6"
+
+  - do:
+      allowed_warnings:
+        - "index template [app-test-some] has index patterns [app-test-*-some-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some] will take precedence during new index creation"
+      indices.put_index_template:
+        name: app-test-some
+        body:
+          index_patterns: ["app-test-*-some-*"]
+          priority: 0
+
+  # The second template has a literal segment after the first wildcard that differs from the first
+  # template (-some_other- vs -some-), so the two patterns do not practically overlap and both
+  # templates should be accepted at the same priority.
+  - do:
+      allowed_warnings:
+        - "index template [app-test-some-other] has index patterns [app-test-*-some_other-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some-other] will take precedence during new index creation"
+      indices.put_index_template:
+        name: app-test-some-other
+        body:
+          index_patterns: ["app-test-*-some_other-*"]
+          priority: 0
+
+  - do:
+      indices.get_index_template:
+        name: app-test-some
+  - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some-*"]}
+
+  - do:
+      indices.get_index_template:
+        name: app-test-some-other
+  - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some_other-*"]}
+
+  - do:
+      indices.delete_index_template:
+        name: app-test-some
+  - do:
+      indices.delete_index_template:
+        name: app-test-some-other
+
 ---
 "Put index template without index_patterns":
 

server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -34,8 +34,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.Operations;
 import org.opensearch.Version;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
@@ -799,13 +797,11 @@ public static Map<String, List<String>> findConflictingV1Templates(
         final String candidateName,
         final List<String> indexPatterns
     ) {
-        Automaton v2automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY));
         Map<String, List<String>> overlappingTemplates = new HashMap<>();
         for (final Map.Entry<String, IndexTemplateMetadata> cursor : state.metadata().templates().entrySet()) {
             String name = cursor.getKey();
             IndexTemplateMetadata template = cursor.getValue();
-            Automaton v1automaton = Regex.simpleMatchToAutomaton(template.patterns().toArray(Strings.EMPTY_ARRAY));
-            if (Operations.isEmpty(Operations.intersection(v2automaton, v1automaton)) == false) {
+            if (patternsActuallyOverlap(indexPatterns, template.patterns())) {
                 logger.debug(
                     "composable template {} and legacy template {} would overlap: {} <=> {}",
                     candidateName,
@@ -848,13 +844,11 @@ static Map<String, List<String>> findConflictingV2Templates(
         boolean checkPriority,
         long priority
     ) {
-        Automaton v1automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY));
         Map<String, List<String>> overlappingTemplates = new HashMap<>();
         for (Map.Entry<String, ComposableIndexTemplate> entry : state.metadata().templatesV2().entrySet()) {
             String name = entry.getKey();
             ComposableIndexTemplate template = entry.getValue();
-            Automaton v2automaton = Regex.simpleMatchToAutomaton(template.indexPatterns().toArray(Strings.EMPTY_ARRAY));
-            if (Operations.isEmpty(Operations.intersection(v1automaton, v2automaton)) == false) {
+            if (patternsActuallyOverlap(indexPatterns, template.indexPatterns())) {
                 if (checkPriority == false || priority == template.priorityOrZero()) {
                     logger.debug(
                         "legacy template {} and composable template {} would overlap: {} <=> {}",
@@ -873,6 +867,36 @@ static Map<String, List<String>> findConflictingV2Templates(
         return overlappingTemplates;
     }
 
+    /**
+     * Checks whether two sets of index patterns practically overlap, using a minimum-string heuristic.
+     * <p>
+     * For each pattern in either set, the wildcard character {@code *} is replaced with the empty string
+     * to produce its "minimum matching string". If that minimum string matches any pattern in the other
+     * set, the two pattern sets are considered to overlap.
+     * <p>
+     * This approach is intentionally more lenient than a full automaton intersection. Full automaton
+     * intersection correctly identifies theoretical overlaps that arise only from creative wildcard usage
+     * (e.g. {@code app-test-*-some-*} and {@code app-test-*-some_other-*} share the synthetic string
+     * {@code app-test--some_other--some-}), but these synthetic strings would never appear as real index
+     * names. The minimum-string heuristic restricts conflict detection to cases where "natural" index
+     * names would match both patterns simultaneously.
+     */
+    static boolean patternsActuallyOverlap(List<String> patterns1, List<String> patterns2) {
+        String[] p2Array = patterns2.toArray(Strings.EMPTY_ARRAY);
+        for (String p1 : patterns1) {
+            if (Regex.simpleMatch(p2Array, p1.replace("*", ""))) {
+                return true;
+            }
+        }
+        String[] p1Array = patterns1.toArray(Strings.EMPTY_ARRAY);
+        for (String p2 : patterns2) {
+            if (Regex.simpleMatch(p1Array, p2.replace("*", ""))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Remove the given index template from the cluster state. The index template name
      * supports simple regex wildcards for removing multiple index templates at a time.

server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -1320,6 +1320,55 @@ public void testPuttingOverlappingV2Template() throws Exception {
         }
     }
 
+    public void testPatternsActuallyOverlap() {
+        // Patterns that share a genuine conflict: baz matches baz*
+        assertTrue(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("egg*", "baz"), Arrays.asList("abc", "baz*")));
+
+        // Patterns with multi-wildcards where literal segments after the first wildcard distinguish them
+        assertFalse(
+            MetadataIndexTemplateService.patternsActuallyOverlap(
+                Arrays.asList("app-test-*-some-*"),
+                Arrays.asList("app-test-*-some_other-*")
+            )
+        );
+
+        // Patterns with a common prefix and different literal suffixes after a wildcard
+        assertFalse(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("foo-*-bar"), Arrays.asList("foo-*-baz")));
+
+        // Patterns where one is a superset of the other (logs-* matches any logs-prod-* index name)
+        assertTrue(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("logs-*"), Arrays.asList("logs-prod-*")));
+
+        // Completely disjoint literal prefixes do not overlap
+        assertFalse(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("foo-*"), Arrays.asList("bar-*")));
+
+        // Identical patterns overlap with themselves
+        assertTrue(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("app-*"), Arrays.asList("app-*")));
+    }
+
+    public void testDistinguishableMultiWildcardTemplatesAccepted() throws Exception {
+        MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+        ClusterState state = ClusterState.EMPTY_STATE;
+
+        ComposableIndexTemplate t1 = new ComposableIndexTemplate(Arrays.asList("app-test-*-some-*"), null, null, 0L, null, null, null);
+        state = service.addIndexTemplateV2(state, false, "app-test-some", t1);
+
+        // This must not throw: the second pattern is distinguishable from the first because
+        // the literal segment after the first wildcard differs (_other vs nothing).
+        ComposableIndexTemplate t2 = new ComposableIndexTemplate(
+            Arrays.asList("app-test-*-some_other-*"),
+            null,
+            null,
+            0L,
+            null,
+            null,
+            null
+        );
+        state = service.addIndexTemplateV2(state, false, "app-test-some-other", t2);
+
+        assertNotNull(state.metadata().templatesV2().get("app-test-some"));
+        assertNotNull(state.metadata().templatesV2().get("app-test-some-other"));
+    }
+
     public void testFindV2Templates() throws Exception {
         final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
         ClusterState state = ClusterState.EMPTY_STATE;