draft for gbh:

- Re-name SPI contract to ReaderManagerProvider to load in DataFormatAwareEngineFactory.
- Deletes EngineBridge - replace with SearchExecEngine
- add delegation framework api - sketch with just filter for now
- removes non reader functionality from DataFormatAwareEngineFactory/Engine.

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>

Author: mch2

.gitignore

@@ -1,7 +1,8 @@
 .claude
 CLAUDE.md
 .cursor*
-
+.kiro/
+**/target/**
 # intellij files
 .idea/
 *.iml

gradle/run.gradle

@@ -60,7 +60,9 @@ testClusters {
       for (String p : installedPlugins) {
         // check if its a local plugin first
         if (project.findProject(':plugins:' + p) != null) {
-          plugin('plugins:' + p)
+          plugin(':plugins:' + p)
+        } else if (project.findProject(':sandbox:plugins:' + p) != null) {
+          plugin(':sandbox:plugins:' + p)
         } else {
           // attempt to fetch it from maven
           project.repositories.mavenLocal()

sandbox/libs/analytics-framework/build.gradle

@@ -14,7 +14,15 @@
 
 def calciteVersion = '1.41.0'
 
+// Guava comes transitively from calcite-core — forbidden on compile classpaths by OpenSearch.
+// Bypass via custom config for classes that extend Calcite types referencing ImmutableList.
+configurations {
+  calciteCompile
+}
+sourceSets.main.compileClasspath += configurations.calciteCompile
+
 dependencies {
+  calciteCompile "com.google.guava:guava:${versions.guava}"
   compileOnly project(':server')
   api "org.apache.calcite:calcite-core:${calciteVersion}"
   // Calcite's expression tree and Enumerable runtime — required by calcite-core API
@@ -27,6 +35,30 @@ dependencies {
   // SLF4J — Calcite's logging facade
   runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
 
+  // Calcite optional deps required at runtime — BuiltInMethod.<clinit> reflectively loads ALL
+  // methods which triggers class loading for every type referenced in Calcite's SqlFunctions.
+  // Every single one of these is needed or the class initializer fails with NoClassDefFoundError.
+  runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
+  runtimeOnly "org.codehaus.janino:janino:3.1.12"
+  runtimeOnly "org.codehaus.janino:commons-compiler:3.1.12"
+  runtimeOnly 'org.jooq:joou-java-6:0.9.4'
+  runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+  runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
+  runtimeOnly "org.apache.commons:commons-lang3:${versions.commonslang}"
+  runtimeOnly 'org.apache.commons:commons-text:1.11.0'
+  runtimeOnly 'org.apache.commons:commons-math3:3.6.1'
+  runtimeOnly 'org.immutables:value-annotations:2.8.8'
+  runtimeOnly 'com.jayway.jsonpath:json-path:2.9.0'
+  runtimeOnly "net.minidev:json-smart:${versions.json_smart}"
+  runtimeOnly 'net.minidev:accessors-smart:2.5.2'
+  runtimeOnly 'org.ow2.asm:asm:9.7.1'
+  runtimeOnly 'org.apache.calcite.avatica:avatica-metrics:1.27.0'
+  runtimeOnly "org.locationtech.jts:jts-core:${versions.jts}"
+  runtimeOnly 'org.locationtech.jts.io:jts-io-common:1.19.0'
+  runtimeOnly 'org.locationtech.proj4j:proj4j:1.2.2'
+  runtimeOnly 'com.google.uzaygezen:uzaygezen-core:0.2'
+
   // Calcite bytecode references annotations from apiguardian (@API) and
   // checker-framework (@EnsuresNonNullIf). compileOnlyApi propagates to
   // consumers' compile/javadoc classpath without becoming a runtime dep.

sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/EngineBridge.java

@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.backend;
+
+import org.opensearch.analytics.delegation.DelegationContext;
+import org.opensearch.analytics.plan.ResolvedPlan;
+import org.opensearch.index.engine.DataFormatAwareEngine;
+import org.opensearch.plugins.ReaderManagerProvider;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Execution context carrying plan, reader, and delegation state through
+ * the query execution lifecycle.
+ *
+ * @opensearch.internal
+ */
+public class ExecutionContext {
+
+    private final ResolvedPlan plan;
+    private final String tableName;
+    private DelegationContext delegationContext;
+    private ReaderProvider readerProvider;
+
+    public ExecutionContext(ResolvedPlan plan, String tableName) {
+        this.plan = plan;
+        this.tableName = tableName;
+    }
+
+    public ResolvedPlan plan() {
+        return plan;
+    }
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    public void setDelegationContext(DelegationContext delegationContext) {
+        this.delegationContext = delegationContext;
+    }
+
+    public boolean hasDelegation() {
+        return delegationContext != null && delegationContext.hasDelegation();
+    }
+
+    public DelegationContext getDelegationContext() {
+        return delegationContext;
+    }
+}

sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java

@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.backend;
+
+import org.opensearch.analytics.plan.ResolvedPlan;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Shard-level search execution engine interface.
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface SearchExecEngine extends Closeable {
+
+    /**
+     * Creates an execution context from a resolved plan.
+     *
+     * @param context               ExecutionContext
+     */
+    void prepare(ExecutionContext context);
+
+    /** Executes the context and returns a result stream. */
+    EngineResultStream execute(ExecutionContext context) throws IOException;
+
+    @Override
+    default void close() throws IOException {}
+}

sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/SearchExecEngine.java

@@ -0,0 +1,91 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.delegation;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.analytics.delegation.filter.FilterDelegationTarget;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Registry mapping delegation context IDs to {@link DelegationTarget} instances.
+ * Each target gets its own ID. A single query may register multiple targets.
+ *
+ * <p>Rust JNI callbacks resolve targets via the static {@link #delegateFilter} entry point.
+ *
+ * @opensearch.internal
+ */
+@ExperimentalApi
+public class DelegationBroker {
+
+    private static final Logger logger = LogManager.getLogger(DelegationBroker.class);
+    private static final DelegationBroker INSTANCE = new DelegationBroker();
+
+    private final AtomicLong nextId = new AtomicLong(1);
+    private final ConcurrentHashMap<Long, DelegationTarget> targets = new ConcurrentHashMap<>();
+
+    public static DelegationBroker getInstance() {
+        return INSTANCE;
+    }
+
+    /**
+     * Registers a delegation target and returns its context ID.
+     */
+    public long register(DelegationTarget target) {
+        long id = nextId.getAndIncrement();
+        targets.put(id, target);
+        logger.info("[DelegationBroker] register: id={}, type={}", id, target.type());
+        return id;
+    }
+
+    /**
+     * Releases a delegation context.
+     */
+    public void release(long delegationContextId) {
+        targets.remove(delegationContextId);
+    }
+
+    /**
+     * Resolves a {@link FilterDelegationTarget} by context ID.
+     */
+    FilterDelegationTarget resolveFilterTarget(long delegationContextId) {
+        DelegationTarget target = targets.get(delegationContextId);
+        return target instanceof FilterDelegationTarget ? (FilterDelegationTarget) target : null;
+    }
+
+    /**
+     * Called from Rust via JNI to delegate a filter predicate.
+     *
+     * @param delegationContextId the context ID
+     * @param targetBackend       the backend name (for logging/routing)
+     * @param segmentOrd          0-based segment ordinal
+     * @param minDocId            inclusive min doc ID
+     * @param maxDocId            exclusive max doc ID
+     * @return matching doc IDs as BitSet.toLongArray(), or empty on error
+     */
+    public static long[] delegateFilter(
+        long delegationContextId, String targetBackend,
+        int segmentOrd, int minDocId, int maxDocId) {
+        logger.info("[DelegationBroker] delegateFilter: ctxId={}, backend={}, segment={}, docs=[{}, {})",
+            delegationContextId, targetBackend, segmentOrd, minDocId, maxDocId);
+
+        FilterDelegationTarget target = INSTANCE.resolveFilterTarget(delegationContextId);
+        if (target == null) {
+            logger.warn("[DelegationBroker] No FilterDelegationTarget for ctxId={}", delegationContextId);
+            return new long[0];
+        }
+
+        long[] result = target.delegateFilter(targetBackend, segmentOrd, minDocId, maxDocId);
+        logger.info("[DelegationBroker] delegateFilter result: segment={}, bitsetWords={}", segmentOrd, result.length);
+        return result;
+    }
+}

sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/delegation/DelegationBroker.java

@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.delegation;
+
+import java.util.List;
+
+/**
+ * Carries delegation state for a query. Holds the broker-assigned context IDs
+ * for all registered delegation targets.
+ *
+ * @opensearch.internal
+ */
+public class DelegationContext {
+
+    public static final DelegationContext NONE = new DelegationContext(List.of());
+
+    private final List<Long> ids;
+
+    public DelegationContext(List<Long> ids) {
+        this.ids = List.copyOf(ids);
+    }
+
+    /** All delegation context IDs for this query. */
+    public List<Long> getIds() {
+        return ids;
+    }
+
+    /** Returns true if this context carries active delegations. */
+    public boolean hasDelegation() {
+        return !ids.isEmpty();
+    }
+
+    /** Releases all delegation targets from the broker. */
+    public void release() {
+        DelegationBroker broker = DelegationBroker.getInstance();
+        for (long id : ids) {
+            broker.release(id);
+        }
+    }
+}