Merge branch 'main' into fix_148138

Author: lkts

docs/changelog/148261.yaml

@@ -0,0 +1,5 @@
+area: ES|QL
+issues: []
+pr: 148261
+summary: Skip filter eval when row group stats prove all rows pass
+type: enhancement

muted-tests.yml

@@ -548,3 +548,9 @@ tests:
 - class: org.elasticsearch.xpack.watcher.test.integration.SingleNodeTests
   method: testThatLoadingWithNonExistingIndexWorks
   issue: https://github.com/elastic/elasticsearch/issues/148323
+- class: org.elasticsearch.xpack.stateless.lucene.BlobCacheIndexInputTests
+  method: testWithByteBufferSlicesReturnsFalseAfterEviction
+  issue: https://github.com/elastic/elasticsearch/issues/148330
+- class: org.elasticsearch.xpack.security.authc.esnative.ReservedRealmElasticAutoconfigIntegTests
+  method: testAutoconfigFailedPasswordPromotion
+  issue: https://github.com/elastic/elasticsearch/issues/122668

server/src/main/java/org/elasticsearch/index/codec/tsdb/AbstractTSDBDocValuesProducer.java

@@ -387,13 +387,12 @@ public BlockLoader.Block tryRead(
                             }
                         } else if (isDense(firstDocId, lastDocId, count)) {
                             try (var builder = factory.singletonBytesRefs(count)) {
-                                long[] offsets = new long[count + 1];
+                                int[] offsets = new int[count + 1];
 
                                 long startOffset = addresses.get(firstDocId);
                                 for (int i = offset, j = 1; i < docs.count(); i++, j++) {
                                     int docId = docs.get(i);
-                                    long nextOffset = addresses.get(docId + 1) - startOffset;
-                                    offsets[j] = nextOffset;
+                                    offsets[j] = Math.toIntExact(addresses.get(docId + 1) - startOffset);
                                 }
 
                                 int length = Math.toIntExact(addresses.get(lastDocId + 1L) - startOffset);
@@ -887,7 +886,7 @@ void decodeBulk(int numBlocks, int firstDocId, int lastDocId, int count, BlockLo
             final int bufferSize = computeMultipleBlockBufferSize(firstBlockId, endBlockId);
 
             int offsetBufferIndex = 0;
-            final long[] offsetBuffer = new long[count + 1];
+            final int[] offsetBuffer = new int[count + 1];
             int valuesBufferIndex = 0;
             final byte[] valuesBuffer = new byte[bufferSize];
 

server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java

@@ -798,12 +798,12 @@ interface SingletonBytesRefBuilder extends Builder {
          * Append multiple BytesRef. Offsets contains offsets of each BytesRef in the byte array.
          * The length of the offsets array is one more than the number of BytesRefs.
          */
-        SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, long[] offsets) throws IOException;
+        SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, int[] offsets) throws IOException;
 
         /**
          * Append multiple BytesRefs, all with the same length.
          */
-        SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, long bytesRefLengths) throws IOException;
+        SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, int bytesRefLengths) throws IOException;
     }
 
     interface FloatBuilder extends Builder {

test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java

@@ -124,18 +124,18 @@ private BytesRefsBuilder() {
                     }
 
                     @Override
-                    public BlockLoader.SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, long[] offsets) throws IOException {
+                    public BlockLoader.SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, int[] offsets) {
                         for (int i = 0; i < offsets.length - 1; i++) {
-                            BytesRef ref = new BytesRef(bytes, (int) offsets[i], (int) (offsets[i + 1] - offsets[i]));
+                            BytesRef ref = new BytesRef(bytes, offsets[i], offsets[i + 1] - offsets[i]);
                             add(BytesRef.deepCopyOf(ref));
                         }
                         return this;
                     }
 
                     @Override
-                    public BlockLoader.SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, long bytesRefLengths) throws IOException {
+                    public BlockLoader.SingletonBytesRefBuilder appendBytesRefs(byte[] bytes, int bytesRefLengths) {
                         for (int i = 0; i < count; i++) {
-                            BytesRef ref = new BytesRef(bytes, (int) (i * bytesRefLengths), (int) bytesRefLengths);
+                            BytesRef ref = new BytesRef(bytes, (int) (i * bytesRefLengths), bytesRefLengths);
                             add(BytesRef.deepCopyOf(ref));
                         }
                         return this;

x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/OptimizedParquetColumnIterator.java

@@ -12,6 +12,7 @@
 import org.apache.parquet.column.impl.ColumnReadStoreImpl;
 import org.apache.parquet.column.page.PageReader;
 import org.apache.parquet.compression.CompressionCodecFactory;
+import org.apache.parquet.filter2.predicate.FilterPredicate;
 import org.apache.parquet.hadoop.ParquetFileReader;
 import org.apache.parquet.hadoop.metadata.BlockMetaData;
 import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -62,6 +63,13 @@
  * {@link BlockFactory#breaker()}) before async I/O starts. The reservation is released
  * when prefetched data is consumed and cleared. If the breaker would trip, prefetch is
  * skipped and the query falls back to synchronous I/O for that row group.
+ *
+ * <p><b>Trivially-passes guard:</b> when late materialization is enabled and row-group
+ * statistics prove every row satisfies the pushed filter ({@link TriviallyPassesChecker}),
+ * the iterator routes that row group through {@code nextStandard} for the remainder of the
+ * row group, skipping per-row filter evaluation and survivor compaction. This benefits queries
+ * that mix selective and non-selective row groups (e.g., time-bucketed data with skewed
+ * filter selectivity).
  */
 final class OptimizedParquetColumnIterator implements CloseableIterator<Page> {
 
@@ -111,8 +119,28 @@ final class OptimizedParquetColumnIterator implements CloseableIterator<Page> {
     private final boolean lateMaterialization;
     private final boolean[] isPredicateColumn;
     private final ParquetPushedExpressions pushedExpressions;
+    /**
+     * The parquet-mr {@link FilterPredicate} resolved by {@code ParquetFormatReader} for this scan,
+     * passed through unchanged so the trivially-passes check sees the same predicate that drove
+     * row-group pruning and ColumnIndex {@link RowRanges} computation. Translating again here
+     * would be wasted work and could subtly diverge if the caller's schema differs from
+     * {@link #projectedSchema}.
+     *
+     * <p>{@code null} when the trivially-passes guard is inactive: late materialization is off,
+     * the reader did not resolve a file-level predicate, or predicate resolution failed earlier
+     * (in which case the reader logged a warning and passed {@code null} through).
+     */
+    private final FilterPredicate triviallyPassesPredicate;
     private final WordMask survivorMask;
     private long rowsEliminatedByLateMaterialization;
+    /**
+     * When {@code true}, row-group statistics prove every row in the current row group satisfies
+     * the pushed filter, so late materialization is bypassed for this row group: filter
+     * evaluation and survivor compaction are skipped, and the standard read path is used.
+     */
+    private boolean currentRowGroupTriviallyPasses;
+    /** Diagnostic counter: number of row groups for which the filter was proven to trivially pass. */
+    private long rowGroupsWithTrivialFilter;
 
     OptimizedParquetColumnIterator(
         ParquetFileReader reader,
@@ -129,7 +157,8 @@ final class OptimizedParquetColumnIterator implements CloseableIterator<Page> {
         RowRanges[] allRowRanges,
         boolean[] survivingRowGroups,
         CompressionCodecFactory codecFactory,
-        ParquetPushedExpressions pushedExpressions
+        ParquetPushedExpressions pushedExpressions,
+        FilterPredicate triviallyPassesPredicate
     ) {
         this.reader = reader;
         this.projectedSchema = projectedSchema;
@@ -151,6 +180,9 @@ final class OptimizedParquetColumnIterator implements CloseableIterator<Page> {
         this.isPredicateColumn = classifyPredicateColumns(attributes, columnInfos, pushedExpressions);
         this.lateMaterialization = pushedExpressions != null && hasProjectionOnlyColumns(isPredicateColumn, columnInfos);
         this.survivorMask = lateMaterialization ? new WordMask() : null;
+        // Caller supplies null when late materialization is off; defensively also drop it here so
+        // the trivially-passes check is gated by a single condition below.
+        this.triviallyPassesPredicate = lateMaterialization ? triviallyPassesPredicate : null;
 
         this.projectedColumnPaths = buildProjectedColumnPaths(columnInfos);
         this.prefetchDepth = computePrefetchDepth(reader.getRowGroups(), this.projectedColumnPaths);
@@ -333,18 +365,34 @@ private boolean advanceRowGroup() throws IOException {
             if (rowsEliminatedByLateMaterialization > 0) {
                 logger.debug("Late materialization eliminated [{}] rows in [{}]", rowsEliminatedByLateMaterialization, fileLocation);
             }
+            if (rowGroupsWithTrivialFilter > 0) {
+                logger.debug(
+                    "Trivially-passes guard skipped late-materialization for [{}] row groups in [{}]",
+                    rowGroupsWithTrivialFilter,
+                    fileLocation
+                );
+            }
             return false;
         }
         rowGroupOrdinal = nextOrdinal;
         pageBatchIndexInRowGroup = 0;
 
         BlockMetaData block = reader.getRowGroups().get(rowGroupOrdinal);
+        // Per-row-group trivially-passes check: when stats prove every row matches the filter,
+        // the late-materialization machinery (decode predicate columns → evaluate filter → compact
+        // survivors) is pure overhead. Switching to the standard path eliminates filter evaluation.
+        currentRowGroupTriviallyPasses = triviallyPassesPredicate != null && TriviallyPassesChecker.check(triviallyPassesPredicate, block);
+        if (currentRowGroupTriviallyPasses) {
+            rowGroupsWithTrivialFilter++;
+        }
         NavigableMap<Long, ColumnChunkPrefetcher.PrefetchedChunk> chunks = takePendingPrefetch(rowGroupOrdinal);
         try {
             RowRanges currentRowRanges = resolveCurrentRowRanges(block);
             // When late materialization is active, skip ColumnIndex page filtering — late-mat handles
             // row-level filtering itself via the survivor mask. Applying both ColumnIndex RowRanges
             // AND late-mat evaluation causes double-filtering that drops rows incorrectly.
+            // The trivially-passes case is handled the same way: we already know all rows match,
+            // so leaving page filtering off is consistent and safe (RowRanges would be all() anyway).
             RowRanges buildRowRanges = lateMaterialization ? null : currentRowRanges;
             rowGroup = PrefetchedRowGroupBuilder.build(
                 block,
@@ -551,12 +599,13 @@ public Page next() {
         }
         int rowsToRead = (int) Math.min(effectiveBatch, rowsRemainingInGroup);
 
-        Page result = lateMaterialization ? nextWithLateMaterialization(rowsToRead) : nextStandard(rowsToRead);
+        boolean useLateMaterialization = lateMaterialization && currentRowGroupTriviallyPasses == false;
+        Page result = useLateMaterialization ? nextWithLateMaterialization(rowsToRead) : nextStandard(rowsToRead);
 
         pageBatchIndexInRowGroup++;
         rowsRemainingInGroup -= rowsToRead;
         if (rowBudget != FormatReader.NO_LIMIT) {
-            rowBudget -= lateMaterialization ? result.getPositionCount() : rowsToRead;
+            rowBudget -= useLateMaterialization ? result.getPositionCount() : rowsToRead;
         }
         return result;
     }

x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReader.java

@@ -813,6 +813,12 @@ private CloseableIterator<Page> createOptimizedIterator(
             }
         }
 
+        // Reuse the FilterPredicate already resolved at the file level so the trivially-passes
+        // guard sees the same predicate that drove row-group pruning and column-index RowRanges.
+        // Only pass it through when late materialization is actually active; otherwise the
+        // iterator has no use for it.
+        FilterPredicate triviallyPassesPredicate = effectivePushed != null ? filterPredicate : null;
+
         return new OptimizedParquetColumnIterator(
             reader,
             projectedSchema,
@@ -828,7 +834,8 @@ private CloseableIterator<Page> createOptimizedIterator(
             allRowRanges,
             survivingRowGroups,
             codecFactory,
-            effectivePushed
+            effectivePushed,
+            triviallyPassesPredicate
         );
     }