From e217312cec7eeed4f9fb5057b40a732e10e5db4b Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Mon, 4 Aug 2025 20:51:36 -0700 Subject: [PATCH 1/2] ip field changes Signed-off-by: Sandesh Kumar --- CHANGELOG.md | 1 + .../index/mapper/IpFieldMapper.java | 2 +- .../provider/DimensionFilterMapper.java | 65 +++++++++++++++---- .../DimensionFilterAndMapperTests.java | 22 +++++-- .../startree/MetricAggregatorTests.java | 32 +++++++++ 5 files changed, 103 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e905c964ec03..ca3f441abc759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Upgrade to protobufs 0.6.0 and clean up deprecated TermQueryProtoUtils code ([#18880](https://github.com/opensearch-project/OpenSearch/pull/18880)) - Prevent shard initialization failure due to streaming consumer errors ([#18877](https://github.com/opensearch-project/OpenSearch/pull/18877)) - APIs for stream transport and new stream-based search api action ([#18722](https://github.com/opensearch-project/OpenSearch/pull/18722)) +- [Star-Tree] Add search support for ip field type ([#18671](https://github.com/opensearch-project/OpenSearch/pull/18671)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index b2e8f75a4f444..d3a604444c10f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -237,7 +237,7 @@ public String typeName() { return CONTENT_TYPE; } - private static InetAddress parse(Object value) { + public static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; } else { diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java index 62a1388315770..f176ada1d6193 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java @@ -10,6 +10,7 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.BytesRef; @@ -32,6 +33,7 @@ import org.opensearch.search.startree.filter.RangeMatchDimFilter; import java.io.IOException; +import java.net.InetAddress; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -161,7 +163,9 @@ class Factory { org.opensearch.index.mapper.KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper(), UNSIGNED_LONG.typeName(), - new UnsignedLongFieldMapperNumeric() + new UnsignedLongFieldMapperNumeric(), + org.opensearch.index.mapper.IpFieldMapper.CONTENT_TYPE, + new IpFieldMapper() ); public static DimensionFilterMapper fromMappedFieldType(MappedFieldType mappedFieldType, SearchContext searchContext) { @@ -406,25 +410,25 @@ Number getNextHigh(Number parsedValue) { } } -class KeywordFieldMapper implements DimensionFilterMapper { +abstract class OrdinalFieldMapper implements DimensionFilterMapper { + + abstract Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldType) throws IllegalArgumentException; @Override public DimensionFilter getExactMatchFilter(MappedFieldType mappedFieldType, List rawValues) { - KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType; List convertedValues = new ArrayList<>(rawValues.size()); for (Object rawValue : rawValues) { - convertedValues.add(parseRawKeyword(mappedFieldType.name(), rawValue, keywordFieldType)); + convertedValues.add(parseRawField(mappedFieldType.name(), rawValue, mappedFieldType)); } return new ExactMatchDimFilter(mappedFieldType.name(), convertedValues); } @Override public DimensionFilter getRangeMatchFilter(MappedFieldType mappedFieldType, StarTreeRangeQuery rangeQuery) { - KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType; return new RangeMatchDimFilter( mappedFieldType.name(), - parseRawKeyword(mappedFieldType.name(), rangeQuery.from(), keywordFieldType), - parseRawKeyword(mappedFieldType.name(), rangeQuery.to(), keywordFieldType), + parseRawField(mappedFieldType.name(), rangeQuery.from(), mappedFieldType), + parseRawField(mappedFieldType.name(), rangeQuery.to(), mappedFieldType), rangeQuery.includeLower(), rangeQuery.includeUpper() ); @@ -484,8 +488,20 @@ public Optional getMatchingOrdinal( } } + @Override + public int compareValues(Object v1, Object v2) { + if (!(v1 instanceof BytesRef) || !(v2 instanceof BytesRef)) { + throw new IllegalArgumentException("Expected BytesRef values for comparison"); + } + return ((BytesRef) v1).compareTo((BytesRef) v2); + } +} + +class KeywordFieldMapper extends OrdinalFieldMapper { + // TODO : Think around making TermBasedFT#indexedValueForSearch() accessor public for reuse here. - private Object parseRawKeyword(String field, Object rawValue, KeywordFieldType keywordFieldType) { + Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldType) { + KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType; Object parsedValue = null; if (rawValue != null) { if (keywordFieldType.getTextSearchInfo().getSearchAnalyzer() == Lucene.KEYWORD_ANALYZER) { @@ -499,12 +515,35 @@ private Object parseRawKeyword(String field, Object rawValue, KeywordFieldType k } return parsedValue; } +} - @Override - public int compareValues(Object v1, Object v2) { - if (!(v1 instanceof BytesRef) || !(v2 instanceof BytesRef)) { - throw new IllegalArgumentException("Expected BytesRef values for keyword comparison"); +/** + * This class provides functionality to map IP address values for exact and range-based + * filtering within a Star-Tree index. It handles the conversion of IP address + * objects into sortable {@link BytesRef}. + */ +class IpFieldMapper extends OrdinalFieldMapper { + + /** + * Parses a raw IP address value into a sortable {@link BytesRef}. + * + * This method handles various input types, including {@link InetAddress}, {@link BytesRef}, + * and {@link String}, converting them into a binary representation using + * {@link InetAddressPoint#encode(InetAddress)}. + * + * @param field The name of the field being processed. + * @param rawValue The raw IP address value. + * @return A {@link BytesRef} representation of the IP address, or null if the input is null. + */ + Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldType) throws IllegalArgumentException { + Object parsedValue = null; + if (rawValue != null) { + try { + return org.opensearch.index.mapper.IpFieldMapper.IpFieldType.parse(rawValue); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse IP value for field [" + field + "]", e); + } } - return ((BytesRef) v1).compareTo((BytesRef) v2); + return parsedValue; } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java index f92515aced43e..bb704ee9dc231 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java @@ -8,6 +8,7 @@ package org.opensearch.search.aggregations.startree; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.opensearch.index.compositeindex.datacube.Metric; @@ -18,7 +19,9 @@ import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.IpFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.StarTreeMapper; @@ -38,6 +41,7 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.net.InetAddress; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -47,12 +51,21 @@ public class DimensionFilterAndMapperTests extends OpenSearchTestCase { + public void testIpMapping() throws Exception { + MappedFieldType mappedFieldType = new IpFieldMapper.IpFieldType("ip_field"); + BytesRef ipAsBytes = new BytesRef(InetAddressPoint.encode(InetAddress.getByName("192.168.1.1"))); + testOrdinalMapping(mappedFieldType, ipAsBytes); + } + public void testKeywordOrdinalMapping() throws IOException { + MappedFieldType mappedFieldType = new KeywordFieldMapper.KeywordFieldType("keyword"); + BytesRef bytesRef = new BytesRef(new byte[] { 17, 29 }); + testOrdinalMapping(mappedFieldType, bytesRef); + } + + private void testOrdinalMapping(final MappedFieldType mappedFieldType, final BytesRef bytesRef) throws IOException { SearchContext searchContext = mock(SearchContext.class); - DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType( - new KeywordFieldMapper.KeywordFieldType("keyword"), - searchContext - ); + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType, searchContext); StarTreeValues starTreeValues = mock(StarTreeValues.class); SortedSetStarTreeValuesIterator sortedSetStarTreeValuesIterator = mock(SortedSetStarTreeValuesIterator.class); TermsEnum termsEnum = mock(TermsEnum.class); @@ -61,7 +74,6 @@ public void testKeywordOrdinalMapping() throws IOException { Optional matchingOrdinal; // Case Exact Match and found - BytesRef bytesRef = new BytesRef(new byte[] { 17, 29 }); when(sortedSetStarTreeValuesIterator.lookupTerm(bytesRef)).thenReturn(1L); matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.EXACT); assertTrue(matchingOrdinal.isPresent()); diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java index f03ea308bfc95..2cd9b02e41599 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java @@ -18,10 +18,12 @@ import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.IntField; import org.apache.lucene.document.KeywordField; import org.apache.lucene.document.LongField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; @@ -32,6 +34,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; @@ -47,6 +50,7 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.OrdinalDimension; +import org.opensearch.index.mapper.IpFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; @@ -77,6 +81,7 @@ import java.io.IOException; import java.math.BigInteger; +import java.net.InetAddress; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -136,6 +141,7 @@ public void testStarTreeDocValues() throws IOException { new DimensionFieldData("half_float_field", () -> random().nextFloat(50), DimensionTypes.HALF_FLOAT), new DimensionFieldData("float_field", () -> random().nextFloat(50), DimensionTypes.FLOAT), new DimensionFieldData("double_field", () -> random().nextDouble(50), DimensionTypes.DOUBLE), + new DimensionFieldData("ip_field", this::randomIp, DimensionTypes.IP), new DimensionFieldData("unsigned_long_field", () -> { long queryValue = randomBoolean() ? 9223372036854775807L - random().nextInt(100000) @@ -658,6 +664,28 @@ NumberFieldMapper.NumberType numberType() { public IndexableField getField(String fieldName, Supplier valueSupplier) { return new BigIntegerField(fieldName, (BigInteger) valueSupplier.get(), Field.Store.YES); } + }), + IP(new DimensionFieldDataSupplier() { + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + try { + InetAddress address = InetAddress.getByName((String) valueSupplier.get()); + return new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(address))); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public MappedFieldType getMappedField(String fieldName) { + return new IpFieldMapper.IpFieldType(fieldName); + } + + @Override + public Dimension getDimension(String fieldName) { + return new OrdinalDimension(fieldName); + } }); private final DimensionFieldDataSupplier dimensionFieldDataSupplier; @@ -680,4 +708,8 @@ private String asUnsignedDecimalString(long l) { return b.toString(); } + private String randomIp() { + Random r = random(); + return r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256); + } } From 9377d86b9a316532149469903adcced26b9b3de3 Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Mon, 4 Aug 2025 23:07:29 -0700 Subject: [PATCH 2/2] increase coverage Signed-off-by: Sandesh Kumar --- .../filter/provider/DimensionFilterMapper.java | 18 +++++++++++++++++- .../DimensionFilterAndMapperTests.java | 15 +++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java index f176ada1d6193..ab1c151e51c04 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java @@ -539,7 +539,23 @@ Object parseRawField(String field, Object rawValue, MappedFieldType mappedFieldT Object parsedValue = null; if (rawValue != null) { try { - return org.opensearch.index.mapper.IpFieldMapper.IpFieldType.parse(rawValue); + switch (rawValue) { + case InetAddress inetAddress -> { + parsedValue = new BytesRef(InetAddressPoint.encode(inetAddress)); + } + case BytesRef bytesRef -> { + return bytesRef; + } + case String s -> { + InetAddress addr = InetAddress.getByName(s); + parsedValue = new BytesRef(InetAddressPoint.encode(addr)); + } + default -> { + throw new IllegalArgumentException( + "Unsupported value type for IP field [" + field + "]: " + rawValue.getClass().getName() + ); + } + } } catch (Exception e) { throw new IllegalArgumentException("Failed to parse IP value for field [" + field + "]", e); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java index bb704ee9dc231..c97c7c7341174 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java @@ -42,6 +42,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -57,6 +58,20 @@ public void testIpMapping() throws Exception { testOrdinalMapping(mappedFieldType, ipAsBytes); } + public void testRawValuesIpParsing() throws UnknownHostException { + SearchContext searchContext = mock(SearchContext.class); + MappedFieldType mappedFieldType = new IpFieldMapper.IpFieldType("ip_field"); + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType, searchContext); + + assertThrows(IllegalArgumentException.class, () -> dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of(1.0f))); + assertThrows( + IllegalArgumentException.class, + () -> dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of("not.a.valid.ip")) + ); + DimensionFilter df = dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of(InetAddress.getByName("192.168.1.1"))); + assertEquals("ip_field", df.getMatchingDimension()); + } + public void testKeywordOrdinalMapping() throws IOException { MappedFieldType mappedFieldType = new KeywordFieldMapper.KeywordFieldType("keyword"); BytesRef bytesRef = new BytesRef(new byte[] { 17, 29 });