diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index ccfd3c4393d3f..3c9cb0fa90017 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -42,12 +42,13 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; @@ -423,6 +424,7 @@ protected void doRun() { Metadata metadata = clusterState.metadata(); // Group the requests by ShardId -> Operations mapping Map> requestsByShard = new HashMap<>(); + Map indexRoutings = new HashMap<>(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docWriteRequest = bulkRequest.requests.get(i); //the request can only be null because we set it to null in the previous step, so it gets ignored @@ -474,8 +476,13 @@ protected void doRun() { break; default: throw new AssertionError("request type not supported: [" + docWriteRequest.opType() + "]"); } - ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex.getName(), - docWriteRequest.id(), docWriteRequest.routing()).shardId(); + IndexRouting indexRouting = indexRoutings.computeIfAbsent( + concreteIndex, + idx -> IndexRouting.fromIndexMetadata(clusterState.metadata().getIndexSafe(idx)) + ); + ShardId shardId = clusterService.operationRouting() + .indexShards(clusterState, concreteIndex.getName(), indexRouting, docWriteRequest.id(), docWriteRequest.routing()) + .shardId(); List shardRequests = requestsByShard.computeIfAbsent(shardId, shard -> new ArrayList<>()); shardRequests.add(new BulkItemRequest(i, docWriteRequest)); } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException e) { diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 5383f2804a101..fdc668b881f3e 100644 --- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -25,19 +25,21 @@ import org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.VersionConflictEngineException; @@ -157,8 +159,13 @@ protected ShardIterator shards(ClusterState clusterState, UpdateRequest request) if (request.getShardId() != null) { return clusterState.routingTable().index(request.concreteIndex()).shard(request.getShardId().getId()).primaryShardIt(); } + IndexMetadata indexMetadata = clusterState.metadata().index(request.concreteIndex()); + if (indexMetadata == null) { + throw new IndexNotFoundException(request.concreteIndex()); + } + IndexRouting indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); ShardIterator shardIterator = clusterService.operationRouting() - .indexShards(clusterState, request.concreteIndex(), request.id(), request.routing()); + .indexShards(clusterState, request.concreteIndex(), indexRouting, request.id(), request.routing()); ShardRouting shard; while ((shard = shardIterator.nextOrNull()) != null) { if (shard.primary()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index c80bb40250f40..b06a0a7c27841 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -12,6 +12,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.elasticsearch.Assertions; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; @@ -22,15 +23,8 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContentFragment; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; @@ -41,6 +35,14 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; @@ -1623,7 +1625,7 @@ public IndexMetadata fromXContent(XContentParser parser) throws IOException { /** * Returns the number of shards that should be used for routing. This basically defines the hash space we use in - * {@link org.elasticsearch.cluster.routing.OperationRouting#generateShardId(IndexMetadata, String, String)} to route documents + * {@link IndexRouting#shardId(String, String)} to route documents * to shards based on their ID or their specific routing value. The default value is {@link #getNumberOfShards()}. This value only * changes if and index is shrunk. */ @@ -1736,7 +1738,7 @@ public static Set selectShrinkShards(int shardId, IndexMetadata sourceI /** * Returns the routing factor for and shrunk index with the given number of target shards. * This factor is used in the hash function in - * {@link org.elasticsearch.cluster.routing.OperationRouting#generateShardId(IndexMetadata, String, String)} to guarantee consistent + * {@link IndexRouting#shardId(String, String)} to guarantee consistent * hashing / routing of documents even if the number of shards changed (ie. a shrunk index). * * @param sourceNumberOfShards the total number of shards in the source index diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java new file mode 100644 index 0000000000000..efd34c4079194 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.core.Nullable; + +import java.util.function.IntConsumer; + +/** + * Generates the shard id for {@code (id, routing)} pairs. + */ +public abstract class IndexRouting { + /** + * Build the routing from {@link IndexMetadata}. + */ + public static IndexRouting fromIndexMetadata(IndexMetadata indexMetadata) { + if (indexMetadata.isRoutingPartitionedIndex()) { + return new Partitioned( + indexMetadata.getRoutingNumShards(), + indexMetadata.getRoutingFactor(), + indexMetadata.getRoutingPartitionSize() + ); + } + return new Unpartitioned(indexMetadata.getRoutingNumShards(), indexMetadata.getRoutingFactor()); + } + + private final int routingNumShards; + private final int routingFactor; + + private IndexRouting(int routingNumShards, int routingFactor) { + this.routingNumShards = routingNumShards; + this.routingFactor = routingFactor; + } + + /** + * Generate the single shard id that should contain a document with the + * provided {@code id} and {@code routing}. + */ + public abstract int shardId(String id, @Nullable String routing); + + /** + * Collect all of the shard ids that *may* contain documents with the + * provided {@code routing}. Indices with a {@code routing_partition} + * will collect more than one shard. Indices without a partition + * will collect the same shard id as would be returned + * by {@link #shardId}. + */ + public abstract void collectSearchShards(String routing, IntConsumer consumer); + + /** + * Convert a hash generated from an {@code (id, routing}) pair into a + * shard id. + */ + protected final int hashToShardId(int hash) { + return Math.floorMod(hash, routingNumShards) / routingFactor; + } + + /** + * Convert a routing value into a hash. + */ + private static int effectiveRoutingToHash(String effectiveRouting) { + return Murmur3HashFunction.hash(effectiveRouting); + } + + /** + * Strategy for indices that are not partitioned. + */ + private static class Unpartitioned extends IndexRouting { + Unpartitioned(int routingNumShards, int routingFactor) { + super(routingNumShards, routingFactor); + } + + @Override + public int shardId(String id, @Nullable String routing) { + return hashToShardId(effectiveRoutingToHash(routing == null ? id : routing)); + } + + @Override + public void collectSearchShards(String routing, IntConsumer consumer) { + consumer.accept(hashToShardId(effectiveRoutingToHash(routing))); + } + } + + /** + * Strategy for partitioned indices. + */ + private static class Partitioned extends IndexRouting { + private final int routingPartitionSize; + + Partitioned(int routingNumShards, int routingFactor, int routingPartitionSize) { + super(routingNumShards, routingFactor); + this.routingPartitionSize = routingPartitionSize; + } + + @Override + public int shardId(String id, @Nullable String routing) { + if (routing == null) { + throw new IllegalArgumentException("A routing value is required for gets from a partitioned index"); + } + int offset = Math.floorMod(effectiveRoutingToHash(id), routingPartitionSize); + return hashToShardId(effectiveRoutingToHash(routing) + offset); + } + + @Override + public void collectSearchShards(String routing, IntConsumer consumer) { + int hash = effectiveRoutingToHash(routing); + for (int i = 0; i < routingPartitionSize; i++) { + consumer.accept(hashToShardId(hash + i)); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 9672c7a382c4e..322db24e8b85b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -11,11 +11,11 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.node.ResponseCollectorService; @@ -45,14 +45,30 @@ void setUseAdaptiveReplicaSelection(boolean useAdaptiveReplicaSelection) { this.useAdaptiveReplicaSelection = useAdaptiveReplicaSelection; } - public ShardIterator indexShards(ClusterState clusterState, String index, String id, @Nullable String routing) { - return shards(clusterState, index, id, routing).shardsIt(); + public ShardIterator indexShards( + ClusterState clusterState, + String index, + IndexRouting indexRouting, + String id, + @Nullable String routing + ) { + return shards(clusterState, index, indexRouting, id, routing).shardsIt(); } + /** + * Shards to use for a {@code GET} operation. + */ public ShardIterator getShards(ClusterState clusterState, String index, String id, @Nullable String routing, @Nullable String preference) { - return preferenceActiveShardIterator(shards(clusterState, index, id, routing), clusterState.nodes().getLocalNodeId(), - clusterState.nodes(), preference, null, null); + IndexRouting indexRouting = IndexRouting.fromIndexMetadata(indexMetadata(clusterState, index)); + return preferenceActiveShardIterator( + shards(clusterState, index, indexRouting, id, routing), + clusterState.nodes().getLocalNodeId(), + clusterState.nodes(), + preference, + null, + null + ); } public ShardIterator getShards(ClusterState clusterState, String index, int shardId, @Nullable String preference) { @@ -100,18 +116,16 @@ private Set computeTargetedShards(ClusterState clusterSt final Set set = new HashSet<>(); // we use set here and not list since we might get duplicates for (String index : concreteIndices) { - final IndexRoutingTable indexRouting = indexRoutingTable(clusterState, index); + final IndexRoutingTable indexRoutingTable = indexRoutingTable(clusterState, index); final IndexMetadata indexMetadata = indexMetadata(clusterState, index); - final Set effectiveRouting = routing.get(index); - if (effectiveRouting != null) { - for (String r : effectiveRouting) { - final int routingPartitionSize = indexMetadata.getRoutingPartitionSize(); - for (int partitionOffset = 0; partitionOffset < routingPartitionSize; partitionOffset++) { - set.add(RoutingTable.shardRoutingTable(indexRouting, calculateScaledShardId(indexMetadata, r, partitionOffset))); - } + final Set indexSearchRouting = routing.get(index); + if (indexSearchRouting != null) { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); + for (String r : indexSearchRouting) { + indexRouting.collectSearchShards(r, s -> set.add(RoutingTable.shardRoutingTable(indexRoutingTable, s))); } } else { - for (IndexShardRoutingTable indexShard : indexRouting) { + for (IndexShardRoutingTable indexShard : indexRoutingTable) { set.add(indexShard); } } @@ -198,7 +212,7 @@ protected IndexRoutingTable indexRoutingTable(ClusterState clusterState, String return indexRouting; } - protected IndexMetadata indexMetadata(ClusterState clusterState, String index) { + private IndexMetadata indexMetadata(ClusterState clusterState, String index) { IndexMetadata indexMetadata = clusterState.metadata().index(index); if (indexMetadata == null) { throw new IndexNotFoundException(index); @@ -206,43 +220,12 @@ protected IndexMetadata indexMetadata(ClusterState clusterState, String index) { return indexMetadata; } - protected IndexShardRoutingTable shards(ClusterState clusterState, String index, String id, String routing) { - int shardId = generateShardId(indexMetadata(clusterState, index), id, routing); - return clusterState.getRoutingTable().shardRoutingTable(index, shardId); + private IndexShardRoutingTable shards(ClusterState clusterState, String index, IndexRouting indexRouting, String id, String routing) { + return clusterState.getRoutingTable().shardRoutingTable(index, indexRouting.shardId(id, routing)); } public ShardId shardId(ClusterState clusterState, String index, String id, @Nullable String routing) { IndexMetadata indexMetadata = indexMetadata(clusterState, index); - return new ShardId(indexMetadata.getIndex(), generateShardId(indexMetadata, id, routing)); - } - - public static int generateShardId(IndexMetadata indexMetadata, @Nullable String id, @Nullable String routing) { - final String effectiveRouting; - final int partitionOffset; - - if (routing == null) { - assert(indexMetadata.isRoutingPartitionedIndex() == false) : "A routing value is required for gets from a partitioned index"; - effectiveRouting = id; - } else { - effectiveRouting = routing; - } - - if (indexMetadata.isRoutingPartitionedIndex()) { - partitionOffset = Math.floorMod(Murmur3HashFunction.hash(id), indexMetadata.getRoutingPartitionSize()); - } else { - // we would have still got 0 above but this check just saves us an unnecessary hash calculation - partitionOffset = 0; - } - - return calculateScaledShardId(indexMetadata, effectiveRouting, partitionOffset); + return new ShardId(indexMetadata.getIndex(), IndexRouting.fromIndexMetadata(indexMetadata).shardId(id, routing)); } - - private static int calculateScaledShardId(IndexMetadata indexMetadata, String effectiveRouting, int partitionOffset) { - final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset; - - // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size - // of original index to hash documents - return Math.floorMod(hash, indexMetadata.getRoutingNumShards()) / indexMetadata.getRoutingFactor(); - } - } diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index 2afc267bff7a3..d59a755993e9e 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -29,7 +29,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.mapper.IdFieldMapper; @@ -48,11 +48,13 @@ */ final class ShardSplittingQuery extends Query { private final IndexMetadata indexMetadata; + private final IndexRouting indexRouting; private final int shardId; private final BitSetProducer nestedParentBitSetProducer; ShardSplittingQuery(IndexMetadata indexMetadata, int shardId, boolean hasNested) { this.indexMetadata = indexMetadata; + this.indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); this.shardId = shardId; this.nestedParentBitSetProducer = hasNested ? newParentDocBitSetProducer() : null; } @@ -70,8 +72,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { FixedBitSet bitSet = new FixedBitSet(leafReader.maxDoc()); Terms terms = leafReader.terms(RoutingFieldMapper.NAME); Predicate includeInShard = ref -> { - int targetShardId = OperationRouting.generateShardId(indexMetadata, - Uid.decodeId(ref.bytes, ref.offset, ref.length), null); + int targetShardId = indexRouting.shardId(Uid.decodeId(ref.bytes, ref.offset, ref.length), null); return shardId == targetShardId; }; if (terms == null) { @@ -114,7 +115,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { }; // in the _routing case we first go and find all docs that have a routing value and mark the ones we have to delete findSplitDocs(RoutingFieldMapper.NAME, ref -> { - int targetShardId = OperationRouting.generateShardId(indexMetadata, null, ref.utf8ToString()); + int targetShardId = indexRouting.shardId(null, ref.utf8ToString()); return shardId == targetShardId; }, leafReader, maybeWrapConsumer.apply(bitSet::set)); @@ -258,7 +259,7 @@ boolean matches(int doc) throws IOException { leftToVisit = 2; leafReader.document(doc, this); assert id != null : "docID must not be null - we might have hit a nested document"; - int targetShardId = OperationRouting.generateShardId(indexMetadata, id, routing); + int targetShardId = indexRouting.shardId(id, routing); return targetShardId != shardId; } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java new file mode 100644 index 0000000000000..0c5e050887951 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -0,0 +1,356 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.cluster.routing; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static org.hamcrest.Matchers.hasSize; + +public class IndexRoutingTests extends ESTestCase{ + public void testGenerateShardId() { + int[][] possibleValues = new int[][] { + {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1} + }; + for (int i = 0; i < 10; i++) { + int[] shardSplits = randomFrom(possibleValues); + assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]); + assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]); + IndexMetadata metadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0]) + .numberOfReplicas(1).build(); + String term = randomAlphaOfLength(10); + final int shard = IndexRouting.fromIndexMetadata(metadata).shardId(term, null); + IndexMetadata shrunk = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1]) + .numberOfReplicas(1) + .setRoutingNumShards(shardSplits[0]).build(); + int shrunkShard = IndexRouting.fromIndexMetadata(shrunk).shardId(term, null); + + Set shardIds = IndexMetadata.selectShrinkShards(shrunkShard, metadata, shrunk.getNumberOfShards()); + assertEquals(1, shardIds.stream().filter((sid) -> sid.id() == shard).count()); + + shrunk = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1) + .setRoutingNumShards(shardSplits[0]).build(); + shrunkShard = IndexRouting.fromIndexMetadata(shrunk).shardId(term, null); + shardIds = IndexMetadata.selectShrinkShards(shrunkShard, metadata, shrunk.getNumberOfShards()); + assertEquals(Arrays.toString(shardSplits), 1, shardIds.stream().filter((sid) -> sid.id() == shard).count()); + } + } + + public void testGenerateShardIdSplit() { + int[][] possibleValues = new int[][] { + {2,4,8}, {2, 10, 20}, {3, 12, 36}, {1,5,15} + }; + for (int i = 0; i < 10; i++) { + int[] shardSplits = randomFrom(possibleValues); + assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); + assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + IndexMetadata metadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0]) + .numberOfReplicas(1).setRoutingNumShards(shardSplits[2]).build(); + String term = randomAlphaOfLength(10); + final int shard = IndexRouting.fromIndexMetadata(metadata).shardId(term, null); + IndexMetadata split = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1]) + .numberOfReplicas(1) + .setRoutingNumShards(shardSplits[2]).build(); + int shrunkShard = IndexRouting.fromIndexMetadata(split).shardId(term, null); + + ShardId shardId = IndexMetadata.selectSplitShard(shrunkShard, metadata, split.getNumberOfShards()); + assertNotNull(shardId); + assertEquals(shard, shardId.getId()); + + split = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1) + .setRoutingNumShards(shardSplits[2]).build(); + shrunkShard = IndexRouting.fromIndexMetadata(split).shardId(term, null); + shardId = IndexMetadata.selectSplitShard(shrunkShard, metadata, split.getNumberOfShards()); + assertNotNull(shardId); + assertEquals(shard, shardId.getId()); + } + } + + public void testCollectSearchShardsInStandardIndex() { + for (int shards = 1; shards < 5; shards++) { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shards).numberOfReplicas(1).build() + ); + + for (int i = 0; i < 20; i++) { + String routing = randomUnicodeOfLengthBetween(1, 50); + + Set searchShardSet = new HashSet<>(); + indexRouting.collectSearchShards(routing, searchShardSet::add); + assertThat(searchShardSet, hasSize(1)); + } + } + } + + public void testPartitionedIndex() { + // make sure the same routing value always has each _id fall within the configured partition size + for (int shards = 1; shards < 5; shards++) { + for (int partitionSize = 1; partitionSize == 1 || partitionSize < shards; partitionSize++) { + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(shards) + .routingPartitionSize(partitionSize) + .numberOfReplicas(1) + .build() + ); + + for (int i = 0; i < 20; i++) { + String routing = randomUnicodeOfLengthBetween(1, 50); + + Set shardSet = new HashSet<>(); + for (int k = 0; k < 150; k++) { + String id = randomUnicodeOfLengthBetween(1, 50); + shardSet.add(indexRouting.shardId(id, routing)); + } + assertThat(shardSet, hasSize(partitionSize)); + + Set searchShardSet = new HashSet<>(); + indexRouting.collectSearchShards(routing, searchShardSet::add); + assertThat(searchShardSet, hasSize(partitionSize)); + } + } + } + } + + public void testPartitionedIndexShrunk() { + Map> routingIdToShard = new HashMap<>(); + + Map routingA = new HashMap<>(); + routingA.put("a_0", 1); + routingA.put("a_1", 2); + routingA.put("a_2", 2); + routingA.put("a_3", 2); + routingA.put("a_4", 1); + routingA.put("a_5", 2); + routingIdToShard.put("a", routingA); + + Map routingB = new HashMap<>(); + routingB.put("b_0", 0); + routingB.put("b_1", 0); + routingB.put("b_2", 0); + routingB.put("b_3", 0); + routingB.put("b_4", 3); + routingB.put("b_5", 3); + routingIdToShard.put("b", routingB); + + Map routingC = new HashMap<>(); + routingC.put("c_0", 1); + routingC.put("c_1", 1); + routingC.put("c_2", 0); + routingC.put("c_3", 0); + routingC.put("c_4", 0); + routingC.put("c_5", 1); + routingIdToShard.put("c", routingC); + + Map routingD = new HashMap<>(); + routingD.put("d_0", 2); + routingD.put("d_1", 2); + routingD.put("d_2", 3); + routingD.put("d_3", 3); + routingD.put("d_4", 3); + routingD.put("d_5", 3); + routingIdToShard.put("d", routingD); + + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .setRoutingNumShards(8) + .numberOfShards(4) + .routingPartitionSize(3) + .numberOfReplicas(1) + .build() + ); + + for (Map.Entry> routingIdEntry : routingIdToShard.entrySet()) { + String routing = routingIdEntry.getKey(); + + for (Map.Entry idEntry : routingIdEntry.getValue().entrySet()) { + String id = idEntry.getKey(); + int shard = idEntry.getValue(); + + assertEquals(shard, indexRouting.shardId(id, routing)); + } + } + } + + public void testPartitionedIndexBWC() { + Map> routingIdToShard = new HashMap<>(); + + Map routingA = new HashMap<>(); + routingA.put("a_0", 3); + routingA.put("a_1", 2); + routingA.put("a_2", 2); + routingA.put("a_3", 3); + routingIdToShard.put("a", routingA); + + Map routingB = new HashMap<>(); + routingB.put("b_0", 5); + routingB.put("b_1", 0); + routingB.put("b_2", 0); + routingB.put("b_3", 0); + routingIdToShard.put("b", routingB); + + Map routingC = new HashMap<>(); + routingC.put("c_0", 4); + routingC.put("c_1", 4); + routingC.put("c_2", 3); + routingC.put("c_3", 4); + routingIdToShard.put("c", routingC); + + Map routingD = new HashMap<>(); + routingD.put("d_0", 3); + routingD.put("d_1", 4); + routingD.put("d_2", 4); + routingD.put("d_3", 4); + routingIdToShard.put("d", routingD); + + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(6) + .routingPartitionSize(2) + .numberOfReplicas(1) + .build() + ); + + for (Map.Entry> routingIdEntry : routingIdToShard.entrySet()) { + String routing = routingIdEntry.getKey(); + + for (Map.Entry idEntry : routingIdEntry.getValue().entrySet()) { + String id = idEntry.getKey(); + int shard = idEntry.getValue(); + + assertEquals(shard, indexRouting.shardId(id, routing)); + } + } + } + + /** + * Ensures that all changes to the hash-function / shard selection are BWC + */ + public void testBWC() { + Map termToShard = new TreeMap<>(); + termToShard.put("sEERfFzPSI", 1); + termToShard.put("cNRiIrjzYd", 7); + termToShard.put("BgfLBXUyWT", 5); + termToShard.put("cnepjZhQnb", 3); + termToShard.put("OKCmuYkeCK", 6); + termToShard.put("OutXGRQUja", 5); + termToShard.put("yCdyocKWou", 1); + termToShard.put("KXuNWWNgVj", 2); + termToShard.put("DGJOYrpESx", 4); + termToShard.put("upLDybdTGs", 5); + termToShard.put("yhZhzCPQby", 1); + termToShard.put("EyCVeiCouA", 1); + termToShard.put("tFyVdQauWR", 6); + termToShard.put("nyeRYDnDQr", 6); + termToShard.put("hswhrppvDH", 0); + termToShard.put("BSiWvDOsNE", 5); + termToShard.put("YHicpFBSaY", 1); + termToShard.put("EquPtdKaBZ", 4); + termToShard.put("rSjLZHCDfT", 5); + termToShard.put("qoZALVcite", 7); + termToShard.put("yDCCPVBiCm", 7); + termToShard.put("ngizYtQgGK", 5); + termToShard.put("FYQRIBcNqz", 0); + termToShard.put("EBzEDAPODe", 2); + termToShard.put("YePigbXgKb", 1); + termToShard.put("PeGJjomyik", 3); + termToShard.put("cyQIvDmyYD", 7); + termToShard.put("yIEfZrYfRk", 5); + termToShard.put("kblouyFUbu", 7); + termToShard.put("xvIGbRiGJF", 3); + termToShard.put("KWimwsREPf", 4); + termToShard.put("wsNavvIcdk", 7); + termToShard.put("xkWaPcCmpT", 0); + termToShard.put("FKKTOnJMDy", 7); + termToShard.put("RuLzobYixn", 2); + termToShard.put("mFohLeFRvF", 4); + termToShard.put("aAMXnamRJg", 7); + termToShard.put("zKBMYJDmBI", 0); + termToShard.put("ElSVuJQQuw", 7); + termToShard.put("pezPtTQAAm", 7); + termToShard.put("zBjjNEjAex", 2); + termToShard.put("PGgHcLNPYX", 7); + termToShard.put("hOkpeQqTDF", 3); + termToShard.put("chZXraUPBH", 7); + termToShard.put("FAIcSmmNXq", 5); + termToShard.put("EZmDicyayC", 0); + termToShard.put("GRIueBeIyL", 7); + termToShard.put("qCChjGZYLp", 3); + termToShard.put("IsSZQwwnUT", 3); + termToShard.put("MGlxLFyyCK", 3); + termToShard.put("YmscwrKSpB", 0); + termToShard.put("czSljcjMop", 5); + termToShard.put("XhfGWwNlng", 1); + termToShard.put("cWpKJjlzgj", 7); + termToShard.put("eDzIfMKbvk", 1); + termToShard.put("WFFWYBfnTb", 0); + termToShard.put("oDdHJxGxja", 7); + termToShard.put("PDOQQqgIKE", 1); + termToShard.put("bGEIEBLATe", 6); + termToShard.put("xpRkJPWVpu", 2); + termToShard.put("kTwZnPEeIi", 2); + termToShard.put("DifcuqSsKk", 1); + termToShard.put("CEmLmljpXe", 5); + termToShard.put("cuNKtLtyJQ", 7); + termToShard.put("yNjiAnxAmt", 5); + termToShard.put("bVDJDCeaFm", 2); + termToShard.put("vdnUhGLFtl", 0); + termToShard.put("LnqSYezXbr", 5); + termToShard.put("EzHgydDCSR", 3); + termToShard.put("ZSKjhJlcpn", 1); + termToShard.put("WRjUoZwtUz", 3); + termToShard.put("RiBbcCdIgk", 4); + termToShard.put("yizTqyjuDn", 4); + termToShard.put("QnFjcpcZUT", 4); + termToShard.put("agYhXYUUpl", 7); + termToShard.put("UOjiTugjNC", 7); + termToShard.put("nICGuWTdfV", 0); + termToShard.put("NrnSmcnUVF", 2); + termToShard.put("ZSzFcbpDqP", 3); + termToShard.put("YOhahLSzzE", 5); + termToShard.put("iWswCilUaT", 1); + termToShard.put("zXAamKsRwj", 2); + termToShard.put("aqGsrUPHFq", 5); + termToShard.put("eDItImYWTS", 1); + termToShard.put("JAYDZMRcpW", 4); + termToShard.put("lmvAaEPflK", 7); + termToShard.put("IKuOwPjKCx", 5); + termToShard.put("schsINzlYB", 1); + termToShard.put("OqbFNxrKrF", 2); + termToShard.put("QrklDfvEJU", 6); + termToShard.put("VLxKRKdLbx", 4); + termToShard.put("imoydNTZhV", 1); + termToShard.put("uFZyTyOMRO", 4); + termToShard.put("nVAZVMPNNx", 3); + termToShard.put("rPIdESYaAO", 5); + termToShard.put("nbZWPWJsIM", 0); + termToShard.put("wRZXPSoEgd", 3); + termToShard.put("nGzpgwsSBc", 4); + termToShard.put("AITyyoyLLs", 4); + IndexRouting indexRouting = IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(8).numberOfReplicas(1).build() + ); + for (Map.Entry entry : termToShard.entrySet()) { + String key = entry.getKey(); + int shard = randomBoolean() ? indexRouting.shardId(key, null) : indexRouting.shardId(randomAlphaOfLength(5), key); + assertEquals(shard, entry.getValue().intValue()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java index eb91982bd3501..4255aba26b2c9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java @@ -7,10 +7,8 @@ */ package org.elasticsearch.cluster.routing; -import org.elasticsearch.Version; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -25,13 +23,11 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -40,315 +36,6 @@ import static org.hamcrest.object.HasToString.hasToString; public class OperationRoutingTests extends ESTestCase{ - - public void testGenerateShardId() { - int[][] possibleValues = new int[][] { - {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1} - }; - for (int i = 0; i < 10; i++) { - int[] shardSplits = randomFrom(possibleValues); - assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]); - assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]); - IndexMetadata metadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0]) - .numberOfReplicas(1).build(); - String term = randomAlphaOfLength(10); - final int shard = OperationRouting.generateShardId(metadata, term, null); - IndexMetadata shrunk = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1]) - .numberOfReplicas(1) - .setRoutingNumShards(shardSplits[0]).build(); - int shrunkShard = OperationRouting.generateShardId(shrunk, term, null); - - Set shardIds = IndexMetadata.selectShrinkShards(shrunkShard, metadata, shrunk.getNumberOfShards()); - assertEquals(1, shardIds.stream().filter((sid) -> sid.id() == shard).count()); - - shrunk = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1) - .setRoutingNumShards(shardSplits[0]).build(); - shrunkShard = OperationRouting.generateShardId(shrunk, term, null); - shardIds = IndexMetadata.selectShrinkShards(shrunkShard, metadata, shrunk.getNumberOfShards()); - assertEquals(Arrays.toString(shardSplits), 1, shardIds.stream().filter((sid) -> sid.id() == shard).count()); - } - } - - public void testGenerateShardIdSplit() { - int[][] possibleValues = new int[][] { - {2,4,8}, {2, 10, 20}, {3, 12, 36}, {1,5,15} - }; - for (int i = 0; i < 10; i++) { - int[] shardSplits = randomFrom(possibleValues); - assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); - assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); - IndexMetadata metadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0]) - .numberOfReplicas(1).setRoutingNumShards(shardSplits[2]).build(); - String term = randomAlphaOfLength(10); - final int shard = OperationRouting.generateShardId(metadata, term, null); - IndexMetadata split = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1]) - .numberOfReplicas(1) - .setRoutingNumShards(shardSplits[2]).build(); - int shrunkShard = OperationRouting.generateShardId(split, term, null); - - ShardId shardId = IndexMetadata.selectSplitShard(shrunkShard, metadata, split.getNumberOfShards()); - assertNotNull(shardId); - assertEquals(shard, shardId.getId()); - - split = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1) - .setRoutingNumShards(shardSplits[2]).build(); - shrunkShard = OperationRouting.generateShardId(split, term, null); - shardId = IndexMetadata.selectSplitShard(shrunkShard, metadata, split.getNumberOfShards()); - assertNotNull(shardId); - assertEquals(shard, shardId.getId()); - } - } - - public void testPartitionedIndex() { - // make sure the same routing value always has each _id fall within the configured partition size - for (int shards = 1; shards < 5; shards++) { - for (int partitionSize = 1; partitionSize == 1 || partitionSize < shards; partitionSize++) { - IndexMetadata metadata = IndexMetadata.builder("test") - .settings(settings(Version.CURRENT)) - .numberOfShards(shards) - .routingPartitionSize(partitionSize) - .numberOfReplicas(1) - .build(); - - for (int i = 0; i < 20; i++) { - String routing = randomUnicodeOfLengthBetween(1, 50); - Set shardSet = new HashSet<>(); - - for (int k = 0; k < 150; k++) { - String id = randomUnicodeOfLengthBetween(1, 50); - - shardSet.add(OperationRouting.generateShardId(metadata, id, routing)); - } - - assertEquals(partitionSize, shardSet.size()); - } - } - } - } - - public void testPartitionedIndexShrunk() { - Map> routingIdToShard = new HashMap<>(); - - Map routingA = new HashMap<>(); - routingA.put("a_0", 1); - routingA.put("a_1", 2); - routingA.put("a_2", 2); - routingA.put("a_3", 2); - routingA.put("a_4", 1); - routingA.put("a_5", 2); - routingIdToShard.put("a", routingA); - - Map routingB = new HashMap<>(); - routingB.put("b_0", 0); - routingB.put("b_1", 0); - routingB.put("b_2", 0); - routingB.put("b_3", 0); - routingB.put("b_4", 3); - routingB.put("b_5", 3); - routingIdToShard.put("b", routingB); - - Map routingC = new HashMap<>(); - routingC.put("c_0", 1); - routingC.put("c_1", 1); - routingC.put("c_2", 0); - routingC.put("c_3", 0); - routingC.put("c_4", 0); - routingC.put("c_5", 1); - routingIdToShard.put("c", routingC); - - Map routingD = new HashMap<>(); - routingD.put("d_0", 2); - routingD.put("d_1", 2); - routingD.put("d_2", 3); - routingD.put("d_3", 3); - routingD.put("d_4", 3); - routingD.put("d_5", 3); - routingIdToShard.put("d", routingD); - - IndexMetadata metadata = IndexMetadata.builder("test") - .settings(settings(Version.CURRENT)) - .setRoutingNumShards(8) - .numberOfShards(4) - .routingPartitionSize(3) - .numberOfReplicas(1) - .build(); - - for (Map.Entry> routingIdEntry : routingIdToShard.entrySet()) { - String routing = routingIdEntry.getKey(); - - for (Map.Entry idEntry : routingIdEntry.getValue().entrySet()) { - String id = idEntry.getKey(); - int shard = idEntry.getValue(); - - assertEquals(shard, OperationRouting.generateShardId(metadata, id, routing)); - } - } - } - - public void testPartitionedIndexBWC() { - Map> routingIdToShard = new HashMap<>(); - - Map routingA = new HashMap<>(); - routingA.put("a_0", 3); - routingA.put("a_1", 2); - routingA.put("a_2", 2); - routingA.put("a_3", 3); - routingIdToShard.put("a", routingA); - - Map routingB = new HashMap<>(); - routingB.put("b_0", 5); - routingB.put("b_1", 0); - routingB.put("b_2", 0); - routingB.put("b_3", 0); - routingIdToShard.put("b", routingB); - - Map routingC = new HashMap<>(); - routingC.put("c_0", 4); - routingC.put("c_1", 4); - routingC.put("c_2", 3); - routingC.put("c_3", 4); - routingIdToShard.put("c", routingC); - - Map routingD = new HashMap<>(); - routingD.put("d_0", 3); - routingD.put("d_1", 4); - routingD.put("d_2", 4); - routingD.put("d_3", 4); - routingIdToShard.put("d", routingD); - - IndexMetadata metadata = IndexMetadata.builder("test") - .settings(settings(Version.CURRENT)) - .numberOfShards(6) - .routingPartitionSize(2) - .numberOfReplicas(1) - .build(); - - for (Map.Entry> routingIdEntry : routingIdToShard.entrySet()) { - String routing = routingIdEntry.getKey(); - - for (Map.Entry idEntry : routingIdEntry.getValue().entrySet()) { - String id = idEntry.getKey(); - int shard = idEntry.getValue(); - - assertEquals(shard, OperationRouting.generateShardId(metadata, id, routing)); - } - } - } - - /** - * Ensures that all changes to the hash-function / shard selection are BWC - */ - public void testBWC() { - Map termToShard = new TreeMap<>(); - termToShard.put("sEERfFzPSI", 1); - termToShard.put("cNRiIrjzYd", 7); - termToShard.put("BgfLBXUyWT", 5); - termToShard.put("cnepjZhQnb", 3); - termToShard.put("OKCmuYkeCK", 6); - termToShard.put("OutXGRQUja", 5); - termToShard.put("yCdyocKWou", 1); - termToShard.put("KXuNWWNgVj", 2); - termToShard.put("DGJOYrpESx", 4); - termToShard.put("upLDybdTGs", 5); - termToShard.put("yhZhzCPQby", 1); - termToShard.put("EyCVeiCouA", 1); - termToShard.put("tFyVdQauWR", 6); - termToShard.put("nyeRYDnDQr", 6); - termToShard.put("hswhrppvDH", 0); - termToShard.put("BSiWvDOsNE", 5); - termToShard.put("YHicpFBSaY", 1); - termToShard.put("EquPtdKaBZ", 4); - termToShard.put("rSjLZHCDfT", 5); - termToShard.put("qoZALVcite", 7); - termToShard.put("yDCCPVBiCm", 7); - termToShard.put("ngizYtQgGK", 5); - termToShard.put("FYQRIBcNqz", 0); - termToShard.put("EBzEDAPODe", 2); - termToShard.put("YePigbXgKb", 1); - termToShard.put("PeGJjomyik", 3); - termToShard.put("cyQIvDmyYD", 7); - termToShard.put("yIEfZrYfRk", 5); - termToShard.put("kblouyFUbu", 7); - termToShard.put("xvIGbRiGJF", 3); - termToShard.put("KWimwsREPf", 4); - termToShard.put("wsNavvIcdk", 7); - termToShard.put("xkWaPcCmpT", 0); - termToShard.put("FKKTOnJMDy", 7); - termToShard.put("RuLzobYixn", 2); - termToShard.put("mFohLeFRvF", 4); - termToShard.put("aAMXnamRJg", 7); - termToShard.put("zKBMYJDmBI", 0); - termToShard.put("ElSVuJQQuw", 7); - termToShard.put("pezPtTQAAm", 7); - termToShard.put("zBjjNEjAex", 2); - termToShard.put("PGgHcLNPYX", 7); - termToShard.put("hOkpeQqTDF", 3); - termToShard.put("chZXraUPBH", 7); - termToShard.put("FAIcSmmNXq", 5); - termToShard.put("EZmDicyayC", 0); - termToShard.put("GRIueBeIyL", 7); - termToShard.put("qCChjGZYLp", 3); - termToShard.put("IsSZQwwnUT", 3); - termToShard.put("MGlxLFyyCK", 3); - termToShard.put("YmscwrKSpB", 0); - termToShard.put("czSljcjMop", 5); - termToShard.put("XhfGWwNlng", 1); - termToShard.put("cWpKJjlzgj", 7); - termToShard.put("eDzIfMKbvk", 1); - termToShard.put("WFFWYBfnTb", 0); - termToShard.put("oDdHJxGxja", 7); - termToShard.put("PDOQQqgIKE", 1); - termToShard.put("bGEIEBLATe", 6); - termToShard.put("xpRkJPWVpu", 2); - termToShard.put("kTwZnPEeIi", 2); - termToShard.put("DifcuqSsKk", 1); - termToShard.put("CEmLmljpXe", 5); - termToShard.put("cuNKtLtyJQ", 7); - termToShard.put("yNjiAnxAmt", 5); - termToShard.put("bVDJDCeaFm", 2); - termToShard.put("vdnUhGLFtl", 0); - termToShard.put("LnqSYezXbr", 5); - termToShard.put("EzHgydDCSR", 3); - termToShard.put("ZSKjhJlcpn", 1); - termToShard.put("WRjUoZwtUz", 3); - termToShard.put("RiBbcCdIgk", 4); - termToShard.put("yizTqyjuDn", 4); - termToShard.put("QnFjcpcZUT", 4); - termToShard.put("agYhXYUUpl", 7); - termToShard.put("UOjiTugjNC", 7); - termToShard.put("nICGuWTdfV", 0); - termToShard.put("NrnSmcnUVF", 2); - termToShard.put("ZSzFcbpDqP", 3); - termToShard.put("YOhahLSzzE", 5); - termToShard.put("iWswCilUaT", 1); - termToShard.put("zXAamKsRwj", 2); - termToShard.put("aqGsrUPHFq", 5); - termToShard.put("eDItImYWTS", 1); - termToShard.put("JAYDZMRcpW", 4); - termToShard.put("lmvAaEPflK", 7); - termToShard.put("IKuOwPjKCx", 5); - termToShard.put("schsINzlYB", 1); - termToShard.put("OqbFNxrKrF", 2); - termToShard.put("QrklDfvEJU", 6); - termToShard.put("VLxKRKdLbx", 4); - termToShard.put("imoydNTZhV", 1); - termToShard.put("uFZyTyOMRO", 4); - termToShard.put("nVAZVMPNNx", 3); - termToShard.put("rPIdESYaAO", 5); - termToShard.put("nbZWPWJsIM", 0); - termToShard.put("wRZXPSoEgd", 3); - termToShard.put("nGzpgwsSBc", 4); - termToShard.put("AITyyoyLLs", 4); - IndexMetadata metadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(8) - .numberOfReplicas(1).build(); - for (Map.Entry entry : termToShard.entrySet()) { - String key = entry.getKey(); - int shard = randomBoolean() ? - OperationRouting.generateShardId(metadata, key, null) : OperationRouting.generateShardId(metadata, "foobar", key); - assertEquals(shard, entry.getValue().intValue()); - } - } - public void testPreferNodes() throws InterruptedException, IOException { TestThreadPool threadPool = null; ClusterService clusterService = null; diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java index 9a5449e224835..11fdbbe7dccd7 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java @@ -25,7 +25,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.NestedPathFieldMapper; @@ -55,7 +55,7 @@ public void testSplitOnID() throws IOException { int targetShardId = randomIntBetween(0, numShards-1); boolean hasNested = randomBoolean(); for (int j = 0; j < numDocs; j++) { - int shardId = OperationRouting.generateShardId(metadata, Integer.toString(j), null); + int shardId = IndexRouting.fromIndexMetadata(metadata).shardId(Integer.toString(j), null); if (hasNested) { List> docs = new ArrayList<>(); int numNested = randomIntBetween(0, 10); @@ -103,7 +103,7 @@ public void testSplitOnRouting() throws IOException { int targetShardId = randomIntBetween(0, numShards-1); for (int j = 0; j < numDocs; j++) { String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); - final int shardId = OperationRouting.generateShardId(metadata, null, routing); + final int shardId = IndexRouting.fromIndexMetadata(metadata).shardId(null, routing); if (hasNested) { List> docs = new ArrayList<>(); int numNested = randomIntBetween(0, 10); @@ -154,7 +154,7 @@ public void testSplitOnIdOrRouting() throws IOException { final int shardId; if (randomBoolean()) { String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); - shardId = OperationRouting.generateShardId(metadata, null, routing); + shardId = IndexRouting.fromIndexMetadata(metadata).shardId(null, routing); rootDoc = Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), @@ -162,7 +162,7 @@ public void testSplitOnIdOrRouting() throws IOException { sequenceIDFields.primaryTerm ); } else { - shardId = OperationRouting.generateShardId(metadata, Integer.toString(j), null); + shardId = IndexRouting.fromIndexMetadata(metadata).shardId(Integer.toString(j), null); rootDoc = Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new SortedNumericDocValuesField("shard_id", shardId), @@ -209,7 +209,7 @@ public void testSplitOnRoutingPartitioned() throws IOException { int targetShardId = randomIntBetween(0, numShards-1); for (int j = 0; j < numDocs; j++) { String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); - final int shardId = OperationRouting.generateShardId(metadata, Integer.toString(j), routing); + final int shardId = IndexRouting.fromIndexMetadata(metadata).shardId(Integer.toString(j), routing); if (hasNested) { List> docs = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java index 0edcb21ec1e56..3e4642df368df 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java @@ -30,7 +30,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.engine.Engine; @@ -195,11 +195,15 @@ public void testSplitShard() throws IOException { BytesRef ref; while((ref = iterator.next()) != null) { String value = ref.utf8ToString(); - assertEquals("value has wrong shards: " + value, targetShardId, OperationRouting.generateShardId(metadata, value, null)); + assertEquals( + "value has wrong shards: " + value, + targetShardId, + IndexRouting.fromIndexMetadata(metadata).shardId(value, null) + ); } for (int i = 0; i < numDocs; i++) { ref = new BytesRef(Integer.toString(i)); - int shardId = OperationRouting.generateShardId(metadata, ref.utf8ToString(), null); + int shardId = IndexRouting.fromIndexMetadata(metadata).shardId(ref.utf8ToString(), null); if (shardId == targetShardId) { assertTrue(ref.utf8ToString() + " is missing", terms.iterator().seekExact(ref)); } else { diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 29ece3c35791c..fe87536438061 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.OperationRouting; @@ -2172,11 +2173,13 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) { if (indexService != null) { assertThat(indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, -1), greaterThan(shard)); + ClusterState clusterState = clusterService.state(); OperationRouting operationRouting = clusterService.operationRouting(); + IndexRouting indexRouting = IndexRouting.fromIndexMetadata(clusterState.metadata().getIndexSafe(index)); while (true) { String routing = RandomStrings.randomAsciiLettersOfLength(random, 10); final int targetShard = operationRouting - .indexShards(clusterService.state(), index.getName(), null, routing) + .indexShards(clusterState, index.getName(), indexRouting, null, routing) .shardId().getId(); if (shard == targetShard) { return routing;