Skip to content

Commit 2bca9bb

Browse files
mgodwanmgodwan
committed
Create Index using context (#15290)
* Create Index using context Signed-off-by: Mohit Godwani <mgodwan@amazon.com>
1 parent a2b6f8c commit 2bca9bb

29 files changed

Lines changed: 934 additions & 42 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3333
- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630))
3434
- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
3535
- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
36+
- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))
3637

3738
### Dependencies
3839
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

‎client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import org.apache.lucene.util.CollectionUtil;
3636
import org.opensearch.cluster.metadata.AliasMetadata;
37+
import org.opensearch.cluster.metadata.Context;
3738
import org.opensearch.cluster.metadata.MappingMetadata;
3839
import org.opensearch.common.settings.Settings;
3940
import org.opensearch.core.xcontent.XContentParser;
@@ -61,6 +62,7 @@ public class GetIndexResponse {
6162
private Map<String, Settings> settings;
6263
private Map<String, Settings> defaultSettings;
6364
private Map<String, String> dataStreams;
65+
private Map<String, Context> contexts;
6466
private String[] indices;
6567

6668
GetIndexResponse(
@@ -69,7 +71,8 @@ public class GetIndexResponse {
6971
Map<String, List<AliasMetadata>> aliases,
7072
Map<String, Settings> settings,
7173
Map<String, Settings> defaultSettings,
72-
Map<String, String> dataStreams
74+
Map<String, String> dataStreams,
75+
Map<String, Context> contexts
7376
) {
7477
this.indices = indices;
7578
// to have deterministic order
@@ -89,6 +92,9 @@ public class GetIndexResponse {
8992
if (dataStreams != null) {
9093
this.dataStreams = dataStreams;
9194
}
95+
if (contexts != null) {
96+
this.contexts = contexts;
97+
}
9298
}
9399

94100
public String[] getIndices() {
@@ -123,6 +129,10 @@ public Map<String, String> getDataStreams() {
123129
return dataStreams;
124130
}
125131

132+
public Map<String, Context> contexts() {
133+
return contexts;
134+
}
135+
126136
/**
127137
* Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to
128138
* false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set
@@ -167,6 +177,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
167177
Settings indexSettings = null;
168178
Settings indexDefaultSettings = null;
169179
String dataStream = null;
180+
Context context = null;
170181
// We start at START_OBJECT since fromXContent ensures that
171182
while (parser.nextToken() != Token.END_OBJECT) {
172183
ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser);
@@ -185,6 +196,9 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
185196
case "defaults":
186197
indexDefaultSettings = Settings.fromXContent(parser);
187198
break;
199+
case "context":
200+
context = Context.fromXContent(parser);
201+
break;
188202
default:
189203
parser.skipChildren();
190204
}
@@ -197,7 +211,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
197211
parser.skipChildren();
198212
}
199213
}
200-
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream);
214+
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream, context);
201215
}
202216

203217
// This is just an internal container to make stuff easier for returning
@@ -207,19 +221,22 @@ private static class IndexEntry {
207221
Settings indexSettings = Settings.EMPTY;
208222
Settings indexDefaultSettings = Settings.EMPTY;
209223
String dataStream;
224+
Context context;
210225

211226
IndexEntry(
212227
List<AliasMetadata> indexAliases,
213228
MappingMetadata indexMappings,
214229
Settings indexSettings,
215230
Settings indexDefaultSettings,
216-
String dataStream
231+
String dataStream,
232+
Context context
217233
) {
218234
if (indexAliases != null) this.indexAliases = indexAliases;
219235
if (indexMappings != null) this.indexMappings = indexMappings;
220236
if (indexSettings != null) this.indexSettings = indexSettings;
221237
if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings;
222238
if (dataStream != null) this.dataStream = dataStream;
239+
if (context != null) this.context = context;
223240
}
224241
}
225242

@@ -229,6 +246,7 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
229246
Map<String, Settings> settings = new HashMap<>();
230247
Map<String, Settings> defaultSettings = new HashMap<>();
231248
Map<String, String> dataStreams = new HashMap<>();
249+
Map<String, Context> contexts = new HashMap<>();
232250
List<String> indices = new ArrayList<>();
233251

234252
if (parser.currentToken() == null) {
@@ -254,12 +272,15 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
254272
if (indexEntry.dataStream != null) {
255273
dataStreams.put(indexName, indexEntry.dataStream);
256274
}
275+
if (indexEntry.context != null) {
276+
contexts.put(indexName, indexEntry.context);
277+
}
257278
} else if (parser.currentToken() == Token.START_ARRAY) {
258279
parser.skipChildren();
259280
} else {
260281
parser.nextToken();
261282
}
262283
}
263-
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams);
284+
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams, contexts);
264285
}
265286
}

‎client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.opensearch.client.AbstractResponseTestCase;
3737
import org.opensearch.client.GetAliasesResponseTests;
3838
import org.opensearch.cluster.metadata.AliasMetadata;
39+
import org.opensearch.cluster.metadata.Context;
3940
import org.opensearch.cluster.metadata.MappingMetadata;
4041
import org.opensearch.common.settings.IndexScopedSettings;
4142
import org.opensearch.common.settings.Settings;
@@ -66,6 +67,7 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
6667
final Map<String, Settings> settings = new HashMap<>();
6768
final Map<String, Settings> defaultSettings = new HashMap<>();
6869
final Map<String, String> dataStreams = new HashMap<>();
70+
final Map<String, Context> contexts = new HashMap<>();
6971
IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
7072
boolean includeDefaults = randomBoolean();
7173
for (String index : indices) {
@@ -90,14 +92,19 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
9092
if (randomBoolean()) {
9193
dataStreams.put(index, randomAlphaOfLength(5).toLowerCase(Locale.ROOT));
9294
}
95+
96+
if (randomBoolean()) {
97+
contexts.put(index, new Context(randomAlphaOfLength(5).toLowerCase(Locale.ROOT)));
98+
}
9399
}
94100
return new org.opensearch.action.admin.indices.get.GetIndexResponse(
95101
indices,
96102
mappings,
97103
aliases,
98104
settings,
99105
defaultSettings,
100-
dataStreams
106+
dataStreams,
107+
null
101108
);
102109
}
103110

@@ -116,6 +123,7 @@ protected void assertInstances(
116123
assertEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
117124
assertEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
118125
assertEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
126+
assertEquals(serverTestInstance.contexts(), clientInstance.contexts());
119127
}
120128

121129
private static MappingMetadata createMappingsForIndex() {

‎server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,23 @@
4141
import org.opensearch.action.support.IndicesOptions;
4242
import org.opensearch.action.support.master.AcknowledgedResponse;
4343
import org.opensearch.cluster.ClusterState;
44+
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
45+
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
46+
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
47+
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
48+
import org.opensearch.cluster.metadata.Context;
4449
import org.opensearch.cluster.metadata.IndexMetadata;
4550
import org.opensearch.cluster.metadata.MappingMetadata;
4651
import org.opensearch.cluster.metadata.Metadata;
52+
import org.opensearch.cluster.service.ClusterService;
4753
import org.opensearch.common.settings.Settings;
4854
import org.opensearch.common.unit.TimeValue;
4955
import org.opensearch.common.xcontent.XContentFactory;
5056
import org.opensearch.core.action.ActionListener;
57+
import org.opensearch.core.common.bytes.BytesReference;
5158
import org.opensearch.index.IndexNotFoundException;
5259
import org.opensearch.index.IndexService;
60+
import org.opensearch.index.IndexSettings;
5361
import org.opensearch.index.mapper.MapperParsingException;
5462
import org.opensearch.index.mapper.MapperService;
5563
import org.opensearch.index.query.RangeQueryBuilder;
@@ -58,7 +66,10 @@
5866
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
5967
import org.opensearch.test.OpenSearchIntegTestCase.Scope;
6068

69+
import java.nio.ByteBuffer;
70+
import java.nio.charset.StandardCharsets;
6171
import java.util.Map;
72+
import java.util.UUID;
6273
import java.util.concurrent.CountDownLatch;
6374
import java.util.concurrent.atomic.AtomicInteger;
6475
import java.util.function.BiFunction;
@@ -429,4 +440,53 @@ public void testCreateIndexWithNullReplicaCountPickUpClusterReplica() {
429440
);
430441
}
431442
}
443+
444+
public void testCreateIndexWithContextSettingsAndTemplate() throws Exception {
445+
int numReplicas = 1;
446+
String indexName = "test-idx-1";
447+
Settings settings = Settings.builder()
448+
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
449+
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), (String) null)
450+
.build();
451+
Context context = new Context("test");
452+
453+
String templateContent = "{\n"
454+
+ " \"template\": {\n"
455+
+ " \"settings\": {\n"
456+
+ " \"merge.policy\": \"log_byte_size\"\n"
457+
+ " }\n"
458+
+ " },\n"
459+
+ " \"_meta\": {\n"
460+
+ " \"_type\": \"@abc_template\",\n"
461+
+ " \"_version\": 1\n"
462+
+ " },\n"
463+
+ " \"version\": 1\n"
464+
+ "}\n";
465+
466+
ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
467+
internalCluster().clusterManagerClient(),
468+
() -> internalCluster().getInstance(ClusterService.class).state()
469+
);
470+
loader.loadTemplate(
471+
new SystemTemplate(
472+
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
473+
SystemTemplateMetadata.fromComponentTemplateInfo("test", 1L),
474+
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
475+
)
476+
);
477+
478+
assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settings).setContext(context).get());
479+
480+
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, internalCluster().getClusterManagerName());
481+
482+
for (IndexService indexService : indicesService) {
483+
assertEquals(indexName, indexService.index().getName());
484+
assertEquals(
485+
numReplicas,
486+
(int) indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null)
487+
);
488+
assertEquals(context, indexService.getMetadata().context());
489+
assertEquals("log_byte_size", indexService.getMetadata().getSettings().get(IndexSettings.INDEX_MERGE_POLICY.getKey()));
490+
}
491+
}
432492
}

‎server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,18 @@
3535
import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
3636
import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
3737
import org.opensearch.cluster.ClusterState;
38+
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
39+
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
40+
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
41+
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
42+
import org.opensearch.cluster.metadata.Context;
3843
import org.opensearch.cluster.metadata.IndexMetadata;
3944
import org.opensearch.cluster.service.ClusterService;
4045
import org.opensearch.common.Priority;
4146
import org.opensearch.common.settings.Setting;
4247
import org.opensearch.common.settings.Settings;
4348
import org.opensearch.common.unit.TimeValue;
49+
import org.opensearch.core.common.bytes.BytesReference;
4450
import org.opensearch.index.IndexModule;
4551
import org.opensearch.index.IndexService;
4652
import org.opensearch.index.VersionType;
@@ -50,10 +56,14 @@
5056
import org.opensearch.test.OpenSearchIntegTestCase;
5157
import org.opensearch.threadpool.ThreadPool;
5258

59+
import java.io.IOException;
60+
import java.nio.ByteBuffer;
61+
import java.nio.charset.StandardCharsets;
5362
import java.util.Arrays;
5463
import java.util.Collection;
5564
import java.util.Collections;
5665
import java.util.List;
66+
import java.util.UUID;
5767

5868
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA;
5969
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ;
@@ -98,6 +108,58 @@ public void testInvalidDynamicUpdate() {
98108
assertNotEquals(indexMetadata.getSettings().get("index.dummy"), "invalid dynamic value");
99109
}
100110

111+
public void testDynamicUpdateWithContextSettingOverlap() throws IOException {
112+
String templateContent = "{\n"
113+
+ " \"template\": {\n"
114+
+ " \"settings\": {\n"
115+
+ " \"index.merge.policy\": \"log_byte_size\"\n"
116+
+ " }\n"
117+
+ " },\n"
118+
+ " \"_meta\": {\n"
119+
+ " \"_type\": \"@abc_template\",\n"
120+
+ " \"_version\": 1\n"
121+
+ " },\n"
122+
+ " \"version\": 1\n"
123+
+ "}\n";
124+
125+
ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
126+
internalCluster().clusterManagerClient(),
127+
() -> internalCluster().getInstance(ClusterService.class).state()
128+
);
129+
loader.loadTemplate(
130+
new SystemTemplate(
131+
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
132+
SystemTemplateMetadata.fromComponentTemplateInfo("testcontext", 1L),
133+
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
134+
)
135+
);
136+
137+
createIndex("test", new Context("testcontext"));
138+
139+
IllegalArgumentException validationException = expectThrows(
140+
IllegalArgumentException.class,
141+
() -> client().admin()
142+
.indices()
143+
.prepareUpdateSettings("test")
144+
.setSettings(Settings.builder().put("index.merge.policy", "tiered"))
145+
.execute()
146+
.actionGet()
147+
);
148+
assertTrue(
149+
validationException.getMessage()
150+
.contains("Cannot apply context template as user provide settings have overlap with the included context template")
151+
);
152+
153+
assertAcked(
154+
client().admin()
155+
.indices()
156+
.prepareUpdateSettings("test")
157+
.setSettings(Settings.builder().put("index.refresh_interval", "60s"))
158+
.execute()
159+
.actionGet()
160+
);
161+
}
162+
101163
@Override
102164
protected Collection<Class<? extends Plugin>> nodePlugins() {
103165
return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class);

‎server/src/main/java/org/opensearch/OpenSearchServerException.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle;
1515
import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED;
1616
import static org.opensearch.Version.V_2_10_0;
17+
import static org.opensearch.Version.V_2_17_0;
1718
import static org.opensearch.Version.V_2_1_0;
1819
import static org.opensearch.Version.V_2_3_0;
1920
import static org.opensearch.Version.V_2_4_0;
@@ -1179,6 +1180,14 @@ public static void registerExceptions() {
11791180
)
11801181
);
11811182
registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_2_10_0));
1183+
registerExceptionHandle(
1184+
new OpenSearchExceptionHandle(
1185+
org.opensearch.indices.InvalidIndexContextException.class,
1186+
org.opensearch.indices.InvalidIndexContextException::new,
1187+
174,
1188+
V_2_17_0
1189+
)
1190+
);
11821191
registerExceptionHandle(
11831192
new OpenSearchExceptionHandle(
11841193
org.opensearch.cluster.block.IndexCreateBlockException.class,

0 commit comments

Comments
 (0)