From 176af3698b9373ade97d4cc75344efcee97d6a4b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 24 Nov 2020 08:03:38 -0800 Subject: [PATCH 01/22] Add Rollup ILM Action this commit introduces a new Rollup ILM Action that allows indices to be rolled up according to a specific rollup config. The action also allows for the new rolled up index to be associated with a different policy than the original/source index. Optionally, the original index can be deleted. Relates #42720. Closes #48003. --- .../xpack/core/XPackClientPlugin.java | 11 +- .../xpack/core/ilm/RollupILMAction.java | 153 ++++++++++++ .../xpack/core/ilm/RollupStep.java | 89 +++++++ .../core/ilm/TimeseriesLifecycleType.java | 28 ++- .../ilm/LifecyclePolicyMetadataTests.java | 6 +- .../xpack/core/ilm/LifecyclePolicyTests.java | 8 +- .../xpack/core/ilm/RollupILMActionTests.java | 112 +++++++++ .../xpack/core/ilm/RollupStepTests.java | 226 ++++++++++++++++++ .../ilm/TimeseriesLifecycleTypeTests.java | 12 + .../ilm/action/PutLifecycleRequestTests.java | 7 +- x-pack/plugin/ilm/qa/multi-node/build.gradle | 2 + .../ilm/TimeSeriesLifecycleActionsIT.java | 62 +++++ .../xpack/ilm/IndexLifecycle.java | 14 +- .../ilm/IndexLifecycleMetadataTests.java | 7 +- 14 files changed, 716 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index e15606c902e71..abdd270eefb06 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.core.ilm.MigrateAction; import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.RollupILMAction; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; @@ -420,7 +421,7 @@ public List> getClientActions() { @Override public List getNamedWriteables() { - return Arrays.asList( + List namedWriteables = new ArrayList<>(Arrays.asList( // graph new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new), // logstash @@ -523,7 +524,13 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_STREAMS, DataStreamFeatureSetUsage::new), // Data Tiers new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_TIERS, DataTiersFeatureSetUsage::new) - ); + )); + + if (RollupV2.isEnabled()) { + namedWriteables.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, RollupILMAction.NAME, RollupILMAction::new)); + } + + return namedWriteables; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java new file mode 100644 index 0000000000000..c06b7314c2ddb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ilm.Step.StepKey; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * A {@link LifecycleAction} which calls {@link org.elasticsearch.xpack.core.rollup.v2.RollupAction} on an index + */ +public class RollupILMAction implements LifecycleAction { + public static final String NAME = "rollup"; + + private static final ParseField CONFIG_FIELD = new ParseField("config"); + private static final ParseField DELETE_FIELD = new ParseField("delete_original"); + private static final ParseField POLICY_FIELD = new ParseField("rollup_policy"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + a -> new RollupILMAction((RollupActionConfig) a[0], (boolean) a[1], (String) a[2])); + + private final RollupActionConfig config; + private final boolean deleteOriginalIndex; + private final String rollupPolicy; + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> RollupActionConfig.fromXContent(p), CONFIG_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DELETE_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), POLICY_FIELD); + } + + public static RollupILMAction parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public RollupILMAction(RollupActionConfig config, boolean deleteOriginalIndex, @Nullable String rollupPolicy) { + this.config = config; + this.deleteOriginalIndex = deleteOriginalIndex; + this.rollupPolicy = rollupPolicy; + } + + public RollupILMAction(StreamInput in) throws IOException { + this(new RollupActionConfig(in), in.readBoolean(), in.readOptionalString()); + } + + @Override + public String getWriteableName() { + return NAME; + } + + RollupActionConfig config() { + return config; + } + + boolean deleteOriginalIndex() { + return deleteOriginalIndex; + } + + String rollupPolicy() { + return rollupPolicy; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONFIG_FIELD.getPreferredName(), config); + builder.field(DELETE_FIELD.getPreferredName(), deleteOriginalIndex); + if (rollupPolicy != null) { + builder.field(POLICY_FIELD.getPreferredName(), rollupPolicy); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + config.writeTo(out); + out.writeBoolean(deleteOriginalIndex); + out.writeOptionalString(rollupPolicy); + } + + @Override + public boolean isSafeAction() { + return false; + } + + @Override + public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); + StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); + StepKey rollupKey = new StepKey(phase, NAME, NAME); + StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); + StepKey deleteStepKey = new StepKey(phase, NAME, DeleteStep.NAME); + + Settings readOnlySettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build(); + + CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, + readOnlyKey); + UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, rollupKey, client, readOnlySettings); + WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, deleteStepKey, client); + + if (deleteOriginalIndex) { + RollupStep rollupStep = new RollupStep(rollupKey, waitForNoFollowerStepKey, client, config, rollupPolicy); + DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep, waitForNoFollowersStep, deleteStep); + } else { + RollupStep rollupStep = new RollupStep(rollupKey, nextStepKey, client, config, rollupPolicy); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RollupILMAction that = (RollupILMAction) o; + + return Objects.equals(this.config, that.config) + && Objects.equals(this.deleteOriginalIndex, that.deleteOriginalIndex) + && Objects.equals(this.rollupPolicy, that.rollupPolicy); + } + + @Override + public int hashCode() { + return Objects.hash(config, deleteOriginalIndex, rollupPolicy); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java new file mode 100644 index 0000000000000..b6568ae45fcef --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.rollup.v2.RollupAction; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; + +import java.util.Objects; + +/** + * Rolls up index using a {@link RollupActionConfig} + */ +public class RollupStep extends AsyncActionStep { + public static final String NAME = "rollup"; + public static final String ROLLUP_INDEX_NAME_POSTFIX = "-rollup"; + + private final RollupActionConfig config; + private final String rollupPolicy; + + public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionConfig config, String rollupPolicy) { + super(key, nextStepKey, client); + this.config = config; + this.rollupPolicy = rollupPolicy; + } + + @Override + public boolean isRetryable() { + return false; + } + + @Override + public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) { + String originalIndex = indexMetadata.getIndex().getName(); + String rollupIndex = originalIndex + ROLLUP_INDEX_NAME_POSTFIX; + // TODO(talevy): change config to be immutable + config.setRollupIndex(rollupIndex); + RollupAction.Request request = new RollupAction.Request(originalIndex, config); + if (rollupPolicy == null) { + getClient().execute(RollupAction.INSTANCE, request, + ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); + } else { + Settings setPolicySettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, rollupPolicy).build(); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(rollupIndex) + .masterNodeTimeout(getMasterTimeout(currentState)).settings(setPolicySettings); + getClient().execute(RollupAction.INSTANCE, request, + ActionListener.wrap(rollupResponse -> { + getClient().admin().indices().updateSettings(updateSettingsRequest, + ActionListener.wrap(settingsResponse -> listener.onResponse(true), listener::onFailure)); + }, listener::onFailure)); + } + } + + public RollupActionConfig getConfig() { + return config; + } + + public String getRollupPolicy() { + return rollupPolicy; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), config, rollupPolicy); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + RollupStep other = (RollupStep) obj; + return super.equals(obj) + && Objects.equals(config, other.config) + && Objects.equals(rollupPolicy, other.rollupPolicy); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index 343dd1a1c9336..12dad017a17c1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.rollup.RollupV2; import java.io.IOException; import java.util.ArrayList; @@ -44,18 +45,13 @@ public class TimeseriesLifecycleType implements LifecycleType { ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); - static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, - MigrateAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME); + static final List ORDERED_VALID_COLD_ACTIONS; static final List ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(WaitForSnapshotAction.NAME, DeleteAction.NAME); static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS); - static final Set VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS); + static final Set VALID_COLD_ACTIONS; static final Set VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS); - private static final Map> ALLOWED_ACTIONS = Map.of( - HOT_PHASE, VALID_HOT_ACTIONS, - WARM_PHASE, VALID_WARM_ACTIONS, - COLD_PHASE, VALID_COLD_ACTIONS, - DELETE_PHASE, VALID_DELETE_ACTIONS); + private static final Map> ALLOWED_ACTIONS; static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Sets.newHashSet(ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); @@ -63,6 +59,22 @@ public class TimeseriesLifecycleType implements LifecycleType { static final Set ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT = Sets.newHashSet(ShrinkAction.NAME, ForceMergeAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME); + static { + if (RollupV2.isEnabled()) { + ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, + MigrateAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME, RollupILMAction.NAME); + } else { + ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, + MigrateAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME); + } + VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS); + ALLOWED_ACTIONS = new HashMap<>(); + ALLOWED_ACTIONS.put(HOT_PHASE, VALID_HOT_ACTIONS); + ALLOWED_ACTIONS.put(WARM_PHASE, VALID_WARM_ACTIONS); + ALLOWED_ACTIONS.put(COLD_PHASE, VALID_COLD_ACTIONS); + ALLOWED_ACTIONS.put(DELETE_PHASE, VALID_DELETE_ACTIONS); + } + private TimeseriesLifecycleType() { } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java index 2b048b51d743b..b6795bd47a954 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java @@ -49,7 +49,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, FreezeAction.NAME, FreezeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, MigrateAction.NAME, MigrateAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, RollupILMAction.NAME, RollupILMAction::new) )); } @@ -72,7 +73,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SetPriorityAction.NAME), SetPriorityAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MigrateAction.NAME), MigrateAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RollupILMAction.NAME), RollupILMAction::parse) )); return new NamedXContentRegistry(entries); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 04461e556fa20..52428ca199b30 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -59,7 +59,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, MigrateAction.NAME, MigrateAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, RollupILMAction.NAME, RollupILMAction::new) )); } @@ -82,7 +83,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MigrateAction.NAME), MigrateAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SearchableSnapshotAction.NAME), - SearchableSnapshotAction::parse) + SearchableSnapshotAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RollupILMAction.NAME), RollupILMAction::parse) )); return new NamedXContentRegistry(entries); } @@ -205,6 +207,8 @@ private static Function getNameToActionFunction() { return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); case MigrateAction.NAME: return new MigrateAction(false); + case RollupILMAction.NAME: + return RollupILMActionTests.randomInstance(); default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java new file mode 100644 index 0000000000000..e083c11b74e9d --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.xpack.core.ilm.Step.StepKey; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfigTests; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class RollupILMActionTests extends AbstractActionTestCase { + + static RollupILMAction randomInstance() { + return new RollupILMAction(RollupActionConfigTests.randomConfig(random()), randomBoolean(), + randomBoolean() ? randomAlphaOfLength(5) : null); + } + + @Override + protected RollupILMAction doParseInstance(XContentParser parser) { + return RollupILMAction.parse(parser); + } + + @Override + protected RollupILMAction createTestInstance() { + return randomInstance(); + } + + @Override + protected Reader instanceReader() { + return RollupILMAction::new; + } + + @Override + public boolean isSafeAction() { + return false; + } + + @Override + public void testToSteps() { + RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig(random()), false, null); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + assertNotNull(steps); + assertEquals(3, steps.size()); + assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); + assertThat(steps.get(0).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(1).getKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(1).getNextStepKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(2).getKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(2).getNextStepKey(), equalTo(nextStepKey)); + } + + public void testToStepsWithDelete() { + RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig(random()), true, null); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + assertNotNull(steps); + assertEquals(5, steps.size()); + assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); + assertThat(steps.get(0).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(1).getKey().getName(), equalTo(ReadOnlyStep.NAME)); + assertThat(steps.get(1).getNextStepKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(2).getKey().getName(), equalTo(RollupStep.NAME)); + assertThat(steps.get(2).getNextStepKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); + assertThat(steps.get(3).getKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); + assertThat(steps.get(3).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); + assertThat(steps.get(4).getKey().getName(), equalTo(DeleteStep.NAME)); + assertThat(steps.get(4).getNextStepKey(), equalTo(nextStepKey)); + } + + public void testEqualsAndHashCode() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copy, this::notCopy); + } + + RollupILMAction copy(RollupILMAction rollupILMAction) { + return new RollupILMAction(rollupILMAction.config(), rollupILMAction.deleteOriginalIndex(), rollupILMAction.rollupPolicy()); + } + + RollupILMAction notCopy(RollupILMAction rollupILMAction) { + RollupActionConfig newConfig = rollupILMAction.config(); + boolean newDeleteOriginalIndex = rollupILMAction.deleteOriginalIndex(); + String newRollupPolicy = rollupILMAction.rollupPolicy(); + switch (randomIntBetween(0, 2)) { + case 0: + newConfig = new RollupActionConfig(rollupILMAction.config().getGroupConfig(), + rollupILMAction.config().getMetricsConfig(), rollupILMAction.config().getTimeout(), + rollupILMAction.config().getRollupIndex() + "not"); + break; + case 1: + newDeleteOriginalIndex = !newDeleteOriginalIndex; + break; + case 2: + newRollupPolicy = randomAlphaOfLength(3); + break; + default: + throw new IllegalStateException("unreachable branch"); + } + return new RollupILMAction(newConfig, newDeleteOriginalIndex, newRollupPolicy); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java new file mode 100644 index 0000000000000..ab8e5b48c4317 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.ilm.Step.StepKey; +import org.elasticsearch.xpack.core.rollup.v2.RollupAction; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfigTests; +import org.mockito.Mockito; + +import java.util.List; + +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.hamcrest.Matchers.equalTo; + +public class RollupStepTests extends AbstractStepTestCase { + + @Override + public RollupStep createRandomInstance() { + if (randomBoolean()) { + return createRandomInstanceWithPolicy(); + } + return createRandomInstanceWithoutPolicy(); + } + + private RollupStep createRandomInstanceWithoutPolicy() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + RollupActionConfig config = RollupActionConfigTests.randomConfig(random()); + return new RollupStep(stepKey, nextStepKey, client, config, null); + } + + private RollupStep createRandomInstanceWithPolicy() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + RollupActionConfig config = RollupActionConfigTests.randomConfig(random()); + return new RollupStep(stepKey, nextStepKey, client, config, randomAlphaOfLength(5)); + } + + @Override + public RollupStep mutateInstance(RollupStep instance) { + StepKey key = instance.getKey(); + StepKey nextKey = instance.getNextStepKey(); + String rollupPolicy = instance.getRollupPolicy(); + + switch (between(0, 2)) { + case 0: + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 1: + nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 2: + rollupPolicy = randomAlphaOfLength(3); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + + return new RollupStep(key, nextKey, instance.getClient(), instance.getConfig(), rollupPolicy); + } + + @Override + public RollupStep copyInstance(RollupStep instance) { + return new RollupStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), + instance.getConfig(), instance.getRollupPolicy()); + } + + private IndexMetadata getIndexMetadata(String index) { + return IndexMetadata.builder(index) + .settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + } + + private static void assertRollupActionRequest(RollupAction.Request request, String sourceIndex) { + assertNotNull(request); + assertThat(request.getSourceIndex(), equalTo(sourceIndex)); + } + + public void testPerformAction() { + String index = randomAlphaOfLength(5); + IndexMetadata indexMetadata = getIndexMetadata(index); + + RollupStep step = createRandomInstanceWithoutPolicy(); + + mockClientRollupCall(index); + + SetOnce actionCompleted = new SetOnce<>(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata( + Metadata.builder() + .put(indexMetadata, true) + ) + .build(); + step.performAction(indexMetadata, clusterState, null, new AsyncActionStep.Listener() { + + @Override + public void onResponse(boolean complete) { + actionCompleted.set(complete); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }); + + assertEquals(true, actionCompleted.get()); + } + + public void testPerformActionWithPolicy() { + String index = randomAlphaOfLength(5); + IndexMetadata indexMetadata = getIndexMetadata(index); + IndexMetadata rollupIndexMetadata = getIndexMetadata(index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX); + + RollupStep step = createRandomInstanceWithPolicy(); + + Mockito.doAnswer(invocation -> { + UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0]; + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + Settings expectedSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, step.getRollupPolicy()).build(); + assertThat(request.settings(), equalTo(expectedSettings)); + assertThat(request.indices(), equalTo(new String[] {rollupIndexMetadata.getIndex().getName()})); + listener.onResponse(AcknowledgedResponse.TRUE); + return null; + }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); + + Mockito.doAnswer(invocation -> { + RollupAction.Request request = (RollupAction.Request) invocation.getArguments()[1]; + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + assertRollupActionRequest(request, index); + listener.onResponse(new RollupAction.Response(true)); + return null; + }).when(client).execute(Mockito.any(), Mockito.any(), Mockito.any()); + + SetOnce actionCompleted = new SetOnce<>(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata( + Metadata.builder() + .put(indexMetadata, true) + ) + .build(); + step.performAction(indexMetadata, clusterState, null, new AsyncActionStep.Listener() { + + @Override + public void onResponse(boolean complete) { + actionCompleted.set(complete); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }); + + assertEquals(true, actionCompleted.get()); + + Mockito.verify(client, Mockito.times(1)).execute(Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(client, Mockito.times(1)).admin(); + Mockito.verify(adminClient, Mockito.only()).indices(); + Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any()); + } + + public void testPerformActionOnDataStream() { + String dataStreamName = "test-datastream"; + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); + IndexMetadata indexMetadata = IndexMetadata.builder(backingIndexName) + .settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + + RollupStep step = createRandomInstanceWithoutPolicy(); + + mockClientRollupCall(backingIndexName); + + SetOnce actionCompleted = new SetOnce<>(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata( + Metadata.builder() + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + List.of(indexMetadata.getIndex()))) + .put(indexMetadata, true) + ) + .build(); + step.performAction(indexMetadata, clusterState, null, new AsyncActionStep.Listener() { + + @Override + public void onResponse(boolean complete) { + actionCompleted.set(complete); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }); + + assertEquals(true, actionCompleted.get()); + } + + private void mockClientRollupCall(String sourceIndex) { + Mockito.doAnswer(invocation -> { + RollupAction.Request request = (RollupAction.Request) invocation.getArguments()[1]; + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + assertRollupActionRequest(request, sourceIndex); + listener.onResponse(new RollupAction.Response(true)); + return null; + }).when(client).execute(Mockito.any(), Mockito.any(), Mockito.any()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index df0017eadd492..910a0c571790c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -7,7 +7,11 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig; +import org.elasticsearch.xpack.core.rollup.job.GroupConfig; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; import java.util.Arrays; import java.util.Collection; @@ -56,6 +60,9 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { // keeping the migrate action disabled as otherwise it could conflict with the allocate action if both are randomly selected for the // same phase private static final MigrateAction TEST_MIGRATE_ACTION = new MigrateAction(false); + private static final RollupILMAction TEST_ROLLUP_ACTION =new RollupILMAction(new RollupActionConfig( + new GroupConfig(new DateHistogramGroupConfig.FixedInterval("field", DateHistogramInterval.DAY)), + Collections.emptyList(), null, "rollup"), false, null); public void testValidatePhases() { boolean invalid = randomBoolean(); @@ -216,6 +223,7 @@ public void testGetOrderedPhases() { phaseMap.put(phaseName, new Phase(phaseName, TimeValue.ZERO, Collections.emptyMap())); } + assertTrue(isSorted(TimeseriesLifecycleType.INSTANCE.getOrderedPhases(phaseMap), Phase::getName, VALID_PHASES)); } @@ -620,6 +628,8 @@ private ConcurrentMap convertActionNamesToActions(Strin return new UnfollowAction(); case MigrateAction.NAME: return new MigrateAction(true); + case RollupILMAction.NAME: + return TEST_ROLLUP_ACTION; } return new DeleteAction(); }).collect(Collectors.toConcurrentMap(LifecycleAction::getWriteableName, Function.identity())); @@ -693,6 +703,8 @@ private LifecycleAction getTestAction(String actionName) { return TEST_SEARCHABLE_SNAPSHOT_ACTION; case MigrateAction.NAME: return TEST_MIGRATE_ACTION; + case RollupILMAction.NAME: + return TEST_ROLLUP_ACTION; default: throw new IllegalArgumentException("unsupported timeseries phase action [" + actionName + "]"); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java index 54a34217a8e4d..71f4fb524fbf2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.core.ilm.MigrateAction; import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.RollupILMAction; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; @@ -77,7 +78,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, MigrateAction.NAME, MigrateAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, RollupILMAction.NAME, RollupILMAction::new) )); } @@ -100,7 +102,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SearchableSnapshotAction.NAME), SearchableSnapshotAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MigrateAction.NAME), MigrateAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RollupILMAction.NAME), RollupILMAction::parse) )); return new NamedXContentRegistry(entries); } diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index 71ffb394bb627..0dda1bedd9892 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -15,6 +15,7 @@ File repoDir = file("$buildDir/testclusters/repo") javaRestTest { /* To support taking index snapshots, we have to set path.repo setting */ systemProperty 'tests.path.repo', repoDir + systemProperty 'es.rollup_v2_feature_flag_enabled', 'true' } testClusters.all { @@ -29,4 +30,5 @@ testClusters.all { setting 'indices.lifecycle.poll_interval', '1000ms' setting 'logger.org.elasticsearch.xpack.core.ilm', 'TRACE' setting 'logger.org.elasticsearch.xpack.ilm', 'TRACE' + systemProperty 'es.rollup_v2_feature_flag_enabled', 'true' } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index df6ecbe3b1a22..cd6cd21b22299 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; @@ -40,6 +41,8 @@ import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.RollupILMAction; +import org.elasticsearch.xpack.core.ilm.RollupStep; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.SetSingleNodeAllocateStep; @@ -49,12 +52,17 @@ import org.elasticsearch.xpack.core.ilm.WaitForActiveShardsStep; import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep; import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; +import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig; +import org.elasticsearch.xpack.core.rollup.job.GroupConfig; +import org.elasticsearch.xpack.core.rollup.job.MetricConfig; +import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; import org.hamcrest.Matchers; import org.junit.Before; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -1469,6 +1477,60 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { }, 30, TimeUnit.SECONDS)); } + public void testRollupIndex() throws Exception { + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); + String rollupIndex = index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX; + index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); + RollupActionConfig rollupConfig = new RollupActionConfig( + new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), + null, randomAlphaOfLength(5)); + + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, false, null)); + updatePolicy(index, policy); + + assertBusy(() -> assertTrue(indexExists(rollupIndex))); + assertBusy(() -> assertFalse(getOnlyIndexSettings(client(), rollupIndex).containsKey(LifecycleSettings.LIFECYCLE_NAME))); + assertBusy(() -> assertTrue(indexExists(index))); + } + + public void testRollupIndexAndDeleteOriginal() throws Exception { + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); + String rollupIndex = index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX; + index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); + RollupActionConfig rollupConfig = new RollupActionConfig( + new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), + null, randomAlphaOfLength(5)); + + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, true, null)); + updatePolicy(index, policy); + + assertBusy(() -> assertTrue(indexExists(rollupIndex))); + assertBusy(() -> assertFalse(getOnlyIndexSettings(client(), rollupIndex).containsKey(LifecycleSettings.LIFECYCLE_NAME))); + assertBusy(() -> assertFalse(indexExists(index))); + } + + public void testRollupIndexAndSetNewRollupPolicy() throws Exception { + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); + String rollupIndex = index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX; + index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); + RollupActionConfig rollupConfig = new RollupActionConfig( + new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), + null, randomAlphaOfLength(5)); + + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, false, policy)); + updatePolicy(index, policy); + + assertBusy(() -> assertTrue(indexExists(rollupIndex))); + assertBusy(() -> assertThat(getOnlyIndexSettings(client(), rollupIndex).get(LifecycleSettings.LIFECYCLE_NAME), equalTo(policy))); + assertBusy(() -> assertTrue(indexExists(index))); + } + // This method should be called inside an assertBusy, it has no retry logic of its own private void assertHistoryIsPresent(String policyName, String indexName, boolean success, String stepName) throws IOException { assertHistoryIsPresent(policyName, indexName, success, null, null, stepName); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 39f4fe65c4382..95d143f423b3d 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -33,6 +33,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rollup.RollupV2; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; @@ -49,6 +50,7 @@ import org.elasticsearch.xpack.core.ilm.MigrateAction; import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.RollupILMAction; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; @@ -210,7 +212,7 @@ public List getNamedWriteables() { @Override public List getNamedXContent() { - return Arrays.asList( + List entries = new ArrayList<>(Arrays.asList( // Custom Metadata new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexLifecycleMetadata.TYPE), parser -> IndexLifecycleMetadata.PARSER.parse(parser, null)), @@ -234,8 +236,14 @@ public List getNa new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SearchableSnapshotAction.NAME), SearchableSnapshotAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MigrateAction.NAME), - MigrateAction::parse) - ); + MigrateAction::parse))); + + if (RollupV2.isEnabled()) { + entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, + new ParseField(RollupILMAction.NAME), RollupILMAction::parse)); + } + + return entries; } @Override diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java index 22ddcd1fdd477..bc0e39eb8c907 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.RollupILMAction; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; @@ -94,7 +95,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, MigrateAction.NAME, MigrateAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, RollupILMAction.NAME, RollupILMAction::new) )); } @@ -117,7 +119,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MigrateAction.NAME), MigrateAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SearchableSnapshotAction.NAME), - SearchableSnapshotAction::parse) + SearchableSnapshotAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RollupILMAction.NAME), RollupILMAction::parse) )); return new NamedXContentRegistry(entries); } From 434f49cfdebf188fa2ccc21464baecf252965a43 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 9 Dec 2020 10:19:38 -0800 Subject: [PATCH 02/22] respond to review --- .../org/elasticsearch/xpack/core/ilm/RollupILMAction.java | 2 +- .../elasticsearch/xpack/core/ilm/RollupILMActionTests.java | 4 ++-- .../main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index c06b7314c2ddb..50c8e124da505 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -72,7 +72,7 @@ RollupActionConfig config() { return config; } - boolean deleteOriginalIndex() { + boolean shouldDeleteOriginalIndex() { return deleteOriginalIndex; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index e083c11b74e9d..7f9a5d8bd936e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -85,12 +85,12 @@ public void testEqualsAndHashCode() { } RollupILMAction copy(RollupILMAction rollupILMAction) { - return new RollupILMAction(rollupILMAction.config(), rollupILMAction.deleteOriginalIndex(), rollupILMAction.rollupPolicy()); + return new RollupILMAction(rollupILMAction.config(), rollupILMAction.shouldDeleteOriginalIndex(), rollupILMAction.rollupPolicy()); } RollupILMAction notCopy(RollupILMAction rollupILMAction) { RollupActionConfig newConfig = rollupILMAction.config(); - boolean newDeleteOriginalIndex = rollupILMAction.deleteOriginalIndex(); + boolean newDeleteOriginalIndex = rollupILMAction.shouldDeleteOriginalIndex(); String newRollupPolicy = rollupILMAction.rollupPolicy(); switch (randomIntBetween(0, 2)) { case 0: diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 95d143f423b3d..dbe91959ef2b2 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -211,8 +211,8 @@ public List getNamedWriteables() { } @Override - public List getNamedXContent() { - List entries = new ArrayList<>(Arrays.asList( + public List getNamedXContent() { + List entries = new ArrayList<>(Arrays.asList( // Custom Metadata new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexLifecycleMetadata.TYPE), parser -> IndexLifecycleMetadata.PARSER.parse(parser, null)), From 638cbe7762635caef46320750168dba7e5108f15 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 14 Dec 2020 10:48:22 -0800 Subject: [PATCH 03/22] fix for latest master --- .../org/elasticsearch/xpack/core/ilm/RollupStep.java | 4 +--- .../xpack/core/ilm/RollupILMActionTests.java | 3 +-- .../xpack/core/ilm/TimeseriesLifecycleTypeTests.java | 2 +- .../xpack/ilm/TimeSeriesLifecycleActionsIT.java | 9 +++------ 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index b6568ae45fcef..2905f142781a7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -42,9 +42,7 @@ public boolean isRetryable() { public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) { String originalIndex = indexMetadata.getIndex().getName(); String rollupIndex = originalIndex + ROLLUP_INDEX_NAME_POSTFIX; - // TODO(talevy): change config to be immutable - config.setRollupIndex(rollupIndex); - RollupAction.Request request = new RollupAction.Request(originalIndex, config); + RollupAction.Request request = new RollupAction.Request(originalIndex, rollupIndex, config); if (rollupPolicy == null) { getClient().execute(RollupAction.INSTANCE, request, ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 7f9a5d8bd936e..8dc4ad8c7f103 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -95,8 +95,7 @@ RollupILMAction notCopy(RollupILMAction rollupILMAction) { switch (randomIntBetween(0, 2)) { case 0: newConfig = new RollupActionConfig(rollupILMAction.config().getGroupConfig(), - rollupILMAction.config().getMetricsConfig(), rollupILMAction.config().getTimeout(), - rollupILMAction.config().getRollupIndex() + "not"); + rollupILMAction.config().getMetricsConfig(), rollupILMAction.config().getTimeout()); break; case 1: newDeleteOriginalIndex = !newDeleteOriginalIndex; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 910a0c571790c..379e69eb6689a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -62,7 +62,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { private static final MigrateAction TEST_MIGRATE_ACTION = new MigrateAction(false); private static final RollupILMAction TEST_ROLLUP_ACTION =new RollupILMAction(new RollupActionConfig( new GroupConfig(new DateHistogramGroupConfig.FixedInterval("field", DateHistogramInterval.DAY)), - Collections.emptyList(), null, "rollup"), false, null); + Collections.emptyList(), null), false, null); public void testValidatePhases() { boolean invalid = randomBoolean(); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index cd6cd21b22299..8a92a10c34835 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1484,8 +1484,7 @@ public void testRollupIndex() throws Exception { index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), - null, randomAlphaOfLength(5)); + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, false, null)); updatePolicy(index, policy); @@ -1502,8 +1501,7 @@ public void testRollupIndexAndDeleteOriginal() throws Exception { index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), - null, randomAlphaOfLength(5)); + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, true, null)); updatePolicy(index, policy); @@ -1520,8 +1518,7 @@ public void testRollupIndexAndSetNewRollupPolicy() throws Exception { index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), - null, randomAlphaOfLength(5)); + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, false, policy)); updatePolicy(index, policy); From e46d4ec99c6d411670a2772060517ee5dc5b8f2c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 21 Dec 2020 14:09:12 -0800 Subject: [PATCH 04/22] remove delete option and add to hot before snapshotting --- .../xpack/core/ilm/RollupILMAction.java | 36 +++------------ .../core/ilm/TimeseriesLifecycleType.java | 8 ++-- .../xpack/core/ilm/RollupILMActionTests.java | 44 +++++-------------- .../ilm/TimeseriesLifecycleTypeTests.java | 9 ++-- .../ilm/TimeSeriesLifecycleActionsIT.java | 6 +-- 5 files changed, 31 insertions(+), 72 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 50c8e124da505..10140d3a0f92a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -31,21 +31,18 @@ public class RollupILMAction implements LifecycleAction { public static final String NAME = "rollup"; private static final ParseField CONFIG_FIELD = new ParseField("config"); - private static final ParseField DELETE_FIELD = new ParseField("delete_original"); private static final ParseField POLICY_FIELD = new ParseField("rollup_policy"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, - a -> new RollupILMAction((RollupActionConfig) a[0], (boolean) a[1], (String) a[2])); + a -> new RollupILMAction((RollupActionConfig) a[0], (String) a[1])); private final RollupActionConfig config; - private final boolean deleteOriginalIndex; private final String rollupPolicy; static { PARSER.declareField(ConstructingObjectParser.constructorArg(), (p, c) -> RollupActionConfig.fromXContent(p), CONFIG_FIELD, ObjectParser.ValueType.OBJECT); - PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DELETE_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), POLICY_FIELD); } @@ -53,14 +50,13 @@ public static RollupILMAction parse(XContentParser parser) { return PARSER.apply(parser, null); } - public RollupILMAction(RollupActionConfig config, boolean deleteOriginalIndex, @Nullable String rollupPolicy) { + public RollupILMAction(RollupActionConfig config, @Nullable String rollupPolicy) { this.config = config; - this.deleteOriginalIndex = deleteOriginalIndex; this.rollupPolicy = rollupPolicy; } public RollupILMAction(StreamInput in) throws IOException { - this(new RollupActionConfig(in), in.readBoolean(), in.readOptionalString()); + this(new RollupActionConfig(in), in.readOptionalString()); } @Override @@ -72,10 +68,6 @@ RollupActionConfig config() { return config; } - boolean shouldDeleteOriginalIndex() { - return deleteOriginalIndex; - } - String rollupPolicy() { return rollupPolicy; } @@ -84,7 +76,6 @@ String rollupPolicy() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(CONFIG_FIELD.getPreferredName(), config); - builder.field(DELETE_FIELD.getPreferredName(), deleteOriginalIndex); if (rollupPolicy != null) { builder.field(POLICY_FIELD.getPreferredName(), rollupPolicy); } @@ -95,7 +86,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { config.writeTo(out); - out.writeBoolean(deleteOriginalIndex); out.writeOptionalString(rollupPolicy); } @@ -109,24 +99,13 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); StepKey rollupKey = new StepKey(phase, NAME, NAME); - StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); - StepKey deleteStepKey = new StepKey(phase, NAME, DeleteStep.NAME); - Settings readOnlySettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build(); - CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, rollupKey, client, readOnlySettings); - WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, deleteStepKey, client); - - if (deleteOriginalIndex) { - RollupStep rollupStep = new RollupStep(rollupKey, waitForNoFollowerStepKey, client, config, rollupPolicy); - DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client); - return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep, waitForNoFollowersStep, deleteStep); - } else { - RollupStep rollupStep = new RollupStep(rollupKey, nextStepKey, client, config, rollupPolicy); - return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); - } + + RollupStep rollupStep = new RollupStep(rollupKey, nextStepKey, client, config, rollupPolicy); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); } @Override @@ -137,13 +116,12 @@ public boolean equals(Object o) { RollupILMAction that = (RollupILMAction) o; return Objects.equals(this.config, that.config) - && Objects.equals(this.deleteOriginalIndex, that.deleteOriginalIndex) && Objects.equals(this.rollupPolicy, that.rollupPolicy); } @Override public int hashCode() { - return Objects.hash(config, deleteOriginalIndex, rollupPolicy); + return Objects.hash(config, rollupPolicy); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index 12dad017a17c1..ad110f1835cde 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -42,7 +42,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; static final List VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, DELETE_PHASE); static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, - ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); + ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS; @@ -54,15 +54,15 @@ public class TimeseriesLifecycleType implements LifecycleType { private static final Map> ALLOWED_ACTIONS; static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Sets.newHashSet(ReadOnlyAction.NAME, ShrinkAction.NAME, - ForceMergeAction.NAME, SearchableSnapshotAction.NAME); + ForceMergeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); // a set of actions that cannot be defined (executed) after the managed index has been mounted as searchable snapshot static final Set ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT = Sets.newHashSet(ShrinkAction.NAME, ForceMergeAction.NAME, - FreezeAction.NAME, SearchableSnapshotAction.NAME); + FreezeAction.NAME, SearchableSnapshotAction.NAME, RollupILMAction.NAME); static { if (RollupV2.isEnabled()) { ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, - MigrateAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME, RollupILMAction.NAME); + MigrateAction.NAME, FreezeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); } else { ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, MigrateAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 8dc4ad8c7f103..2e97a83a4c10b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -9,9 +9,11 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.xpack.core.ilm.Step.StepKey; +import org.elasticsearch.xpack.core.rollup.job.MetricConfig; import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfigTests; +import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -19,8 +21,8 @@ public class RollupILMActionTests extends AbstractActionTestCase { static RollupILMAction randomInstance() { - return new RollupILMAction(RollupActionConfigTests.randomConfig(random()), randomBoolean(), - randomBoolean() ? randomAlphaOfLength(5) : null); + return new RollupILMAction(RollupActionConfigTests.randomConfig(random()), + randomBoolean() ? randomAlphaOfLength(5) : null); } @Override @@ -45,7 +47,7 @@ public boolean isSafeAction() { @Override public void testToSteps() { - RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig(random()), false, null); + RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig(random()), null); String phase = randomAlphaOfLengthBetween(1, 10); StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)); @@ -60,52 +62,30 @@ public void testToSteps() { assertThat(steps.get(2).getNextStepKey(), equalTo(nextStepKey)); } - public void testToStepsWithDelete() { - RollupILMAction action = new RollupILMAction(RollupActionConfigTests.randomConfig(random()), true, null); - String phase = randomAlphaOfLengthBetween(1, 10); - StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), - randomAlphaOfLengthBetween(1, 10)); - List steps = action.toSteps(null, phase, nextStepKey); - assertNotNull(steps); - assertEquals(5, steps.size()); - assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME)); - assertThat(steps.get(0).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME)); - assertThat(steps.get(1).getKey().getName(), equalTo(ReadOnlyStep.NAME)); - assertThat(steps.get(1).getNextStepKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(2).getKey().getName(), equalTo(RollupStep.NAME)); - assertThat(steps.get(2).getNextStepKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); - assertThat(steps.get(3).getKey().getName(), equalTo(WaitForNoFollowersStep.NAME)); - assertThat(steps.get(3).getNextStepKey().getName(), equalTo(DeleteStep.NAME)); - assertThat(steps.get(4).getKey().getName(), equalTo(DeleteStep.NAME)); - assertThat(steps.get(4).getNextStepKey(), equalTo(nextStepKey)); - } - public void testEqualsAndHashCode() { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copy, this::notCopy); } RollupILMAction copy(RollupILMAction rollupILMAction) { - return new RollupILMAction(rollupILMAction.config(), rollupILMAction.shouldDeleteOriginalIndex(), rollupILMAction.rollupPolicy()); + return new RollupILMAction(rollupILMAction.config(), rollupILMAction.rollupPolicy()); } RollupILMAction notCopy(RollupILMAction rollupILMAction) { RollupActionConfig newConfig = rollupILMAction.config(); - boolean newDeleteOriginalIndex = rollupILMAction.shouldDeleteOriginalIndex(); String newRollupPolicy = rollupILMAction.rollupPolicy(); - switch (randomIntBetween(0, 2)) { + switch (randomIntBetween(0, 1)) { case 0: - newConfig = new RollupActionConfig(rollupILMAction.config().getGroupConfig(), - rollupILMAction.config().getMetricsConfig(), rollupILMAction.config().getTimeout()); + List metricConfigs = rollupILMAction.config().getMetricsConfig(); + metricConfigs.add(new MetricConfig(randomAlphaOfLength(4), Collections.singletonList("max"))); + newConfig = new RollupActionConfig(rollupILMAction.config().getGroupConfig(), metricConfigs, + rollupILMAction.config().getTimeout()); break; case 1: - newDeleteOriginalIndex = !newDeleteOriginalIndex; - break; - case 2: newRollupPolicy = randomAlphaOfLength(3); break; default: throw new IllegalStateException("unreachable branch"); } - return new RollupILMAction(newConfig, newDeleteOriginalIndex, newRollupPolicy); + return new RollupILMAction(newConfig, newRollupPolicy); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 379e69eb6689a..8d8b7ebf71699 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -62,7 +62,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { private static final MigrateAction TEST_MIGRATE_ACTION = new MigrateAction(false); private static final RollupILMAction TEST_ROLLUP_ACTION =new RollupILMAction(new RollupActionConfig( new GroupConfig(new DateHistogramGroupConfig.FixedInterval("field", DateHistogramInterval.DAY)), - Collections.emptyList(), null), false, null); + Collections.emptyList(), null), null); public void testValidatePhases() { boolean invalid = randomBoolean(); @@ -201,9 +201,9 @@ public void testValidateConflictingDataMigrationConfigurations() { } public void testActionsThatCannotFollowSearchableSnapshot() { - assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT.size(), is(4)); + assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT.size(), is(5)); assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT, containsInAnyOrder(ShrinkAction.NAME, FreezeAction.NAME, - ForceMergeAction.NAME, SearchableSnapshotAction.NAME)); + ForceMergeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME)); } public void testValidateActionsFollowingSearchableSnapshot() { @@ -213,7 +213,8 @@ public void testValidateActionsFollowingSearchableSnapshot() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.validateActionsFollowingSearchableSnapshot(List.of(hotPhase, warmPhase, coldPhase))); - assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink] actions" + + assertThat(e.getMessage(), is( + "phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink, rollup] actions" + " which are not allowed after a managed index is mounted as a searchable snapshot")); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 8a92a10c34835..3506466e038ef 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1486,7 +1486,7 @@ public void testRollupIndex() throws Exception { new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, false, null)); + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, null)); updatePolicy(index, policy); assertBusy(() -> assertTrue(indexExists(rollupIndex))); @@ -1503,7 +1503,7 @@ public void testRollupIndexAndDeleteOriginal() throws Exception { new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, true, null)); + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, null)); updatePolicy(index, policy); assertBusy(() -> assertTrue(indexExists(rollupIndex))); @@ -1520,7 +1520,7 @@ public void testRollupIndexAndSetNewRollupPolicy() throws Exception { new GroupConfig(new DateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, false, policy)); + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, policy)); updatePolicy(index, policy); assertBusy(() -> assertTrue(indexExists(rollupIndex))); From 3e38fd65495c67ee4e88619bb20467de327c3745 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Jan 2021 10:12:32 -0800 Subject: [PATCH 05/22] enable rollup_v2 flag for ilm tests --- x-pack/plugin/ilm/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle index 402acc53eef52..3cc544b807377 100644 --- a/x-pack/plugin/ilm/build.gradle +++ b/x-pack/plugin/ilm/build.gradle @@ -18,3 +18,7 @@ dependencies { } addQaCheckDependencies() + +test { + systemProperty 'es.rollup_v2_feature_flag_enabled', 'true' +} From 6f7f6f3239da37d42903707bf2d6014557dcc19d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Jan 2021 11:40:26 -0800 Subject: [PATCH 06/22] put Rollup behind feature flag in ilm hot phase actions --- .../xpack/core/ilm/TimeseriesLifecycleType.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index ad110f1835cde..a4ba3c01907e6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -41,13 +41,12 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String COLD_PHASE = "cold"; static final String DELETE_PHASE = "delete"; static final List VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, DELETE_PHASE); - static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, - ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); + static final List ORDERED_VALID_HOT_ACTIONS; static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS; static final List ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(WaitForSnapshotAction.NAME, DeleteAction.NAME); - static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); + static final Set VALID_HOT_ACTIONS; static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS); static final Set VALID_COLD_ACTIONS; static final Set VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS); @@ -61,12 +60,17 @@ public class TimeseriesLifecycleType implements LifecycleType { static { if (RollupV2.isEnabled()) { + ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, + ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, MigrateAction.NAME, FreezeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); } else { + ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, + ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, MigrateAction.NAME, FreezeAction.NAME, SearchableSnapshotAction.NAME); } + VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS); ALLOWED_ACTIONS = new HashMap<>(); ALLOWED_ACTIONS.put(HOT_PHASE, VALID_HOT_ACTIONS); From ff38007898682c672b122f03fd00be85467fa435 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Jan 2021 12:08:18 -0800 Subject: [PATCH 07/22] fix rollup ilm action test --- .../org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 2e97a83a4c10b..59bab7062181e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.v2.RollupActionConfigTests; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -76,7 +77,7 @@ RollupILMAction notCopy(RollupILMAction rollupILMAction) { switch (randomIntBetween(0, 1)) { case 0: List metricConfigs = rollupILMAction.config().getMetricsConfig(); - metricConfigs.add(new MetricConfig(randomAlphaOfLength(4), Collections.singletonList("max"))); + metricConfigs.add(new MetricConfig(randomAlphaOfLength(4), new ArrayList<>(Collections.singletonList("max")))); newConfig = new RollupActionConfig(rollupILMAction.config().getGroupConfig(), metricConfigs, rollupILMAction.config().getTimeout()); break; From a5b1bb94e1ee359a092e5d88410e8f7a1d33074e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Jan 2021 16:08:56 -0800 Subject: [PATCH 08/22] really fix it --- .../elasticsearch/xpack/core/ilm/RollupILMActionTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java index 59bab7062181e..23ca4f52bd8f0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java @@ -76,8 +76,8 @@ RollupILMAction notCopy(RollupILMAction rollupILMAction) { String newRollupPolicy = rollupILMAction.rollupPolicy(); switch (randomIntBetween(0, 1)) { case 0: - List metricConfigs = rollupILMAction.config().getMetricsConfig(); - metricConfigs.add(new MetricConfig(randomAlphaOfLength(4), new ArrayList<>(Collections.singletonList("max")))); + List metricConfigs = new ArrayList<>(rollupILMAction.config().getMetricsConfig()); + metricConfigs.add(new MetricConfig(randomAlphaOfLength(4), Collections.singletonList("max"))); newConfig = new RollupActionConfig(rollupILMAction.config().getGroupConfig(), metricConfigs, rollupILMAction.config().getTimeout()); break; From d954f55e98e3861a22b2ad6eb17901f970e56c9b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 12 Jan 2021 15:29:04 -0800 Subject: [PATCH 09/22] update rollup ilm to more steps and retryable --- .../xpack/core/ilm/RollupILMAction.java | 21 +++-- .../xpack/core/ilm/RollupStep.java | 36 ++------ .../xpack/core/ilm/RollupStepTests.java | 89 ++----------------- .../ilm/TimeSeriesLifecycleActionsIT.java | 6 +- 4 files changed, 31 insertions(+), 121 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index 55ea444210185..babaea30422cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -97,15 +96,23 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); - StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); + StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyStep.NAME); StepKey rollupKey = new StepKey(phase, NAME, NAME); - Settings readOnlySettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build(); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); - UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, rollupKey, client, readOnlySettings); - - RollupStep rollupStep = new RollupStep(rollupKey, nextStepKey, client, config, rollupPolicy); - return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); + ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, rollupKey, client); + Settings setPolicySettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, rollupPolicy).build(); + StepKey updateLifecyclePolicyKey = new StepKey(phase, NAME, UpdateSettingsStep.NAME); + UpdateSettingsStep updateLifecycleSettingsStep = new UpdateSettingsStep(updateLifecyclePolicyKey, nextStepKey, + client, setPolicySettings); + final Step rollupStep; + if (rollupPolicy != null) { + rollupStep = new RollupStep(rollupKey, updateLifecyclePolicyKey, client, config); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep, updateLifecycleSettingsStep); + } else { + rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); + } } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index af969e61efbf5..d1c962b0e2e83 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -6,12 +6,10 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.action.RollupAction; @@ -22,53 +20,36 @@ */ public class RollupStep extends AsyncActionStep { public static final String NAME = "rollup"; - public static final String ROLLUP_INDEX_NAME_POSTFIX = "-rollup"; + public static final String ROLLUP_INDEX_NAME_PREFIX = "_rollup"; private final RollupActionConfig config; - private final String rollupPolicy; - public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionConfig config, String rollupPolicy) { + public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionConfig config) { super(key, nextStepKey, client); this.config = config; - this.rollupPolicy = rollupPolicy; } @Override public boolean isRetryable() { - return false; + return true; } @Override public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) { String originalIndex = indexMetadata.getIndex().getName(); - String rollupIndex = originalIndex + ROLLUP_INDEX_NAME_POSTFIX; + String rollupIndex = ROLLUP_INDEX_NAME_PREFIX + originalIndex; RollupAction.Request request = new RollupAction.Request(originalIndex, rollupIndex, config); - if (rollupPolicy == null) { - getClient().execute(RollupAction.INSTANCE, request, - ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); - } else { - Settings setPolicySettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, rollupPolicy).build(); - UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(rollupIndex) - .masterNodeTimeout(getMasterTimeout(currentState)).settings(setPolicySettings); - getClient().execute(RollupAction.INSTANCE, request, - ActionListener.wrap(rollupResponse -> { - getClient().admin().indices().updateSettings(updateSettingsRequest, - ActionListener.wrap(settingsResponse -> listener.onResponse(true), listener::onFailure)); - }, listener::onFailure)); - } + getClient().execute(RollupAction.INSTANCE, request, + ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); } public RollupActionConfig getConfig() { return config; } - public String getRollupPolicy() { - return rollupPolicy; - } - @Override public int hashCode() { - return Objects.hash(super.hashCode(), config, rollupPolicy); + return Objects.hash(super.hashCode(), config); } @Override @@ -81,7 +62,6 @@ public boolean equals(Object obj) { } RollupStep other = (RollupStep) obj; return super.equals(obj) - && Objects.equals(config, other.config) - && Objects.equals(rollupPolicy, other.rollupPolicy); + && Objects.equals(config, other.config); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java index db7ea4093295d..4cf89efe1d84a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupStepTests.java @@ -8,14 +8,11 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.RollupActionConfigTests; @@ -31,53 +28,34 @@ public class RollupStepTests extends AbstractStepTestCase { @Override public RollupStep createRandomInstance() { - if (randomBoolean()) { - return createRandomInstanceWithPolicy(); - } - return createRandomInstanceWithoutPolicy(); - } - - private RollupStep createRandomInstanceWithoutPolicy() { StepKey stepKey = randomStepKey(); StepKey nextStepKey = randomStepKey(); RollupActionConfig config = RollupActionConfigTests.randomConfig(random()); - return new RollupStep(stepKey, nextStepKey, client, config, null); - } - - private RollupStep createRandomInstanceWithPolicy() { - StepKey stepKey = randomStepKey(); - StepKey nextStepKey = randomStepKey(); - RollupActionConfig config = RollupActionConfigTests.randomConfig(random()); - return new RollupStep(stepKey, nextStepKey, client, config, randomAlphaOfLength(5)); + return new RollupStep(stepKey, nextStepKey, client, config); } @Override public RollupStep mutateInstance(RollupStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); - String rollupPolicy = instance.getRollupPolicy(); - switch (between(0, 2)) { + switch (between(0, 1)) { case 0: key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); break; case 1: nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); break; - case 2: - rollupPolicy = randomAlphaOfLength(3); - break; default: throw new AssertionError("Illegal randomisation branch"); } - return new RollupStep(key, nextKey, instance.getClient(), instance.getConfig(), rollupPolicy); + return new RollupStep(key, nextKey, instance.getClient(), instance.getConfig()); } @Override public RollupStep copyInstance(RollupStep instance) { - return new RollupStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), - instance.getConfig(), instance.getRollupPolicy()); + return new RollupStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getConfig()); } private IndexMetadata getIndexMetadata(String index) { @@ -95,7 +73,7 @@ public void testPerformAction() { String index = randomAlphaOfLength(5); IndexMetadata indexMetadata = getIndexMetadata(index); - RollupStep step = createRandomInstanceWithoutPolicy(); + RollupStep step = createRandomInstance(); mockClientRollupCall(index); @@ -122,61 +100,6 @@ public void onFailure(Exception e) { assertEquals(true, actionCompleted.get()); } - public void testPerformActionWithPolicy() { - String index = randomAlphaOfLength(5); - IndexMetadata indexMetadata = getIndexMetadata(index); - IndexMetadata rollupIndexMetadata = getIndexMetadata(index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX); - - RollupStep step = createRandomInstanceWithPolicy(); - - Mockito.doAnswer(invocation -> { - UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0]; - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - Settings expectedSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, step.getRollupPolicy()).build(); - assertThat(request.settings(), equalTo(expectedSettings)); - assertThat(request.indices(), equalTo(new String[] {rollupIndexMetadata.getIndex().getName()})); - listener.onResponse(AcknowledgedResponse.TRUE); - return null; - }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); - - Mockito.doAnswer(invocation -> { - RollupAction.Request request = (RollupAction.Request) invocation.getArguments()[1]; - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - assertRollupActionRequest(request, index); - listener.onResponse(new RollupAction.Response(true)); - return null; - }).when(client).execute(Mockito.any(), Mockito.any(), Mockito.any()); - - SetOnce actionCompleted = new SetOnce<>(); - ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata( - Metadata.builder() - .put(indexMetadata, true) - ) - .build(); - step.performAction(indexMetadata, clusterState, null, new AsyncActionStep.Listener() { - - @Override - public void onResponse(boolean complete) { - actionCompleted.set(complete); - } - - @Override - public void onFailure(Exception e) { - throw new AssertionError("Unexpected method call", e); - } - }); - - assertEquals(true, actionCompleted.get()); - - Mockito.verify(client, Mockito.times(1)).execute(Mockito.any(), Mockito.any(), Mockito.any()); - Mockito.verify(client, Mockito.times(1)).admin(); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any()); - } - public void testPerformActionOnDataStream() { String dataStreamName = "test-datastream"; String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); @@ -184,7 +107,7 @@ public void testPerformActionOnDataStream() { .settings(settings(Version.CURRENT)) .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); - RollupStep step = createRandomInstanceWithoutPolicy(); + RollupStep step = createRandomInstance(); mockClientRollupCall(backingIndexName); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index cf2d2ba7864af..01794fe97b362 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1480,7 +1480,7 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { public void testRollupIndex() throws Exception { createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); - String rollupIndex = index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX; + String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), @@ -1497,7 +1497,7 @@ public void testRollupIndex() throws Exception { public void testRollupIndexAndDeleteOriginal() throws Exception { createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); - String rollupIndex = index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX; + String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), @@ -1514,7 +1514,7 @@ public void testRollupIndexAndDeleteOriginal() throws Exception { public void testRollupIndexAndSetNewRollupPolicy() throws Exception { createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); - String rollupIndex = index + RollupStep.ROLLUP_INDEX_NAME_POSTFIX; + String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), From 4679677d68b09903d04eaedfe831d09f2c768151 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 14 Jan 2021 16:20:23 -0800 Subject: [PATCH 10/22] fix test bugs --- .../xpack/core/ilm/RollupStep.java | 2 +- .../xpack/ilm/TimeSeriesLifecycleActionsIT.java | 17 ----------------- .../xpack/rollup/v2/RollupV2Indexer.java | 15 ++++++++++----- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index d1c962b0e2e83..09c1493d5e390 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -20,7 +20,7 @@ */ public class RollupStep extends AsyncActionStep { public static final String NAME = "rollup"; - public static final String ROLLUP_INDEX_NAME_PREFIX = "_rollup"; + public static final String ROLLUP_INDEX_NAME_PREFIX = "rollup-"; private final RollupActionConfig config; diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 01794fe97b362..0d82a613e563c 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1494,23 +1494,6 @@ public void testRollupIndex() throws Exception { assertBusy(() -> assertTrue(indexExists(index))); } - public void testRollupIndexAndDeleteOriginal() throws Exception { - createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); - String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; - index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); - RollupActionConfig rollupConfig = new RollupActionConfig( - new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); - - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, null)); - updatePolicy(index, policy); - - assertBusy(() -> assertTrue(indexExists(rollupIndex))); - assertBusy(() -> assertFalse(getOnlyIndexSettings(client(), rollupIndex).containsKey(LifecycleSettings.LIFECYCLE_NAME))); - assertBusy(() -> assertFalse(indexExists(index))); - } - public void testRollupIndexAndSetNewRollupPolicy() throws Exception { createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java index 2ef037e9f6d59..358c205f34d3f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java @@ -209,16 +209,21 @@ protected void onFinish(ActionListener listener) { ActionListener.wrap(r -> { ClientHelper.executeWithHeadersAsync(headers, ClientHelper.ROLLUP_ORIGIN, client, ResizeAction.INSTANCE, resizeRequest, - ActionListener.wrap(rr -> { deleteTmpIndex(listener); }, e -> { deleteTmpIndex(listener); })); - }, e -> { deleteTmpIndex(listener); })); + ActionListener.wrap(rr -> { deleteTmpIndex(listener, null); }, e -> { deleteTmpIndex(listener, e); })); + }, e -> { deleteTmpIndex(listener, null); })); } - private void deleteTmpIndex(ActionListener listener) { + private void deleteTmpIndex(ActionListener listener, Exception exception) { ClientHelper.executeWithHeadersAsync(headers, ClientHelper.ROLLUP_ORIGIN, client, DeleteIndexAction.INSTANCE, new DeleteIndexRequest(tmpIndex), ActionListener.wrap(r -> { - listener.onResponse(null); - completionListener.onResponse(null); + if (exception == null) { + listener.onResponse(null); + completionListener.onResponse(null); + } else { + listener.onFailure(exception); + completionListener.onFailure(exception); + } }, e -> { listener.onFailure(e); completionListener.onFailure(e); From d9523a8dcce52071069df7273b63eedb9f9cac38 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 19 Jan 2021 11:02:41 -0800 Subject: [PATCH 11/22] simplify and fix tests --- .../xpack/core/ilm/RollupILMAction.java | 16 ++++------------ .../xpack/ilm/TimeSeriesLifecycleActionsIT.java | 17 ----------------- .../ilm/actions/SearchableSnapshotActionIT.java | 2 +- 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index babaea30422cb..def43fce9642c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -101,18 +101,10 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, rollupKey, client); - Settings setPolicySettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, rollupPolicy).build(); - StepKey updateLifecyclePolicyKey = new StepKey(phase, NAME, UpdateSettingsStep.NAME); - UpdateSettingsStep updateLifecycleSettingsStep = new UpdateSettingsStep(updateLifecyclePolicyKey, nextStepKey, - client, setPolicySettings); - final Step rollupStep; - if (rollupPolicy != null) { - rollupStep = new RollupStep(rollupKey, updateLifecyclePolicyKey, client, config); - return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep, updateLifecycleSettingsStep); - } else { - rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); - return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); - } + Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); + // TODO(talevy): make RollupAction allow setting a policy on the index to make the transition more atomic + // blocked on #66423. + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); } @Override diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index b3f8791fc48c7..dc27e11b83bea 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1493,23 +1493,6 @@ public void testRollupIndex() throws Exception { assertBusy(() -> assertTrue(indexExists(index))); } - public void testRollupIndexAndSetNewRollupPolicy() throws Exception { - createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); - String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; - index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); - RollupActionConfig rollupConfig = new RollupActionConfig( - new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); - - createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, policy)); - updatePolicy(index, policy); - - assertBusy(() -> assertTrue(indexExists(rollupIndex))); - assertBusy(() -> assertThat(getOnlyIndexSettings(client(), rollupIndex).get(LifecycleSettings.LIFECYCLE_NAME), equalTo(policy))); - assertBusy(() -> assertTrue(indexExists(index))); - } - // This method should be called inside an assertBusy, it has no retry logic of its own private void assertHistoryIsPresent(String policyName, String indexName, boolean success, String stepName) throws IOException { assertHistoryIsPresent(policyName, indexName, success, null, null, stepName); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 750d4118fcbc8..063028869fe54 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -229,7 +229,7 @@ public void testCreateInvalidPolicy() { ) ); - assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink]" + + assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink, rollup]" + " actions which are not allowed after a managed index is mounted as a searchable snapshot")); } From d53427eb63b3a3af9491a4a322f94f2d6e27ffea Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 19 Jan 2021 11:45:45 -0800 Subject: [PATCH 12/22] fix checkstyle --- .../java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index def43fce9642c..e1a9677f19ba4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; From 0f8ebcc6bb8cbed7c32e3708f8a0f0fe0d8cea40 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 20 Jan 2021 10:38:28 -0800 Subject: [PATCH 13/22] re-introduce policy updating by way of a new step --- .../xpack/core/ilm/RollupILMAction.java | 14 +- .../xpack/core/ilm/RollupStep.java | 7 +- .../core/ilm/UpdateRollupIndexPolicyStep.java | 69 +++++++++ .../ilm/UpdateRollupIndexPolicyStepTests.java | 144 ++++++++++++++++++ .../ilm/TimeSeriesLifecycleActionsIT.java | 19 ++- 5 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStepTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java index e1a9677f19ba4..6827094119855 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java @@ -100,10 +100,16 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, rollupKey, client); - Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); - // TODO(talevy): make RollupAction allow setting a policy on the index to make the transition more atomic - // blocked on #66423. - return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); + if (rollupPolicy == null) { + Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep); + } else { + StepKey updateRollupIndexPolicyStepKey = new StepKey(phase, NAME, UpdateRollupIndexPolicyStep.NAME); + Step rollupStep = new RollupStep(rollupKey, updateRollupIndexPolicyStepKey, client, config); + Step updateRollupIndexPolicyStep = new UpdateRollupIndexPolicyStep(updateRollupIndexPolicyStepKey, nextStepKey, + client, rollupPolicy); + return List.of(checkNotWriteIndexStep, readOnlyStep, rollupStep, updateRollupIndexPolicyStep); + } } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index 09c1493d5e390..e75a3a58f3c15 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -29,6 +29,10 @@ public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionC this.config = config; } + public static String getRollupIndexName(String index) { + return ROLLUP_INDEX_NAME_PREFIX + index; + } + @Override public boolean isRetryable() { return true; @@ -37,8 +41,7 @@ public boolean isRetryable() { @Override public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) { String originalIndex = indexMetadata.getIndex().getName(); - String rollupIndex = ROLLUP_INDEX_NAME_PREFIX + originalIndex; - RollupAction.Request request = new RollupAction.Request(originalIndex, rollupIndex, config); + RollupAction.Request request = new RollupAction.Request(originalIndex, getRollupIndexName(originalIndex), config); getClient().execute(RollupAction.INSTANCE, request, ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java new file mode 100644 index 0000000000000..0691b8dbfacfd --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +import java.util.Objects; + +/** + * Updates the lifecycle policy for the rollup index for the original/currently managed index + */ +public class UpdateRollupIndexPolicyStep extends AsyncActionStep { + public static final String NAME = "update-rollup-policy"; + + private final String rollupPolicy; + + public UpdateRollupIndexPolicyStep(StepKey key, StepKey nextStepKey, Client client, String rollupPolicy) { + super(key, nextStepKey, client); + this.rollupPolicy = rollupPolicy; + } + + @Override + public boolean isRetryable() { + return true; + } + + public String getRollupPolicy() { + return rollupPolicy; + } + + @Override + public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) { + String rollupIndex = RollupStep.getRollupIndexName(indexMetadata.getIndex().getName()); + Settings settings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, rollupPolicy).build(); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(rollupIndex) + .masterNodeTimeout(getMasterTimeout(currentState)) + .settings(settings); + getClient().admin().indices().updateSettings(updateSettingsRequest, + ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); + } + + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), rollupPolicy); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + UpdateRollupIndexPolicyStep other = (UpdateRollupIndexPolicyStep) obj; + return super.equals(obj) && + Objects.equals(rollupPolicy, other.rollupPolicy); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStepTests.java new file mode 100644 index 0000000000000..0a644db059c26 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStepTests.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ilm; + + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.ilm.AsyncActionStep.Listener; +import org.elasticsearch.xpack.core.ilm.Step.StepKey; +import org.mockito.Mockito; + +import static org.hamcrest.Matchers.equalTo; + +public class UpdateRollupIndexPolicyStepTests extends AbstractStepMasterTimeoutTestCase { + + @Override + public UpdateRollupIndexPolicyStep createRandomInstance() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + String rollupPolicy = randomAlphaOfLength(10); + + return new UpdateRollupIndexPolicyStep(stepKey, nextStepKey, client, rollupPolicy); + } + + @Override + public UpdateRollupIndexPolicyStep mutateInstance(UpdateRollupIndexPolicyStep instance) { + StepKey key = instance.getKey(); + StepKey nextKey = instance.getNextStepKey(); + String rollupPolicy = instance.getRollupPolicy(); + + switch (between(0, 2)) { + case 0: + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 1: + nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 2: + rollupPolicy = randomAlphaOfLength(5); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + + return new UpdateRollupIndexPolicyStep(key, nextKey, client, rollupPolicy); + } + + @Override + public UpdateRollupIndexPolicyStep copyInstance(UpdateRollupIndexPolicyStep instance) { + return new UpdateRollupIndexPolicyStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), + instance.getRollupPolicy()); + } + + @Override + protected IndexMetadata getIndexMetadata() { + return IndexMetadata.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + } + + public void testPerformAction() { + IndexMetadata indexMetadata = getIndexMetadata(); + String rollupIndex = RollupStep.getRollupIndexName(indexMetadata.getIndex().getName()); + UpdateRollupIndexPolicyStep step = createRandomInstance(); + Settings settings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, step.getRollupPolicy()).build(); + + Mockito.doAnswer(invocation -> { + UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0]; + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + assertThat(request.settings(), equalTo(settings)); + assertThat(request.indices(), equalTo(new String[] { rollupIndex })); + listener.onResponse(AcknowledgedResponse.TRUE); + return null; + }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); + + SetOnce actionCompleted = new SetOnce<>(); + + step.performAction(indexMetadata, emptyClusterState(), null, new Listener() { + + @Override + public void onResponse(boolean complete) { + actionCompleted.set(complete); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }); + + assertEquals(true, actionCompleted.get()); + + Mockito.verify(client, Mockito.only()).admin(); + Mockito.verify(adminClient, Mockito.only()).indices(); + Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any()); + } + + public void testPerformActionFailure() { + IndexMetadata indexMetadata = getIndexMetadata(); + String rollupIndex = RollupStep.getRollupIndexName(indexMetadata.getIndex().getName()); + Exception exception = new RuntimeException(); + UpdateRollupIndexPolicyStep step = createRandomInstance(); + Settings settings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, step.getRollupPolicy()).build(); + + Mockito.doAnswer(invocation -> { + UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0]; + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + assertThat(request.settings(), equalTo(settings)); + assertThat(request.indices(), equalTo(new String[] { rollupIndex })); + listener.onFailure(exception); + return null; + }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); + + SetOnce exceptionThrown = new SetOnce<>(); + step.performAction(indexMetadata, emptyClusterState(), null, new Listener() { + + @Override + public void onResponse(boolean complete) { + throw new AssertionError("Unexpected method call"); + } + + @Override + public void onFailure(Exception e) { + assertSame(exception, e); + exceptionThrown.set(true); + } + }); + + assertEquals(true, exceptionThrown.get()); + + Mockito.verify(client, Mockito.only()).admin(); + Mockito.verify(adminClient, Mockito.only()).indices(); + Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any()); + } +} diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index dc27e11b83bea..bcd276cc3bfa0 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1479,7 +1479,7 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { public void testRollupIndex() throws Exception { createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); - String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; + String rollupIndex = RollupStep.getRollupIndexName(index); index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), @@ -1493,6 +1493,23 @@ public void testRollupIndex() throws Exception { assertBusy(() -> assertTrue(indexExists(index))); } + public void testRollupIndexAndSetNewRollupPolicy() throws Exception { + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); + String rollupIndex = RollupStep.ROLLUP_INDEX_NAME_PREFIX + index; + index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); + RollupActionConfig rollupConfig = new RollupActionConfig( + new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); + + createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, policy)); + updatePolicy(index, policy); + + assertBusy(() -> assertTrue(indexExists(rollupIndex))); + assertBusy(() -> assertThat(getOnlyIndexSettings(client(), rollupIndex).get(LifecycleSettings.LIFECYCLE_NAME), equalTo(policy))); + assertBusy(() -> assertTrue(indexExists(index))); + } + // This method should be called inside an assertBusy, it has no retry logic of its own private void assertHistoryIsPresent(String policyName, String indexName, boolean success, String stepName) throws IOException { assertHistoryIsPresent(policyName, indexName, success, null, null, stepName); From b622f39144026b310e2515cf8ceb1b7ccfda1420 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 21 Jan 2021 14:36:27 -0800 Subject: [PATCH 14/22] fix merge exceptions --- .../elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index bcd276cc3bfa0..83237c50b3b26 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -1483,7 +1483,7 @@ public void testRollupIndex() throws Exception { index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max")))); createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, null)); updatePolicy(index, policy); @@ -1500,7 +1500,7 @@ public void testRollupIndexAndSetNewRollupPolicy() throws Exception { index(client(), index, "_id", "timestamp", "2020-01-01T05:10:00Z", "volume", 11.0); RollupActionConfig rollupConfig = new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("timestamp", DateHistogramInterval.DAY)), - Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max"))), null); + Collections.singletonList(new MetricConfig("volume", Collections.singletonList("max")))); createNewSingletonPolicy(client(), policy, "cold", new RollupILMAction(rollupConfig, policy)); updatePolicy(index, policy); From 968e7aa646641635c9d4e5d1e159f993ad8acdec Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 25 Jan 2021 08:01:34 -0800 Subject: [PATCH 15/22] fix test setup --- .../xpack/core/ilm/TimeseriesLifecycleTypeTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 9531d9903a20a..303377ee6cc8c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.core.rollup.RollupActionConfig; import org.elasticsearch.xpack.core.rollup.RollupActionDateHistogramGroupConfig; import org.elasticsearch.xpack.core.rollup.RollupActionGroupConfig; +import org.elasticsearch.xpack.core.rollup.job.MetricConfig; import java.util.Arrays; import java.util.Collection; @@ -62,7 +63,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { private static final MigrateAction TEST_MIGRATE_ACTION = new MigrateAction(false); private static final RollupILMAction TEST_ROLLUP_ACTION =new RollupILMAction(new RollupActionConfig( new RollupActionGroupConfig(new RollupActionDateHistogramGroupConfig.FixedInterval("field", DateHistogramInterval.DAY)), - Collections.emptyList()), null); + Collections.singletonList(new MetricConfig("field", Collections.singletonList("max")))), null); public void testValidatePhases() { boolean invalid = randomBoolean(); From c9e31d52239ddd5a93de4d0d76b6d711da3673d8 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 26 Jan 2021 14:09:41 -0800 Subject: [PATCH 16/22] docs and cleanup --- .../reference/ilm/actions/ilm-rollup.asciidoc | 56 +++++++++++++++++++ .../xpack/core/ilm/RollupStep.java | 1 + .../core/ilm/UpdateRollupIndexPolicyStep.java | 10 +++- .../xpack/ilm/IndexLifecycleService.java | 4 +- 4 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 docs/reference/ilm/actions/ilm-rollup.asciidoc diff --git a/docs/reference/ilm/actions/ilm-rollup.asciidoc b/docs/reference/ilm/actions/ilm-rollup.asciidoc new file mode 100644 index 0000000000000..17b536da9e15c --- /dev/null +++ b/docs/reference/ilm/actions/ilm-rollup.asciidoc @@ -0,0 +1,56 @@ +[role="xpack"] +[[ilm-rollup]] +=== Rollup + +Phases allowed: hot, cold. + +Aggregates an index's time series data and stores the results in a new read-only +index. For example, you can roll up hourly data into daily or weekly summaries. + +For more information about Rollup, check out the <> + +The name of the rolled up index will be the original index name of the managed index prefixed +with `rollup-`. + +[[ilm-rollup-options]] +==== Rollup options +`config`:: +(Required, integer) +The Rollup configuration, a more detailed description of the +Rollup configuration specification can be found <>. + +`rollup_policy`:: +(Optional, string) +The name of an <> ({ilm-init}) policy to associate +with the newly created rollup index. + +[[ilm-rollup-ex]] +==== Example + +[source,console] +-------------------------------------------------- +PUT _ilm/policy/my_policy +{ + "policy": { + "phases": { + "cold": { + "actions": { + "rollup" : { + "config": { + "groups": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "1y" + } + }, + "metrics": [ + { "field": "temperature", "metrics": [ "avg" ] } + ] + } + } + } + } + } + } +} +-------------------------------------------------- diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java index e75a3a58f3c15..48aada3f9d72a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java @@ -42,6 +42,7 @@ public boolean isRetryable() { public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) { String originalIndex = indexMetadata.getIndex().getName(); RollupAction.Request request = new RollupAction.Request(originalIndex, getRollupIndexName(originalIndex), config); + // currently RollupAction always acknowledges action was complete when no exceptions are thrown. getClient().execute(RollupAction.INSTANCE, request, ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java index 0691b8dbfacfd..45758f62a7693 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.client.Client; @@ -44,8 +45,13 @@ public void performAction(IndexMetadata indexMetadata, ClusterState currentState UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(rollupIndex) .masterNodeTimeout(getMasterTimeout(currentState)) .settings(settings); - getClient().admin().indices().updateSettings(updateSettingsRequest, - ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure)); + getClient().admin().indices().updateSettings(updateSettingsRequest, ActionListener.wrap(response -> { + if (response.isAcknowledged()) { + listener.onResponse(true); + } else { + listener.onFailure(new ElasticsearchException("settings update not acknowledged in step [" + getKey().toString() + "]")); + } + }, listener::onFailure)); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java index 11baa27d72dad..f35d416a47eff 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.RollupStep; import org.elasticsearch.xpack.core.ilm.ShrinkStep; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.scheduler.SchedulerEngine; @@ -40,7 +41,6 @@ import java.io.Closeable; import java.time.Clock; -import java.util.Collections; import java.util.Set; import java.util.function.LongSupplier; @@ -53,7 +53,7 @@ public class IndexLifecycleService implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, IndexEventListener { private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class); - private static final Set IGNORE_STEPS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkStep.NAME); + private static final Set IGNORE_STEPS_MAINTENANCE_REQUESTED = Set.of(ShrinkStep.NAME, RollupStep.NAME); private volatile boolean isMaster = false; private volatile TimeValue pollInterval; From 1d1fc31f0c3207e48e62a648fe2153f6dcff2234 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 26 Jan 2021 14:11:00 -0800 Subject: [PATCH 17/22] add docs to ILM Actions page --- docs/reference/ilm/ilm-actions.asciidoc | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/reference/ilm/ilm-actions.asciidoc b/docs/reference/ilm/ilm-actions.asciidoc index 4d04e38a40018..32f4d37bbf10d 100644 --- a/docs/reference/ilm/ilm-actions.asciidoc +++ b/docs/reference/ilm/ilm-actions.asciidoc @@ -5,7 +5,7 @@ <>:: Move shards to nodes with different performance characteristics -and reduce the number of replicas. +and reduce the number of replicas. <>:: Permanently remove the index. @@ -22,10 +22,10 @@ Move the index shards to the <> that corresponds to the current {ilm-init} phase. <>:: -Block write operations to the index. - +Block write operations to the index. + <>:: -Remove the index as the write index for the rollover alias and +Remove the index as the write index for the rollover alias and start indexing to a new index. <>:: @@ -35,17 +35,21 @@ and mount it as a searchable snapshot. <>:: Lower the priority of an index as it moves through the lifecycle -to ensure that hot indices are recovered first. +to ensure that hot indices are recovered first. <>:: Reduce the number of primary shards by shrinking the index into a new index. <>:: Convert a follower index to a regular index. -Performed automatically before a rollover, shrink, or searchable snapshot action. +Performed automatically before a rollover, shrink, or searchable snapshot action. <>:: -Ensure that a snapshot exists before deleting the index. +Ensure that a snapshot exists before deleting the index. + +<>:: +Aggregates an index's time series data and stores the results in a new read-only +index. For example, you can roll up hourly data into daily or weekly summaries. include::actions/ilm-allocate.asciidoc[] include::actions/ilm-delete.asciidoc[] @@ -59,3 +63,4 @@ include::actions/ilm-set-priority.asciidoc[] include::actions/ilm-shrink.asciidoc[] include::actions/ilm-unfollow.asciidoc[] include::actions/ilm-wait-for-snapshot.asciidoc[] +include::actions/ilm-rollup.asciidoc[] From 49b58022be6b6b197dbafe25f58ed3fc23474aca Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 26 Jan 2021 14:19:32 -0800 Subject: [PATCH 18/22] place ILM Rollup Docs behind unreleased branch conditional --- docs/reference/ilm/ilm-actions.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/reference/ilm/ilm-actions.asciidoc b/docs/reference/ilm/ilm-actions.asciidoc index 32f4d37bbf10d..9c103c6c7b906 100644 --- a/docs/reference/ilm/ilm-actions.asciidoc +++ b/docs/reference/ilm/ilm-actions.asciidoc @@ -47,10 +47,14 @@ Performed automatically before a rollover, shrink, or searchable snapshot action <>:: Ensure that a snapshot exists before deleting the index. +ifdef::permanently-unreleased-branch[] + <>:: Aggregates an index's time series data and stores the results in a new read-only index. For example, you can roll up hourly data into daily or weekly summaries. +endif::[] + include::actions/ilm-allocate.asciidoc[] include::actions/ilm-delete.asciidoc[] include::actions/ilm-forcemerge.asciidoc[] @@ -63,4 +67,7 @@ include::actions/ilm-set-priority.asciidoc[] include::actions/ilm-shrink.asciidoc[] include::actions/ilm-unfollow.asciidoc[] include::actions/ilm-wait-for-snapshot.asciidoc[] + +ifdef::permanently-unreleased-branch[] include::actions/ilm-rollup.asciidoc[] +endif::[] From e340f07b9f38779daffd2868b7a96d8f8987b002 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 27 Jan 2021 12:34:20 -0800 Subject: [PATCH 19/22] Update docs/reference/ilm/actions/ilm-rollup.asciidoc Co-authored-by: James Rodewig <40268737+jrodewig@users.noreply.github.com> --- docs/reference/ilm/actions/ilm-rollup.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ilm/actions/ilm-rollup.asciidoc b/docs/reference/ilm/actions/ilm-rollup.asciidoc index 17b536da9e15c..703e13518522e 100644 --- a/docs/reference/ilm/actions/ilm-rollup.asciidoc +++ b/docs/reference/ilm/actions/ilm-rollup.asciidoc @@ -21,7 +21,7 @@ Rollup configuration specification can be found <> `rollup_policy`:: (Optional, string) -The name of an <> ({ilm-init}) policy to associate +The name of an <> ({ilm-init}) policy to associate with the newly created rollup index. [[ilm-rollup-ex]] From 7efb1a30ea9dc776b4678ade68dbe44ab08a4970 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 27 Jan 2021 12:34:30 -0800 Subject: [PATCH 20/22] Update docs/reference/ilm/actions/ilm-rollup.asciidoc Co-authored-by: James Rodewig <40268737+jrodewig@users.noreply.github.com> --- docs/reference/ilm/actions/ilm-rollup.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/ilm/actions/ilm-rollup.asciidoc b/docs/reference/ilm/actions/ilm-rollup.asciidoc index 703e13518522e..d61de95177586 100644 --- a/docs/reference/ilm/actions/ilm-rollup.asciidoc +++ b/docs/reference/ilm/actions/ilm-rollup.asciidoc @@ -16,8 +16,8 @@ with `rollup-`. ==== Rollup options `config`:: (Required, integer) -The Rollup configuration, a more detailed description of the -Rollup configuration specification can be found <>. +The rollup configuration, a more detailed description of the +rollup configuration specification can be found <>. `rollup_policy`:: (Optional, string) From 10487aab499e4bfa5d19b9a5eb48dd7141d22b3d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 27 Jan 2021 12:34:38 -0800 Subject: [PATCH 21/22] Update docs/reference/ilm/actions/ilm-rollup.asciidoc Co-authored-by: James Rodewig <40268737+jrodewig@users.noreply.github.com> --- docs/reference/ilm/actions/ilm-rollup.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ilm/actions/ilm-rollup.asciidoc b/docs/reference/ilm/actions/ilm-rollup.asciidoc index d61de95177586..acb29f909ab94 100644 --- a/docs/reference/ilm/actions/ilm-rollup.asciidoc +++ b/docs/reference/ilm/actions/ilm-rollup.asciidoc @@ -7,7 +7,7 @@ Phases allowed: hot, cold. Aggregates an index's time series data and stores the results in a new read-only index. For example, you can roll up hourly data into daily or weekly summaries. -For more information about Rollup, check out the <> +For more information about rollup, see the <> The name of the rolled up index will be the original index name of the managed index prefixed with `rollup-`. From ec30764aa31ee8910b7d850492246198e201e70c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 28 Jan 2021 16:58:27 -0800 Subject: [PATCH 22/22] prioritize rollup before shrink/forcemerge in hot phase --- .../elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index a4ba3c01907e6..522e51ef64570 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -61,7 +61,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static { if (RollupV2.isEnabled()) { ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, - ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); + ReadOnlyAction.NAME, RollupILMAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, MigrateAction.NAME, FreezeAction.NAME, RollupILMAction.NAME, SearchableSnapshotAction.NAME); } else {