From e04b9142aaa4624c608277bfd66c8a1b0c331e0b Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 15 Jun 2018 13:58:03 +0100 Subject: [PATCH 1/2] Adds ability to update a policy as long as no indexes are in the shrink action --- .../indexlifecycle/IndexLifecycleRunner.java | 17 ++ .../action/TransportPutLifecycleAction.java | 6 +- .../IndexLifecycleRunnerTests.java | 156 ++++++++++++++++++ .../test/index_lifecycle/10_basic.yml | 106 ++++++++++++ 4 files changed, 283 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java b/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java index 118855ccc4669..74c1c75884f06 100644 --- a/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java +++ b/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.indexlifecycle; +import com.carrotsearch.hppc.cursors.ObjectCursor; + import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.shrink.ShrinkAction; @@ -342,6 +344,21 @@ private static boolean canSetPolicy(StepKey currentStepKey, String currentPolicy } } + public static boolean canUpdatePolicy(String policyName, LifecyclePolicy newPolicy, ClusterState currentState) { + for (ObjectCursor cursor : currentState.getMetaData().indices().values()) { + IndexMetaData idxMetadata = cursor.value; + Settings idxSettings = idxMetadata.getSettings(); + String currentPolicyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxSettings); + if (policyName.equals(currentPolicyName)) { + StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(idxSettings); + if (canSetPolicy(currentStepKey, policyName, newPolicy) == false) { + return false; + } + } + } + return true; + } + public static ClusterState removePolicyForIndexes(final Index[] indices, ClusterState currentState, List failedIndexes) { MetaData.Builder newMetadata = MetaData.builder(currentState.getMetaData()); boolean clusterStateChanged = false; diff --git a/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java b/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java index b9f9814e9c0af..0d1096c6e23b5 100644 --- a/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java +++ b/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request; import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response; +import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleRunner; import java.util.Map; import java.util.SortedMap; @@ -72,7 +73,8 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (currentMetadata == null) { // first time using index-lifecycle feature, bootstrap metadata currentMetadata = IndexLifecycleMetadata.EMPTY; } - if (currentMetadata.getPolicyMetadatas().containsKey(request.getPolicy().getName())) { + if (currentMetadata.getPolicyMetadatas().containsKey(request.getPolicy().getName()) && IndexLifecycleRunner + .canUpdatePolicy(request.getPolicy().getName(), request.getPolicy(), currentState) == false) { throw new ResourceAlreadyExistsException("Lifecycle policy already exists: {}", request.getPolicy().getName()); } @@ -95,4 +97,4 @@ public ClusterState execute(ClusterState currentState) throws Exception { protected ClusterBlockException checkBlock(Request request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java b/x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java index dc833388043f3..327a7373a9c77 100644 --- a/x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java +++ b/x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java @@ -911,6 +911,162 @@ public void testSetPolicyForIndexIndexInShrink() { assertSame(clusterState, newClusterState); } + public void testCanUpdatePolicy() { + String indexName = randomAlphaOfLength(10); + String oldPolicyName = "old_policy"; + String newPolicyName = "new_policy"; + LifecyclePolicy newPolicy = new LifecyclePolicy(TestLifecycleType.INSTANCE, newPolicyName, Collections.emptyMap()); + StepKey currentStep = AbstractStepTestCase.randomStepKey(); + Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName) + .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder); + + boolean canUpdatePolicy = IndexLifecycleRunner.canUpdatePolicy(oldPolicyName, newPolicy, clusterState); + + assertTrue(canUpdatePolicy); + } + + public void testCanUpdatePolicyIndexInShrink() { + String indexName = randomAlphaOfLength(10); + String oldPolicyName = "old_policy"; + String newPolicyName = "new_policy"; + LifecyclePolicy newPolicy = new LifecyclePolicy(TestLifecycleType.INSTANCE, newPolicyName, Collections.emptyMap()); + StepKey currentStep = new StepKey(randomAlphaOfLength(10), ShrinkAction.NAME, randomAlphaOfLength(10)); + Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName) + .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder); + + boolean canUpdatePolicy = IndexLifecycleRunner.canUpdatePolicy(oldPolicyName, newPolicy, clusterState); + + assertFalse(canUpdatePolicy); + } + + public void testCanUpdatePolicyIndexNotManaged() { + String indexName = randomAlphaOfLength(10); + String oldPolicyName = "old_policy"; + String newPolicyName = "new_policy"; + LifecyclePolicy newPolicy = new LifecyclePolicy(TestLifecycleType.INSTANCE, newPolicyName, Collections.emptyMap()); + Settings.Builder indexSettingsBuilder = Settings.builder(); + ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder); + + boolean canUpdatePolicy = IndexLifecycleRunner.canUpdatePolicy(oldPolicyName, newPolicy, clusterState); + + assertTrue(canUpdatePolicy); + } + + public void testCanUpdatePolicyDifferentPolicy() { + String indexName = randomAlphaOfLength(10); + String oldPolicyName = "old_policy"; + String newPolicyName = "new_policy"; + LifecyclePolicy newPolicy = new LifecyclePolicy(TestLifecycleType.INSTANCE, newPolicyName, Collections.emptyMap()); + StepKey currentStep = new StepKey(randomAlphaOfLength(10), ShrinkAction.NAME, randomAlphaOfLength(10)); + Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "different_policy") + .put(LifecycleSettings.LIFECYCLE_PHASE, currentStep.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder); + + boolean canUpdatePolicy = IndexLifecycleRunner.canUpdatePolicy(oldPolicyName, newPolicy, clusterState); + + assertTrue(canUpdatePolicy); + } + + public void testCanUpdatePolicyMultipleIndexesUpdateAllowed() { + String oldPolicyName = "old_policy"; + String newPolicyName = "new_policy"; + LifecyclePolicy newPolicy = new LifecyclePolicy(TestLifecycleType.INSTANCE, newPolicyName, Collections.emptyMap()); + + String index1Name = randomAlphaOfLength(10); + StepKey currentStep1 = AbstractStepTestCase.randomStepKey(); + Settings.Builder indexSettingsBuilder1 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName).put(LifecycleSettings.LIFECYCLE_PHASE, currentStep1.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep1.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep1.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + IndexMetaData indexMetadata1 = IndexMetaData.builder(index1Name).settings(indexSettingsBuilder1).build(); + + String index2Name = randomAlphaOfLength(10); + StepKey currentStep2 = AbstractStepTestCase.randomStepKey(); + Settings.Builder indexSettingsBuilder2 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName).put(LifecycleSettings.LIFECYCLE_PHASE, currentStep2.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep2.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep2.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + IndexMetaData indexMetadata2 = IndexMetaData.builder(index2Name).settings(indexSettingsBuilder2).build(); + + String index3Name = randomAlphaOfLength(10); + StepKey currentStep3 = new StepKey(randomAlphaOfLength(10), ShrinkAction.NAME, randomAlphaOfLength(10)); + Settings.Builder indexSettingsBuilder3 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, "different_policy").put(LifecycleSettings.LIFECYCLE_PHASE, currentStep3.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep3.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep3.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + IndexMetaData indexMetadata3 = IndexMetaData.builder(index3Name).settings(indexSettingsBuilder3).build(); + + String index4Name = randomAlphaOfLength(10); + Settings.Builder indexSettingsBuilder4 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + IndexMetaData indexMetadata4 = IndexMetaData.builder(index4Name).settings(indexSettingsBuilder4).build(); + + MetaData metadata = MetaData.builder().put(indexMetadata1, true).put(indexMetadata2, true).put(indexMetadata3, true) + .put(indexMetadata4, true).build(); + ClusterState clusterState = ClusterState.builder(new ClusterName("my_cluster")).metaData(metadata).build(); + + boolean canUpdatePolicy = IndexLifecycleRunner.canUpdatePolicy(oldPolicyName, newPolicy, clusterState); + + assertTrue(canUpdatePolicy); + } + + public void testCanUpdatePolicyMultipleIndexesUpdateForbidden() { + String oldPolicyName = "old_policy"; + String newPolicyName = "new_policy"; + LifecyclePolicy newPolicy = new LifecyclePolicy(TestLifecycleType.INSTANCE, newPolicyName, Collections.emptyMap()); + + String index1Name = randomAlphaOfLength(10); + StepKey currentStep1 = AbstractStepTestCase.randomStepKey(); + Settings.Builder indexSettingsBuilder1 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName).put(LifecycleSettings.LIFECYCLE_PHASE, currentStep1.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep1.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep1.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + IndexMetaData indexMetadata1 = IndexMetaData.builder(index1Name).settings(indexSettingsBuilder1).build(); + + String index2Name = randomAlphaOfLength(10); + StepKey currentStep2 = new StepKey(randomAlphaOfLength(10), ShrinkAction.NAME, randomAlphaOfLength(10)); + Settings.Builder indexSettingsBuilder2 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName).put(LifecycleSettings.LIFECYCLE_PHASE, currentStep2.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep2.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep2.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + IndexMetaData indexMetadata2 = IndexMetaData.builder(index2Name).settings(indexSettingsBuilder2).build(); + + String index3Name = randomAlphaOfLength(10); + StepKey currentStep3 = new StepKey(randomAlphaOfLength(10), ShrinkAction.NAME, randomAlphaOfLength(10)); + Settings.Builder indexSettingsBuilder3 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(LifecycleSettings.LIFECYCLE_NAME, "different_policy").put(LifecycleSettings.LIFECYCLE_PHASE, currentStep3.getPhase()) + .put(LifecycleSettings.LIFECYCLE_ACTION, currentStep3.getAction()) + .put(LifecycleSettings.LIFECYCLE_STEP, currentStep3.getName()).put(LifecycleSettings.LIFECYCLE_SKIP, true); + IndexMetaData indexMetadata3 = IndexMetaData.builder(index3Name).settings(indexSettingsBuilder3).build(); + + String index4Name = randomAlphaOfLength(10); + Settings.Builder indexSettingsBuilder4 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + IndexMetaData indexMetadata4 = IndexMetaData.builder(index4Name).settings(indexSettingsBuilder4).build(); + + MetaData metadata = MetaData.builder().put(indexMetadata1, true).put(indexMetadata2, true).put(indexMetadata3, true) + .put(indexMetadata4, true).build(); + ClusterState clusterState = ClusterState.builder(new ClusterName("my_cluster")).metaData(metadata).build(); + + boolean canUpdatePolicy = IndexLifecycleRunner.canUpdatePolicy(oldPolicyName, newPolicy, clusterState); + + assertFalse(canUpdatePolicy); + } + public void testRemovePolicyForIndex() { String indexName = randomAlphaOfLength(10); String oldPolicyName = "old_policy"; diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/10_basic.yml index 61435af9f5f82..c12c2ff6bc704 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/10_basic.yml @@ -61,6 +61,112 @@ setup: xpack.index_lifecycle.get_lifecycle: lifecycle: "my_timeseries_lifecycle" +--- +"Test Policy Update": + - do: + acknowlege: true + xpack.index_lifecycle.put_lifecycle: + lifecycle: "my_timeseries_lifecycle" + body: | + { + "policy": { + "type": "timeseries", + "phases": { + "warm": { + "after": "10s", + "actions": { + "forcemerge": { + "max_num_segments": 10000 + } + } + }, + "delete": { + "after": "30s", + "actions": { + "delete": {} + } + } + } + } + } + + - do: + acknowledge: true + xpack.index_lifecycle.get_lifecycle: + lifecycle: "my_timeseries_lifecycle" + - match: { my_timeseries_lifecycle.type: "timeseries" } + - match: { my_timeseries_lifecycle.phases.warm.after: "10s" } + - match: { my_timeseries_lifecycle.phases.delete.after: "30s" } + + + - do: + indices.create: + index: my_index + body: + settings: + index.lifecycle.name: "my_moveable_timeseries_lifecycle" + + - do: + indices.create: + index: my_index2 + body: + settings: + index.lifecycle.name: "my_moveable_timeseries_lifecycle" + + - do: + acknowlege: true + xpack.index_lifecycle.put_lifecycle: + lifecycle: "my_timeseries_lifecycle" + body: | + { + "policy": { + "type": "timeseries", + "phases": { + "warm": { + "after": "300s", + "actions": { + "forcemerge": { + "max_num_segments": 10000 + } + } + }, + "delete": { + "after": "600s", + "actions": { + "delete": {} + } + } + } + } + } + + - do: + acknowledge: true + xpack.index_lifecycle.get_lifecycle: + lifecycle: "my_timeseries_lifecycle" + - match: { my_timeseries_lifecycle.type: "timeseries" } + - match: { my_timeseries_lifecycle.phases.warm.after: "300s" } + - match: { my_timeseries_lifecycle.phases.delete.after: "600s" } + + - do: + acknowledge: true + indices.delete: + index: my_index + - do: + acknowledge: true + indices.delete: + index: my_index2 + + - do: + acknowledge: true + xpack.index_lifecycle.delete_lifecycle: + lifecycle: "my_timeseries_lifecycle" + + - do: + catch: missing + xpack.index_lifecycle.get_lifecycle: + lifecycle: "my_timeseries_lifecycle" + --- "Test Undeletable Policy In Use": - do: From 843cc8b4284c45978369ed1b4b99378434116d1b Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 20 Jun 2018 10:47:44 +0100 Subject: [PATCH 2/2] Address review comments --- .../indexlifecycle/IndexLifecycleRunner.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java b/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java index 74c1c75884f06..7a866d5905fa1 100644 --- a/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java +++ b/x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java @@ -344,6 +344,21 @@ private static boolean canSetPolicy(StepKey currentStepKey, String currentPolicy } } + /** + * Returns true if the provided policy is allowed to be updated + * given the current {@link ClusterState}. In practice this method checks + * that all the indexes using the provided policyName is in a + * state where it is able to deal with the policy being updated to + * newPolicy. If any of these indexes is not in a state wheree + * it can deal with the update the method will return false. + * + * @param policyName + * the name of the policy being updated + * @param newPolicy + * the new version of the {@link LifecyclePolicy} + * @param currentState + * the current {@link ClusterState} + */ public static boolean canUpdatePolicy(String policyName, LifecyclePolicy newPolicy, ClusterState currentState) { for (ObjectCursor cursor : currentState.getMetaData().indices().values()) { IndexMetaData idxMetadata = cursor.value;