diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 8e2f3414d58b1..bbadc54c40032 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -198,6 +198,7 @@ import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction; import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.core.spatial.SpatialFeatureSetUsage; import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; @@ -381,6 +382,7 @@ public List> getClientActions() { GetSnapshotLifecycleAction.INSTANCE, DeleteSnapshotLifecycleAction.INSTANCE, ExecuteSnapshotLifecycleAction.INSTANCE, + GetSnapshotLifecycleStatsAction.INSTANCE, // Freeze FreezeIndexAction.INSTANCE, // Data Frame diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java index 2786fac735b43..84e8f288564da 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStats; import java.io.IOException; import java.util.Collections; @@ -38,36 +39,52 @@ public class SnapshotLifecycleMetadata implements MetaData.Custom { public static final String TYPE = "snapshot_lifecycle"; - public static final ParseField OPERATION_MODE_FIELD = new ParseField("operation_mode"); - public static final ParseField POLICIES_FIELD = new ParseField("policies"); - public static final SnapshotLifecycleMetadata EMPTY = new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING); + private static final ParseField OPERATION_MODE_FIELD = new ParseField("operation_mode"); + private static final ParseField POLICIES_FIELD = new ParseField("policies"); + private static final ParseField STATS_FIELD = new ParseField("stats"); + + public static final SnapshotLifecycleMetadata EMPTY = + new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats()); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(TYPE, a -> new SnapshotLifecycleMetadata( ((List) a[0]).stream() .collect(Collectors.toMap(m -> m.getPolicy().getId(), Function.identity())), - OperationMode.valueOf((String) a[1]))); + OperationMode.valueOf((String) a[1]), + (SnapshotLifecycleStats) a[2])); static { PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> SnapshotLifecyclePolicyMetadata.parse(p, n), v -> { throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported"); }, POLICIES_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), OPERATION_MODE_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (v, o) -> SnapshotLifecycleStats.parse(v), STATS_FIELD); } private final Map snapshotConfigurations; private final OperationMode operationMode; + private final SnapshotLifecycleStats slmStats; - public SnapshotLifecycleMetadata(Map snapshotConfigurations, OperationMode operationMode) { + public SnapshotLifecycleMetadata(Map snapshotConfigurations, + OperationMode operationMode, + SnapshotLifecycleStats slmStats) { this.snapshotConfigurations = new HashMap<>(snapshotConfigurations); this.operationMode = operationMode; + this.slmStats = slmStats; } public SnapshotLifecycleMetadata(StreamInput in) throws IOException { this.snapshotConfigurations = in.readMap(StreamInput::readString, SnapshotLifecyclePolicyMetadata::new); this.operationMode = in.readEnum(OperationMode.class); + // TODO: version qualify this with the correct version (7.5) once available + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.slmStats = new SnapshotLifecycleStats(in); + } else { + this.slmStats = new SnapshotLifecycleStats(); + } } public Map getSnapshotConfigurations() { @@ -78,6 +95,10 @@ public OperationMode getOperationMode() { return operationMode; } + public SnapshotLifecycleStats getStats() { + return this.slmStats; + } + @Override public EnumSet context() { return MetaData.ALL_CONTEXTS; @@ -102,12 +123,17 @@ public Version getMinimalSupportedVersion() { public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.snapshotConfigurations, StreamOutput::writeString, (out1, value) -> value.writeTo(out1)); out.writeEnum(this.operationMode); + // TODO: version qualify this with the correct version (7.5) once available + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + this.slmStats.writeTo(out); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations); builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode); + builder.field(STATS_FIELD.getPreferredName(), this.slmStats); return builder; } @@ -118,7 +144,7 @@ public String toString() { @Override public int hashCode() { - return Objects.hash(this.snapshotConfigurations, this.operationMode); + return Objects.hash(this.snapshotConfigurations, this.operationMode, this.slmStats); } @Override @@ -131,18 +157,21 @@ public boolean equals(Object obj) { } SnapshotLifecycleMetadata other = (SnapshotLifecycleMetadata) obj; return this.snapshotConfigurations.equals(other.snapshotConfigurations) && - this.operationMode.equals(other.operationMode); + this.operationMode.equals(other.operationMode) && + this.slmStats.equals(other.slmStats); } public static class SnapshotLifecycleMetadataDiff implements NamedDiff { final Diff> lifecycles; final OperationMode operationMode; + final SnapshotLifecycleStats slmStats; SnapshotLifecycleMetadataDiff(SnapshotLifecycleMetadata before, SnapshotLifecycleMetadata after) { this.lifecycles = DiffableUtils.diff(before.snapshotConfigurations, after.snapshotConfigurations, DiffableUtils.getStringKeySerializer()); this.operationMode = after.operationMode; + this.slmStats = after.slmStats; } public SnapshotLifecycleMetadataDiff(StreamInput in) throws IOException { @@ -150,13 +179,19 @@ public SnapshotLifecycleMetadataDiff(StreamInput in) throws IOException { SnapshotLifecyclePolicyMetadata::new, SnapshotLifecycleMetadataDiff::readLifecyclePolicyDiffFrom); this.operationMode = in.readEnum(OperationMode.class); + // TODO: version qualify this with the correct version (7.5) once available + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.slmStats = new SnapshotLifecycleStats(in); + } else { + this.slmStats = new SnapshotLifecycleStats(); + } } @Override public MetaData.Custom apply(MetaData.Custom part) { TreeMap newLifecycles = new TreeMap<>( lifecycles.apply(((SnapshotLifecycleMetadata) part).snapshotConfigurations)); - return new SnapshotLifecycleMetadata(newLifecycles, this.operationMode); + return new SnapshotLifecycleMetadata(newLifecycles, this.operationMode, this.slmStats); } @Override @@ -168,6 +203,10 @@ public String getWriteableName() { public void writeTo(StreamOutput out) throws IOException { lifecycles.writeTo(out); out.writeEnum(this.operationMode); + // TODO: version qualify this with the correct version (7.5) once available + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + this.slmStats.writeTo(out); + } } static Diff readLifecyclePolicyDiffFrom(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java new file mode 100644 index 0000000000000..ff37feb11642e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.slm.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStats; + +import java.io.IOException; +import java.util.Objects; + +/** + * This class represents the action of retriving the stats for snapshot lifecycle management. + * These are retrieved from the master's cluster state and contain numbers related to the count of + * snapshots taken or deleted, as well as retention runs and time spent deleting snapshots. + */ +public class GetSnapshotLifecycleStatsAction extends ActionType { + public static final GetSnapshotLifecycleStatsAction INSTANCE = new GetSnapshotLifecycleStatsAction(); + public static final String NAME = "cluster:admin/slm/stats"; + + protected GetSnapshotLifecycleStatsAction() { + super(NAME, GetSnapshotLifecycleStatsAction.Response::new); + } + + public static class Request extends AcknowledgedRequest { + + public Request() { } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private SnapshotLifecycleStats slmStats; + + public Response() { } + + public Response(SnapshotLifecycleStats slmStats) { + this.slmStats = slmStats; + } + + public Response(StreamInput in) throws IOException { + this.slmStats = new SnapshotLifecycleStats(in); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return this.slmStats.toXContent(builder, params); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + this.slmStats.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(this.slmStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + GetSnapshotLifecycleStatsAction.Response other = (GetSnapshotLifecycleStatsAction.Response) obj; + return this.slmStats.equals(other.slmStats); + } + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleStats.java new file mode 100644 index 0000000000000..dcf4bfe502236 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleStats.java @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.slm; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.metrics.CounterMetric; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * SnapshotLifecycleStats contains metrics and stats about snapshot lifecycle policy execution - how + * many snapshots were taken, deleted, how many failures, etc. It contains both global stats + * (snapshots taken, retention runs), and per-policy stats. + */ +public class SnapshotLifecycleStats implements Writeable, ToXContentObject { + + private final CounterMetric retentionRunCount = new CounterMetric(); + private final CounterMetric retentionFailedCount = new CounterMetric(); + private final CounterMetric retentionTimedOut = new CounterMetric(); + private final CounterMetric retentionTimeMs = new CounterMetric(); + private final Map policyStats; + + public static final ParseField RETENTION_RUNS = new ParseField("retention_runs"); + public static final ParseField RETENTION_FAILED = new ParseField("retention_failed"); + public static final ParseField RETENTION_TIMED_OUT = new ParseField("retention_timed_out"); + public static final ParseField RETENTION_TIME = new ParseField("retention_deletion_time"); + public static final ParseField RETENTION_TIME_MILLIS = new ParseField("retention_deletion_time_millis"); + public static final ParseField POLICY_STATS = new ParseField("policy_stats"); + public static final ParseField TOTAL_TAKEN = new ParseField("total_snapshots_taken"); + public static final ParseField TOTAL_FAILED = new ParseField("total_snapshots_failed"); + public static final ParseField TOTAL_DELETIONS = new ParseField("total_snapshots_deleted"); + public static final ParseField TOTAL_DELETION_FAILURES = new ParseField("total_snapshot_deletion_failures"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("snapshot_policy_stats", true, + a -> { + long runs = (long) a[0]; + long failed = (long) a[1]; + long timedOut = (long) a[2]; + long timeMs = (long) a[3]; + Map policyStatsMap = ((List) a[4]).stream() + .collect(Collectors.toMap(m -> m.policyId, Function.identity())); + return new SnapshotLifecycleStats(runs, failed, timedOut, timeMs, policyStatsMap); + }); + + static { + PARSER.declareLong(ConstructingObjectParser.constructorArg(), RETENTION_RUNS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), RETENTION_FAILED); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), RETENTION_TIMED_OUT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), RETENTION_TIME_MILLIS); + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> SnapshotPolicyStats.parse(p, n), POLICY_STATS); + } + + public SnapshotLifecycleStats() { + this.policyStats = new ConcurrentHashMap<>(); + } + + // Package visible for testing + SnapshotLifecycleStats(long retentionRuns, long retentionFailed, long retentionTimedOut, long retentionTimeMs, + Map policyStats) { + this.retentionRunCount.inc(retentionRuns); + this.retentionFailedCount.inc(retentionFailed); + this.retentionTimedOut.inc(retentionTimedOut); + this.retentionTimeMs.inc(retentionTimeMs); + this.policyStats = policyStats; + } + + public SnapshotLifecycleStats(StreamInput in) throws IOException { + this.policyStats = new ConcurrentHashMap<>(in.readMap(StreamInput::readString, SnapshotPolicyStats::new)); + this.retentionRunCount.inc(in.readVLong()); + this.retentionFailedCount.inc(in.readVLong()); + this.retentionTimedOut.inc(in.readVLong()); + this.retentionTimeMs.inc(in.readVLong()); + } + + public static SnapshotLifecycleStats parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public SnapshotLifecycleStats merge(SnapshotLifecycleStats other) { + + HashMap newPolicyStats = new HashMap<>(this.policyStats); + // Merges the per-run stats (the stats in "other") with the stats already present + other.policyStats + .forEach((policyId, perRunPolicyStats) -> { + newPolicyStats.compute(policyId, (k, existingPolicyMetrics) -> { + if (existingPolicyMetrics == null) { + return perRunPolicyStats; + } else { + return existingPolicyMetrics.merge(perRunPolicyStats); + } + }); + }); + + return new SnapshotLifecycleStats(this.retentionRunCount.count() + other.retentionRunCount.count(), + this.retentionFailedCount.count() + other.retentionFailedCount.count(), + this.retentionTimedOut.count() + other.retentionTimedOut.count(), + this.retentionTimeMs.count() + other.retentionTimeMs.count(), + newPolicyStats); + } + + public SnapshotLifecycleStats removePolicy(String policyId) { + Map policyStats = new HashMap<>(this.policyStats); + policyStats.remove(policyId); + return new SnapshotLifecycleStats(this.retentionRunCount.count(), this.retentionFailedCount.count(), + this.retentionTimedOut.count(), this.retentionTimeMs.count(), + policyStats); + } + + /** + * @return a map of per-policy stats for each SLM policy + */ + public Map getMetrics() { + return Collections.unmodifiableMap(this.policyStats); + } + + /** + * Increment the number of times SLM retention has been run + */ + public void retentionRun() { + this.retentionRunCount.inc(); + } + + /** + * Increment the number of times SLM retention has failed + */ + public void retentionFailed() { + this.retentionFailedCount.inc(); + } + + /** + * Increment the number of times that SLM retention timed out due to the max delete time + * window being exceeded. + */ + public void retentionTimedOut() { + this.retentionTimedOut.inc(); + } + + /** + * Register the amount of time taken for deleting snapshots during SLM retention + */ + public void deletionTime(TimeValue elapsedTime) { + this.retentionTimeMs.inc(elapsedTime.millis()); + } + + /** + * Increment the per-policy snapshot taken count for the given policy id + */ + public void snapshotTaken(String slmPolicy) { + this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotTaken(); + } + + /** + * Increment the per-policy snapshot failure count for the given policy id + */ + public void snapshotFailed(String slmPolicy) { + this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotFailed(); + } + + /** + * Increment the per-policy snapshot deleted count for the given policy id + */ + public void snapshotDeleted(String slmPolicy) { + this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotDeleted(); + } + + /** + * Increment the per-policy snapshot deletion failure count for the given policy id + */ + public void snapshotDeleteFailure(String slmPolicy) { + this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotDeleteFailure(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(policyStats, StreamOutput::writeString, (v, o) -> o.writeTo(v)); + out.writeVLong(retentionRunCount.count()); + out.writeVLong(retentionFailedCount.count()); + out.writeVLong(retentionTimedOut.count()); + out.writeVLong(retentionTimeMs.count()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(RETENTION_RUNS.getPreferredName(), this.retentionRunCount.count()); + builder.field(RETENTION_FAILED.getPreferredName(), this.retentionFailedCount.count()); + builder.field(RETENTION_TIMED_OUT.getPreferredName(), this.retentionTimedOut.count()); + TimeValue retentionTime = TimeValue.timeValueMillis(this.retentionTimeMs.count()); + builder.field(RETENTION_TIME.getPreferredName(), retentionTime); + builder.field(RETENTION_TIME_MILLIS.getPreferredName(), retentionTime.millis()); + + Map metrics = getMetrics(); + long totalTaken = metrics.values().stream().mapToLong(s -> s.snapshotsTaken.count()).sum(); + long totalFailed = metrics.values().stream().mapToLong(s -> s.snapshotsFailed.count()).sum(); + long totalDeleted = metrics.values().stream().mapToLong(s -> s.snapshotsDeleted.count()).sum(); + long totalDeleteFailures = metrics.values().stream().mapToLong(s -> s.snapshotDeleteFailures.count()).sum(); + builder.field(TOTAL_TAKEN.getPreferredName(), totalTaken); + builder.field(TOTAL_FAILED.getPreferredName(), totalFailed); + builder.field(TOTAL_DELETIONS.getPreferredName(), totalDeleted); + builder.field(TOTAL_DELETION_FAILURES.getPreferredName(), totalDeleteFailures); + builder.startObject(POLICY_STATS.getPreferredName()); + for (Map.Entry policy : metrics.entrySet()) { + SnapshotPolicyStats perPolicyMetrics = policy.getValue(); + perPolicyMetrics.toXContent(builder, params); + } + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(retentionRunCount.count(), retentionFailedCount.count(), + retentionTimedOut.count(), retentionTimeMs.count(), policyStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + SnapshotLifecycleStats other = (SnapshotLifecycleStats) obj; + return Objects.equals(retentionRunCount.count(), other.retentionRunCount.count()) && + Objects.equals(retentionFailedCount.count(), other.retentionFailedCount.count()) && + Objects.equals(retentionTimedOut.count(), other.retentionTimedOut.count()) && + Objects.equals(retentionTimeMs.count(), other.retentionTimeMs.count()) && + Objects.equals(policyStats, other.policyStats); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public static class SnapshotPolicyStats implements Writeable, ToXContentFragment { + private final String policyId; + private final CounterMetric snapshotsTaken = new CounterMetric(); + private final CounterMetric snapshotsFailed = new CounterMetric(); + private final CounterMetric snapshotsDeleted = new CounterMetric(); + private final CounterMetric snapshotDeleteFailures = new CounterMetric(); + + public static final ParseField SNAPSHOTS_TAKEN = new ParseField("snapshots_taken"); + public static final ParseField SNAPSHOTS_FAILED = new ParseField("snapshots_failed"); + public static final ParseField SNAPSHOTS_DELETED = new ParseField("snapshots_deleted"); + public static final ParseField SNAPSHOT_DELETION_FAILURES = new ParseField("snapshot_deletion_failures"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("snapshot_policy_stats", true, + (a, id) -> { + long taken = (long) a[0]; + long failed = (long) a[1]; + long deleted = (long) a[2]; + long deleteFailed = (long) a[3]; + return new SnapshotPolicyStats(id, taken, failed, deleted, deleteFailed); + }); + + static { + PARSER.declareLong(ConstructingObjectParser.constructorArg(), SNAPSHOTS_TAKEN); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), SNAPSHOTS_FAILED); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), SNAPSHOTS_DELETED); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), SNAPSHOT_DELETION_FAILURES); + } + + SnapshotPolicyStats(String slmPolicy) { + this.policyId = slmPolicy; + } + + SnapshotPolicyStats(String policyId, long snapshotsTaken, long snapshotsFailed, long deleted, long failedDeletes) { + this.policyId = policyId; + this.snapshotsTaken.inc(snapshotsTaken); + this.snapshotsFailed.inc(snapshotsFailed); + this.snapshotsDeleted.inc(deleted); + this.snapshotDeleteFailures.inc(failedDeletes); + } + + SnapshotPolicyStats(StreamInput in) throws IOException { + this.policyId = in.readString(); + this.snapshotsTaken.inc(in.readVLong()); + this.snapshotsFailed.inc(in.readVLong()); + this.snapshotsDeleted.inc(in.readVLong()); + this.snapshotDeleteFailures.inc(in.readVLong()); + } + + public static SnapshotPolicyStats parse(XContentParser parser, String policyId) { + return PARSER.apply(parser, policyId); + } + + public SnapshotPolicyStats merge(SnapshotPolicyStats other) { + return new SnapshotPolicyStats( + this.policyId, + this.snapshotsTaken.count() + other.snapshotsTaken.count(), + this.snapshotsFailed.count() + other.snapshotsFailed.count(), + this.snapshotsDeleted.count() + other.snapshotsDeleted.count(), + this.snapshotDeleteFailures.count() + other.snapshotDeleteFailures.count()); + } + + void snapshotTaken() { + snapshotsTaken.inc(); + } + + void snapshotFailed() { + snapshotsFailed.inc(); + } + + void snapshotDeleted() { + snapshotsDeleted.inc(); + } + + void snapshotDeleteFailure() { + snapshotDeleteFailures.inc(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(policyId); + out.writeVLong(snapshotsTaken.count()); + out.writeVLong(snapshotsFailed.count()); + out.writeVLong(snapshotsDeleted.count()); + out.writeVLong(snapshotDeleteFailures.count()); + } + + @Override + public int hashCode() { + return Objects.hash(policyId, snapshotsTaken.count(), snapshotsFailed.count(), + snapshotsDeleted.count(), snapshotDeleteFailures.count()); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + SnapshotPolicyStats other = (SnapshotPolicyStats) obj; + return Objects.equals(policyId, other.policyId) && + Objects.equals(snapshotsTaken.count(), other.snapshotsTaken.count()) && + Objects.equals(snapshotsFailed.count(), other.snapshotsFailed.count()) && + Objects.equals(snapshotsDeleted.count(), other.snapshotsDeleted.count()) && + Objects.equals(snapshotDeleteFailures.count(), other.snapshotDeleteFailures.count()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(policyId); + builder.field(SnapshotPolicyStats.SNAPSHOTS_TAKEN.getPreferredName(), snapshotsTaken.count()); + builder.field(SnapshotPolicyStats.SNAPSHOTS_FAILED.getPreferredName(), snapshotsFailed.count()); + builder.field(SnapshotPolicyStats.SNAPSHOTS_DELETED.getPreferredName(), snapshotsDeleted.count()); + builder.field(SnapshotPolicyStats.SNAPSHOT_DELETION_FAILURES.getPreferredName(), snapshotDeleteFailures.count()); + builder.endObject(); + return null; + } + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java new file mode 100644 index 0000000000000..1e9b1fa717809 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.slm; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStatsTests; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class SnapshotLifecycleMetadataTests extends AbstractSerializingTestCase { + @Override + protected SnapshotLifecycleMetadata doParseInstance(XContentParser parser) throws IOException { + return SnapshotLifecycleMetadata.PARSER.apply(parser, null); + } + + @Override + protected SnapshotLifecycleMetadata createTestInstance() { + int policyCount = randomIntBetween(0, 3); + Map policies = new HashMap<>(policyCount); + for (int i = 0; i < policyCount; i++) { + String id = "policy-" + randomAlphaOfLength(3); + policies.put(id, SnapshotLifecyclePolicyMetadataTests.createRandomPolicyMetadata(id)); + } + return new SnapshotLifecycleMetadata(policies, randomFrom(OperationMode.values()), + SnapshotLifecycleStatsTests.randomLifecycleStats()); + } + + @Override + protected Writeable.Reader instanceReader() { + return SnapshotLifecycleMetadata::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleStatsTests.java new file mode 100644 index 0000000000000..25b6c26998d21 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleStatsTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.slm; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class SnapshotLifecycleStatsTests extends AbstractSerializingTestCase { + @Override + protected SnapshotLifecycleStats doParseInstance(XContentParser parser) throws IOException { + return SnapshotLifecycleStats.parse(parser); + } + + public static SnapshotLifecycleStats.SnapshotPolicyStats randomPolicyStats(String policyId) { + return new SnapshotLifecycleStats.SnapshotPolicyStats(policyId, + randomBoolean() ? 0 : randomNonNegativeLong(), + randomBoolean() ? 0 : randomNonNegativeLong(), + randomBoolean() ? 0 : randomNonNegativeLong(), + randomBoolean() ? 0 : randomNonNegativeLong()); + } + + public static SnapshotLifecycleStats randomLifecycleStats() { + int policies = randomIntBetween(0, 5); + Map policyStats = new HashMap<>(policies); + for (int i = 0; i < policies; i++) { + String policy = "policy-" + randomAlphaOfLength(4); + policyStats.put(policy, randomPolicyStats(policy)); + } + return new SnapshotLifecycleStats( + randomBoolean() ? 0 : randomNonNegativeLong(), + randomBoolean() ? 0 : randomNonNegativeLong(), + randomBoolean() ? 0 : randomNonNegativeLong(), + randomBoolean() ? 0 : randomNonNegativeLong(), + policyStats); + } + + @Override + protected SnapshotLifecycleStats createTestInstance() { + return randomLifecycleStats(); + } + + @Override + protected SnapshotLifecycleStats mutateInstance(SnapshotLifecycleStats instance) throws IOException { + return randomValueOtherThan(instance, () -> instance.merge(createTestInstance())); + } + + @Override + protected Writeable.Reader instanceReader() { + return SnapshotLifecycleStats::new; + } +} diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleIT.java index 6b6a340c6b008..d29f5bc29cf26 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.slm; import org.apache.http.util.EntityUtils; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.Request; @@ -120,6 +121,14 @@ public void testFullPolicySnapshot() throws Exception { assertThat(lastSnapshotName, startsWith("snap-")); assertHistoryIsPresent(policyName, true, repoId); + + Map stats = getSLMStats(); + Map policyStats = (Map) stats.get(SnapshotLifecycleStats.POLICY_STATS.getPreferredName()); + Map policyIdStats = (Map) policyStats.get(policyName); + int snapsTaken = (int) policyIdStats.get(SnapshotLifecycleStats.SnapshotPolicyStats.SNAPSHOTS_TAKEN.getPreferredName()); + int totalTaken = (int) stats.get(SnapshotLifecycleStats.TOTAL_TAKEN.getPreferredName()); + assertThat(snapsTaken, greaterThanOrEqualTo(1)); + assertThat(totalTaken, greaterThanOrEqualTo(1)); }); Request delReq = new Request("DELETE", "/_slm/policy/" + policyName); @@ -171,12 +180,21 @@ public void testPolicyFailure() throws Exception { assertThat(snapshotName, startsWith("snap-")); } assertHistoryIsPresent(policyName, false, repoName); + + Map stats = getSLMStats(); + Map policyStats = (Map) stats.get(SnapshotLifecycleStats.POLICY_STATS.getPreferredName()); + Map policyIdStats = (Map) policyStats.get(policyName); + int snapsFailed = (int) policyIdStats.get(SnapshotLifecycleStats.SnapshotPolicyStats.SNAPSHOTS_FAILED.getPreferredName()); + int totalFailed = (int) stats.get(SnapshotLifecycleStats.TOTAL_FAILED.getPreferredName()); + assertThat(snapsFailed, greaterThanOrEqualTo(1)); + assertThat(totalFailed, greaterThanOrEqualTo(1)); }); Request delReq = new Request("DELETE", "/_slm/policy/" + policyName); assertOK(client().performRequest(delReq)); } + @SuppressWarnings("unchecked") public void testPolicyManualExecution() throws Exception { final String indexName = "test"; final String policyName = "test-policy"; @@ -218,9 +236,18 @@ public void testPolicyManualExecution() throws Exception { } catch (ResponseException e) { fail("expected snapshot to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity())); } + + Map stats = getSLMStats(); + Map policyStats = (Map) stats.get(SnapshotLifecycleStats.POLICY_STATS.getPreferredName()); + Map policyIdStats = (Map) policyStats.get(policyName); + int snapsTaken = (int) policyIdStats.get(SnapshotLifecycleStats.SnapshotPolicyStats.SNAPSHOTS_TAKEN.getPreferredName()); + int totalTaken = (int) stats.get(SnapshotLifecycleStats.TOTAL_TAKEN.getPreferredName()); + assertThat(snapsTaken, equalTo(1)); + assertThat(totalTaken, equalTo(1)); }); } + Request delReq = new Request("DELETE", "/_slm/policy/" + policyName); assertOK(client().performRequest(delReq)); @@ -235,6 +262,7 @@ public void testPolicyManualExecution() throws Exception { }); } + @SuppressWarnings("unchecked") public void testBasicTimeBasedRetenion() throws Exception { final String indexName = "test"; final String policyName = "test-policy"; @@ -300,8 +328,23 @@ public void testBasicTimeBasedRetenion() throws Exception { } catch (ResponseException e) { assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("snapshot_missing_exception")); } + + Map stats = getSLMStats(); + Map policyStats = (Map) stats.get(SnapshotLifecycleStats.POLICY_STATS.getPreferredName()); + Map policyIdStats = (Map) policyStats.get(policyName); + int snapsTaken = (int) policyIdStats.get(SnapshotLifecycleStats.SnapshotPolicyStats.SNAPSHOTS_TAKEN.getPreferredName()); + int snapsDeleted = (int) policyIdStats.get(SnapshotLifecycleStats.SnapshotPolicyStats.SNAPSHOTS_DELETED.getPreferredName()); + int retentionRun = (int) stats.get(SnapshotLifecycleStats.RETENTION_RUNS.getPreferredName()); + int totalTaken = (int) stats.get(SnapshotLifecycleStats.TOTAL_TAKEN.getPreferredName()); + int totalDeleted = (int) stats.get(SnapshotLifecycleStats.TOTAL_DELETIONS.getPreferredName()); + assertThat(snapsTaken, equalTo(1)); + assertThat(totalTaken, equalTo(1)); + assertThat(retentionRun, greaterThanOrEqualTo(1)); + assertThat(snapsDeleted, equalTo(1)); + assertThat(totalDeleted, equalTo(1)); }, 60, TimeUnit.SECONDS); + Request delReq = new Request("DELETE", "/_slm/policy/" + policyName); assertOK(client().performRequest(delReq)); @@ -402,6 +445,18 @@ private static Map extractMetadata(Map snapshotR .orElse(null); } + private Map getSLMStats() { + try { + Response response = client().performRequest(new Request("GET", "/_slm/stats")); + try (InputStream content = response.getEntity().getContent()) { + return XContentHelper.convertToMap(XContentType.JSON.xContent(), content, true); + } + } catch (Exception e) { + fail("exception retrieving stats: " + e); + throw new ElasticsearchException(e); + } + } + // This method should be called inside an assertBusy, it has no retry logic of its own private void assertHistoryIsPresent(String policyName, boolean success, String repository) throws IOException { final Request historySearchRequest = new Request("GET", ".slm-history*/_search"); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 03f3f66b8485f..57c49af8bad7e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -64,6 +64,7 @@ import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleAction; +import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction; import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore; import org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry; @@ -94,10 +95,12 @@ import org.elasticsearch.xpack.slm.action.RestDeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.action.RestExecuteSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.action.RestGetSnapshotLifecycleAction; +import org.elasticsearch.xpack.slm.action.RestGetSnapshotLifecycleStatsAction; import org.elasticsearch.xpack.slm.action.RestPutSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.action.TransportDeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.action.TransportExecuteSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.action.TransportGetSnapshotLifecycleAction; +import org.elasticsearch.xpack.slm.action.TransportGetSnapshotLifecycleStatsAction; import org.elasticsearch.xpack.slm.action.TransportPutSnapshotLifecycleAction; import java.io.IOException; @@ -212,7 +215,8 @@ public List getRestHandlers(Settings settings, RestController restC new RestPutSnapshotLifecycleAction(restController), new RestDeleteSnapshotLifecycleAction(restController), new RestGetSnapshotLifecycleAction(restController), - new RestExecuteSnapshotLifecycleAction(restController) + new RestExecuteSnapshotLifecycleAction(restController), + new RestGetSnapshotLifecycleStatsAction(restController) ); } @@ -241,6 +245,7 @@ public List getRestHandlers(Settings settings, RestController restC new ActionHandler<>(DeleteSnapshotLifecycleAction.INSTANCE, TransportDeleteSnapshotLifecycleAction.class), new ActionHandler<>(GetSnapshotLifecycleAction.INSTANCE, TransportGetSnapshotLifecycleAction.class), new ActionHandler<>(ExecuteSnapshotLifecycleAction.INSTANCE, TransportExecuteSnapshotLifecycleAction.class), + new ActionHandler<>(GetSnapshotLifecycleStatsAction.INSTANCE, TransportGetSnapshotLifecycleStatsAction.class), usageAction, infoAction); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java index c899a51c28f60..53d4a5307b0dc 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java @@ -74,7 +74,8 @@ private ClusterState updateSLMState(final ClusterState currentState) { return ClusterState.builder(currentState) .metaData(MetaData.builder(currentState.metaData()) .putCustom(SnapshotLifecycleMetadata.TYPE, - new SnapshotLifecycleMetadata(currentMetadata.getSnapshotConfigurations(), newMode))) + new SnapshotLifecycleMetadata(currentMetadata.getSnapshotConfigurations(), + newMode, currentMetadata.getStats()))) .build(); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java index 7b5f2bcec415a..10898493fb8e1 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java @@ -192,15 +192,19 @@ public ClusterState execute(ClusterState currentState) throws Exception { } SnapshotLifecyclePolicyMetadata.Builder newPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder(policyMetadata); + final SnapshotLifecycleStats stats = snapMeta.getStats(); if (exception.isPresent()) { + stats.snapshotFailed(policyName); newPolicyMetadata.setLastFailure(new SnapshotInvocationRecord(snapshotName, timestamp, exceptionToString())); } else { + stats.snapshotTaken(policyName); newPolicyMetadata.setLastSuccess(new SnapshotInvocationRecord(snapshotName, timestamp, null)); } snapLifecycles.put(policyName, newPolicyMetadata.build()); - SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, snapMeta.getOperationMode()); + SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, + snapMeta.getOperationMode(), stats); MetaData currentMeta = currentState.metaData(); return ClusterState.builder(currentState) .metaData(MetaData.builder(currentMeta) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java index 27cd0163ad713..0d3f70851e402 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java @@ -68,11 +68,25 @@ public void triggered(SchedulerEngine.Event event) { assert event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_JOB_ID) : "expected id to be " + SnapshotRetentionService.SLM_RETENTION_JOB_ID + " but it was " + event.getJobName(); if (running.compareAndSet(false, true)) { + final SnapshotLifecycleStats slmStats = new SnapshotLifecycleStats(); + + // Defined here so it can be re-used without having to repeat it + final Consumer failureHandler = e -> { + try { + logger.error("error during snapshot retention task", e); + slmStats.retentionFailed(); + updateStateWithStats(slmStats); + } finally { + running.set(false); + } + }; + try { - logger.info("starting SLM retention snapshot cleanup task"); final ClusterState state = clusterService.state(); final TimeValue maxDeletionTime = LifecycleSettings.SLM_RETENTION_DURATION_SETTING.get(state.metaData().settings()); + logger.info("starting SLM retention snapshot cleanup task"); + slmStats.retentionRun(); // Find all SLM policies that have retention enabled final Map policiesWithRetention = getAllPoliciesWithRetentionEnabled(state); @@ -82,30 +96,41 @@ public void triggered(SchedulerEngine.Event event) { .map(SnapshotLifecyclePolicy::getRepository) .collect(Collectors.toSet()); + if (repositioriesToFetch.isEmpty()) { + running.set(false); + return; + } + + // Finally, asynchronously retrieve all the snapshots, deleting them serially, + // before updating the cluster state with the new metrics and setting 'running' + // back to false getAllSuccessfulSnapshots(repositioriesToFetch, new ActionListener<>() { @Override public void onResponse(Map> allSnapshots) { - // Find all the snapshots that are past their retention date - final Map> snapshotsToBeDeleted = allSnapshots.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, - e -> e.getValue().stream() - .filter(snapshot -> snapshotEligibleForDeletion(snapshot, allSnapshots, policiesWithRetention)) - .collect(Collectors.toList()))); - - // Finally, delete the snapshots that need to be deleted - deleteSnapshots(snapshotsToBeDeleted, maxDeletionTime); - running.set(false); + try { + // Find all the snapshots that are past their retention date + final Map> snapshotsToBeDeleted = allSnapshots.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> e.getValue().stream() + .filter(snapshot -> snapshotEligibleForDeletion(snapshot, allSnapshots, policiesWithRetention)) + .collect(Collectors.toList()))); + + // Finally, delete the snapshots that need to be deleted + deleteSnapshots(snapshotsToBeDeleted, maxDeletionTime, slmStats); + + updateStateWithStats(slmStats); + } finally { + running.set(false); + } } @Override public void onFailure(Exception e) { - running.set(false); + failureHandler.accept(e); } - }, err -> running.set(false)); - + }, failureHandler); } catch (Exception e) { - running.set(false); - throw e; + failureHandler.accept(e); } } else { logger.trace("snapshot lifecycle retention task started, but a task is already running, skipping"); @@ -202,7 +227,18 @@ public void onFailure(Exception e) { }); } - void deleteSnapshots(Map> snapshotsToDelete, TimeValue maximumTime) { + static String getPolicyId(SnapshotInfo snapshotInfo) { + return Optional.ofNullable(snapshotInfo.userMetadata()) + .filter(meta -> meta.get(SnapshotLifecyclePolicy.POLICY_ID_METADATA_FIELD) != null) + .filter(meta -> meta.get(SnapshotLifecyclePolicy.POLICY_ID_METADATA_FIELD) instanceof String) + .map(meta -> (String) meta.get(SnapshotLifecyclePolicy.POLICY_ID_METADATA_FIELD)) + .orElseThrow(() -> new IllegalStateException("expected snapshot " + snapshotInfo + + " to have a policy in its metadata, but it did not")); + } + + void deleteSnapshots(Map> snapshotsToDelete, + TimeValue maximumTime, + SnapshotLifecycleStats slmStats) { int count = snapshotsToDelete.values().stream().mapToInt(List::size).sum(); if (count == 0) { logger.debug("no snapshots are eligible for deletion"); @@ -216,7 +252,7 @@ void deleteSnapshots(Map> snapshotsToDelete, TimeValu String repo = entry.getKey(); List snapshots = entry.getValue(); for (SnapshotInfo info : snapshots) { - deleteSnapshot(repo, info.snapshotId()); + deleteSnapshot(getPolicyId(info), repo, info.snapshotId(), slmStats); deleted++; // Check whether we have exceeded the maximum time allowed to spend deleting // snapshots, if we have, short-circuit the rest of the deletions @@ -226,16 +262,21 @@ void deleteSnapshots(Map> snapshotsToDelete, TimeValu logger.info("maximum snapshot retention deletion time reached, time spent: [{}]," + " maximum allowed time: [{}], deleted {} out of {} snapshots scheduled for deletion", elapsedDeletionTime, maximumTime, deleted, count); + slmStats.deletionTime(elapsedDeletionTime); + slmStats.retentionTimedOut(); return; } } } + TimeValue totalElapsedTime = TimeValue.timeValueNanos(nowNanoSupplier.getAsLong() - startTime); + logger.debug("total elapsed time for deletion of [{}] snapshots: {}", deleted, totalElapsedTime); + slmStats.deletionTime(totalElapsedTime); } /** * Delete the given snapshot from the repository in blocking manner */ - void deleteSnapshot(String repo, SnapshotId snapshot) { + void deleteSnapshot(String slmPolicy, String repo, SnapshotId snapshot, SnapshotLifecycleStats slmStats) { logger.info("[{}] snapshot retention deleting snapshot [{}]", repo, snapshot); CountDownLatch latch = new CountDownLatch(1); client.admin().cluster().prepareDeleteSnapshot(repo, snapshot.getName()) @@ -244,13 +285,17 @@ void deleteSnapshot(String repo, SnapshotId snapshot) { public void onResponse(AcknowledgedResponse acknowledgedResponse) { if (acknowledgedResponse.isAcknowledged()) { logger.debug("[{}] snapshot [{}] deleted successfully", repo, snapshot); + } else { + logger.warn("[{}] snapshot [{}] delete issued but the request was not acknowledged", repo, snapshot); } + slmStats.snapshotDeleted(slmPolicy); } @Override public void onFailure(Exception e) { logger.warn(new ParameterizedMessage("[{}] failed to delete snapshot [{}] for retention", repo, snapshot), e); + slmStats.snapshotDeleteFailure(slmPolicy); } }, latch)); try { @@ -260,6 +305,11 @@ public void onFailure(Exception e) { } catch (InterruptedException e) { logger.error(new ParameterizedMessage("[{}] deletion of snapshot [{}] interrupted", repo, snapshot), e); + slmStats.snapshotDeleteFailure(slmPolicy); } } + + void updateStateWithStats(SnapshotLifecycleStats newStats) { + clusterService.submitStateUpdateTask("update_slm_stats", new UpdateSnapshotLifecycleStatsTask(newStats)); + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java new file mode 100644 index 0000000000000..7d3946b57ceab --- /dev/null +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.slm; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; + +/** + * {@link UpdateSnapshotLifecycleStatsTask} is a cluster state update task that retrieves the + * current SLM stats, merges them with the newly produced stats (non-mutating), and then updates + * the cluster state with the new stats numbers + */ +public class UpdateSnapshotLifecycleStatsTask extends ClusterStateUpdateTask { + private static final Logger logger = LogManager.getLogger(SnapshotRetentionTask.class); + + private final SnapshotLifecycleStats runStats; + + UpdateSnapshotLifecycleStatsTask(SnapshotLifecycleStats runStats) { + this.runStats = runStats; + } + + @Override + public ClusterState execute(ClusterState currentState) { + final MetaData currentMeta = currentState.metaData(); + final SnapshotLifecycleMetadata currentSlmMeta = currentMeta.custom(SnapshotLifecycleMetadata.TYPE); + + if (currentSlmMeta == null) { + return currentState; + } + + SnapshotLifecycleStats newMetrics = currentSlmMeta.getStats().merge(runStats); + SnapshotLifecycleMetadata newSlmMeta = new SnapshotLifecycleMetadata(currentSlmMeta.getSnapshotConfigurations(), + currentSlmMeta.getOperationMode(), newMetrics); + + return ClusterState.builder(currentState) + .metaData(MetaData.builder(currentMeta) + .putCustom(SnapshotLifecycleMetadata.TYPE, newSlmMeta)) + .build(); + } + + @Override + public void onFailure(String source, Exception e) { + logger.error(new ParameterizedMessage("failed to update cluster state with snapshot lifecycle stats, " + + "source: [{}], missing stats: [{}]", source, runStats), + e); + } +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java new file mode 100644 index 0000000000000..b8629c2db5760 --- /dev/null +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.slm.action; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction; + +public class RestGetSnapshotLifecycleStatsAction extends BaseRestHandler { + + public RestGetSnapshotLifecycleStatsAction(RestController controller) { + controller.registerHandler(RestRequest.Method.GET, "/_slm/stats", this); + } + + @Override + public String getName() { + return "slm_get_lifecycle_stats"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + GetSnapshotLifecycleStatsAction.Request req = new GetSnapshotLifecycleStatsAction.Request(); + req.timeout(request.paramAsTime("timeout", req.timeout())); + req.masterNodeTimeout(request.paramAsTime("master_timeout", req.masterNodeTimeout())); + + return channel -> client.execute(GetSnapshotLifecycleStatsAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java index 443bc7696822c..39b85c712e906 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java @@ -55,7 +55,7 @@ protected void masterOperation(Task task, DeleteSnapshotLifecycleAction.Request ClusterState state, ActionListener listener) throws Exception { clusterService.submitStateUpdateTask("delete-snapshot-lifecycle-" + request.getLifecycleId(), - new AckedClusterStateUpdateTask(request, listener) { + new AckedClusterStateUpdateTask<>(request, listener) { @Override protected DeleteSnapshotLifecycleAction.Response newResponse(boolean acknowledged) { return new DeleteSnapshotLifecycleAction.Response(acknowledged); @@ -82,7 +82,8 @@ public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState) .metaData(MetaData.builder(metaData) .putCustom(SnapshotLifecycleMetadata.TYPE, - new SnapshotLifecycleMetadata(newConfigs, snapMeta.getOperationMode()))) + new SnapshotLifecycleMetadata(newConfigs, + snapMeta.getOperationMode(), snapMeta.getStats().removePolicy(request.getLifecycleId())))) .build(); } }); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java new file mode 100644 index 0000000000000..9e677e13f025d --- /dev/null +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.slm.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStats; + +import java.io.IOException; + +public class TransportGetSnapshotLifecycleStatsAction extends + TransportMasterNodeAction { + + @Inject + public TransportGetSnapshotLifecycleStatsAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(GetSnapshotLifecycleStatsAction.NAME, transportService, clusterService, threadPool, actionFilters, + GetSnapshotLifecycleStatsAction.Request::new, indexNameExpressionResolver); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected GetSnapshotLifecycleStatsAction.Response read(StreamInput in) throws IOException { + return new GetSnapshotLifecycleStatsAction.Response(in); + } + + @Override + protected void masterOperation(Task task, GetSnapshotLifecycleStatsAction.Request request, + ClusterState state, ActionListener listener) { + SnapshotLifecycleMetadata slmMeta = state.metaData().custom(SnapshotLifecycleMetadata.TYPE); + if (slmMeta == null) { + listener.onResponse(new GetSnapshotLifecycleStatsAction.Response(new SnapshotLifecycleStats())); + } else { + listener.onResponse(new GetSnapshotLifecycleStatsAction.Response(slmMeta.getStats())); + } + } + + @Override + protected ClusterBlockException checkBlock(GetSnapshotLifecycleStatsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java index 1bcc700433965..e487ba3709127 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.SnapshotLifecycleService; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStats; import java.io.IOException; import java.time.Instant; @@ -93,7 +94,8 @@ public ClusterState execute(ClusterState currentState) { OperationMode mode = Optional.ofNullable(ilmMeta) .map(IndexLifecycleMetadata::getOperationMode) .orElse(OperationMode.RUNNING); - lifecycleMetadata = new SnapshotLifecycleMetadata(Collections.singletonMap(id, meta), mode); + lifecycleMetadata = new SnapshotLifecycleMetadata(Collections.singletonMap(id, meta), + mode, new SnapshotLifecycleStats()); logger.info("adding new snapshot lifecycle [{}]", id); } else { Map snapLifecycles = new HashMap<>(snapMeta.getSnapshotConfigurations()); @@ -105,7 +107,8 @@ public ClusterState execute(ClusterState currentState) { .setModifiedDate(Instant.now().toEpochMilli()) .build(); snapLifecycles.put(id, newLifecycle); - lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, snapMeta.getOperationMode()); + lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, + snapMeta.getOperationMode(), snapMeta.getStats()); if (oldLifecycle == null) { logger.info("adding new snapshot lifecycle [{}]", id); } else { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java index bcd268b8b3e92..f3ed5924cfebc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.OperationMode; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.slm.SnapshotLifecycleStats; import java.util.Collections; @@ -58,7 +59,8 @@ private void assertNoMove(OperationMode currentMode, OperationMode requestedMode private OperationMode executeUpdate(boolean metadataInstalled, OperationMode currentMode, OperationMode requestMode, boolean assertSameClusterState) { IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(Collections.emptyMap(), currentMode); - SnapshotLifecycleMetadata snapshotLifecycleMetadata = new SnapshotLifecycleMetadata(Collections.emptyMap(), currentMode); + SnapshotLifecycleMetadata snapshotLifecycleMetadata = + new SnapshotLifecycleMetadata(Collections.emptyMap(), currentMode, new SnapshotLifecycleStats()); ImmutableOpenMap.Builder customsMapBuilder = ImmutableOpenMap.builder(); MetaData.Builder metaData = MetaData.builder() .persistentSettings(settings(Version.CURRENT).build()); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java index 1627e819024ac..2a8868c480c14 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java @@ -91,7 +91,8 @@ public void testNothingScheduledWhenNotRunning() { .setModifiedDate(1) .build(); ClusterState initialState = createState(new SnapshotLifecycleMetadata( - Collections.singletonMap(initialPolicy.getPolicy().getId(), initialPolicy), OperationMode.RUNNING)); + Collections.singletonMap(initialPolicy.getPolicy().getId(), initialPolicy), + OperationMode.RUNNING, new SnapshotLifecycleStats())); try (ThreadPool threadPool = new TestThreadPool("test"); ClusterService clusterService = ClusterServiceUtils.createClusterService(initialState, threadPool); SnapshotLifecycleService sls = new SnapshotLifecycleService(Settings.EMPTY, @@ -107,8 +108,10 @@ public void testNothingScheduledWhenNotRunning() { .build(); Map policies = new HashMap<>(); policies.put(newPolicy.getPolicy().getId(), newPolicy); - ClusterState emptyState = createState(new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING)); - ClusterState state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING)); + ClusterState emptyState = + createState(new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats())); + ClusterState state = + createState(new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats())); sls.clusterChanged(new ClusterChangedEvent("1", state, emptyState)); @@ -118,13 +121,13 @@ public void testNothingScheduledWhenNotRunning() { sls.onMaster(); assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.singleton("initial-1"))); - state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.STOPPING)); + state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.STOPPING, new SnapshotLifecycleStats())); sls.clusterChanged(new ClusterChangedEvent("2", state, emptyState)); // Since the service is stopping, jobs should have been cancelled assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.emptySet())); - state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.STOPPED)); + state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.STOPPED, new SnapshotLifecycleStats())); sls.clusterChanged(new ClusterChangedEvent("3", state, emptyState)); // Since the service is stopped, jobs should have been cancelled @@ -149,7 +152,8 @@ public void testPolicyCRUD() throws Exception { () -> new FakeSnapshotTask(e -> trigger.get().accept(e)), clusterService, clock)) { sls.offMaster(); - SnapshotLifecycleMetadata snapMeta = new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING); + SnapshotLifecycleMetadata snapMeta = + new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats()); ClusterState previousState = createState(snapMeta); Map policies = new HashMap<>(); @@ -159,7 +163,7 @@ public void testPolicyCRUD() throws Exception { .setModifiedDate(1) .build(); policies.put(policy.getPolicy().getId(), policy); - snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING); + snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats()); ClusterState state = createState(snapMeta); ClusterChangedEvent event = new ClusterChangedEvent("1", state, previousState); trigger.set(e -> { @@ -188,7 +192,7 @@ public void testPolicyCRUD() throws Exception { .setModifiedDate(2) .build(); policies.put(policy.getPolicy().getId(), newPolicy); - state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING)); + state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats())); event = new ClusterChangedEvent("2", state, previousState); sls.clusterChanged(event); assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.singleton("foo-2"))); @@ -205,7 +209,8 @@ public void testPolicyCRUD() throws Exception { final int currentCount2 = triggerCount.get(); previousState = state; // Create a state simulating the policy being deleted - state = createState(new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING)); + state = + createState(new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats())); event = new ClusterChangedEvent("2", state, previousState); sls.clusterChanged(event); clock.fastForwardSeconds(2); @@ -222,7 +227,7 @@ public void testPolicyCRUD() throws Exception { .setModifiedDate(1) .build(); policies.put(policy.getPolicy().getId(), policy); - snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING); + snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats()); previousState = state; state = createState(snapMeta); event = new ClusterChangedEvent("1", state, previousState); @@ -255,7 +260,8 @@ public void testPolicyNamesEndingInNumbers() throws Exception { () -> new FakeSnapshotTask(e -> trigger.get().accept(e)), clusterService, clock)) { sls.onMaster(); - SnapshotLifecycleMetadata snapMeta = new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING); + SnapshotLifecycleMetadata snapMeta = + new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats()); ClusterState previousState = createState(snapMeta); Map policies = new HashMap<>(); @@ -266,7 +272,7 @@ public void testPolicyNamesEndingInNumbers() throws Exception { .setModifiedDate(1) .build(); policies.put(policy.getPolicy().getId(), policy); - snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING); + snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats()); ClusterState state = createState(snapMeta); ClusterChangedEvent event = new ClusterChangedEvent("1", state, previousState); sls.clusterChanged(event); @@ -281,7 +287,7 @@ public void testPolicyNamesEndingInNumbers() throws Exception { .setModifiedDate(1) .build(); policies.put(secondPolicy.getPolicy().getId(), secondPolicy); - snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING); + snapMeta = new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats()); state = createState(snapMeta); event = new ClusterChangedEvent("2", state, previousState); sls.clusterChanged(event); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java index 8ba12e433fb25..69b6467e72bec 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java @@ -56,7 +56,8 @@ public class SnapshotLifecycleTaskTests extends ESTestCase { public void testGetSnapMetadata() { final String id = randomAlphaOfLength(4); final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); - final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm), OperationMode.RUNNING); + final SnapshotLifecycleMetadata meta = + new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm), OperationMode.RUNNING, new SnapshotLifecycleStats()); final ClusterState state = ClusterState.builder(new ClusterName("test")) .metaData(MetaData.builder() @@ -76,7 +77,8 @@ public void testGetSnapMetadata() { public void testSkipCreatingSnapshotWhenJobDoesNotMatch() { final String id = randomAlphaOfLength(4); final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); - final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm), OperationMode.RUNNING); + final SnapshotLifecycleMetadata meta = + new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm), OperationMode.RUNNING, new SnapshotLifecycleStats()); final ClusterState state = ClusterState.builder(new ClusterName("test")) .metaData(MetaData.builder() @@ -106,7 +108,8 @@ public void testSkipCreatingSnapshotWhenJobDoesNotMatch() { public void testCreateSnapshotOnTrigger() { final String id = randomAlphaOfLength(4); final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id); - final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm), OperationMode.RUNNING); + final SnapshotLifecycleMetadata meta = + new SnapshotLifecycleMetadata(Collections.singletonMap(id, slpm), OperationMode.RUNNING, new SnapshotLifecycleStats()); final ClusterState state = ClusterState.builder(new ClusterName("test")) .metaData(MetaData.builder() diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index a56dd55d8a404..b137473551bac 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -67,7 +67,8 @@ public void testGetAllPoliciesWithRetentionEnabled() { // Test with empty SLM metadata MetaData metaData = MetaData.builder() - .putCustom(SnapshotLifecycleMetadata.TYPE, new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING)) + .putCustom(SnapshotLifecycleMetadata.TYPE, + new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats())) .build(); state = ClusterState.builder(new ClusterName("cluster")).metaData(metaData).build(); assertThat(SnapshotRetentionTask.getAllPoliciesWithRetentionEnabled(state), equalTo(Collections.emptyMap())); @@ -251,7 +252,8 @@ public ClusterState createState(SnapshotLifecyclePolicy... policies) { .collect(Collectors.toMap(pm -> pm.getPolicy().getId(), pm -> pm)); MetaData metaData = MetaData.builder() - .putCustom(SnapshotLifecycleMetadata.TYPE, new SnapshotLifecycleMetadata(policyMetadataMap, OperationMode.RUNNING)) + .putCustom(SnapshotLifecycleMetadata.TYPE, + new SnapshotLifecycleMetadata(policyMetadataMap, OperationMode.RUNNING, new SnapshotLifecycleStats())) .build(); return ClusterState.builder(new ClusterName("cluster")) .metaData(metaData) @@ -280,7 +282,7 @@ void getAllSuccessfulSnapshots(Collection repositories, } @Override - void deleteSnapshots(Map> snapshotsToDelete, TimeValue maxDeleteTime) { + void deleteSnapshots(Map> snapshotsToDelete, TimeValue maxDeleteTime, SnapshotLifecycleStats slmStats) { this.snapshotDeleter.accept(snapshotsToDelete); } } @@ -307,7 +309,7 @@ void getAllSuccessfulSnapshots(Collection repositories, } @Override - void deleteSnapshot(String repo, SnapshotId snapshot) { + void deleteSnapshot(String policyId, String repo, SnapshotId snapshot, SnapshotLifecycleStats slmStats) { deleteRunner.accept(repo, snapshot); } }