Skip to content

Allow to delete indices related to index templates #1692

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danielmitterdorfer merged 2 commits into from
Mar 29, 2023
Merged

Allow to delete indices related to index templates #1692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5b50f7b
Allow to delete indices related to index templates
danielmitterdorfer Mar 27, 2023
21bbdf6
Merge remote-tracking branch 'origin/master' into delete-idx-wildcard
pquentin Mar 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 49 additions & 22 deletions esrally/driver/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1528,17 +1528,30 @@ async def __call__(self, es, params):
request_params = mandatory(params, "request-params", self)
ops_count = 0

for template_name, delete_matching_indices, index_pattern in templates:
if not only_if_exists:
await es.indices.delete_index_template(name=template_name, params=request_params, ignore=[404])
ops_count += 1
elif only_if_exists and await es.indices.exists_index_template(name=template_name):
self.logger.info("Composable Index template [%s] already exists. Deleting it.", template_name)
await es.indices.delete_index_template(name=template_name, params=request_params)
ops_count += 1
# ensure that we do not provide an empty index pattern by accident
if delete_matching_indices and index_pattern:
await es.indices.delete(index=index_pattern)
prior_destructive_setting = None
current_destructive_setting = None
try:
for template_name, delete_matching_indices, index_pattern in templates:
if not only_if_exists:
await es.indices.delete_index_template(name=template_name, params=request_params, ignore=[404])
ops_count += 1
elif only_if_exists and await es.indices.exists_index_template(name=template_name):
self.logger.info("Composable Index template [%s] already exists. Deleting it.", template_name)
await es.indices.delete_index_template(name=template_name, params=request_params)
ops_count += 1
# ensure that we do not provide an empty index pattern by accident
if delete_matching_indices and index_pattern:
# only set if really required
if current_destructive_setting is None:
Copy link
Copy Markdown
Contributor

@dliappis dliappis Mar 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For anyone else wondering what's up here, it's related to the delete-matching-indices property of templates and composable-templates.

current_destructive_setting = False
prior_destructive_setting = await set_destructive_requires_name(es, current_destructive_setting)
ops_count += 1

await es.indices.delete(index=index_pattern)
ops_count += 1
finally:
if current_destructive_setting is not None:
await set_destructive_requires_name(es, prior_destructive_setting)
ops_count += 1

return {
Expand Down Expand Up @@ -1583,17 +1596,31 @@ async def __call__(self, es, params):
request_params = params.get("request-params", {})
ops_count = 0

for template_name, delete_matching_indices, index_pattern in template_names:
if not only_if_exists:
await es.indices.delete_template(name=template_name, params=request_params)
ops_count += 1
elif only_if_exists and await es.indices.exists_template(name=template_name):
self.logger.info("Index template [%s] already exists. Deleting it.", template_name)
await es.indices.delete_template(name=template_name, params=request_params)
ops_count += 1
# ensure that we do not provide an empty index pattern by accident
if delete_matching_indices and index_pattern:
await es.indices.delete(index=index_pattern)
prior_destructive_setting = None
current_destructive_setting = None

try:
for template_name, delete_matching_indices, index_pattern in template_names:
if not only_if_exists:
await es.indices.delete_template(name=template_name, params=request_params)
ops_count += 1
elif only_if_exists and await es.indices.exists_template(name=template_name):
self.logger.info("Index template [%s] already exists. Deleting it.", template_name)
await es.indices.delete_template(name=template_name, params=request_params)
ops_count += 1
# ensure that we do not provide an empty index pattern by accident
if delete_matching_indices and index_pattern:
# only set if really required
if current_destructive_setting is None:
current_destructive_setting = False
prior_destructive_setting = await set_destructive_requires_name(es, current_destructive_setting)
ops_count += 1

await es.indices.delete(index=index_pattern)
ops_count += 1
finally:
if current_destructive_setting is not None:
await set_destructive_requires_name(es, prior_destructive_setting)
ops_count += 1

return {
Expand Down
47 changes: 40 additions & 7 deletions tests/driver/runner_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2896,29 +2896,47 @@ class TestDeleteIndexTemplateRunner:
async def test_deletes_all_index_templates(self, es):
es.indices.delete_template = mock.AsyncMock()
es.indices.delete = mock.AsyncMock()
es.cluster.get_settings = mock.AsyncMock(return_value={"persistent": {}, "transient": {"action.destructive_requires_name": True}})
es.cluster.put_settings = mock.AsyncMock()

r = runner.DeleteIndexTemplate()

params = {
"templates": [
("templateA", False, None),
("templateB", True, "logs-*"),
("templateC", True, "metrics-*"),
],
"request-params": {"timeout": 60},
}
result = await r(es, params)

# 2 times delete index template, one time delete matching indices
# 3 times delete index template, 2 times to set/reset transient cluster settings, 2 times delete matching indices
assert result == {
"weight": 3,
"weight": 7,
"unit": "ops",
"success": True,
}

es.indices.delete_template.assert_has_awaits(
[mock.call(name="templateA", params=params["request-params"]), mock.call(name="templateB", params=params["request-params"])]
[
mock.call(name="templateA", params=params["request-params"]),
mock.call(name="templateB", params=params["request-params"]),
mock.call(name="templateC", params=params["request-params"]),
]
)
es.indices.delete.assert_has_awaits(
[
mock.call(index="logs-*"),
mock.call(index="metrics-*"),
]
)
es.cluster.put_settings.assert_has_awaits(
[
mock.call(body={"transient": {"action.destructive_requires_name": False}}),
mock.call(body={"transient": {"action.destructive_requires_name": True}}),
]
)
es.indices.delete.assert_awaited_once_with(index="logs-*")

@mock.patch("elasticsearch.Elasticsearch")
@pytest.mark.asyncio
Expand Down Expand Up @@ -3183,22 +3201,25 @@ class TestDeleteComposableTemplateRunner:
async def test_deletes_all_index_templates(self, es):
es.indices.delete_index_template = mock.AsyncMock()
es.indices.delete = mock.AsyncMock()
es.cluster.get_settings = mock.AsyncMock(return_value={"persistent": {}, "transient": {"action.destructive_requires_name": True}})
es.cluster.put_settings = mock.AsyncMock()

r = runner.DeleteComposableTemplate()

params = {
"templates": [
("templateA", False, None),
("templateB", True, "logs-*"),
("templateC", True, "metrics-*"),
],
"request-params": {"timeout": 60},
"only-if-exists": False,
}
result = await r(es, params)

# 2 times delete index template, one time delete matching indices
# 3 times delete index template, 2 times to set/reset transient cluster settings, 2 times delete matching indices
assert result == {
"weight": 3,
"weight": 7,
"unit": "ops",
"success": True,
}
Expand All @@ -3207,9 +3228,21 @@ async def test_deletes_all_index_templates(self, es):
[
mock.call(name="templateA", params=params["request-params"], ignore=[404]),
mock.call(name="templateB", params=params["request-params"], ignore=[404]),
mock.call(name="templateC", params=params["request-params"], ignore=[404]),
]
)
es.cluster.put_settings.assert_has_awaits(
[
mock.call(body={"transient": {"action.destructive_requires_name": False}}),
mock.call(body={"transient": {"action.destructive_requires_name": True}}),
]
)
es.indices.delete.assert_has_awaits(
[
mock.call(index="logs-*"),
mock.call(index="metrics-*"),
]
)
es.indices.delete.assert_awaited_once_with(index="logs-*")

@mock.patch("elasticsearch.Elasticsearch")
@pytest.mark.asyncio
Expand Down