diff --git a/docs/advanced.rst b/docs/advanced.rst index 6bdb39f4e..c74a401cb 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -10,6 +10,8 @@ Rally uses `Jinja2 `_ as a template language s Elasticsearch utilizes Mustache formatting in a few places, notably in `search templates `_ and `Watcher templates `_. If you are using Mustache in your Rally tracks you must `escape them properly `_. See :ref:`put_pipeline` for an example. +.. _advanced_extensions: + Extensions """""""""" @@ -85,6 +87,8 @@ Assuming we pass ``--track-params="es_snapshot_restore_recovery_max_bytes_per_se The parameter ``default_value`` controls the value to use for the setting if it is undefined. If the setting is undefined and there is no default value, nothing will be added. +* ``build_flavor``: a global variable that holds build flavor reported by Elasticsearch cluster targetted by Rally. + .. _adding_tracks_custom_param_sources: Controlling Operation Parameters Using Custom Parameter Sources diff --git a/docs/migrate.rst b/docs/migrate.rst index 5401bfadd..f3a03bf72 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -1,11 +1,25 @@ Migration Guide =============== +Migrating to Rally 2.9.0 +------------------------ + +``build_flavor`` becomes a global variable in track templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rally now treats ``build_flavor`` as a :ref:`global variable` in Jinja2 templates. The value of the variable depends on the build flavor reported by Elasticsearch cluster targetted by Rally. +The ``build_flavor`` becomes a reserved name so user-supplied track parameters must be named differently. Rename your track parameters before upgrade if needed. + +Example error when running with ``--track-params="build_flavor:test"``:: + + Some of your track parameter(s) "build_flavor" are defined by Rally and cannot be modified. + Migrating to Rally 2.7.1 ------------------------ Elasticsearch client logs are now captured by the `elastic_transport `_ logger ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Rally migrated to the 8.x version of the ``elasticsearch-py`` library which uses a new logger named ``elastic_transport``. Rally will automatically configure this logger to only emit logs of level ``WARNING`` and above, even if a past Rally version configured logging using the ``~./rally/logging.json`` file without that logger. Snapshot repository plugins are no longer built from source diff --git a/esrally/track/loader.py b/esrally/track/loader.py index f6e6fe17b..ae846f577 100644 --- a/esrally/track/loader.py +++ b/esrally/track/loader.py @@ -732,13 +732,13 @@ def read_glob_files(self, pattern): return ",\n".join(source) -def default_internal_template_vars(glob_helper=lambda f: [], clock=time.Clock): +def default_internal_template_vars(glob_helper=lambda f: [], clock=time.Clock, build_flavor=None): """ Dict of internal global variables used by our jinja2 renderers """ - return { "globals": { + "build_flavor": build_flavor, "now": clock.now(), "glob": glob_helper, }, @@ -806,7 +806,7 @@ def register_all_params_in_track(assembled_source, complete_track_params=None): complete_track_params.populate_track_defined_params(j2_variables) -def render_template_from_file(template_file_name, template_vars, complete_track_params=None): +def render_template_from_file(template_file_name, template_vars, complete_track_params=None, build_flavor=None): def relative_glob(start, f): result = glob.glob(os.path.join(start, f)) if result: @@ -823,7 +823,7 @@ def relative_glob(start, f): loader=jinja2.FileSystemLoader(base_path), template_source=template_source.assembled_source, template_vars=template_vars, - template_internal_vars=default_internal_template_vars(glob_helper=lambda f: relative_glob(base_path, f)), + template_internal_vars=default_internal_template_vars(glob_helper=lambda f: relative_glob(base_path, f), build_flavor=build_flavor), ) @@ -971,6 +971,11 @@ def __init__(self, user_specified_track_params=None): self.track_defined_params = set() self.user_specified_track_params = user_specified_track_params if user_specified_track_params else {} + def internal_user_defined_track_params(self): + set_user_params = set(list(self.user_specified_track_params.keys())) + set_internal_params = set(default_internal_template_vars()["globals"].keys()) + return list(set_user_params & set_internal_params) + def populate_track_defined_params(self, list_of_track_params=None): self.track_defined_params.update(set(list_of_track_params)) @@ -996,12 +1001,14 @@ def __init__(self, cfg): track_schema_file = os.path.join(cfg.opts("node", "rally.root"), "resources", "track-schema.json") with open(track_schema_file, encoding="utf-8") as f: self.track_schema = json.loads(f.read()) + self.build_flavor = cfg.opts("mechanic", "distribution.flavor", default_value="default", mandatory=False) self.track_params = cfg.opts("track", "params", mandatory=False) self.complete_track_params = CompleteTrackParams(user_specified_track_params=self.track_params) self.read_track = TrackSpecificationReader( track_params=self.track_params, complete_track_params=self.complete_track_params, selected_challenge=cfg.opts("track", "challenge.name", mandatory=False), + build_flavor=self.build_flavor, ) self.logger = logging.getLogger(__name__) @@ -1020,7 +1027,9 @@ def read(self, track_name, track_spec_file, mapping_dir): # involving lines numbers and it also does not bloat Rally's log file so much. with tempfile.NamedTemporaryFile(delete=False, suffix=".json") as tmp: try: - rendered = render_template_from_file(track_spec_file, self.track_params, complete_track_params=self.complete_track_params) + rendered = render_template_from_file( + track_spec_file, self.track_params, complete_track_params=self.complete_track_params, build_flavor=self.build_flavor + ) with open(tmp.name, "w", encoding="utf-8") as f: f.write(rendered) self.logger.info("Final rendered track for '%s' has been written to '%s'.", track_spec_file, tmp.name) @@ -1076,6 +1085,16 @@ def read(self, track_name, track_spec_file, mapping_dir): current_track = self.read_track(track_name, track_spec, mapping_dir, track_spec_file) + internal_user_defined_track_params = self.complete_track_params.internal_user_defined_track_params() + if len(internal_user_defined_track_params) > 0: + params_list = ",".join(opts.double_quoted_list_of(sorted(internal_user_defined_track_params))) + err_msg = f"Some of your track parameter(s) {params_list} are defined by Rally and cannot be modified.\n" + + self.logger.critical(err_msg) + # also dump the message on the console + console.println(err_msg) + raise exceptions.TrackConfigError(f"Reserved track parameters {sorted(internal_user_defined_track_params)}.") + unused_user_defined_track_params = self.complete_track_params.unused_user_defined_track_params() if len(unused_user_defined_track_params) > 0: err_msg = ( @@ -1161,9 +1180,10 @@ class TrackSpecificationReader: Creates a track instances based on its parsed JSON description. """ - def __init__(self, track_params=None, complete_track_params=None, selected_challenge=None, source=io.FileSource): + def __init__(self, track_params=None, complete_track_params=None, selected_challenge=None, source=io.FileSource, build_flavor=None): self.name = None self.base_path = None + self.build_flavor = build_flavor self.track_params = track_params if track_params else {} self.complete_track_params = complete_track_params self.selected_challenge = selected_challenge @@ -1291,7 +1311,11 @@ def _load_template(self, contents, description): self.logger.info("Loading template [%s].", description) register_all_params_in_track(contents, self.complete_track_params) try: - rendered = render_template(template_source=contents, template_vars=self.track_params) + rendered = render_template( + template_source=contents, + template_vars=self.track_params, + template_internal_vars={"globals": {"build_flavor": self.build_flavor}}, + ) return json.loads(rendered) except Exception as e: self.logger.exception("Could not load file template for %s.", description) diff --git a/tests/track/loader_test.py b/tests/track/loader_test.py index 1d79e49a0..fbbdeb0d4 100644 --- a/tests/track/loader_test.py +++ b/tests/track/loader_test.py @@ -955,13 +955,14 @@ def assert_equal_ignore_whitespace(expected, actual): class TestTemplateRender: - unittest_template_internal_vars = loader.default_internal_template_vars(clock=StaticClock) + unittest_template_internal_vars = loader.default_internal_template_vars(clock=StaticClock, build_flavor="test") def test_render_simple_template(self): template = """ { "key": {{'01-01-2000' | days_ago(now)}}, - "key2": "static value" + "key2": "static value", + "key3": "{{build_flavor}}" } """ @@ -970,7 +971,8 @@ def test_render_simple_template(self): expected = """ { "key": 5864, - "key2": "static value" + "key2": "static value", + "key3": "test" } """ assert rendered == expected @@ -1063,6 +1065,29 @@ def test_render_template_with_variables(self): """ assert_equal_ignore_whitespace(expected, rendered) + def test_render_template_with_conditions(self): + template = """ + { + {%- if build_flavor != "test" %} + "key1": "build_flavor" + {%- elif lifecycle == "ilm" %} + "key1": "ilm" + {%- else %} + "key1": "else" + {%- endif %} + } + """ + rendered = loader.render_template( + template, template_vars={"lifecycle": "ilm"}, template_internal_vars=self.unittest_template_internal_vars + ) + + expected = """ + { + "key1": "ilm" + } + """ + assert_equal_ignore_whitespace(expected, rendered) + class TestCompleteTrackParams: assembled_source = textwrap.dedent( @@ -1071,6 +1096,7 @@ class TestCompleteTrackParams: "key2": {{ value2 | default(3) }}, "key3": {{ value3 | default("default_value3") }} "key4": {{ value2 | default(3) }} + "key5": {{ build_flavor }} """ ) @@ -1086,6 +1112,13 @@ def test_check_complete_track_params_does_not_fail_with_no_track_params(self): assert complete_track_params.sorted_track_defined_params == [] + def test_internal_user_defined_track_params(self): + # track params that deliberatly collide with internal variables + track_params = {"now": "test", "build_flavor": "test"} + complete_track_params = loader.CompleteTrackParams(user_specified_track_params=track_params) + + assert sorted(complete_track_params.internal_user_defined_track_params()) == ["build_flavor", "now"] + def test_unused_user_defined_track_params(self): track_params = {"number_of_repliacs": 1, "enable_source": True, "number_of_shards": 5} # deliberate typo # unknown parameter @@ -2960,7 +2993,11 @@ def test_parse_valid_track_specification_with_composable_template(self): { "template": { "settings": { + {%- if build_flavor == "test" %} "index.number_of_replicas": {{ number_of_replicas }} + {%- else %} + "index.number_of_replicas": 99 + {%- endif %} }, "mappings": { "properties": { @@ -2975,6 +3012,7 @@ def test_parse_valid_track_specification_with_composable_template(self): ], } ), + build_flavor="test", ) resulting_track = reader("unittest", track_specification, "/mappings") assert ["index_pattern", "number_of_replicas", "number_of_shards"] == complete_track_params.sorted_track_defined_params