diff --git a/docs/source/_static/vr_full_model_configuration.toml b/docs/source/_static/vr_full_model_configuration.toml index f54f4861e..eee7498eb 100644 --- a/docs/source/_static/vr_full_model_configuration.toml +++ b/docs/source/_static/vr_full_model_configuration.toml @@ -1,278 +1,278 @@ [core.data_output_options] -save_initial_state = true +out_continuous_file_name = "all_continuous_data.nc" +out_final_file_name = "final_state.nc" +out_initial_file_name = "initial_state.nc" +out_merge_file_name = "vr_full_model_configuration.toml" out_path = "/tmp/ve_example/out" save_continuous_data = true save_final_state = true +save_initial_state = true save_merged_config = true -out_initial_file_name = "initial_state.nc" -out_continuous_file_name = "all_continuous_data.nc" -out_final_file_name = "final_state.nc" -out_merge_file_name = "vr_full_model_configuration.toml" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "air_temperature_ref" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "relative_humidity_ref" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "atmospheric_pressure_ref" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "precipitation" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "atmospheric_co2_ref" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "mean_annual_temperature" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_climate_data.nc" +file_path = "/private/tmp/ve_example/data/example_climate_data.nc" var_name = "wind_speed_ref" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_elevation_data.nc" +file_path = "/private/tmp/ve_example/data/example_elevation_data.nc" var_name = "elevation" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_surface_runoff_data.nc" +file_path = "/private/tmp/ve_example/data/example_surface_runoff_data.nc" var_name = "surface_runoff" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "pH" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "bulk_density" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "clay_fraction" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_c_pool_lmwc" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_c_pool_maom" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_c_pool_microbe" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_c_pool_pom" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_c_pool_necromass" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_enzyme_pom" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_soil_data.nc" +file_path = "/private/tmp/ve_example/data/example_soil_data.nc" var_name = "soil_enzyme_maom" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "litter_pool_above_metabolic" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "litter_pool_above_structural" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "litter_pool_woody" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "litter_pool_below_metabolic" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "litter_pool_below_structural" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "lignin_above_structural" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "lignin_woody" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_litter_data.nc" +file_path = "/private/tmp/ve_example/data/example_litter_data.nc" var_name = "lignin_below_structural" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_plant_data.nc" +file_path = "/private/tmp/ve_example/data/example_plant_data.nc" var_name = "plant_cohorts_n" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_plant_data.nc" +file_path = "/private/tmp/ve_example/data/example_plant_data.nc" var_name = "plant_cohorts_pft" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_plant_data.nc" +file_path = "/private/tmp/ve_example/data/example_plant_data.nc" var_name = "plant_cohorts_cell_id" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_plant_data.nc" +file_path = "/private/tmp/ve_example/data/example_plant_data.nc" var_name = "plant_cohorts_dbh" [[core.data.variable]] -file = "/private/tmp/ve_example/data/example_plant_data.nc" +file_path = "/private/tmp/ve_example/data/example_plant_data.nc" var_name = "photosynthetic_photon_flux_density" [core.grid] -grid_type = "square" cell_area = 8100 cell_nx = 9 cell_ny = 9 +grid_type = "square" xoff = -45.0 yoff = -45.0 [core.timing] +run_length = "2 years" start_date = "2013-01-01" update_interval = "1 month" -run_length = "2 years" [core.layers] +above_canopy_height_offset = 2.0 +canopy_layers = 10 soil_layers = [ - -0.25, - -1.0, + -0.25, + -1.0, ] -canopy_layers = 10 -above_canopy_height_offset = 2.0 -surface_layer_height = 0.1 subcanopy_layer_height = 1.5 +surface_layer_height = 0.1 [hydrology] -initial_soil_moisture = 0.5 initial_groundwater_saturation = 0.9 +initial_soil_moisture = 0.5 [abiotic_simple] [[animal.functional_groups]] -name = "carnivorous_bird" -taxa = "bird" +adult_mass = 1.0 +birth_mass = 0.1 +development_status = "adult" +development_type = "direct" diet = "carnivore" +excretion_type = "uricotelic" metabolic_type = "endothermic" -reproductive_type = "iteroparous" -development_type = "direct" -development_status = "adult" +name = "carnivorous_bird" offspring_functional_group = "carnivorous_bird" -excretion_type = "uricotelic" -birth_mass = 0.1 -adult_mass = 1.0 +reproductive_type = "iteroparous" +taxa = "bird" [[animal.functional_groups]] -name = "herbivorous_bird" -taxa = "bird" +adult_mass = 0.5 +birth_mass = 0.05 +development_status = "adult" +development_type = "direct" diet = "herbivore" +excretion_type = "uricotelic" metabolic_type = "endothermic" -reproductive_type = "iteroparous" -development_type = "direct" -development_status = "adult" +name = "herbivorous_bird" offspring_functional_group = "herbivorous_bird" -excretion_type = "uricotelic" -birth_mass = 0.05 -adult_mass = 0.5 +reproductive_type = "iteroparous" +taxa = "bird" [[animal.functional_groups]] -name = "carnivorous_mammal" -taxa = "mammal" +adult_mass = 40.0 +birth_mass = 4.0 +development_status = "adult" +development_type = "direct" diet = "carnivore" +excretion_type = "ureotelic" metabolic_type = "endothermic" -reproductive_type = "iteroparous" -development_type = "direct" -development_status = "adult" +name = "carnivorous_mammal" offspring_functional_group = "carnivorous_mammal" -excretion_type = "ureotelic" -birth_mass = 4.0 -adult_mass = 40.0 +reproductive_type = "iteroparous" +taxa = "mammal" [[animal.functional_groups]] -name = "herbivorous_mammal" -taxa = "mammal" +adult_mass = 10.0 +birth_mass = 1.0 +development_status = "adult" +development_type = "direct" diet = "herbivore" +excretion_type = "ureotelic" metabolic_type = "endothermic" -reproductive_type = "iteroparous" -development_type = "direct" -development_status = "adult" +name = "herbivorous_mammal" offspring_functional_group = "herbivorous_mammal" -excretion_type = "ureotelic" -birth_mass = 1.0 -adult_mass = 10.0 +reproductive_type = "iteroparous" +taxa = "mammal" [[animal.functional_groups]] -name = "carnivorous_insect" -taxa = "insect" +adult_mass = 0.01 +birth_mass = 0.001 +development_status = "adult" +development_type = "direct" diet = "carnivore" +excretion_type = "uricotelic" metabolic_type = "ectothermic" -reproductive_type = "iteroparous" -development_type = "direct" -development_status = "adult" +name = "carnivorous_insect" offspring_functional_group = "carnivorous_insect" -excretion_type = "uricotelic" -birth_mass = 0.001 -adult_mass = 0.01 +reproductive_type = "iteroparous" +taxa = "insect" [[animal.functional_groups]] -name = "herbivorous_insect" -taxa = "insect" +adult_mass = 0.005 +birth_mass = 0.0005 +development_status = "adult" +development_type = "direct" diet = "herbivore" +excretion_type = "uricotelic" metabolic_type = "ectothermic" -reproductive_type = "semelparous" -development_type = "direct" -development_status = "adult" +name = "herbivorous_insect" offspring_functional_group = "herbivorous_insect" -excretion_type = "uricotelic" -birth_mass = 0.0005 -adult_mass = 0.005 +reproductive_type = "semelparous" +taxa = "insect" [[animal.functional_groups]] -name = "butterfly" -taxa = "insect" +adult_mass = 0.005 +birth_mass = 0.0005 +development_status = "adult" +development_type = "indirect" diet = "herbivore" +excretion_type = "uricotelic" metabolic_type = "ectothermic" -reproductive_type = "semelparous" -development_type = "indirect" -development_status = "adult" +name = "butterfly" offspring_functional_group = "caterpillar" -excretion_type = "uricotelic" -birth_mass = 0.0005 -adult_mass = 0.005 +reproductive_type = "semelparous" +taxa = "insect" [[animal.functional_groups]] -name = "caterpillar" -taxa = "insect" +adult_mass = 0.005 +birth_mass = 0.0005 +development_status = "larval" +development_type = "indirect" diet = "herbivore" +excretion_type = "uricotelic" metabolic_type = "ectothermic" -reproductive_type = "nonreproductive" -development_type = "indirect" -development_status = "larval" +name = "caterpillar" offspring_functional_group = "butterfly" -excretion_type = "uricotelic" -birth_mass = 0.0005 -adult_mass = 0.005 +reproductive_type = "nonreproductive" +taxa = "insect" [plants] a_plant_integer = 12 ftypes = [ - { pft_name = "shrub", max_height = 1.0 }, - { pft_name = "broadleaf", max_height = 50.0 }, -] \ No newline at end of file + {pft_name = "shrub", max_height = 1.0}, + {pft_name = "broadleaf", max_height = 50.0}, +] diff --git a/docs/source/development/design/defining_new_models.md b/docs/source/development/design/defining_new_models.md index 7c53ca745..9e73fcb9b 100644 --- a/docs/source/development/design/defining_new_models.md +++ b/docs/source/development/design/defining_new_models.md @@ -322,12 +322,14 @@ Writing JSONSchema documents can be very tedious. The following tools may be of Both of those tools take data documents formatted as JSON as inputs, where we use TOML configuration files, but there are lots of web tools to convert TOML to JSON and back. -As an example, the `FreshwaterModel` above might need two configuration options. +As an example, the `FreshwaterModel` above might need the following configuration +options. ```toml [freshwater] update_interval = "1 month" no_of_ponds = 3 +pond_classification_path = "../path/to/pond_classification.csv" ``` The JSON Schema document generated from the JSON Schema app above is shown below. Some @@ -336,6 +338,28 @@ Virtual Ecosystem configuration and so can be deleted. You may also need to edit properties are required and which provide defaults that will be used to fill missing properties. +#### Paths in model schema + +You may want your configuration file to point to resources stored in an external file, +as in the example above. You should not be loading core data in this way, but you may +want to point to a file that defines model specific configuration data. For example, the +plants model uses definitions of different plant functional types: the most convenient +way to provide these for the model initialisation is as a small CSV file containing a +data frame. This isn't data that is needed by the other models, but it is easier to +maintain and edit in a small CSV, so it is passed in via the `pft_definitions_path` +configuration option. + +However, users may provide a configuration with an absolute file, but could also provide +a path relative to the configuration file itself. This can become a problem when a +complete configuration is compiled from sections in multiple configuration files, +possibly in different locations. + +For this reason, the Virtual Ecosystem resolves configured paths when the configuration +is compiled and checked. In order to trigger this path resolution, you **must** use the +`_path` suffix on configuration options that set file paths. This naming convention +allows the Virtual Ecosystem configuration to manage file paths to ensure that file +paths are preserved when configuration files are compiled. + ```json { "$schema": "https://json-schema.org/draft/2019-09/schema", @@ -363,11 +387,20 @@ properties. "examples": [ 3 ] + }, + "pond_classification_path": { + "type": "string", + "default": "", + "title": "The pond_classification_path Schema", + "examples": [ + "../path/to/pond_classification.csv" + ] } }, "examples": [{ "update_interval": "1 month", - "no_of_ponds": 3 + "no_of_ponds": 3, + "pond_classification_path": "../path/to/pond_classification.csv" }] } ``` diff --git a/docs/source/using_the_ve/data/data.md b/docs/source/using_the_ve/data/data.md index dccbaed8a..31f6ab821 100644 --- a/docs/source/using_the_ve/data/data.md +++ b/docs/source/using_the_ve/data/data.md @@ -206,8 +206,8 @@ like the example below for each variable to be loaded. ```toml [[core.data.variable]] -file="'../../data/xy_dim.nc'" -var_name="temp" +file_path = "'../../data/xy_dim.nc'" +var_name = "temp" ``` You can include `core.data.variable` tags in different files. This can be useful to @@ -222,8 +222,8 @@ object: ```{code-cell} ipython3 data_toml = """[[core.data.variable]] -file="../../data/xy_dim.nc" -var_name="temp" +file_path = "../../data/xy_dim.nc" +var_name = "temp" """ config = Config(cfg_strings=data_toml) diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 6642a6459..fd94f7baa 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -215,7 +215,10 @@ def test_config_merge(dest, source, exp_result, exp_conflicts): ), pytest.param( None, - '[[core.data.variable]]\nfile = "cellid_coords.nc\nvar_name = "temp\n"', + """[[core.data.variable]] + file_path = "cellid_coords.nc + var_name = "temp" + """, [], does_not_raise(), None, @@ -224,8 +227,14 @@ def test_config_merge(dest, source, exp_result, exp_conflicts): pytest.param( None, [ - '[[core.data.variable]]\nfile = "cellid_coords.nc\nvar_name = "temp\n"', - '[[core.data.variable]]\nfile = "cellid_coords.nc\nvar_name = "patm\n"', + """[[core.data.variable]] + file_path = "cellid_coords.nc + var_name = "temp" + """, + """[[core.data.variable]] + file_path = "cellid_coords.nc + var_name = "patm" + """, ], [], does_not_raise(), @@ -242,7 +251,10 @@ def test_config_merge(dest, source, exp_result, exp_conflicts): ), pytest.param( "string1", - '[[core.data.variable]]\nfile = "cellid_coords.nc\nvar_name = "temp\n"', + """[[core.data.variable]] + file_path = "cellid_coords.nc + var_name = "temp" + """, [], pytest.raises(ValueError), "Do not use both cfg_paths and cfg_strings.", @@ -755,52 +767,89 @@ def test_Config_export_config(caplog, shared_datadir, auto, expected_log_entries @pytest.mark.parametrize("cfg_is_relative", (True, False)) @pytest.mark.parametrize("filepath_is_relative", (True, False)) @pytest.mark.parametrize( - "filepaths,expected", + "params_dict_source,expected", ( pytest.param( - ["file.txt", "file2.txt"], - ["path/to/config/file.txt", "path/to/config/file2.txt"], - id="co-located", + { + "file1_path": "file.txt", + "other_path": "file2.txt", + "foo": "bar", + "baz": 6, + }, + { + "file1_path": "path/to/config/file.txt", + "other_path": "path/to/config/file2.txt", + "foo": "bar", + "baz": 6, + }, + id="colocated", ), pytest.param( - ["data/file.txt", "data/file2.txt"], - ["path/to/config/data/file.txt", "path/to/config/data/file2.txt"], + { + "file1_path": "data/file.txt", + "other_path": "data/file2.txt", + "foo": "bar", + "baz": 6, + }, + { + "file1_path": "path/to/config/data/file.txt", + "other_path": "path/to/config/data/file2.txt", + "foo": "bar", + "baz": 6, + }, id="inside_cfg_dir", ), pytest.param( - ["../data/file.txt", "../data/file2.txt"], - ["path/to/data/file.txt", "path/to/data/file2.txt"], - id="outside_cfg_dir", - ), - pytest.param( - ["../../../data/file.txt", "../../../data/file2.txt"], - ["data/file.txt", "data/file2.txt"], - id="moar_outside_cfg_dir", - ), - # pytest.param( - # False, - # [{"file": str(_ABS_PATH / "file.txt"), "var_name": "my_path"}], - # [{"file": str(_ABS_PATH / "file.txt"), "var_name": "my_path"}], - # id="leave_abs_paths_unchanged", - # ), - # pytest.param( - # False, - # [{"var_name": "my_path"}], - # [{"var_name": "my_path"}], - # id="ignore_missing_file_key", - # ), - # pytest.param( - # False, - # {"file": "file.txt", "var_name": "my_path"}, - # {"file": "file.txt", "var_name": "my_path"}, - # id="variable_not_list", - # ), + { + "file1_path": "../data/file.txt", + "other_path": "../data/file2.txt", + "foo": "bar", + "baz": 6, + }, + { + "file1_path": "path/to/data/file.txt", + "other_path": "path/to/data/file2.txt", + "foo": "bar", + "baz": 6, + }, + id="parallel_to_cfg_dir", + ), + pytest.param( + { + "file1_path": "../../data/file.txt", + "other_path": "../../data/file2.txt", + "foo": "bar", + "baz": 6, + }, + { + "file1_path": "path/data/file.txt", + "other_path": "path/data/file2.txt", + "foo": "bar", + "baz": 6, + }, + id="below_cfg_dir", + ), + pytest.param( + { + "file1_path": "../../data/file.txt", + "other_path": "data/file2.txt", + "foo": "bar", + "baz": 6, + }, + { + "file1_path": "path/data/file.txt", + "other_path": "path/to/config/data/file2.txt", + "foo": "bar", + "baz": 6, + }, + id="mixed", + ), ), ) -def test__resolve_config_paths( - tmpdir, cfg_is_relative, filepath_is_relative, filepaths, expected +def test__resolve_config_paths_file_locations( + tmpdir, cfg_is_relative, filepath_is_relative, params_dict_source, expected ): - """Test the _fix_up_variable_entry_paths() function. + """Test the __resolve_config_paths() function can get relative paths correctly. This is using tmpdir to get an OS appropriate base file path - the location is not used for any actual file IO. @@ -813,22 +862,111 @@ def test__resolve_config_paths( cfg_absolute = execution_root / cfg_relative cfg_path = cfg_relative if cfg_is_relative else cfg_absolute - # Package the inputs for testing - vars = [{"file": fn, "var_name": f"v_{idx}"} for idx, fn in enumerate(filepaths)] - params_dict = {"core": {"data": {"variable": vars}}} + # Clone the input to avoid editing the test environment + params_dict = params_dict_source.copy() # For absolute file path entries, construct from the inputs if not filepath_is_relative: - for entry in params_dict["core"]["data"]["variable"]: - entry["file"] = str( - (execution_root / cfg_relative / Path(entry["file"])).resolve() - ) + for key, val in params_dict.items(): + if key.endswith("_path"): + params_dict[key] = str( + (execution_root / cfg_relative / Path(val)).resolve() + ) # Run the function _resolve_config_paths(cfg_path, params_dict) - for result, expected in zip(params_dict["core"]["data"]["variable"], expected): - if cfg_is_relative and filepath_is_relative: - assert Path(result["file"]) == Path(expected) - else: - assert Path(result["file"]) == execution_root / expected + for key, val in params_dict.items(): + # Test that paths have been resolved as expected + # but that the other entries have been left alone + if key.endswith("_path"): + if cfg_is_relative and filepath_is_relative: + assert Path(val) == Path(expected[key]) + else: + assert Path(val) == execution_root / expected[key] + + elif key == "foo": + assert val == "bar" + elif key == "baz": + assert val == 6 + + +@pytest.mark.parametrize( + "params_dict,raises,expected,err_msg", + # The str(Path(x)) pattern in the expected values below is to ensure that + # the expected paths are converted to the file system of the test machine. + ( + pytest.param( + { + "file1_path": "file.txt", + "other_path": "file2.txt", + "foo": "bar", + "baz": 6, + }, + does_not_raise(), + { + "file1_path": str(Path("path/to/config/file.txt")), + "other_path": str(Path("path/to/config/file2.txt")), + "foo": "bar", + "baz": 6, + }, + None, + id="all_good_flat_colocated", + ), + pytest.param( + { + "file1_path": "file.txt", + "other_path": "../../file2.txt", + "foo": "bar", + "baz": 6, + }, + does_not_raise(), + { + "file1_path": str(Path("path/to/config/file.txt")), + "other_path": str(Path("path/file2.txt")), + "foo": "bar", + "baz": 6, + }, + None, + id="all_good_flat_mixed", + ), + pytest.param( + { + "file1_path": "file.txt", + "nested": {"other_path": "../../file2.txt", "foo": "bar"}, + "baz": 6, + }, + does_not_raise(), + { + "file1_path": str(Path("path/to/config/file.txt")), + "nested": {"other_path": str(Path("path/file2.txt")), "foo": "bar"}, + "baz": 6, + }, + None, + id="all_good_nested_mixed", + ), + pytest.param( + { + "file1_path": 42, + "nested": {"other_path": "../../file2.txt", "foo": "bar"}, + "baz": 6, + }, + pytest.raises(ValueError), + None, + "The value for config key 'file1_path' is not a string: 42", + id="bad_path_value", + ), + ), +) +def test__resolve_config_paths_values(tmpdir, params_dict, raises, expected, err_msg): + """Test the _resolve_config_paths_values handles different inputs as expected.""" + from virtual_ecosystem.core.config import _resolve_config_paths + + with raises as excep: + # Run the function + _resolve_config_paths(Path("path/to/config"), params_dict) + + assert params_dict == expected + + if excep is not None: + assert str(excep.value) == err_msg diff --git a/tests/core/test_data.py b/tests/core/test_data.py index dd7472c8a..581f36a66 100644 --- a/tests/core/test_data.py +++ b/tests/core/test_data.py @@ -530,16 +530,16 @@ def test_Data_load_to_dataarray_data_handling( pytest.param( """[core] [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "temp" [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "prec" [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "elev" [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "vapd" """, does_not_raise(), @@ -570,16 +570,16 @@ def test_Data_load_to_dataarray_data_handling( pytest.param( """[core] [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "temp" [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "prec" [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "elev" [[core.data.variable]] - file = "cellid_coords.nc" + file_path = "cellid_coords.nc" var_name = "elev" """, pytest.raises(ConfigurationError), @@ -636,7 +636,7 @@ def test_Data_load_from_config( # Note that the no data test gets the default empty dict for cfg["core"]["data"] if "variable" in cfg["core"]["data"]: for each_var in cfg["core"]["data"]["variable"]: - each_var["file"] = shared_datadir / each_var["file"] + each_var["file_path"] = shared_datadir / each_var["file_path"] with exp_error as err: data.load_data_config(config=cfg) diff --git a/virtual_ecosystem/core/config.py b/virtual_ecosystem/core/config.py index cd9733450..87f9fc6d7 100644 --- a/virtual_ecosystem/core/config.py +++ b/virtual_ecosystem/core/config.py @@ -97,36 +97,44 @@ def config_merge( return dest, conflicts -def _resolve_config_paths(config_dir: Path, params: dict[str, Any]) -> None: +def _resolve_config_paths(config_dir: Path, config_dict: dict[str, Any]) -> None: """Resolve paths in a configuration file. - Takes the path of a directory containing a given configuration file and resolves any - file paths in the configuration file contents, relative to that file location. + Configuration files may contain keys providing file paths for data and other + settings: these paths may be absolute but also could be relative to the specific + configuration file. This becomes a problem when configurations are compiled across + multiple configuration files, possibly in different locations, so this function + searches the configuration dictionary loaded from a single file and updates + configure paths to be congruent from the directory in which Virtual Ecosystem is + being run. - Todo: - At present, this only targets `core.data.variable` configuration entries and may - want to resolve additional paths in the future. + At present, the configuration schema does not have an explicit mechanism to type a + configuration option as being a path, so we currently use the `_path` suffix to + indicate configuration options setting a path. So, this function recursively search + a configuration file payload for values stored under keys ending in `_path` and + resolves the paths. Args: config_dir: A folder containing a configuration file. - params: A dictionary of contents of the configuration file, which may contain - file paths to resolve. + config_dict: A dictionary of contents of the configuration file, which may + contain file paths to resolve. + + Raises: + ValueError: if a key ending in ``_path`` has a non-string value. """ - try: - var_entries = params["core"]["data"]["variable"] - except KeyError: - # No variable entries - return - - if not isinstance(var_entries, list): - # Must be an array - return - - for entry in var_entries: - # Though all variable entries should have a file attribute according to the - # schema, the config has not been verified at this stage so we need to check - if "file" in entry: - file_path = Path(entry["file"]) + + for key, item in config_dict.items(): + if isinstance(item, dict): + _resolve_config_paths(config_dir=config_dir, config_dict=item) + elif isinstance(item, list): + for list_entry in item: + _resolve_config_paths(config_dir=config_dir, config_dict=list_entry) + elif key.endswith("_path"): + if not isinstance(item, str): + raise ValueError( + f"The value for config key '{key}' is not a string: {item}" + ) + file_path = Path(item) if not file_path.is_absolute(): # The resolve method is used here because it is the only method to # resolve ../ entries from relative file paths. However, it also makes @@ -139,7 +147,7 @@ def _resolve_config_paths(config_dir: Path, params: dict[str, Any]) -> None: config_absolute = Path(config_dir.root).absolute() file_resolved = file_resolved.relative_to(config_absolute) - entry["file"] = str(file_resolved) + config_dict[key] = str(file_resolved) class Config(dict): @@ -404,7 +412,13 @@ def resolve_config_file_paths(self) -> None: for config_file, contents in self.toml_contents.items(): if isinstance(config_file, Path): - _resolve_config_paths(config_file.parent, contents) + try: + _resolve_config_paths( + config_dir=config_file.parent, config_dict=contents + ) + except ValueError as excep: + LOGGER.critical(excep) + raise excep def build_config(self) -> None: """Build a combined configuration from the loaded files. diff --git a/virtual_ecosystem/core/data.py b/virtual_ecosystem/core/data.py index 8d80808f2..560445fc0 100644 --- a/virtual_ecosystem/core/data.py +++ b/virtual_ecosystem/core/data.py @@ -99,19 +99,19 @@ .. code-block:: toml [[core.data.variable]] - file="/path/to/file.nc" + file_path="/path/to/file.nc" var_name="precip" [[core.data.variable]] - file="/path/to/file.nc" + file_path="/path/to/file.nc" var_name="temperature" [[core.data.variable]] + file_path="/path/to/a/different/file.nc" var_name="elev" -Data configurations must not contain repeated data variable names. - You can include ```core.data.variable``` tags in different files. This can be useful to group model-specific data with other model configuration options, and allow -configuration files to be swapped in a more modular fashion. +configuration files to be swapped in a more modular fashion. However, the data +configurations across all files **must not** contain repeated data variable names. .. code-block:: python @@ -343,7 +343,7 @@ def load_data_config(self, config: Config) -> None: # processed try: self[each_var["var_name"]] = load_to_dataarray( - file=Path(each_var["file"]), + file=Path(each_var["file_path"]), var_name=each_var["var_name"], ) except Exception as err: diff --git a/virtual_ecosystem/core/module_schema.json b/virtual_ecosystem/core/module_schema.json index ba7513764..b573a5d3f 100644 --- a/virtual_ecosystem/core/module_schema.json +++ b/virtual_ecosystem/core/module_schema.json @@ -96,7 +96,7 @@ "items": { "type": "object", "properties": { - "file": { + "file_path": { "type": "string" }, "var_name": { @@ -104,7 +104,7 @@ } }, "required": [ - "file", + "file_path", "var_name" ] } diff --git a/virtual_ecosystem/example_data/config/data_config.toml b/virtual_ecosystem/example_data/config/data_config.toml index 5af3cb14b..c353e2579 100644 --- a/virtual_ecosystem/example_data/config/data_config.toml +++ b/virtual_ecosystem/example_data/config/data_config.toml @@ -3,30 +3,30 @@ save_initial_state = true # Climate data [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "air_temperature_ref" [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "relative_humidity_ref" [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "atmospheric_pressure_ref" [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "precipitation" [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "atmospheric_co2_ref" [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "mean_annual_temperature" [[core.data.variable]] -file = "../data/example_climate_data.nc" +file_path = "../data/example_climate_data.nc" var_name = "wind_speed_ref" # Elevation [[core.data.variable]] -file = "../data/example_elevation_data.nc" +file_path = "../data/example_elevation_data.nc" var_name = "elevation" # # Hydrology @@ -36,144 +36,144 @@ var_name = "elevation" # Soil [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "pH" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "bulk_density" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "clay_fraction" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_c_pool_lmwc" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_c_pool_maom" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_c_pool_microbe" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_c_pool_pom" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_c_pool_necromass" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_enzyme_pom" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_enzyme_maom" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_n_pool_don" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_n_pool_particulate" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_n_pool_necromass" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_n_pool_maom" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_n_pool_ammonium" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_n_pool_nitrate" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_dop" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_particulate" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_necromass" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_maom" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_primary" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_secondary" [[core.data.variable]] -file = "../data/example_soil_data.nc" +file_path = "../data/example_soil_data.nc" var_name = "soil_p_pool_labile" # Litter [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "litter_pool_above_metabolic" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "litter_pool_above_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "litter_pool_woody" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "litter_pool_below_metabolic" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "litter_pool_below_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "lignin_above_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "lignin_woody" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "lignin_below_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_n_ratio_above_metabolic" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_n_ratio_above_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_n_ratio_woody" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_n_ratio_below_metabolic" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_n_ratio_below_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_p_ratio_above_metabolic" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_p_ratio_above_structural" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_p_ratio_woody" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_p_ratio_below_metabolic" [[core.data.variable]] -file = "../data/example_litter_data.nc" +file_path = "../data/example_litter_data.nc" var_name = "c_p_ratio_below_structural" # Plants [[core.data.variable]] -file = "../data/example_plant_data.nc" +file_path = "../data/example_plant_data.nc" var_name = "plant_cohorts_n" [[core.data.variable]] -file = "../data/example_plant_data.nc" +file_path = "../data/example_plant_data.nc" var_name = "plant_cohorts_pft" [[core.data.variable]] -file = "../data/example_plant_data.nc" +file_path = "../data/example_plant_data.nc" var_name = "plant_cohorts_cell_id" [[core.data.variable]] -file = "../data/example_plant_data.nc" +file_path = "../data/example_plant_data.nc" var_name = "plant_cohorts_dbh" [[core.data.variable]] -file = "../data/example_plant_data.nc" +file_path = "../data/example_plant_data.nc" var_name = "photosynthetic_photon_flux_density"