diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 3a46e9b7..714f7219 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -1,10 +1,10 @@ [default] case = v2.LR.historical_0201 constraint = "" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands = "" input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist -mapping_file = "/home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201" partition = "debug" @@ -83,10 +83,8 @@ years = "1850:1852:2", [e3sm_diags] active = True grid = '180x360_aave' -obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/" ref_final_yr = 2014 ref_start_yr = 1985 -reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/" sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis", short_name = 'v2.LR.historical_0201' ts_num_years = 2 diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index f8cc58ec..a2042f34 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -1,7 +1,7 @@ [default] case = v2.LR.historical_0201 constraint = "" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" +environment_commands = "" input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" @@ -74,11 +74,9 @@ years = "1850:1854:2", [e3sm_diags] active = True grid = '180x360_aave' -obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/" ref_final_yr = 2014 ref_start_yr = 1985 # TODO: this directory is missing OMI-MLS -reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/" sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis", short_name = 'v2.LR.historical_0201' ts_num_years = 2 @@ -101,8 +99,6 @@ years = "1850:1854:2", "1850:1854:4", [[ atm_monthly_180x360_aave_tc_analysis ]] # Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing sets = "tc_analysis", - # TODO: The data from this path doesn't appear to be in /lcrc/group/e3sm/diagnostics/observations/Atm - tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/" years = "1850:1852:2", [[ atm_monthly_180x360_aave_mvm ]] diff --git a/tests/integration/template_bundles.cfg b/tests/integration/template_bundles.cfg index b10d7a2c..594b6fcc 100644 --- a/tests/integration/template_bundles.cfg +++ b/tests/integration/template_bundles.cfg @@ -4,7 +4,7 @@ constraint = "#expand constraint#" environment_commands = "#expand environment_commands#" input = "#expand user_input#E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist -mapping_file = "#expand mapping_path#map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py output = "#expand user_output#zppy_test_bundles_output/v2.LR.historical_0201" partition = "#expand partition_short#" @@ -83,10 +83,8 @@ years = "1850:1852:2", [e3sm_diags] active = True grid = '180x360_aave' -obs_ts = "#expand diags_obs_ts#" ref_final_yr = 2014 ref_start_yr = 1985 -reference_data_path = "#expand diags_obs_climo#" sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis", short_name = 'v2.LR.historical_0201' ts_num_years = 2 diff --git a/tests/integration/template_complete_run.cfg b/tests/integration/template_complete_run.cfg index d6d1f5d1..221f3701 100644 --- a/tests/integration/template_complete_run.cfg +++ b/tests/integration/template_complete_run.cfg @@ -74,11 +74,9 @@ years = "1850:1854:2", [e3sm_diags] active = True grid = '180x360_aave' -obs_ts = "#expand diags_obs_ts#" ref_final_yr = 2014 ref_start_yr = 1985 # TODO: this directory is missing OMI-MLS -reference_data_path = "#expand diags_obs_climo#" sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis", short_name = 'v2.LR.historical_0201' ts_num_years = 2 @@ -101,8 +99,6 @@ years = "1850:1854:2", "1850:1854:4", [[ atm_monthly_180x360_aave_tc_analysis ]] # Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing sets = "tc_analysis", - # TODO: The data from this path doesn't appear to be in /lcrc/group/e3sm/diagnostics/observations/Atm - tc_obs = "#expand diags_obs_tc#" years = "1850:1852:2", [[ atm_monthly_180x360_aave_mvm ]] diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 8231ebbf..58c09271 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -123,8 +123,6 @@ def check_mismatched_images( def get_chyrsalis_expansions(config): - diags_base_path = config.get("diagnostics", "base_path") - unified_path = config.get("e3sm_unified", "base_path") # Note: `os.environ.get("USER")` also works. Here we're already using mache but not os, so using mache. username = config.get("web_portal", "username") web_base_path = config.get("web_portal", "base_path") @@ -133,14 +131,9 @@ def get_chyrsalis_expansions(config): "constraint": "", # To run this test, replace conda environment with your e3sm_diags dev environment "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_20220614", - "diags_obs_climo": f"{diags_base_path}/observations/Atm/climatology/", - "diags_obs_tc": f"{diags_base_path}/observations/Atm/tc-analysis/", - "diags_obs_ts": f"{diags_base_path}/observations/Atm/time-series/", "diags_walltime": "2:00:00", - "environment_commands": f"source {unified_path}/load_latest_e3sm_unified_chrysalis.sh", "environment_commands_test": "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.8.0rc6_chrysalis.sh", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", - "mapping_path": "/home/ac.zender/data/maps/", "partition_long": "compute", "partition_short": "debug", "qos_long": "regular", @@ -154,8 +147,6 @@ def get_chyrsalis_expansions(config): def get_compy_expansions(config): - diags_base_path = config.get("diagnostics", "base_path") - unified_path = config.get("e3sm_unified", "base_path") # Note: `os.environ.get("USER")` also works. Here we're already using mache but not os, so using mache. username = config.get("web_portal", "username") web_base_path = config.get("web_portal", "base_path") @@ -164,14 +155,9 @@ def get_compy_expansions(config): "constraint": "", # To run this test, replace conda environment with your e3sm_diags dev environment "diags_environment_commands": "source /qfs/people/fors729/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_20220722", - "diags_obs_climo": f"{diags_base_path}/observations/Atm/climatology/", - "diags_obs_tc": f"{diags_base_path}/observations/Atm/tc-analysis/", - "diags_obs_ts": f"{diags_base_path}/observations/Atm/time-series/", "diags_walltime": "03:00:00", - "environment_commands": f"source {unified_path}/load_latest_e3sm_unified_compy.sh", "environment_commands_test": "source /share/apps/E3SM/conda_envs/test_e3sm_unified_1.8.0rc6_compy.sh", "expected_dir": "/compyfs/www/zppy_test_resources/", - "mapping_path": "/compyfs/zender/maps/", "partition_long": "slurm", "partition_short": "short", "qos_long": "regular", @@ -185,8 +171,6 @@ def get_compy_expansions(config): def get_perlmutter_expansions(config): - diags_base_path = config.get("diagnostics", "base_path") - unified_path = config.get("e3sm_unified", "base_path") # Note: `os.environ.get("USER")` also works. Here we're already using mache but not os, so using mache. username = config.get("web_portal", "username") web_base_path = config.get("web_portal", "base_path") @@ -195,14 +179,9 @@ def get_perlmutter_expansions(config): "constraint": "cpu", # To run this test, replace conda environment with your e3sm_diags dev environment "diags_environment_commands": "source /global/homes/f/forsyth/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_20220715", - "diags_obs_climo": f"{diags_base_path}/observations/Atm/climatology/", - "diags_obs_tc": f"{diags_base_path}/observations/Atm/tc-analysis/", - "diags_obs_ts": f"{diags_base_path}/observations/Atm/time-series/", "diags_walltime": "6:00:00", - "environment_commands": f"source {unified_path}/load_latest_e3sm_unified_pm-cpu.sh", "environment_commands_test": "source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.8.0rc6_pm-cpu.sh", "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/", - "mapping_path": "/global/homes/z/zender/data/maps/", "partition_long": "", "partition_short": "", "qos_long": "regular", @@ -257,6 +236,10 @@ def generate_cfgs(unified_testing=False): expansions = get_expansions() if unified_testing: expansions["environment_commands"] = expansions["environment_commands_test"] + else: + # The cfg doesn't need this line, + # but it would be difficult to only write environment_commands in the unified_testing case. + expansions["environment_commands"] = "" machine = expansions["machine"] cfg_names = ["bundles", "complete_run"] diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index 17716191..3f702c74 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -57,6 +57,19 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 c["sub"] = c["subsection"] else: c["sub"] = c["grid"] + # Make a guess for observation paths, if need be + if c["reference_data_path"] == "": + c[ + "reference_data_path" + ] = f"{c['diagnostics_base_path']}/observations/Atm/climatology/" + if ("tc_analysis" in c["sets"]) and (c["tc_obs"] == ""): + c[ + "tc_obs" + ] = f"{c['diagnostics_base_path']}/observations/Atm/tc-analysis/" + if ("ts_num_years" in c.keys()) and (c["obs_ts"] == ""): + c[ + "obs_ts" + ] = f"{c['diagnostics_base_path']}/observations/Atm/time-series/" if c["run_type"] == "model_vs_obs": prefix = "e3sm_diags_%s_%s_%04d-%04d" % ( c["sub"],