Allow more concise cfg

tests/integration/generated/test_bundles_chrysalis.cfg

@@ -1,10 +1,10 @@
 [default]
 case = v2.LR.historical_0201
 constraint = ""
-environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
+environment_commands = ""
 input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
-mapping_file = "/home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
 output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201"
 partition = "debug"
@@ -83,10 +83,8 @@ years = "1850:1852:2",
 [e3sm_diags]
 active = True
 grid = '180x360_aave'
-obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"
 ref_final_yr = 2014
 ref_start_yr = 1985
-reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/"
 sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis",
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -1,7 +1,7 @@
 [default]
 case = v2.LR.historical_0201
 constraint = ""
-environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
+environment_commands = ""
 input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
@@ -74,11 +74,9 @@ years = "1850:1854:2",
 [e3sm_diags]
 active = True
 grid = '180x360_aave'
-obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"
 ref_final_yr = 2014
 ref_start_yr = 1985
 # TODO: this directory is missing OMI-MLS
-reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/"
 sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis",
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2
@@ -101,8 +99,6 @@ years = "1850:1854:2", "1850:1854:4",
   [[ atm_monthly_180x360_aave_tc_analysis ]]
   # Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing
   sets = "tc_analysis",
-  # TODO: The data from this path doesn't appear to be in /lcrc/group/e3sm/diagnostics/observations/Atm
-  tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   years = "1850:1852:2",
 
   [[ atm_monthly_180x360_aave_mvm ]]

tests/integration/template_bundles.cfg

@@ -4,7 +4,7 @@ constraint = "#expand constraint#"
 environment_commands = "#expand environment_commands#"
 input = "#expand user_input#E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
-mapping_file = "#expand mapping_path#map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
 output = "#expand user_output#zppy_test_bundles_output/v2.LR.historical_0201"
 partition = "#expand partition_short#"
@@ -83,10 +83,8 @@ years = "1850:1852:2",
 [e3sm_diags]
 active = True
 grid = '180x360_aave'
-obs_ts = "#expand diags_obs_ts#"
 ref_final_yr = 2014
 ref_start_yr = 1985
-reference_data_path = "#expand diags_obs_climo#"
 sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis",
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2

tests/integration/template_complete_run.cfg

@@ -74,11 +74,9 @@ years = "1850:1854:2",
 [e3sm_diags]
 active = True
 grid = '180x360_aave'
-obs_ts = "#expand diags_obs_ts#"
 ref_final_yr = 2014
 ref_start_yr = 1985
 # TODO: this directory is missing OMI-MLS
-reference_data_path = "#expand diags_obs_climo#"
 sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis",
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2
@@ -101,8 +99,6 @@ years = "1850:1854:2", "1850:1854:4",
   [[ atm_monthly_180x360_aave_tc_analysis ]]
   # Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing
   sets = "tc_analysis",
-  # TODO: The data from this path doesn't appear to be in /lcrc/group/e3sm/diagnostics/observations/Atm
-  tc_obs = "#expand diags_obs_tc#"
   years = "1850:1852:2",
 
   [[ atm_monthly_180x360_aave_mvm ]]

tests/integration/utils.py

@@ -123,8 +123,6 @@ def check_mismatched_images(
 
 
 def get_chyrsalis_expansions(config):
-    diags_base_path = config.get("diagnostics", "base_path")
-    unified_path = config.get("e3sm_unified", "base_path")
     # Note: `os.environ.get("USER")` also works. Here we're already using mache but not os, so using mache.
     username = config.get("web_portal", "username")
     web_base_path = config.get("web_portal", "base_path")
@@ -133,14 +131,9 @@ def get_chyrsalis_expansions(config):
         "constraint": "",
         # To run this test, replace conda environment with your e3sm_diags dev environment
         "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_20220614",
-        "diags_obs_climo": f"{diags_base_path}/observations/Atm/climatology/",
-        "diags_obs_tc": f"{diags_base_path}/observations/Atm/tc-analysis/",
-        "diags_obs_ts": f"{diags_base_path}/observations/Atm/time-series/",
         "diags_walltime": "2:00:00",
-        "environment_commands": f"source {unified_path}/load_latest_e3sm_unified_chrysalis.sh",
         "environment_commands_test": "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.8.0rc6_chrysalis.sh",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
-        "mapping_path": "/home/ac.zender/data/maps/",
         "partition_long": "compute",
         "partition_short": "debug",
         "qos_long": "regular",
@@ -154,8 +147,6 @@ def get_chyrsalis_expansions(config):
 
 
 def get_compy_expansions(config):
-    diags_base_path = config.get("diagnostics", "base_path")
-    unified_path = config.get("e3sm_unified", "base_path")
     # Note: `os.environ.get("USER")` also works. Here we're already using mache but not os, so using mache.
     username = config.get("web_portal", "username")
     web_base_path = config.get("web_portal", "base_path")
@@ -164,14 +155,9 @@ def get_compy_expansions(config):
         "constraint": "",
         # To run this test, replace conda environment with your e3sm_diags dev environment
         "diags_environment_commands": "source /qfs/people/fors729/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_20220722",
-        "diags_obs_climo": f"{diags_base_path}/observations/Atm/climatology/",
-        "diags_obs_tc": f"{diags_base_path}/observations/Atm/tc-analysis/",
-        "diags_obs_ts": f"{diags_base_path}/observations/Atm/time-series/",
         "diags_walltime": "03:00:00",
-        "environment_commands": f"source {unified_path}/load_latest_e3sm_unified_compy.sh",
         "environment_commands_test": "source /share/apps/E3SM/conda_envs/test_e3sm_unified_1.8.0rc6_compy.sh",
         "expected_dir": "/compyfs/www/zppy_test_resources/",
-        "mapping_path": "/compyfs/zender/maps/",
         "partition_long": "slurm",
         "partition_short": "short",
         "qos_long": "regular",
@@ -185,8 +171,6 @@ def get_compy_expansions(config):
 
 
 def get_perlmutter_expansions(config):
-    diags_base_path = config.get("diagnostics", "base_path")
-    unified_path = config.get("e3sm_unified", "base_path")
     # Note: `os.environ.get("USER")` also works. Here we're already using mache but not os, so using mache.
     username = config.get("web_portal", "username")
     web_base_path = config.get("web_portal", "base_path")
@@ -195,14 +179,9 @@ def get_perlmutter_expansions(config):
         "constraint": "cpu",
         # To run this test, replace conda environment with your e3sm_diags dev environment
         "diags_environment_commands": "source /global/homes/f/forsyth/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_20220715",
-        "diags_obs_climo": f"{diags_base_path}/observations/Atm/climatology/",
-        "diags_obs_tc": f"{diags_base_path}/observations/Atm/tc-analysis/",
-        "diags_obs_ts": f"{diags_base_path}/observations/Atm/time-series/",
         "diags_walltime": "6:00:00",
-        "environment_commands": f"source {unified_path}/load_latest_e3sm_unified_pm-cpu.sh",
         "environment_commands_test": "source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.8.0rc6_pm-cpu.sh",
         "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/",
-        "mapping_path": "/global/homes/z/zender/data/maps/",
         "partition_long": "",
         "partition_short": "",
         "qos_long": "regular",
@@ -257,6 +236,10 @@ def generate_cfgs(unified_testing=False):
     expansions = get_expansions()
     if unified_testing:
         expansions["environment_commands"] = expansions["environment_commands_test"]
+    else:
+        # The cfg doesn't need this line,
+        # but it would be difficult to only write environment_commands in the unified_testing case.
+        expansions["environment_commands"] = ""
     machine = expansions["machine"]
 
     cfg_names = ["bundles", "complete_run"]

zppy/e3sm_diags.py

@@ -57,6 +57,19 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
                 c["sub"] = c["subsection"]
             else:
                 c["sub"] = c["grid"]
+            # Make a guess for observation paths, if need be
+            if c["reference_data_path"] == "":
+                c[
+                    "reference_data_path"
+                ] = f"{c['diagnostics_base_path']}/observations/Atm/climatology/"
+            if ("tc_analysis" in c["sets"]) and (c["tc_obs"] == ""):
+                c[
+                    "tc_obs"
+                ] = f"{c['diagnostics_base_path']}/observations/Atm/tc-analysis/"
+            if ("ts_num_years" in c.keys()) and (c["obs_ts"] == ""):
+                c[
+                    "obs_ts"
+                ] = f"{c['diagnostics_base_path']}/observations/Atm/time-series/"
             if c["run_type"] == "model_vs_obs":
                 prefix = "e3sm_diags_%s_%s_%04d-%04d" % (
                     c["sub"],