Fixes for rc4

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -155,7 +155,6 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   # Reference paths
   reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/"
   # mvo diurnal_cycle only
-  # NOTE: This is NOT the inference zppy would have made!
   dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
   # mvo streamflow only
   streamflow_obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"

tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg

@@ -155,7 +155,6 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   # Reference paths
   reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/"
   # mvo diurnal_cycle only
-  # NOTE: This is NOT the inference zppy would have made!
   dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
   # mvo streamflow only
   streamflow_obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"

tests/integration/image_checker.py

@@ -111,19 +111,60 @@ def check_images(parameters: Parameters, prefix: str):
 def construct_markdown_summary_table(
     test_results_dict: Dict[str, Results], output_file_path: str
 ):
+    machine_info = MachineInfo()
+    config = machine_info.config
+    web_portal_base_path = config.get(
+        "web_portal", "base_path"
+    )  # Does NOT include trailing "/"
+    web_portal_base_url = config.get(
+        "web_portal", "base_url"
+    )  # Does NOT include trailing "/"
     with open(output_file_path, "w") as f:
         f.write("# Summary of test results\n\n")
         f.write(
-            "Diff subdir is where to find the lists of missing/mismatched images, the image diff grid, and the individual diffs.\n"
-        )
-        f.write("Note image diff grids can not yet be constructed automatically.\n")
-        f.write(
-            "| Test name | Total images | Correct images | Missing images | Mismatched images | Diff subdir | \n"
+            "| Test name | Total images | Correct images | Missing images | Mismatched images | \n"
         )
-        f.write("| --- | --- | --- | --- | --- | --- | \n")
+        f.write("| --- | --- | --- | --- | --- | \n")
         for test_name, test_results in test_results_dict.items():
+            missing_str = f"{test_results.image_count_missing}"
+            mismatched_str = f"{test_results.image_count_mismatched}"
+
+            # test_results.diff_dir starts with the file path that is displayed on the web server.
+            # That is, it starts with the web_portal_base_path
+
+            web_link = ""
+            diff_subdir = f"{test_results.diff_dir}/{test_results.prefix}"
+            if test_results.diff_dir.startswith(web_portal_base_path):
+                web_subdir = test_results.diff_dir.removeprefix(web_portal_base_path)
+                web_link = f"{web_portal_base_url}/{web_subdir}/{test_results.prefix}"
+            if web_link:
+                has_missing: bool = test_results.image_count_missing > 0
+                if has_missing:
+                    if os.path.exists(f"{diff_subdir}/missing_images.txt"):
+                        missing_str = f"{test_results.image_count_missing} ([list]({web_link}/missing_images.txt))"
+                    else:
+                        missing_str = (
+                            f"{test_results.image_count_missing} (no list created)"
+                        )
+                has_mismatched: bool = test_results.image_count_mismatched > 0
+                if has_mismatched:
+                    mismatched_list_exists = os.path.exists(
+                        f"{diff_subdir}/mismatched_images.txt"
+                    )
+                    image_diff_grid_exists = os.path.exists(
+                        f"{diff_subdir}/image_diff_grid.pdf"
+                    )
+                    if mismatched_list_exists and image_diff_grid_exists:
+                        mismatched_str = f"{test_results.image_count_mismatched} ([list]({web_link}/mismatched_images.txt), [grid]({web_link}/image_diff_grid.pdf))"
+                    elif mismatched_list_exists:
+                        mismatched_str = f"{test_results.image_count_mismatched} ([list]({web_link}/mismatched_images.txt), no grid created)"
+                    elif image_diff_grid_exists:
+                        mismatched_str = f"{test_results.image_count_mismatched} (no list created, [grid]({web_link}/image_diff_grid.pdf))"
+                    else:
+                        mismatched_str = f"{test_results.image_count_mismatched} (no list/grid created)"
+
             f.write(
-                f"| {test_name} | {test_results.image_count_total} | {test_results.image_count_correct} | {test_results.image_count_missing} | {test_results.image_count_mismatched} | {test_results.diff_dir}/{test_results.prefix} | \n"
+                f"| {test_name} | {test_results.image_count_total} | {test_results.image_count_correct} | {missing_str} | {mismatched_str} | \n"
             )
     print(f"Copy the output of {output_file_path} to a Pull Request comment")
 
@@ -347,7 +388,7 @@ def _make_image_diff_grid(diff_subdir, pdf_name="image_diff_grid.pdf", rows_per_
         fig, axes = plt.subplots(rows_per_page, cols)
         print(f"Page {page}")
         for i, ax_row in enumerate(axes):
-            count = page * 3 + i
+            count = page * rows_per_page + i
             if count > len(prefixes) - 1:
                 break
             # We already know all the files are in `diff_subdir`; no need to repeat it.

tests/integration/template_weekly_comprehensive_v2.cfg

@@ -131,7 +131,7 @@ years = "1982:1984:2",
 
   [[ atm_monthly_180x360_aave ]]
   climo_subsection = "atm_monthly_180x360_aave"
-  dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
+  dc_obs_climo = '#expand path_dc_obs_climo#'
 
   [[ atm_monthly_180x360_aave_mvm ]]
   # Test model-vs-model using the same files as the reference

tests/integration/template_weekly_comprehensive_v3.cfg

@@ -155,8 +155,7 @@ tc_obs = "#expand diagnostics_base_path#/observations/Atm/tc-analysis/"
   # Reference paths
   reference_data_path = "#expand diagnostics_base_path#/observations/Atm/climatology/"
   # mvo diurnal_cycle only
-  # NOTE: This is NOT the inference zppy would have made!
-  dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
+  dc_obs_climo = '#expand path_dc_obs_climo#'
   # mvo streamflow only
   streamflow_obs_ts = "#expand diagnostics_base_path#/observations/Atm/time-series/"
 
@@ -321,7 +320,7 @@ model_tableID = 'Amon'
 e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed
 multiprocessing = True
 num_workers = 24
-obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
+obs_ts = '#expand path_pcmdi_diags_obs_ts#'
 pcmdi_debug = False
 ts_num_years = 2
 ts_years = "1985-1994",

tests/integration/template_weekly_legacy_3.0.0_comprehensive_v2.cfg

@@ -131,7 +131,7 @@ years = "1982:1984:2",
 
   [[ atm_monthly_180x360_aave ]]
   climo_subsection = "atm_monthly_180x360_aave"
-  dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
+  dc_obs_climo = '#expand path_dc_obs_climo#'
 
   [[ atm_monthly_180x360_aave_mvm ]]
   # Test model-vs-model using the same files as the reference

tests/integration/template_weekly_legacy_3.0.0_comprehensive_v3.cfg

@@ -155,8 +155,7 @@ tc_obs = "#expand diagnostics_base_path#/observations/Atm/tc-analysis/"
   # Reference paths
   reference_data_path = "#expand diagnostics_base_path#/observations/Atm/climatology/"
   # mvo diurnal_cycle only
-  # NOTE: This is NOT the inference zppy would have made!
-  dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
+  dc_obs_climo = '#expand path_dc_obs_climo#'
   # mvo streamflow only
   streamflow_obs_ts = "#expand diagnostics_base_path#/observations/Atm/time-series/"
 

tests/integration/utils.py

@@ -12,9 +12,19 @@
 # pytest tests/integration/test_*.py
 
 TEST_SPECIFICS: Dict[str, Any] = {
+    # These are custom environment_commands for specific tasks.
+    # Never set these to "", because they will print the line
+    # `environment_commands = ""` for the corresponding task,
+    # thus overriding the value set higher up in the cfg.
+    # That is, there will be no environment set.
+    # (`environment_commands = ""` only redirects to Unified
+    # if specified under the [default] task)
     "diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
     "global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
     "pcmdi_diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
+    # This is the environment setup for other tasks.
+    # Leave as "" to use the latest Unified environment.
+    "environment_commands": "",
     "cfgs_to_run": [
         "weekly_bundles",
         "weekly_comprehensive_v2",
@@ -47,11 +57,14 @@ def get_chyrsalis_expansions(config):
         "case_name_v2": "v2.LR.historical_0201",
         "constraint": "",
         "diags_walltime": "5:00:00",
-        "environment_commands_test": "",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
         "mpas_analysis_walltime": "00:30:00",
         "partition_long": "compute",
         "partition_short": "debug",
+        # This differs from the default path /lcrc/group/e3sm/diagnostics/observations/Atm/climatology
+        "path_dc_obs_climo": "/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology",
+        # This differs from the default path /lcrc/group/e3sm/diagnostics/observations/Atm/time-series
+        "path_pcmdi_diags_obs_ts": "/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series",
         "qos_long": "regular",
         "qos_short": "regular",
         "user_input_v2": "/lcrc/group/e3sm/ac.forsyth2/",
@@ -71,11 +84,12 @@ def get_compy_expansions(config):
         "case_name_v2": "v2.LR.historical_0201",
         "constraint": "",
         "diags_walltime": "03:00:00",
-        "environment_commands_test": "",
         "expected_dir": "/compyfs/www/zppy_test_resources/",
         "mpas_analysis_walltime": "02:00:00",
         "partition_long": "slurm",
         "partition_short": "short",
+        "path_dc_obs_climo": "/compyfs/diagnostics/observations/Atm/climatology/",
+        "path_pcmdi_diags_obs_ts": "/compyfs/diagnostics/observations/Atm/time-series/",
         "qos_long": "regular",
         "qos_short": "regular",
         "user_input_v2": "/compyfs/fors729/",
@@ -95,11 +109,12 @@ def get_perlmutter_expansions(config):
         "case_name_v2": "v2.LR.historical_0201",
         "constraint": "cpu",
         "diags_walltime": "6:00:00",
-        "environment_commands_test": "",
         "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/",
         "mpas_analysis_walltime": "03:00:00",
         "partition_long": "",
         "partition_short": "",
+        "path_dc_obs_climo": "/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/climatology/",
+        "path_pcmdi_diags_obs_ts": "/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/time-series/",
         "qos_long": "regular",
         "qos_short": "regular",  # debug walltime too short?
         # Use CFS for large datasets
@@ -125,14 +140,6 @@ def get_expansions():
         raise ValueError(f"Unsupported machine={machine}")
 
     # Set up environments
-    # To run this test, replace conda environment with your e3sm_diags dev environment
-    # Or the Unified environment
-    # (The same for `global_time_series_environment_commands` and `pcmdi_diags_environment_commands`)
-    # Never set this to "" because it will print the line
-    # `environment_commands = ""` for the [e3sm_diags] task, overriding zppy's
-    # default of using Unified. That is, there will be no environment set.
-    # `environment_commands = ""` only redirects to Unified if specified under the
-    # [default] task
     expansions["diags_environment_commands"] = TEST_SPECIFICS[
         "diags_environment_commands"
     ]
@@ -142,6 +149,7 @@ def get_expansions():
     expansions["pcmdi_diags_environment_commands"] = TEST_SPECIFICS[
         "pcmdi_diags_environment_commands"
     ]
+    expansions["environment_commands"] = TEST_SPECIFICS["environment_commands"]
 
     # Activate requested tests
     expansions["active_e3sm_to_cmip"] = "False"
@@ -194,32 +202,13 @@ def substitute_expansions(expansions, file_in, file_out):
                 file_write.write(line)
 
 
-def generate_cfgs(unified_testing=False, dry_run=False):
+def generate_cfgs(dry_run=False):
     git_top_level = (
         subprocess.check_output("git rev-parse --show-toplevel".split())
         .strip()
         .decode("utf-8")
     )
     expansions = get_expansions()
-    if unified_testing:
-        expansions["environment_commands"] = expansions["environment_commands_test"]
-        # Use Unified for e3sm_diags, global_time_series, pcmdi_diags unless we specify otherwise
-        if expansions["diags_environment_commands"] == "":
-            expansions["diags_environment_commands"] = expansions[
-                "environment_commands_test"
-            ]
-        if expansions["global_time_series_environment_commands"] == "":
-            expansions["global_time_series_environment_commands"] = expansions[
-                "environment_commands_test"
-            ]
-        if expansions["pcmdi_diags_environment_commands"] == "":
-            expansions["pcmdi_diags_environment_commands"] = expansions[
-                "environment_commands_test"
-            ]
-    else:
-        # The cfg doesn't need this line,
-        # but it would be difficult to only write environment_commands in the unified_testing case.
-        expansions["environment_commands"] = ""
     machine = expansions["machine"]
 
     if dry_run:
@@ -315,7 +304,6 @@ def generate_cfgs(unified_testing=False, dry_run=False):
         substitute_expansions(expansions, script_template, script_generated)
     print("CFG FILES HAVE BEEN GENERATED FROM TEMPLATES WITH THESE SETTINGS:")
     print(f"UNIQUE_ID={TEST_SPECIFICS['unique_id']}")
-    print(f"unified_testing={unified_testing}")
     print(f"diags_environment_commands={expansions['diags_environment_commands']}")
     print(
         f"global_time_series_environment_commands={expansions['global_time_series_environment_commands']}"
@@ -330,4 +318,4 @@ def generate_cfgs(unified_testing=False, dry_run=False):
 
 
 if __name__ == "__main__":
-    generate_cfgs(unified_testing=False, dry_run=False)
+    generate_cfgs(dry_run=False)