Split global_time_series task

conda/dev.yml

@@ -1,3 +1,9 @@
+# Conda development environment for testing local source code changes to zppy before merging them to production (main branch).
+# To set up:
+# conda clean --all --y
+# conda env create -f conda/dev.yml -n env_name
+# pre-commit run --all-files
+# pip install .
 name: zppy_dev
 channels:
   - conda-forge

tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg

@@ -173,7 +173,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True

tests/integration/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg

@@ -46,7 +46,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 # (This cfg is the setup portion only)
 # [global_time_series]

tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg

@@ -34,7 +34,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True

tests/integration/generated/test_min_case_global_time_series_viewers_original_8_chrysalis.cfg

@@ -34,7 +34,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True

tests/integration/generated/test_min_case_mpas_analysis_chrysalis.cfg

@@ -22,4 +22,4 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"

tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg

@@ -176,7 +176,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1980-1984", "1980-1990",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -80,7 +80,7 @@ walltime = "00:30:00"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
   mapping_file = "glb"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  vars = ""
   years = "1985:1995:5",
 
   [[ land_monthly ]]
@@ -206,21 +206,94 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True
 climo_years = "1985-1989", "1990-1995",
 environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
-moc_file=mocTimeSeries_1985-1995.nc
-plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+#moc_file=mocTimeSeries_1985-1995.nc
+# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 ts_num_years = 5
 ts_years = "1985-1989", "1985-1995",
 walltime = "00:30:00"
 years = "1985-1995",
 
+  # Important parameter combinations ##########################################
+
+  [[ viewer_both ]]
+  # 1. make_viewer = True, plots_original set, >= 1 plots_<component> set
+  # NOTE: This is the case displayed in examples/post.v3.LR.historical_zppy_v3.cfg
+  make_viewer = True
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ viewer_original ]]
+  # 2. make_viewer = True, plots_original set, 0 plots_<component> set
+  make_viewer = True
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ viewer_component ]]
+  # 3. make_viewer = True, plots_original set, >= 1 plots_<component> set
+  make_viewer = True
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  plots_original = ""
+
+  # We can ignore this case for image checking.
+  # [[ viewer_none ]]
+  # # 4. make_viewer = True, plots_original not set, 0 plots_<component> set
+  # make_viewer = True
+  # plots_original = ""
+
+  [[ classic_pdf_both ]]
+  # 5. make_viewer = False, plots_original set, >= 1 plots_<component> set
+  # Default make_viewer = False
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ classic_pdf_original ]]
+  # 6. make_viewer = False, plots_original set, 0 plots_<component> set
+  # NOTE: This is the default case -- where all of the above parameters are at their default values.
+  # Default make_viewer = False
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ classic_pdf_component ]]
+  # 7. make_viewer = False, plots_original set, >= 1 plots_<component> set
+  # Default make_viewer = False
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  plots_original = ""
+
+  # We can ignore this case for image checking.
+  # [[ classic_pdf_none ]]
+  # # 8. make_viewer = False, plots_original not set, 0 plots_<component> set
+  # # Default make_viewer = False
+  # plots_original = ""
+
+  # Special cases #############################################################
+
+  [[ all_lnd_var_viewer ]]
+  # 1. plot ALL land variables
+  make_viewer = True
+  partition = "compute"
+  plots_lnd = "all"
+  plots_original = ""
+  walltime = "03:00:00"
+
+  [[ classic_original_8_no_ocn ]]
+  # 2. exclude ocean from original plots
+  plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
+
+
 [ilamb]
 active = True
 e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave"

tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg

@@ -176,7 +176,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1980-1984", "1980-1990",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True

tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg

@@ -206,7 +206,7 @@ partition = "compute"
 qos = "regular"
 shortTermArchive = True
 ts_years = "1985-1989", "1985-1995",
-walltime = "00:30:00"
+walltime = "02:00:00"
 
 [global_time_series]
 active = True

tests/integration/template_weekly_comprehensive_v3.cfg

@@ -80,7 +80,7 @@ walltime = "00:30:00"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
   mapping_file = "glb"
-  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  vars = ""
   years = "1985:1995:5",
 
   [[ land_monthly ]]
@@ -214,13 +214,86 @@ climo_years = "1985-1989", "1990-1995",
 environment_commands = "#expand global_time_series_environment_commands#"
 experiment_name = "#expand case_name#"
 figstr = "#expand case_name#"
-moc_file=mocTimeSeries_1985-1995.nc
-plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+#moc_file=mocTimeSeries_1985-1995.nc
+# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 ts_num_years = 5
 ts_years = "1985-1989", "1985-1995",
 walltime = "00:30:00"
 years = "1985-1995",
 
+  # Important parameter combinations ##########################################
+
+  [[ viewer_both ]]
+  # 1. make_viewer = True, plots_original set, >= 1 plots_<component> set
+  # NOTE: This is the case displayed in examples/post.v3.LR.historical_zppy_v3.cfg
+  make_viewer = True
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ viewer_original ]]
+  # 2. make_viewer = True, plots_original set, 0 plots_<component> set
+  make_viewer = True
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ viewer_component ]]
+  # 3. make_viewer = True, plots_original set, >= 1 plots_<component> set
+  make_viewer = True
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  plots_original = ""
+
+  # We can ignore this case for image checking.
+  # [[ viewer_none ]]
+  # # 4. make_viewer = True, plots_original not set, 0 plots_<component> set
+  # make_viewer = True
+  # plots_original = ""
+
+  [[ classic_pdf_both ]]
+  # 5. make_viewer = False, plots_original set, >= 1 plots_<component> set
+  # Default make_viewer = False
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ classic_pdf_original ]]
+  # 6. make_viewer = False, plots_original set, 0 plots_<component> set
+  # NOTE: This is the default case -- where all of the above parameters are at their default values.
+  # Default make_viewer = False
+  moc_file = "mocTimeSeries_1985-1995.nc"
+  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",
+
+  [[ classic_pdf_component ]]
+  # 7. make_viewer = False, plots_original set, >= 1 plots_<component> set
+  # Default make_viewer = False
+  plots_atm = "TREFHT"
+  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+  plots_original = ""
+
+  # We can ignore this case for image checking.
+  # [[ classic_pdf_none ]]
+  # # 8. make_viewer = False, plots_original not set, 0 plots_<component> set
+  # # Default make_viewer = False
+  # plots_original = ""
+
+  # Special cases #############################################################
+
+  [[ all_lnd_var_viewer ]]
+  # 1. plot ALL land variables
+  make_viewer = True
+  partition = "compute"
+  plots_lnd = "all"
+  plots_original = ""
+  walltime = "03:00:00"
+
+  [[ classic_original_8_no_ocn ]]
+  # 2. exclude ocean from original plots
+  plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
+
+
 [ilamb]
 active = #expand active_ilamb#
 e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave"

zppy/__main__.py

@@ -3,6 +3,7 @@
 import importlib
 import io
 import os
+import pprint
 import shutil
 from datetime import datetime, timezone
 from typing import Any, List, Tuple
@@ -156,6 +157,8 @@ def _validate_config(config):
 
     result = config.validate(validator)
     if result is not True:
+        pp = pprint.PrettyPrinter(indent=2, width=10)
+        pp.pprint(result)
         logger.critical("Validation results={}".format(result))
         raise ValueError(
             "Configuration file validation failed. Parameters listed as false in the validation results have invalid values."

zppy/defaults/default.ini

@@ -318,12 +318,20 @@ color = string(default="Blue")
 experiment_name = string(default="")
 # What the plot files should be named
 figstr = string(default="")
+# Set to True to construct a Viewer page
+make_viewer = boolean(default=False)
+# regions to plot: glb, n, s (global, northern hemisphere, southern hemisphere)
+regions = string(default="glb,n,s")
+ts_years = string_list(default=list(""))
+# `years = "1-100",` would plot years 1 to 100 on the graphs.
+#
+# These are only needed if make_viewer = False (default):
+#
 # The specific subdirectory with the ocean data
 # NOTE: always overrides value in [default]
 input_subdir = string(default="archive/ocn/hist")
-# Set to True to construct a Viewer page
-make_viewer = boolean(default=False)
 moc_file = string(default="")
+# nrows, ncols are DEPRECATED. Specifying them will have no effect.
 # Number of columns per page
 ncols = integer(default=2)
 # Number of rows per page
@@ -337,17 +345,34 @@ nrows = integer(default=4)
 # net_atm_water_imbalance requires PRECC, PRECL, QFLX
 # Remove the 3 ocean plots (change_ohc,max_moc,change_sea_level) if you don't have ocean data.
 plots_original = string(default="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance")
+#
+# These are only needed if make_viewer = True:
+#
 # The names of the extra plots you want displayed (i.e., define specific variables)
 # These should be a subset of the `vars` generated by the `ts` `glb` subtasks.
 plots_atm = string(default="")
 plots_ice = string(default="")
 # Set `plots_lnd = "all"` to run every variable in the land csv file.
 plots_lnd = string(default="")
 plots_ocn = string(default="")
-# regions to plot: glb, n, s (global, northern hemisphere, southern hemisphere)
-regions = string(default="glb,n,s")
-ts_years = string_list(default=list(""))
-# `years = "1-100",` would plot years 1 to 100 on the graphs.
+
+  [[__many__]]
+  climo_years = string_list(default=None)
+  color = string(default=None)
+  experiment_name = string(default=None)
+  figstr = string(default=None)
+  input_subdir = string(default=None)
+  make_viewer = boolean(default=None)
+  moc_file = string(default=None)
+  ncols = integer(default=None)
+  nrows = integer(default=None)
+  plots_original = string(default=None)
+  plots_atm = string(default=None)
+  plots_ice = string(default=None)
+  plots_lnd = string(default=None)
+  plots_ocn = string(default=None)
+  regions = string(default=None)
+  ts_years = string_list(default=None)
 
 [ilamb]
 cfg = string(default="inclusions/ilamb/ilamb.cfg")

zppy/global_time_series.py

@@ -40,7 +40,11 @@ def global_time_series(config, script_dir, existing_bundles, job_ids_file):
             if ("last_year" in c.keys()) and (c["year2"] > c["last_year"]):
                 continue  # Skip this year set
             c["scriptDir"] = script_dir
-            prefix: str = f"global_time_series_{c['year1']:04d}-{c['year2']:04d}"
+            prefix: str
+            if c["subsection"]:
+                prefix = f"global_time_series_{c['subsection']}_{c['year1']:04d}-{c['year2']:04d}"
+            else:
+                prefix = f"global_time_series_{c['year1']:04d}-{c['year2']:04d}"
             print(prefix)
             c["prefix"] = prefix
             bash_file, settings_file, status_file = get_file_names(script_dir, prefix)