diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 9efd7bb7..3a46e9b7 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -1,7 +1,7 @@ [default] case = v2.LR.historical_0201 constraint = "" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.8.0rc6_chrysalis.sh" +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "/home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index 16629b0a..d0187318 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -1,7 +1,7 @@ [default] case = v2.LR.historical_0201 constraint = "" -environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.8.0rc6_chrysalis.sh" +environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh" input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "/home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 5e16308a..8231ebbf 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -281,4 +281,4 @@ def generate_cfgs(unified_testing=False): if __name__ == "__main__": - generate_cfgs(True) + generate_cfgs() diff --git a/zppy/__main__.py b/zppy/__main__.py index f80cfc19..aa8fe811 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -66,6 +66,7 @@ def main(): # noqa: C901 username = os.environ.get("USER") output = output.replace("$USER", username) scriptDir = os.path.join(output, "post/scripts") + job_ids_file = os.path.join(scriptDir, "jobids.txt") try: os.makedirs(scriptDir) except OSError as exc: @@ -134,28 +135,30 @@ def main(): # noqa: C901 existing_bundles = predefined_bundles(config, scriptDir, existing_bundles) # climo tasks - existing_bundles = climo(config, scriptDir, existing_bundles) + existing_bundles = climo(config, scriptDir, existing_bundles, job_ids_file) # time series tasks - existing_bundles = ts(config, scriptDir, existing_bundles) + existing_bundles = ts(config, scriptDir, existing_bundles, job_ids_file) # tc_analysis tasks - existing_bundles = tc_analysis(config, scriptDir, existing_bundles) + existing_bundles = tc_analysis(config, scriptDir, existing_bundles, job_ids_file) # e3sm_diags tasks - existing_bundles = e3sm_diags(config, scriptDir, existing_bundles) + existing_bundles = e3sm_diags(config, scriptDir, existing_bundles, job_ids_file) # amwg tasks - existing_bundles = amwg(config, scriptDir, existing_bundles) + existing_bundles = amwg(config, scriptDir, existing_bundles, job_ids_file) # mpas_analysis tasks - existing_bundles = mpas_analysis(config, scriptDir, existing_bundles) + existing_bundles = mpas_analysis(config, scriptDir, existing_bundles, job_ids_file) # global time series tasks - existing_bundles = global_time_series(config, scriptDir, existing_bundles) + existing_bundles = global_time_series( + config, scriptDir, existing_bundles, job_ids_file + ) # ilamb tasks - existing_bundles = ilamb(config, scriptDir, existing_bundles) + existing_bundles = ilamb(config, scriptDir, existing_bundles, job_ids_file) # Submit bundle jobs for b in existing_bundles: @@ -169,6 +172,7 @@ def main(): # noqa: C901 b.bundle_file, b.bundle_status, b.export, + job_ids_file, dependFiles=b.dependencies_external, ) diff --git a/zppy/amwg.py b/zppy/amwg.py index 912f73a7..40c882bb 100644 --- a/zppy/amwg.py +++ b/zppy/amwg.py @@ -8,7 +8,7 @@ # ----------------------------------------------------------------------------- -def amwg(config, scriptDir, existing_bundles): +def amwg(config, scriptDir, existing_bundles, job_ids_file): # Initialize jinja2 template engine templateLoader = jinja2.FileSystemLoader( @@ -79,7 +79,11 @@ def amwg(config, scriptDir, existing_bundles): if c["bundle"] == "": # Submit job submitScript( - scriptFile, statusFile, export, dependFiles=dependencies + scriptFile, + statusFile, + export, + job_ids_file, + dependFiles=dependencies, ) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/climo.py b/zppy/climo.py index 7b5dd4ad..bcfd21de 100644 --- a/zppy/climo.py +++ b/zppy/climo.py @@ -16,7 +16,7 @@ # ----------------------------------------------------------------------------- -def climo(config, scriptDir, existing_bundles): +def climo(config, scriptDir, existing_bundles, job_ids_file): # --- Initialize jinja2 template engine --- templateLoader = jinja2.FileSystemLoader( @@ -95,7 +95,7 @@ def climo(config, scriptDir, existing_bundles): if not c["dry_run"]: if c["bundle"] == "": # Submit job - submitScript(scriptFile, statusFile, export) + submitScript(scriptFile, statusFile, export, job_ids_file) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index 6f4cefc9..3425e981 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -17,7 +17,7 @@ # ----------------------------------------------------------------------------- # FIXME: C901 'e3sm_diags' is too complex (20) -def e3sm_diags(config, scriptDir, existing_bundles): # noqa: C901 +def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 # Initialize jinja2 template engine templateLoader = jinja2.FileSystemLoader( @@ -214,7 +214,11 @@ def e3sm_diags(config, scriptDir, existing_bundles): # noqa: C901 if c["bundle"] == "": # Submit job submitScript( - scriptFile, statusFile, export, dependFiles=dependencies + scriptFile, + statusFile, + export, + job_ids_file, + dependFiles=dependencies, ) # Due to a `socket.gaierror: [Errno -2] Name or service not known` error when running e3sm_diags with tc_analysis diff --git a/zppy/global_time_series.py b/zppy/global_time_series.py index 4cbaedd8..c46fe992 100644 --- a/zppy/global_time_series.py +++ b/zppy/global_time_series.py @@ -15,7 +15,7 @@ # ----------------------------------------------------------------------------- -def global_time_series(config, scriptDir, existing_bundles): +def global_time_series(config, scriptDir, existing_bundles, job_ids_file): # Initialize jinja2 template engine templateLoader = jinja2.FileSystemLoader( @@ -124,7 +124,11 @@ def global_time_series(config, scriptDir, existing_bundles): if c["bundle"] == "": # Submit job submitScript( - scriptFile, statusFile, export, dependFiles=dependencies + scriptFile, + statusFile, + export, + job_ids_file, + dependFiles=dependencies, ) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/ilamb.py b/zppy/ilamb.py index f924bf0d..ab899472 100644 --- a/zppy/ilamb.py +++ b/zppy/ilamb.py @@ -16,7 +16,7 @@ # ----------------------------------------------------------------------------- -def ilamb(config, scriptDir, existing_bundles): +def ilamb(config, scriptDir, existing_bundles, job_ids_file): # Initialize jinja2 template engine templateLoader = jinja2.FileSystemLoader( @@ -118,7 +118,11 @@ def ilamb(config, scriptDir, existing_bundles): if c["bundle"] == "": # Submit job submitScript( - scriptFile, statusFile, export, dependFiles=dependencies + scriptFile, + statusFile, + export, + job_ids_file, + dependFiles=dependencies, ) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/mpas_analysis.py b/zppy/mpas_analysis.py index 7bfa0faa..fcd2325d 100644 --- a/zppy/mpas_analysis.py +++ b/zppy/mpas_analysis.py @@ -15,7 +15,7 @@ # ----------------------------------------------------------------------------- -def mpas_analysis(config, scriptDir, existing_bundles): +def mpas_analysis(config, scriptDir, existing_bundles, job_ids_file): # Initialize jinja2 template engine templateLoader = jinja2.FileSystemLoader( @@ -123,7 +123,11 @@ def mpas_analysis(config, scriptDir, existing_bundles): if c["bundle"] == "": # Submit job submitScript( - scriptFile, statusFile, export, dependFiles=dependencies + scriptFile, + statusFile, + export, + job_ids_file, + dependFiles=dependencies, ) # Note that this line should still be executed even if jobid == -1 diff --git a/zppy/tc_analysis.py b/zppy/tc_analysis.py index 97fdb3d1..df1f5b8f 100644 --- a/zppy/tc_analysis.py +++ b/zppy/tc_analysis.py @@ -9,7 +9,7 @@ # ----------------------------------------------------------------------------- -def tc_analysis(config, scriptDir, existing_bundles): +def tc_analysis(config, scriptDir, existing_bundles, job_ids_file): # Initialize jinja2 template engine templateLoader = jinja2.FileSystemLoader( @@ -78,7 +78,11 @@ def tc_analysis(config, scriptDir, existing_bundles): if c["bundle"] == "": # Submit job submitScript( - scriptFile, statusFile, export, dependFiles=dependencies + scriptFile, + statusFile, + export, + job_ids_file, + dependFiles=dependencies, ) # Note that this line should still be executed even if jobid == -1 diff --git a/zppy/ts.py b/zppy/ts.py index b5b0730f..de77ce2d 100644 --- a/zppy/ts.py +++ b/zppy/ts.py @@ -17,7 +17,7 @@ # ----------------------------------------------------------------------------- -def ts(config, scriptDir, existing_bundles): +def ts(config, scriptDir, existing_bundles, job_ids_file): # --- Initialize jinja2 template engine --- templateLoader = jinja2.FileSystemLoader( @@ -108,7 +108,7 @@ def ts(config, scriptDir, existing_bundles): if not c["dry_run"]: if c["bundle"] == "": # Submit job - submitScript(scriptFile, statusFile, export) + submitScript(scriptFile, statusFile, export, job_ids_file) else: print("...adding to bundle '%s'" % (c["bundle"])) diff --git a/zppy/utils.py b/zppy/utils.py index ebf8c9ce..3cf9f5a2 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -153,7 +153,7 @@ def getComponent(input_files): # ----------------------------------------------------------------------------- -def submitScript(scriptFile, statusFile, export, dependFiles=[]): +def submitScript(scriptFile, statusFile, export, job_ids_file, dependFiles=[]): # id of submitted job, or -1 if not submitted jobid = None @@ -209,6 +209,11 @@ def submitScript(scriptFile, statusFile, export, dependFiles=[]): print(stderr) raise RuntimeError(error_str) jobid = int(out.split()[-1]) + with open(job_ids_file, "a") as j: + # To include the scriptFile, use this line: + # j.write(f"{scriptFile}: {jobid}\n") + # To cancel all jobs associated with this zppy run, use `xargs scancel < jobids.txt`. + j.write(f"{jobid}\n") # Small pause to avoid overloading queueing system time.sleep(0.2)