From 35b7a991c247f37e2a91bc0034a72a73cd2474df Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Wed, 19 Jun 2024 08:31:24 -0700 Subject: [PATCH 1/6] port yaml model setup into run --- src/ILAMB/run.py | 118 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 13 deletions(-) diff --git a/src/ILAMB/run.py b/src/ILAMB/run.py index 319dcc24..2d6c5f9e 100644 --- a/src/ILAMB/run.py +++ b/src/ILAMB/run.py @@ -2,6 +2,8 @@ import os import pickle from traceback import format_exc +import yaml +import inspect from mpi4py import MPI @@ -9,7 +11,7 @@ from ILAMB.ModelResult import ModelResult logger = logging.getLogger("%i" % MPI.COMM_WORLD.rank) - +rank = 0 def InitializeModels( model_root, @@ -107,14 +109,89 @@ def InitializeModels( return M +def _parse_model_yaml(filename: str, cache_path: str = "./", only_models: list = []): + """Setup models using a yaml file.""" + model_classes = { + "ModelResult": ModelResult, + } + models = [] + with open(filename, encoding="utf-8") as fin: + yml = yaml.safe_load(fin) + for name, opts in yml.items(): + # optionally filter models + if len(only_models) > 0 and name not in only_models: + continue + + if "name" not in opts: + opts["name"] = name + + # if the model_year option is given, convert to lits of floats + if "model_year" in opts: + opts["model_year"] = [ + float(y.strip()) for y in opts["model_year"].split(",") + ] + + # select the class type + cls = model_classes[opts["type"]] if "type" in opts else ModelResult + if cls is None: + typ = opts["type"] + raise ValueError(f"The model type '{typ}' is not available") + fcns = dir(cls) + + # if the pickle file exists, just load it + cache = os.path.join(cache_path, f"{name}.pkl") + if os.path.exists(cache): + if "read_pickle" in fcns: + model = cls().read_pickle(cache) + else: + with open(cache, mode="rb") as fin: + model = pickle.load(fin) + models.append(model) + continue + + # call the constructor using keywords defined in the YAML file + cls = model_classes[opts["type"]] if "type" in opts else ModelResult + model = cls( + **{ + key: opts[key] + for key in inspect.getfullargspec(cls).args + if key in opts + } + ) + + # some model types have a find_files() method, call if present loading + # proper keywords from the YAML file + if "find_files" in fcns: + model.find_files( + **{ + key: opts[key] + for key in inspect.getfullargspec(model.find_files).args + if key in opts + } + ) + + # some model types allow you to specify snynonms + if "add_synonym" in fcns and "synonyms" in opts: + for mvar, syn in opts["synonyms"].items(): + model.add_synonym(mvar, syn) + + # cache the model result + if rank == 0: + if "read_pickle" in fcns: + model.to_pickle(cache) + else: + with open(cache, mode="wb") as fin: + pickle.dump(model, fin) + + models.append(model) + + for model in models: + if isinstance(model.color, str) and model.color.startswith("#"): + model.color = clr.hex2color(model.color) + return models + def ParseModelSetup( - model_setup, - models=[], - verbose=False, - filter="", - regex="", - models_path="./", - comm=MPI.COMM_WORLD, + model_setup, models=[], verbose=False, filter="", regex="", models_path="./", comm=MPI.COMM_WORLD ): """Initializes a list of models @@ -137,12 +214,24 @@ def ParseModelSetup( a list of the model results, sorted alphabetically by name """ - rank = comm.rank + if rank == 0 and verbose: + print("\nSetting up model results from %s\n" % model_setup) + + # intercept if this is a yaml file + if model_setup.endswith(".yaml"): + M = _parse_model_yaml(model_setup, cache_path=models_path, only_models=models) + if rank == 0 and verbose: + for m in M: + print((" {0:>45}").format(m.name)) + if len(M) == 0: + print("No model results found") + comm.Barrier() + comm.Abort(0) + return M + # initialize the models M = [] max_model_name_len = 0 - if rank == 0 and verbose: - print("\nSetting up model results from %s\n" % model_setup) with open(model_setup) as f: for line in f.readlines(): if line.strip().startswith("#"): @@ -151,12 +240,15 @@ def ParseModelSetup( mname = None mdir = None model_year = None + mgrp = "" if len(line) >= 2: mname = line[0].strip() mdir = line[1].strip() # if mdir not a directory, then maybe path is relative to ILAMB_ROOT if not os.path.isdir(mdir): mdir = os.path.join(os.environ["ILAMB_ROOT"], mdir).strip() + if len(line) == 3: + mgrp = line[2].strip() if len(line) == 4: model_year = [float(line[2].strip()), float(line[3].strip())] max_model_name_len = max(max_model_name_len, len(mname)) @@ -174,6 +266,7 @@ def ParseModelSetup( filter=filter, regex=regex, model_year=model_year, + group=mgrp, ) except Exception as ex: logger.debug("[%s]" % mname, format_exc()) @@ -183,8 +276,7 @@ def ParseModelSetup( # assign unique colors clrs = il.GenerateDistinctColors(len(M)) for m in M: - clr = clrs.pop(0) - m.color = clr + m.color = clrs.pop(0) # save model objects as pickle files comm.Barrier() From cdadd201a969aabb08b9ae5cc8fbb682b1e6d722 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Wed, 19 Jun 2024 10:50:18 -0700 Subject: [PATCH 2/6] add to debug log for weird problem --- src/ILAMB/Regions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ILAMB/Regions.py b/src/ILAMB/Regions.py index a2db6672..4c0fa696 100644 --- a/src/ILAMB/Regions.py +++ b/src/ILAMB/Regions.py @@ -218,6 +218,9 @@ def getMask(self, label, var): ).argmax(axis=1) else: # if more globally defined, nearest neighbor is fine + if lat is None or var.lat is None: + msg = f"No latitudes are given either in the variable:\n{var}\n{var.lat}\nor in the region: {label} {lat}" + logger.debug(msg) rows = (np.abs(lat[:, np.newaxis] - var.lat)).argmin(axis=0) cols = (np.abs(lon[:, np.newaxis] - var.lon)).argmin(axis=0) if var.ndata: From 6bd0af2ec34967dd05fd1f5197753d946b993dce Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Wed, 19 Jun 2024 10:51:03 -0700 Subject: [PATCH 3/6] improve logic so regions like 'hilat' don't cause errors --- src/ILAMB/ilamblib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ILAMB/ilamblib.py b/src/ILAMB/ilamblib.py index ea0b0e3c..c433b271 100644 --- a/src/ILAMB/ilamblib.py +++ b/src/ILAMB/ilamblib.py @@ -1004,9 +1004,9 @@ def FromNetCDF4( lat_name = [] lon_name = [] for key in grp.variables.keys(): - if "lat" in key: + if key.lower().startswith("lat"): lat_name.append(key) - if "lon" in key: + if key.lower().startswith("lon"): lon_name.append(key) if "altitude" in key: depth_name = key From dd9c5d2463eb044a98c8a07194684957bb0cae39 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 27 Jun 2024 13:28:58 -0700 Subject: [PATCH 4/6] Wrong directory name --- src/ILAMB/data/ilamb.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ILAMB/data/ilamb.cfg b/src/ILAMB/data/ilamb.cfg index 5311d3f6..74ab63e8 100644 --- a/src/ILAMB/data/ilamb.cfg +++ b/src/ILAMB/data/ilamb.cfg @@ -103,7 +103,7 @@ force_emulation = True [HIPPOAToM] ctype = "ConfGSNF" variable = "gsnf" -source = "DATA/co2/HIPPO_AToM/HIPPO_AToM.nc" +source = "DATA/co2/HIPPOATom/HIPPO_AToM.nc" model_flux = "ra+rh-gpp" #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From cadd476c289485f4ed70f297e4dfe540b634584f Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 27 Jun 2024 13:30:51 -0700 Subject: [PATCH 5/6] add a error handle for scaling --- src/ILAMB/Confrontation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ILAMB/Confrontation.py b/src/ILAMB/Confrontation.py index 28ed9706..c29b4117 100644 --- a/src/ILAMB/Confrontation.py +++ b/src/ILAMB/Confrontation.py @@ -345,7 +345,8 @@ def stageData(self, m): t0=None if len(self.study_limits) != 2 else self.study_limits[0], tf=None if len(self.study_limits) != 2 else self.study_limits[1], ) - obs.data *= self.scale_factor + with np.errstate(all="ignore"): + obs.data *= self.scale_factor if obs.time is None: raise il.NotTemporalVariable() self.pruneRegions(obs) From b37a8bb7a8fec795fdcbe5925d1e6931a2237bf2 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Tue, 23 Jul 2024 12:18:44 -0700 Subject: [PATCH 6/6] duplicate name for colors --- src/ILAMB/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ILAMB/run.py b/src/ILAMB/run.py index 2d6c5f9e..92306e03 100644 --- a/src/ILAMB/run.py +++ b/src/ILAMB/run.py @@ -9,6 +9,7 @@ from ILAMB import ilamblib as il from ILAMB.ModelResult import ModelResult +from matplotlib import colors as mplclrs logger = logging.getLogger("%i" % MPI.COMM_WORLD.rank) rank = 0 @@ -187,7 +188,7 @@ def _parse_model_yaml(filename: str, cache_path: str = "./", only_models: list = for model in models: if isinstance(model.color, str) and model.color.startswith("#"): - model.color = clr.hex2color(model.color) + model.color = mplclrs.hex2color(model.color) return models def ParseModelSetup(