From 4f16a7f7b2cfa27c701ec6e46a543673e3b9055b Mon Sep 17 00:00:00 2001 From: "Jitendra (Jitu) Kumar" <7295155+jitendra-kumar@users.noreply.github.com> Date: Tue, 30 Dec 2025 12:56:24 -0500 Subject: [PATCH 1/3] Adding feature to downloads collections from the ILAMB data server. Currently support the ILAMB-Data (default), ABoVE-Data, and NGEEA-Data collections. --- bin/ilamb-fetch | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/bin/ilamb-fetch b/bin/ilamb-fetch index 158ba901..6ece7282 100755 --- a/bin/ilamb-fetch +++ b/bin/ilamb-fetch @@ -35,6 +35,8 @@ def filehash(filepath): def GenerateSha1sumFile(root, suffix=".nc"): lines = "" + collection=os.path.basename(os.path.normpath(root)) + print(f'Collection: {collection}') for topdir, dirs, files in os.walk(root): if topdir.startswith("_"): continue @@ -46,7 +48,7 @@ def GenerateSha1sumFile(root, suffix=".nc"): size = os.path.getsize(fpath) sha = filehash(fpath) name = os.path.relpath(fpath, root) - lines += "%s %s\n" % (sha, name) + lines += "%s %s/%s\n" % (sha, collection, name) return lines @@ -86,9 +88,17 @@ parser.add_argument( dest="remote_root", metavar="PATH", type=str, - default="https://www.ilamb.org/ILAMB-Data/", + default="https://www.ilamb.org/DATA", help="Location on the remote system.", ) +parser.add_argument( + "-k", + "--collection", + dest="collection", + type=str, + default="ILAMB-Data", + help="Data collection to download. [Options: ILAMB-Data (default); ABoVE-Data; NGEEA-Data;]", +) parser.add_argument( "-c", "--create", @@ -112,7 +122,7 @@ args = parser.parse_args() # use create mode if you want to make a checksum file of a directory if args.create: - with open(args.local_root + "/SHA1SUM", mode="w") as f: + with open(args.local_root + "/SHA1SUM_" + args.collection, mode="w") as f: f.write(GenerateSha1sumFile(args.local_root)) sys.exit() @@ -123,7 +133,7 @@ print( # download and build the sha1sum check files try: - resp = requests.get(args.remote_root + "/SHA1SUM", verify=(not args.check)) + resp = requests.get(args.remote_root + "/SHA1SUM_" + args.collection, verify=(not args.check)) except requests.exceptions.SSLError: print( """ @@ -136,17 +146,17 @@ certificate check step. ) sys.exit(1) -with open(args.local_root + "/SHA1SUM", "wb") as f: +with open(args.local_root + "/SHA1SUM_" + args.collection, "wb") as f: f.write(resp.content) -if "404 Not Found" in open(args.local_root + "/SHA1SUM").read(): +if "404 Not Found" in open(args.local_root + "/SHA1SUM_" + args.collection).read(): raise ValueError( - "Could not find the sha1 sum file: %s" % (args.remote_root + "/SHA1SUM") + "Could not find the sha1 sum file: %s" % (args.remote_root + "/SHA1SUM_" + args.collection) ) -needs_updating = CheckSha1sumFile(args.local_root + "/SHA1SUM", args.local_root) +needs_updating = CheckSha1sumFile(args.local_root + "/SHA1SUM_" + args.collection, args.local_root) if len(needs_updating) == 0: print("\nAll your data is up-to-date and clean.\n") - os.system("rm -f " + args.local_root + "/SHA1SUM") + os.system("rm -f " + args.local_root + "/SHA1SUM_" + args.collection) sys.exit() print("\nI found the following files which are missing, out of date, or corrupt:\n") @@ -200,4 +210,4 @@ if reply[0] == "y": pbar.update(len(ch)) print("\nDownload complete. Rerun ilamb-fetch to check file integrity.\n") -os.system("rm -f " + args.local_root + "/SHA1SUM") +os.system("rm -f " + args.local_root + "/SHA1SUM_" + args.collection) From 8048abff5377c0735718d9c58fac380aaa113261 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Mon, 5 Jan 2026 11:28:01 -0500 Subject: [PATCH 2/3] moving data requires a small fix to basin locations, use pathlib instead --- src/ILAMB/ConfTWSA.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ILAMB/ConfTWSA.py b/src/ILAMB/ConfTWSA.py index ff611dbe..569961dd 100644 --- a/src/ILAMB/ConfTWSA.py +++ b/src/ILAMB/ConfTWSA.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import matplotlib.pyplot as plt import numpy as np @@ -34,11 +35,8 @@ def __init__(self, **keywords): # Adding a member variable called basins, add them as regions r = Regions() nbasins = self.keywords.get("nbasins", 30) - self.basins = r.addRegionNetCDF4( - os.path.join( - "/".join(self.source.split("/")[:-3]), "mrro/Dai/basins_0.5x0.5.nc" - ) - )[:nbasins] + basin_source = Path(self.source).parent.parent / "mrro/Dai/basins_0.5x0.5.nc" + self.basins = r.addRegionNetCDF4(basin_source)[:nbasins] def stageData(self, m): r"""Extracts model data which is comparable to the observations. From fe2f4ec3a9d70db397404bd8d526a96d5a1c2831 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Mon, 5 Jan 2026 11:28:32 -0500 Subject: [PATCH 3/3] move to the Test-Data collection --- test/makefile | 6 +++--- test/test.cfg | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/makefile b/test/makefile index e0a22700..885c6bb2 100644 --- a/test/makefile +++ b/test/makefile @@ -2,12 +2,12 @@ NP = 1 OPTS = all: - @ilamb-fetch --remote_root https://www.ilamb.org/Test-Data --no-check-certificate -y + @ilamb-fetch --no-check-certificate -y --collection Test-Data @mpirun -n ${NP} ilamb-run \ - --define_regions Whittaker.nc \ + --define_regions Test-Data/Whittaker.nc \ --df_errs `python -c "import ILAMB; print(ILAMB.__path__[0] + '/data/quantiles_Whittaker_cmip5v6.parquet')"` \ --config test.cfg \ - --model_root ./MODELS/ \ + --model_root ./Test-Data/MODELS/ \ --regions global shsa \ --title "ILAMB Benchmark: test runs" ${OPTS} @python score_diff.py scores_test.csv.gold _build/scores.csv diff --git a/test/test.cfg b/test/test.cfg index 1e29e331..b2d26ed0 100644 --- a/test/test.cfg +++ b/test/test.cfg @@ -13,14 +13,14 @@ weight = 5 mass_weighting = True [Fluxnet] -source = "DATA/gpp.nc" +source = "Test-Data/DATA/gpp.nc" weight = 9 table_unit = "g m-2 d-1" plot_unit = "g m-2 d-1" skip_iav = False [GBAF] -source = "DATA/gpp_0.5x0.5.nc" +source = "Test-Data/DATA/gpp_0.5x0.5.nc" weight = 15 table_unit = "Pg yr-1" plot_unit = "g m-2 d-1" @@ -35,7 +35,7 @@ weight = 5 ctype = "ConfNBP" [Hoffman] -source = "DATA/nbp_1850-2010.nc" +source = "Test-Data/DATA/nbp_1850-2010.nc" weight = 20 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -47,7 +47,7 @@ weight = 5 mass_weighting = True [GBAF] -source = "DATA/nee_0.5x0.5.nc" +source = "Test-Data/DATA/nee_0.5x0.5.nc" weight = 4 table_unit = "Pg yr-1" plot_unit = "g m-2 d-1" @@ -67,7 +67,7 @@ weight = 5 [Dai] ctype = "ConfRunoff" -source = "DATA/runoff.nc" +source = "Test-Data/DATA/runoff.nc" weight = 15 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -80,7 +80,7 @@ weight = 5 ctype = "ConfTWSA" [GRACE] -source = "DATA/twsa_0.5x0.5.nc" +source = "Test-Data/DATA/twsa_0.5x0.5.nc" weight = 25 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -90,7 +90,7 @@ variable = "tsl" [NSIDC] ctype = "ConfPermafrost" -source = "DATA/Brown2002.nc" +source = "Test-Data/DATA/Brown2002.nc" y0 = 1970. yf = 2000. Teps = 273.15 @@ -109,7 +109,7 @@ derived = "rsus/rsds" weight = 1 [MODIS] -source = "DATA/albedo_0.5x0.5.nc" +source = "Test-Data/DATA/albedo_0.5x0.5.nc" weight = 20 ########################################################################### @@ -124,5 +124,5 @@ variable = "tas" weight = 2 [CRU] -source = "DATA/tas_0.5x0.5.nc" +source = "Test-Data/DATA/tas_0.5x0.5.nc" weight = 25