Skip to content

Adding feature to downloads collections from the ILAMB data server. #121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nocollier merged 3 commits into from
Jan 5, 2026
Merged

Adding feature to downloads collections from the ILAMB data server. #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f16a7f
Adding feature to downloads collections from the ILAMB data server. C…
jitendra-kumar Dec 30, 2025
8048abf
moving data requires a small fix to basin locations, use pathlib instead
nocollier Jan 5, 2026
fe2f4ec
move to the Test-Data collection
nocollier Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 20 additions & 10 deletions bin/ilamb-fetch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ def filehash(filepath):

def GenerateSha1sumFile(root, suffix=".nc"):
lines = ""
collection=os.path.basename(os.path.normpath(root))
print(f'Collection: {collection}')
for topdir, dirs, files in os.walk(root):
if topdir.startswith("_"):
continue
Expand All @@ -46,7 +48,7 @@ def GenerateSha1sumFile(root, suffix=".nc"):
size = os.path.getsize(fpath)
sha = filehash(fpath)
name = os.path.relpath(fpath, root)
lines += "%s %s\n" % (sha, name)
lines += "%s %s/%s\n" % (sha, collection, name)
return lines


Expand Down Expand Up @@ -86,9 +88,17 @@ parser.add_argument(
dest="remote_root",
metavar="PATH",
type=str,
default="https://www.ilamb.org/ILAMB-Data/",
default="https://www.ilamb.org/DATA",
help="Location on the remote system.",
)
parser.add_argument(
"-k",
"--collection",
dest="collection",
type=str,
default="ILAMB-Data",
help="Data collection to download. [Options: ILAMB-Data (default); ABoVE-Data; NGEEA-Data;]",
)
parser.add_argument(
"-c",
"--create",
Expand All @@ -112,7 +122,7 @@ args = parser.parse_args()

# use create mode if you want to make a checksum file of a directory
if args.create:
with open(args.local_root + "/SHA1SUM", mode="w") as f:
with open(args.local_root + "/SHA1SUM_" + args.collection, mode="w") as f:
f.write(GenerateSha1sumFile(args.local_root))
sys.exit()

Expand All @@ -123,7 +133,7 @@ print(

# download and build the sha1sum check files
try:
resp = requests.get(args.remote_root + "/SHA1SUM", verify=(not args.check))
resp = requests.get(args.remote_root + "/SHA1SUM_" + args.collection, verify=(not args.check))
except requests.exceptions.SSLError:
print(
"""
Expand All @@ -136,17 +146,17 @@ certificate check step.
)
sys.exit(1)

with open(args.local_root + "/SHA1SUM", "wb") as f:
with open(args.local_root + "/SHA1SUM_" + args.collection, "wb") as f:
f.write(resp.content)
if "404 Not Found" in open(args.local_root + "/SHA1SUM").read():
if "404 Not Found" in open(args.local_root + "/SHA1SUM_" + args.collection).read():
raise ValueError(
"Could not find the sha1 sum file: %s" % (args.remote_root + "/SHA1SUM")
"Could not find the sha1 sum file: %s" % (args.remote_root + "/SHA1SUM_" + args.collection)
)
needs_updating = CheckSha1sumFile(args.local_root + "/SHA1SUM", args.local_root)
needs_updating = CheckSha1sumFile(args.local_root + "/SHA1SUM_" + args.collection, args.local_root)

if len(needs_updating) == 0:
print("\nAll your data is up-to-date and clean.\n")
os.system("rm -f " + args.local_root + "/SHA1SUM")
os.system("rm -f " + args.local_root + "/SHA1SUM_" + args.collection)
sys.exit()

print("\nI found the following files which are missing, out of date, or corrupt:\n")
Expand Down Expand Up @@ -200,4 +210,4 @@ if reply[0] == "y":
pbar.update(len(ch))
print("\nDownload complete. Rerun ilamb-fetch to check file integrity.\n")

os.system("rm -f " + args.local_root + "/SHA1SUM")
os.system("rm -f " + args.local_root + "/SHA1SUM_" + args.collection)
8 changes: 3 additions & 5 deletions src/ILAMB/ConfTWSA.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
Expand Down Expand Up @@ -34,11 +35,8 @@ def __init__(self, **keywords):
# Adding a member variable called basins, add them as regions
r = Regions()
nbasins = self.keywords.get("nbasins", 30)
self.basins = r.addRegionNetCDF4(
os.path.join(
"/".join(self.source.split("/")[:-3]), "mrro/Dai/basins_0.5x0.5.nc"
)
)[:nbasins]
basin_source = Path(self.source).parent.parent / "mrro/Dai/basins_0.5x0.5.nc"
self.basins = r.addRegionNetCDF4(basin_source)[:nbasins]

def stageData(self, m):
r"""Extracts model data which is comparable to the observations.
Expand Down
6 changes: 3 additions & 3 deletions test/makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ NP = 1
OPTS =

all:
@ilamb-fetch --remote_root https://www.ilamb.org/Test-Data --no-check-certificate -y
@ilamb-fetch --no-check-certificate -y --collection Test-Data
@mpirun -n ${NP} ilamb-run \
--define_regions Whittaker.nc \
--define_regions Test-Data/Whittaker.nc \
--df_errs `python -c "import ILAMB; print(ILAMB.__path__[0] + '/data/quantiles_Whittaker_cmip5v6.parquet')"` \
--config test.cfg \
--model_root ./MODELS/ \
--model_root ./Test-Data/MODELS/ \
--regions global shsa \
--title "ILAMB Benchmark: test runs" ${OPTS}
@python score_diff.py scores_test.csv.gold _build/scores.csv
Expand Down
18 changes: 9 additions & 9 deletions test/test.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ weight = 5
mass_weighting = True

[Fluxnet]
source = "DATA/gpp.nc"
source = "Test-Data/DATA/gpp.nc"
weight = 9
table_unit = "g m-2 d-1"
plot_unit = "g m-2 d-1"
skip_iav = False

[GBAF]
source = "DATA/gpp_0.5x0.5.nc"
source = "Test-Data/DATA/gpp_0.5x0.5.nc"
weight = 15
table_unit = "Pg yr-1"
plot_unit = "g m-2 d-1"
Expand All @@ -35,7 +35,7 @@ weight = 5
ctype = "ConfNBP"

[Hoffman]
source = "DATA/nbp_1850-2010.nc"
source = "Test-Data/DATA/nbp_1850-2010.nc"
weight = 20

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand All @@ -47,7 +47,7 @@ weight = 5
mass_weighting = True

[GBAF]
source = "DATA/nee_0.5x0.5.nc"
source = "Test-Data/DATA/nee_0.5x0.5.nc"
weight = 4
table_unit = "Pg yr-1"
plot_unit = "g m-2 d-1"
Expand All @@ -67,7 +67,7 @@ weight = 5

[Dai]
ctype = "ConfRunoff"
source = "DATA/runoff.nc"
source = "Test-Data/DATA/runoff.nc"
weight = 15

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand All @@ -80,7 +80,7 @@ weight = 5
ctype = "ConfTWSA"

[GRACE]
source = "DATA/twsa_0.5x0.5.nc"
source = "Test-Data/DATA/twsa_0.5x0.5.nc"
weight = 25

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand All @@ -90,7 +90,7 @@ variable = "tsl"

[NSIDC]
ctype = "ConfPermafrost"
source = "DATA/Brown2002.nc"
source = "Test-Data/DATA/Brown2002.nc"
y0 = 1970.
yf = 2000.
Teps = 273.15
Expand All @@ -109,7 +109,7 @@ derived = "rsus/rsds"
weight = 1

[MODIS]
source = "DATA/albedo_0.5x0.5.nc"
source = "Test-Data/DATA/albedo_0.5x0.5.nc"
weight = 20

###########################################################################
Expand All @@ -124,5 +124,5 @@ variable = "tas"
weight = 2

[CRU]
source = "DATA/tas_0.5x0.5.nc"
source = "Test-Data/DATA/tas_0.5x0.5.nc"
weight = 25