Skip to content

Commit d97db74

Browse files
author
dpath2o
committed
some significant additions to classification and metrics that allow for bin_min/win_days and mean_period to be given as optional inputs at command line. This also entailed a new directory stucture in storing/writing the data
1 parent 28924f5 commit d97db74

19 files changed

+29250
-6726
lines changed

‎notebooks/FI_processing.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1807,9 +1807,9 @@
18071807
],
18081808
"metadata": {
18091809
"kernelspec": {
1810-
"display_name": "Python 3 (ipykernel)",
1810+
"display_name": "Python [conda env:afim-geo]",
18111811
"language": "python",
1812-
"name": "python3"
1812+
"name": "conda-env-afim-geo-py"
18131813
},
18141814
"language_info": {
18151815
"codemirror_mode": {
@@ -1821,7 +1821,7 @@
18211821
"name": "python",
18221822
"nbconvert_exporter": "python",
18231823
"pygments_lexer": "ipython3",
1824-
"version": "3.11.12"
1824+
"version": "3.12.13"
18251825
}
18261826
},
18271827
"nbformat": 4,

‎notebooks/paper1.ipynb‎

Lines changed: 26290 additions & 4640 deletions
Large diffs are not rendered by default.

‎notebooks/paper2.ipynb‎

Lines changed: 572 additions & 664 deletions
Large diffs are not rendered by default.

‎notebooks/paper3.ipynb‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "083d7816-7500-4922-bef2-ecbe6865cd51",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import sys, os, pygmt, importlib\n",
11+
"mod_path = '/home/581/da1339/AFIM/src/AFIM/src'\n",
12+
"sys.path.insert(0, mod_path)\n",
13+
"from sea_ice_toolbox import SeaIceToolbox, SeaIceToolboxManager\n",
14+
"import numpy as np\n",
15+
"import pandas as pd\n",
16+
"import xarray as xr\n",
17+
"import xesmf as xe\n",
18+
"import matplotlib.pyplot as plt\n",
19+
"from pathlib import Path"
20+
]
21+
},
22+
{
23+
"cell_type": "code",
24+
"execution_count": null,
25+
"id": "6dc5e1d8-54a0-42f2-8682-e7dd470b4ed5",
26+
"metadata": {},
27+
"outputs": [],
28+
"source": [
29+
"load_vars = [\"aice\", \"tarea\", \"hi\", \"uvel\", \"vvel\", \"strength\",\n",
30+
" \"dvidtt\", \"daidtt\", \"dvidtd\", \"daidtd\",\n",
31+
" \"KuxE\", \"KuxN\", \"KuyE\", \"KuyN\",\n",
32+
" \"earea\", \"narea\", \"uarea\"]\n",
33+
"P_JSON = Path.home() / \"AFIM\" / \"src\" / \"AFIM\" / \"src\" / \"JSONs\" / \"sea_ice_access-om3.json\"\n",
34+
"P_log = Path.home() / \"logs\" / f\"LD_testing.log\"\n",
35+
"mgr = SeaIceToolboxManager(P_log = P_log)\n",
36+
"tb = mgr.get_toolbox(P_json = P_JSON,\n",
37+
" sim_name = \"LD-om3-exp02\", \n",
38+
" dt0_str = \"1993-01-01\",\n",
39+
" dtN_str = \"1993-12-31\",\n",
40+
" list_of_BorC2T = \"Tc\",\n",
41+
" ice_type = \"FI\",\n",
42+
" hemisphere = \"south\")\n",
43+
"CICE_SO = tb.load_cice_zarr(slice_hem=True, variables=load_vars)\n",
44+
"SIA_SH = tb.compute_hemisphere_ice_area(CICE_SO['aice'], CICE_SO['tarea'],\n",
45+
" ice_area_scale=tb.SIC_scale,\n",
46+
" add_grounded_iceberg_area=False)\n",
47+
"tb.define_ice_mask_name(ice_type=\"FI\")\n",
48+
"I_day = tb.load_classified_ice(class_method=\"raw\")[tb.mask_name]\n",
49+
"I_bin = tb.load_classified_ice(class_method=\"binary-days\")[tb.mask_name]"
50+
]
51+
},
52+
{
53+
"cell_type": "code",
54+
"execution_count": null,
55+
"id": "c7fde2c6-5365-4d42-9e85-9dd817b4d44a",
56+
"metadata": {},
57+
"outputs": [],
58+
"source": [
59+
"A = CICE_SO[\"tarea\"].isel(time=0)\n",
60+
"# Apply mask(s)\n",
61+
"I_daily = CICE_SO.where(I_day)\n",
62+
"I_binly = CICE_SO.where(I_bin)\n",
63+
"FIA = tb.compute_hemisphere_ice_area(I_binly['aice'], I_binly['tarea'],\n",
64+
" ice_area_scale=tb.FIC_scale,\n",
65+
" add_grounded_iceberg_area=False)\n",
66+
"FIA.plot()\n",
67+
"# # Build dict(s)\n",
68+
"# I_dy = tb.metrics_data_dict(I_day, I_daily, A)\n",
69+
"# I_bn = tb.metrics_data_dict(I_bin, I_binly, A)\n",
70+
"# tb.define_metrics_zarr(class_method=\"raw\")\n",
71+
"# tb.compute_sea_ice_metrics(I_dy, tb.D_mets_zarr)\n",
72+
"# tb.define_metrics_zarr(class_method=\"binary-days\")\n",
73+
"# tb.compute_sea_ice_metrics(I_bn, tb.D_mets_zarr)"
74+
]
75+
},
76+
{
77+
"cell_type": "code",
78+
"execution_count": null,
79+
"id": "d74b0a65-1bca-4ba1-a00d-9cd9f603ba3d",
80+
"metadata": {},
81+
"outputs": [],
82+
"source": [
83+
"FIT = tb.compute_hemisphere_ice_thickness(I_binly['aice'], I_binly['hi'], I_binly['tarea'])\n",
84+
"FIT.plot()"
85+
]
86+
},
87+
{
88+
"cell_type": "code",
89+
"execution_count": null,
90+
"id": "c7ab7874-824c-4e33-b7e8-4b5ccffb8b71",
91+
"metadata": {},
92+
"outputs": [],
93+
"source": [
94+
"SIA = tb.compute_hemisphere_ice_area(CICE_SO['aice'], CICE_SO['tarea'],\n",
95+
" ice_area_scale=tb.SIC_scale,\n",
96+
" add_grounded_iceberg_area=False)\n",
97+
"SIA.plot()"
98+
]
99+
},
100+
{
101+
"cell_type": "code",
102+
"execution_count": null,
103+
"id": "bcc0c7a2-66e8-41e8-bc47-e7bae358024b",
104+
"metadata": {},
105+
"outputs": [],
106+
"source": [
107+
"SIT = tb.compute_hemisphere_ice_thickness(CICE_SO['aice'], CICE_SO['hi'], CICE_SO['tarea'])\n",
108+
"SIT.plot()"
109+
]
110+
}
111+
],
112+
"metadata": {
113+
"kernelspec": {
114+
"display_name": "Python [conda env:analysis3-26.02]",
115+
"language": "python",
116+
"name": "conda-env-analysis3-26.02-py"
117+
},
118+
"language_info": {
119+
"codemirror_mode": {
120+
"name": "ipython",
121+
"version": 3
122+
},
123+
"file_extension": ".py",
124+
"mimetype": "text/x-python",
125+
"name": "python",
126+
"nbconvert_exporter": "python",
127+
"pygments_lexer": "ipython3",
128+
"version": "3.11.14"
129+
}
130+
},
131+
"nbformat": 4,
132+
"nbformat_minor": 5
133+
}

‎scripts/CICE/re_organise_afim_archive.sh‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
shopt -s nullglob
4+
5+
ARCHIVE_ROOT="${HOME}/AFIM_archive"
6+
BIN_WIN_DAYS="11"
7+
BIN_MIN_DAYS="09"
8+
MEAN_PERIOD="15"
9+
EXECUTE=0
10+
11+
usage() {
12+
cat <<EOF
13+
Usage: $(basename "$0") [options]
14+
15+
Options:
16+
--archive-root PATH Root archive directory (default: ~/AFIM_archive)
17+
--bin-win NN Binary window days, zero-padded or integer (default: 08)
18+
--bin-min NN Binary minimum days (default: 05)
19+
--mean-period NN Rolling mean period days (default: 05)
20+
--execute Actually move files (default is dry-run)
21+
-h, --help Show this help
22+
23+
Moves old stores like:
24+
SH/ispd_thresh_5.0e-4/FI_Tb.zarr
25+
SH/ispd_thresh_5.0e-4/FI_Tb_mets.zarr
26+
SH/ispd_thresh_5.0e-4/FI_Tb_bin.zarr
27+
SH/ispd_thresh_5.0e-4/FI_Tb_bin_mets.zarr
28+
SH/ispd_thresh_5.0e-4/FI_Tb_roll.zarr
29+
SH/ispd_thresh_5.0e-4/FI_Tb_roll_mets.zarr
30+
31+
to:
32+
SH/ispd_thresh_5.0e-4/FI/Tb/raw.zarr
33+
SH/ispd_thresh_5.0e-4/FI/Tb/mets.zarr
34+
SH/ispd_thresh_5.0e-4/FI/Tb/bin-win-08_bin-min-05/raw.zarr
35+
SH/ispd_thresh_5.0e-4/FI/Tb/bin-win-08_bin-min-05/mets.zarr
36+
SH/ispd_thresh_5.0e-4/FI/Tb/roll-days-05/raw.zarr
37+
SH/ispd_thresh_5.0e-4/FI/Tb/roll-days-05/mets.zarr
38+
EOF
39+
}
40+
41+
while [[ $# -gt 0 ]]; do
42+
case "$1" in
43+
--archive-root) ARCHIVE_ROOT="$2"; shift 2 ;;
44+
--bin-win) BIN_WIN_DAYS="$(printf "%02d" "$2")"; shift 2 ;;
45+
--bin-min) BIN_MIN_DAYS="$(printf "%02d" "$2")"; shift 2 ;;
46+
--mean-period) MEAN_PERIOD="$(printf "%02d" "$2")"; shift 2 ;;
47+
--execute) EXECUTE=1; shift ;;
48+
-h|--help) usage; exit 0 ;;
49+
*) echo "Unknown option: $1" >&2; usage; exit 1 ;;
50+
esac
51+
done
52+
53+
move_store() {
54+
local src="$1"
55+
local dst="$2"
56+
[[ -e "$src" ]] || return 0
57+
mkdir -p "$(dirname "$dst")"
58+
if [[ -e "$dst" ]]; then
59+
echo "SKIP (destination exists): $dst"
60+
return 0
61+
fi
62+
if [[ $EXECUTE -eq 1 ]]; then
63+
echo "MOVE: $src -> $dst"
64+
mv "$src" "$dst"
65+
else
66+
echo "DRYRUN: mv '$src' '$dst'"
67+
fi
68+
}
69+
70+
for sim_dir in "${ARCHIVE_ROOT}"/*; do
71+
[[ -d "${sim_dir}/zarr/SH" ]] || continue
72+
sh_root="${sim_dir}/zarr/SH"
73+
echo "=== simulation: $(basename "$sim_dir") ==="
74+
# Optional SI / MIZ legacy layout
75+
move_store "${sh_root}/SI.zarr" "${sh_root}/SI/raw.zarr"
76+
move_store "${sh_root}/SI_mets.zarr" "${sh_root}/SI/mets.zarr"
77+
move_store "${sh_root}/MIZ.zarr" "${sh_root}/MIZ/raw.zarr"
78+
move_store "${sh_root}/MIZ_mets.zarr" "${sh_root}/MIZ/mets.zarr"
79+
for ispd_dir in "${sh_root}"/ispd_thresh_*; do
80+
[[ -d "$ispd_dir" ]] || continue
81+
for src in "${ispd_dir}"/*.zarr; do
82+
base="$(basename "$src")"
83+
if [[ "$base" =~ ^(FI|PI)_([A-Za-z0-9]+)(_bin|_roll)?(_mets)?\.zarr$ ]]; then
84+
ice_type="${BASH_REMATCH[1]}"
85+
reg_type="${BASH_REMATCH[2]}"
86+
method_suffix="${BASH_REMATCH[3]}"
87+
mets_suffix="${BASH_REMATCH[4]}"
88+
case "$method_suffix" in
89+
"")
90+
dst_dir="${ispd_dir}/${ice_type}/${reg_type}/raw"
91+
;;
92+
"_bin")
93+
dst_dir="${ispd_dir}/${ice_type}/${reg_type}/bin-win-${BIN_WIN_DAYS}_bin-min-${BIN_MIN_DAYS}"
94+
;;
95+
"_roll")
96+
dst_dir="${ispd_dir}/${ice_type}/${reg_type}/roll-days-${MEAN_PERIOD}"
97+
;;
98+
*)
99+
echo "SKIP (unhandled method): $src"
100+
continue
101+
;;
102+
esac
103+
if [[ -n "$mets_suffix" ]]; then
104+
dst="${dst_dir}/mets.zarr"
105+
else
106+
dst="${dst_dir}/data.zarr"
107+
fi
108+
move_store "$src" "$dst"
109+
fi
110+
done
111+
done
112+
done
113+
114+
if [[ $EXECUTE -eq 1 ]]; then
115+
find "${ARCHIVE_ROOT}" -depth -type d -empty -delete
116+
else
117+
echo
118+
echo "Dry-run only. Re-run with --execute to perform the moves."
119+
fi

‎scripts/classification/classify_sea_ice.pbs‎

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
#PBS -l jobfs=2GB
66
#PBS -l storage=gdata/gv90+gdata/xp65+gdata/ik11+gdata/vk83
77
#PBS -l wd
8-
#PBS -P jk72
8+
#PBS -P gv90
99
#PBS -q normalbw
1010

11+
set -euo pipefail
12+
1113
module use /g/data/xp65/public/modules
1214
module load conda/analysis3-25.12
1315

@@ -16,10 +18,22 @@ cd ~/AFIM/src/AFIM/scripts/classification/
1618
# Default JSON if not passed
1719
: "${P_JSON:=/home/581/da1339/AFIM/src/AFIM/src/JSONs/sea_ice_config.json}"
1820

19-
echo "Running for ${SIM_NAME}, ${START_DATE} to ${END_DATE}, ${ICE_TYPE}, BorC2T_type=${BorC2T_TYPE:-ALL}, NetCDF Engine=${NC_ENG}"
21+
# Rebuild BorC2T array from qsub-safe serialised string
22+
BORC2T_ARR=()
23+
if [[ -n "${BorC2T_TYPE:-}" ]]; then
24+
IFS=':' read -r -a BORC2T_ARR <<< "${BorC2T_TYPE}"
25+
fi
26+
27+
echo "Running for ${SIM_NAME}, ${START_DATE} to ${END_DATE}, ${ICE_TYPE}"
28+
echo "BorC2T_type=${BorC2T_TYPE:-ALL}"
29+
echo "NetCDF Engine=${NC_ENG}"
2030
echo "P_JSON=${P_JSON}"
31+
echo "BIN_WIN_DAYS=${BIN_WIN_DAYS:-<toolbox default>}"
32+
echo "BIN_MIN_DAYS=${BIN_MIN_DAYS:-<toolbox default>}"
33+
echo "MEAN_PERIOD=${MEAN_PERIOD:-<toolbox default>}"
2134

22-
F_log=/g/data/gv90/da1339/logs/classification/${SIM_NAME}/${START_DATE}_${END_DATE}_${ISPD_THRESH}_procees_${ICE_TYPE}.log
35+
PARAM_TAG="bw${BIN_WIN_DAYS:-def}_bm${BIN_MIN_DAYS:-def}_mp${MEAN_PERIOD:-def}"
36+
F_log="/g/data/gv90/da1339/logs/classification/${SIM_NAME}/${START_DATE}_${END_DATE}_${ISPD_THRESH}_${ICE_TYPE}_${PARAM_TAG}.log"
2337
mkdir -p "$(dirname "$F_log")"
2438

2539
CMD=(python3 classify_sea_ice.py "$SIM_NAME" \
@@ -31,10 +45,54 @@ CMD=(python3 classify_sea_ice.py "$SIM_NAME" \
3145
--ice_type "$ICE_TYPE" \
3246
--netcdf_engine "$NC_ENG")
3347

34-
[ -n "${BorC2T_TYPE}" ] && CMD+=(--BorC2T_type ${BorC2T_TYPE})
35-
[ "${OVERWRITE_ZARR}" = true ] && CMD+=(--overwrite_zarr)
36-
[ "${DELETE_ORIGINAL_ICEH}" = true ] && CMD+=(--delete_original_iceh)
37-
[ "${ROLL_MEAN}" = true ] && CMD+=(--rolling_mean)
48+
[[ ${#BORC2T_ARR[@]} -gt 0 ]] && CMD+=(--BorC2T_type "${BORC2T_ARR[@]}")
49+
[[ -n "${BIN_WIN_DAYS:-}" ]] && CMD+=(--bin_win_days "$BIN_WIN_DAYS")
50+
[[ -n "${BIN_MIN_DAYS:-}" ]] && CMD+=(--bin_min_days "$BIN_MIN_DAYS")
51+
[[ -n "${MEAN_PERIOD:-}" ]] && CMD+=(--mean_period "$MEAN_PERIOD")
52+
[[ "${OVERWRITE_ZARR:-false}" == true ]] && CMD+=(--overwrite_zarr)
53+
[[ "${DELETE_ORIGINAL_ICEH:-false}" == true ]] && CMD+=(--delete_original_iceh)
54+
[[ "${ROLL_MEAN:-false}" == true ]] && CMD+=(--rolling_mean)
3855

3956
echo "Running: ${CMD[*]}"
4057
"${CMD[@]}"
58+
59+
# #!/bin/bash
60+
# #PBS -l walltime=3:00:00
61+
# #PBS -l mem=190GB
62+
# #PBS -l ncpus=28
63+
# #PBS -l jobfs=2GB
64+
# #PBS -l storage=gdata/gv90+gdata/xp65+gdata/ik11+gdata/vk83
65+
# #PBS -l wd
66+
# #PBS -P jk72
67+
# #PBS -q normalbw
68+
69+
# module use /g/data/xp65/public/modules
70+
# module load conda/analysis3-25.12
71+
72+
# cd ~/AFIM/src/AFIM/scripts/classification/
73+
74+
# # Default JSON if not passed
75+
# : "${P_JSON:=/home/581/da1339/AFIM/src/AFIM/src/JSONs/sea_ice_config.json}"
76+
77+
# echo "Running for ${SIM_NAME}, ${START_DATE} to ${END_DATE}, ${ICE_TYPE}, BorC2T_type=${BorC2T_TYPE:-ALL}, NetCDF Engine=${NC_ENG}"
78+
# echo "P_JSON=${P_JSON}"
79+
80+
# F_log=/g/data/gv90/da1339/logs/classification/${SIM_NAME}/${START_DATE}_${END_DATE}_${ISPD_THRESH}_procees_${ICE_TYPE}.log
81+
# mkdir -p "$(dirname "$F_log")"
82+
83+
# CMD=(python3 classify_sea_ice.py "$SIM_NAME" \
84+
# --P_JSON "$P_JSON" \
85+
# --start_date "$START_DATE" \
86+
# --end_date "$END_DATE" \
87+
# --log_file "$F_log" \
88+
# --ispd_thresh "$ISPD_THRESH" \
89+
# --ice_type "$ICE_TYPE" \
90+
# --netcdf_engine "$NC_ENG")
91+
92+
# [ -n "${BorC2T_TYPE}" ] && CMD+=(--BorC2T_type ${BorC2T_TYPE})
93+
# [ "${OVERWRITE_ZARR}" = true ] && CMD+=(--overwrite_zarr)
94+
# [ "${DELETE_ORIGINAL_ICEH}" = true ] && CMD+=(--delete_original_iceh)
95+
# [ "${ROLL_MEAN}" = true ] && CMD+=(--rolling_mean)
96+
97+
# echo "Running: ${CMD[*]}"
98+
# "${CMD[@]}"

0 commit comments

Comments
 (0)