monitor resource consumption on linux

emilyalbini · emilyalbini · commit 43914a10ea33 · 2026-03-25T17:29:50.000+01:00
diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh
@@ -24,6 +24,9 @@ cargo --version
 rustc --version
 curl -sSfL --retry 10 https://get.nexte.st/"$NEXTEST_VERSION"/"$1" | gunzip | tar -xvf - -C ~/.cargo/bin
 
+cargo build --release -p ci-resource-usage
+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv
+
 #
 # Set up a custom temporary directory within whatever one we were given so that
 # we can check later whether we left detritus around.
diff --git a/.github/buildomat/jobs/build-and-test-helios.sh b/.github/buildomat/jobs/build-and-test-helios.sh
@@ -2,7 +2,7 @@
 #:
 #: name = "build-and-test (helios)"
 #: variety = "basic"
-#: target = "helios-2.0-32c256gb"
+#: target = "helios-2.0-20260324-16c128gb"
 #: rust_toolchain = true
 #: output_rules = [
 #:	"%/work/*",
@@ -13,6 +13,7 @@
 #:	"%/var/tmp/omicron_tmp/**/*",
 #:	"!/var/tmp/omicron_tmp/crdb-base*",
 #:	"!/var/tmp/omicron_tmp/rustc*",
+#:  "%/var/tmp/ci-resource-usage.csv",
 #: ]
 #: access_repos = [
 #:	"oxidecomputer/dendrite"
diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh
@@ -12,6 +12,7 @@
 #:	"%/var/tmp/omicron_tmp/**/*",
 #:	"!/var/tmp/omicron_tmp/crdb-base*",
 #:	"!/var/tmp/omicron_tmp/rustc*",
+#:  "%/var/tmp/ci-resource-usage.csv",
 #: ]
 #: access_repos = [
 #:	"oxidecomputer/dendrite",
diff --git a/.github/buildomat/jobs/check-features.sh b/.github/buildomat/jobs/check-features.sh
@@ -2,10 +2,11 @@
 #:
 #: name = "check-features (helios)"
 #: variety = "basic"
-#: target = "helios-2.0"
+#: target = "helios-2.0-20260324"
 #: rust_toolchain = true
 #: output_rules = [
 #:  "/out/*",
+#:  "%/var/tmp/ci-resource-usage.csv",
 #: ]
 
 # Run the check-features `xtask` on illumos, testing compilation of feature combinations.
@@ -20,6 +21,9 @@ source .github/buildomat/ci-env.sh
 cargo --version
 rustc --version
 
+cargo build --release -p ci-resource-usage
+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv
+
 #
 # Set up our PATH for use with this workspace.
 #
diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh
@@ -2,9 +2,11 @@
 #:
 #: name = "clippy (helios)"
 #: variety = "basic"
-#: target = "helios-2.0"
+#: target = "helios-2.0-20260324"
 #: rust_toolchain = true
-#: output_rules = []
+#: output_rules = [
+#:  "%/var/tmp/ci-resource-usage.csv",
+#: ]
 
 # Run clippy on illumos (not just other systems) because a bunch of our code
 # (that we want to check) is conditionally-compiled on illumos only.
@@ -28,6 +30,9 @@ source .github/buildomat/ci-env.sh
 cargo --version
 rustc --version
 
+cargo build --release -p ci-resource-usage
+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv
+
 banner prerequisites
 ptime -m bash ./tools/install_builder_prerequisites.sh -y
 
diff --git a/.github/buildomat/jobs/omicron-common.sh b/.github/buildomat/jobs/omicron-common.sh
@@ -2,9 +2,11 @@
 #:
 #: name = "omicron-common (helios)"
 #: variety = "basic"
-#: target = "helios-2.0"
+#: target = "helios-2.0-20260324"
 #: rust_toolchain = true
-#: output_rules = []
+#: output_rules = [
+#:  "%/var/tmp/ci-resource-usage.csv",
+#: ]
 
 # Verify that omicron-common builds successfully when used as a dependency
 # in an external project. It must not leak anything that requires an external
@@ -20,6 +22,9 @@ source .github/buildomat/ci-env.sh
 cargo --version
 rustc --version
 
+cargo build --release -p ci-resource-usage
+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv
+
 cd /tmp
 cargo new --lib test-project
 cd test-project
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
@@ -2,10 +2,11 @@
 #:
 #: name = "helios / package"
 #: variety = "basic"
-#: target = "helios-2.0-16c64gb"
+#: target = "helios-2.0-20260324-16c64gb"
 #: rust_toolchain = true
 #: output_rules = [
 #:	"=/work/package.tar.gz",
+#:  "%/var/tmp/ci-resource-usage.csv",
 #: ]
 #:
 
@@ -19,6 +20,9 @@ source .github/buildomat/ci-env.sh
 cargo --version
 rustc --version
 
+cargo build --release -p ci-resource-usage
+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv
+
 WORK=/work
 pfexec mkdir -p $WORK && pfexec chown $USER $WORK
 
diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
@@ -2,7 +2,7 @@
 #:
 #: name = "helios / build TUF repo"
 #: variety = "basic"
-#: target = "helios-2.0-16c128gb"
+#: target = "helios-2.0-20260324-16c128gb"
 #: rust_toolchain = true
 #: output_rules = [
 #:	"=/work/manifest.toml",
@@ -12,6 +12,7 @@
 #:	"=/work/incorporation.p5m",
 #:	"=/work/incorporation.p5p",
 #:	"%/work/*.log",
+#:  "%/var/tmp/ci-resource-usage.csv",
 #: ]
 #: access_repos = [
 #:	"oxidecomputer/amd-firmware",
@@ -61,6 +62,9 @@ source .github/buildomat/ci-env.sh
 cargo --version
 rustc --version
 
+cargo build --release -p ci-resource-usage
+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv
+
 # Before we do _anything_, quickly check that Cargo.lock is properly locked.
 # Most of our tools (including releng!) eventually call `cargo xtask`, which
 # runs without `--locked` and will update the lockfile.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -32,6 +32,7 @@ members = [
     "cockroach-metrics",
     "common",
     "dev-tools/cert-dev",
+    "dev-tools/ci-resource-usage",
     "dev-tools/clickana",
     "dev-tools/clickhouse-cluster-dev",
     "dev-tools/ch-dev",
diff --git a/dev-tools/ci-resource-usage/Cargo.toml b/dev-tools/ci-resource-usage/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "ci-resource-usage"
+version = "0.1.0"
+edition.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+
+[target.'cfg(target_os = "illumos")'.dependencies]
+kstat-rs.workspace = true
+
+[lints]
+workspace = true
diff --git a/dev-tools/ci-resource-usage/src/illumos.rs b/dev-tools/ci-resource-usage/src/illumos.rs
@@ -0,0 +1,163 @@
+use crate::{CpuCore, Io, Memory, Swap};
+use anyhow::{Context as _, Error, anyhow, bail};
+use kstat_rs::{Ctl, Data, Kstat, NamedData};
+
+pub(crate) fn gather_cpu_cores() -> Result<Vec<CpuCore>, Error> {
+    let mut cores = Vec::new();
+    let ctl = Ctl::new()?;
+    for mut entry in ctl.filter(Some("cpu"), None, Some("sys")) {
+        let parsed = KstatCpuCore::read(&ctl, &mut entry)?;
+
+        let used_ticks = parsed.cpu_ticks_user + parsed.cpu_ticks_kernel;
+        let wait_ticks = parsed.cpu_ticks_wait;
+        cores.push(CpuCore {
+            total_ticks: used_ticks + wait_ticks + parsed.cpu_ticks_idle,
+            used_ticks,
+            wait_ticks,
+        });
+    }
+    Ok(cores)
+}
+
+pub(crate) fn gather_io() -> Result<Io, Error> {
+    let mut io = Io { read_ops: 0, write_ops: 0 };
+    let ctl = Ctl::new()?;
+    for mut entry in ctl.filter(Some("blkdev"), None, None) {
+        let Data::Io(data) = ctl.read(&mut entry)? else {
+            bail!("expected io for {}", entry.ks_name);
+        };
+        io.read_ops += u64::from(data.reads);
+        io.write_ops += u64::from(data.writes);
+    }
+    Ok(io)
+}
+
+pub(crate) fn gather_load_1min() -> Result<f64, Error> {
+    let data: KstatSystemMisc = get("unix", 0, "system_misc")?;
+    Ok(data.avenrun_1min as f64 / 256.0)
+}
+
+pub(crate) fn gather_memory() -> Result<Memory, Error> {
+    let data: KstatSystemPages = get("unix", 0, "system_pages")?;
+    Ok(Memory { total: data.physmem * 4096, available: data.freemem * 4096 })
+}
+
+pub(crate) fn gather_swap() -> Result<Swap, Error> {
+    let mut swap = Swap { read: 0, write: 0 };
+    let ctl = Ctl::new()?;
+    for mut entry in ctl.filter(Some("cpu"), None, Some("vm")) {
+        let parsed = KstatCpuVm::read(&ctl, &mut entry)?;
+        swap.read += parsed.anonpgin;
+        swap.write += parsed.anonpgout;
+    }
+    Ok(swap)
+}
+
+fn get<T: ReadNamed>(module: &str, inst: i32, name: &str) -> Result<T, Error> {
+    let ctl = Ctl::new()?;
+    let mut kstat = ctl
+        .filter(Some(module), Some(inst), Some(name))
+        .next()
+        .ok_or_else(|| anyhow!("missing {module}:{inst}:{name} kstat"))?;
+    T::read(&ctl, &mut kstat)
+}
+
+macro_rules! kstat_named {
+    (struct $ident:ident { $($field:ident: $ty:ty,)* }) => {
+        struct $ident {
+            $($field: $ty,)*
+        }
+
+        impl ReadNamed for $ident {
+            fn read(ctl: &Ctl, entry: &mut Kstat<'_>) -> Result<Self, Error> {
+                let id = format!(
+                    "{}:{}:{}",
+                    entry.ks_module,
+                    entry.ks_instance,
+                    entry.ks_name,
+                );
+
+                let Data::Named(vec_named) = ctl.read(entry)? else {
+                    bail!("expected named data for {id}");
+                };
+
+                $(let mut $field = None;)*
+                for named in vec_named {
+                    match named.name {
+                        $(stringify!($field) => {
+                            $field = Some(
+                                <$ty>::from_named_data(named.value)
+                                    .with_context(|| format!(
+                                        "failed to parse field {} of {id}",
+                                        named.name,
+                                    ))?
+                            );
+                        })*
+                        _ => {}
+                    }
+                }
+
+                Ok(Self {
+                    $(
+                        $field: $field.ok_or_else(|| anyhow!(
+                            "missing kstat field named {} in {id}",
+                            stringify!($field),
+                        ))?,
+                    )*
+                })
+            }
+        }
+    }
+}
+
+kstat_named! {
+    struct KstatCpuCore {
+        cpu_ticks_idle: u64,
+        cpu_ticks_kernel: u64,
+        cpu_ticks_user: u64,
+        cpu_ticks_wait: u64,
+    }
+}
+
+kstat_named! {
+    struct KstatCpuVm {
+        anonpgin: u64,
+        anonpgout: u64,
+    }
+}
+
+kstat_named! {
+    struct KstatSystemMisc {
+        avenrun_1min: u64,
+    }
+}
+
+kstat_named! {
+    struct KstatSystemPages {
+        physmem: u64,
+        freemem: u64,
+    }
+}
+
+trait ReadNamed: Sized {
+    fn read(ctl: &Ctl, kstat: &mut Kstat<'_>) -> Result<Self, Error>;
+}
+
+trait FromNamedData: Sized {
+    fn from_named_data(named_data: NamedData<'_>) -> Result<Self, Error>;
+}
+
+impl FromNamedData for u64 {
+    fn from_named_data(named_data: NamedData<'_>) -> Result<Self, Error> {
+        match named_data {
+            NamedData::Char(_) => bail!("expected unsigned int, found Char"),
+            NamedData::Int32(_) => bail!("expected unsigned int, found Int32"),
+            NamedData::UInt32(inner) => Ok(inner.into()),
+            NamedData::Int64(_) => bail!("expetced unsigned int, found Int64"),
+            NamedData::UInt64(inner) => Ok(inner),
+            NamedData::String(_) => {
+                bail!("expected unsigned int, found String");
+            }
+        }
+    }
+}
diff --git a/dev-tools/ci-resource-usage/src/linux.rs b/dev-tools/ci-resource-usage/src/linux.rs
diff --git a/dev-tools/ci-resource-usage/src/main.rs b/dev-tools/ci-resource-usage/src/main.rs

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,9 @@ cargo --version`
`24`	`24`	`rustc --version`
`25`	`25`	`curl -sSfL --retry 10 https://get.nexte.st/"$NEXTEST_VERSION"/"$1" \| gunzip \| tar -xvf - -C ~/.cargo/bin`
`26`	`26`
	`27`	`+cargo build --release -p ci-resource-usage`
	`28`	`+bmat process start ci-resource-usage target/release/ci-resource-usage /var/tmp/ci-resource-usage.csv`
	`29`	`+`
`27`	`30`	`#`
`28`	`31`	`# Set up a custom temporary directory within whatever one we were given so that`
`29`	`32`	`# we can check later whether we left detritus around.`