Skip to content

ci(versioning): crates-to-package.sh refactor #1875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iunanua merged 3 commits into from
Apr 14, 2026
Merged

ci(versioning): crates-to-package.sh refactor #1875

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f93bd9b
script refactor
iunanua Apr 14, 2026
2a8434d
fix publish = false crates
iunanua Apr 14, 2026
e2e2df4
Include note
iunanua Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 79 additions & 92 deletions scripts/crates-to-package.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,112 +2,99 @@
# Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
# SPDX-License-Identifier: Apache-2.0
#
# Compares every workspace package's effective [package] version between GIT_VERSION_BASE
# and GIT_VERSION_HEAD. Prints package names suitable for `cargo package -p` when the version
# changed — not when only dependency versions changed.
# Uses git for both sides so results match the refs, not uncommitted working tree edits.
# Compares workspace crate versions between two git refs by running `cargo metadata` at each ref.
# Prints crate names suitable for `cargo package -p` when the crate's *own* package version changed.
#
# CI should set GIT_VERSION_BASE / GIT_VERSION_HEAD to the commits being compared (e.g. PR base
# and head). Defaults below are for local use; comparing to origin/main vs HEAD is not the same
# as "what this PR changed" once main has moved.
# Important: this intentionally ignores dependency version bumps unless they also changed the crate
# version. (Cargo metadata reports package versions, not dependency requirements.)
#
# Excludes are determined from crate metadata: crates marked `publish = false` are excluded.
# (Crates with publish unset or publish = [...] are considered publishable.)
#
# Important: this script is used by libddprof-build gitlab pipeline

set -euo pipefail

GIT_VERSION_BASE="${GIT_VERSION_BASE:-origin/main}"
GIT_VERSION_HEAD="${GIT_VERSION_HEAD:-HEAD}"

# [workspace.package].version from workspace root Cargo.toml (stdin).
workspace_package_version() {
awk '
/^\[workspace\.package\]/ { w = 1; next }
w && /^\[/ { w = 0 }
w && /^version[[:space:]]*=/ {
line = $0
sub(/^[^"]*"/, "", line)
sub(/".*/, "", line)
print line
exit
}
'
}
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd)"

# Effective [package] version: explicit "x.y.z" or "__WS__" if version.workspace = true.
package_version_kind() {
awk '
/^\[package\]/ { p = 1; next }
p && /^\[/ { p = 0 }
p && /^version\.workspace[[:space:]]*=[[:space:]]*true/ { print "__WS__"; exit }
p && /^version[[:space:]]*=/ && $0 !~ /version\.workspace/ {
line = $0
sub(/^[^"]*"/, "", line)
sub(/".*/, "", line)
print line
exit
}
'
}
tmp_root="$(mktemp -d)"
base_dir="${tmp_root}/base"
head_dir="${tmp_root}/head"

effective_package_version() {
local manifest_content="$1"
local workspace_content="$2"
local kind
kind=$(printf '%s' "$manifest_content" | package_version_kind)
if [ "$kind" = "__WS__" ]; then
printf '%s' "$workspace_content" | workspace_package_version
return
fi
printf '%s' "$kind"
cleanup() {
# best-effort cleanup
git -C "${REPO_ROOT}" worktree remove --force "${base_dir}" >/dev/null 2>&1 || true
git -C "${REPO_ROOT}" worktree remove --force "${head_dir}" >/dev/null 2>&1 || true
rm -rf "${tmp_root}" >/dev/null 2>&1 || true
}
trap cleanup EXIT

excluded_crate() {
local n="$1"
case "$n" in
libdd-*-ffi) return 0 ;;
datadog-*) return 0 ;;
bin_tests | tools | sidecar_mockgen | cc_utils | spawn_worker | symbolizer-ffi | test_spawn_from_lib | build_common | build-common | builder)
return 0
;;
esac
return 1
}

workspace_old=$(git show "${GIT_VERSION_BASE}:Cargo.toml" 2>/dev/null || true)
workspace_new=$(git show "${GIT_VERSION_HEAD}:Cargo.toml" 2>/dev/null || { cat Cargo.toml; })

METADATA=$(cargo metadata --format-version=1 --no-deps)
WORKSPACE_ROOT=$(echo "$METADATA" | jq -r '.workspace_root')
# Ensure refs exist locally for worktree add.
git -C "${REPO_ROOT}" fetch --no-tags origin >/dev/null 2>&1 || true

declare -A emit=()
git -C "${REPO_ROOT}" worktree add --detach "${base_dir}" "${GIT_VERSION_BASE}" >/dev/null
git -C "${REPO_ROOT}" worktree add --detach "${head_dir}" "${GIT_VERSION_HEAD}" >/dev/null

while IFS=$'\t' read -r name mpath; do
[ -z "$name" ] && continue
relpath=${mpath#"$WORKSPACE_ROOT/"}
BASE_METADATA="$(cargo metadata --manifest-path "${base_dir}/Cargo.toml" --format-version=1 --no-deps)"
HEAD_METADATA="$(cargo metadata --manifest-path "${head_dir}/Cargo.toml" --format-version=1 --no-deps)"

old_m=$(git show "${GIT_VERSION_BASE}:${relpath}" 2>/dev/null || true)
new_m=$(git show "${GIT_VERSION_HEAD}:${relpath}" 2>/dev/null || true)
[ -n "$new_m" ] || continue

v_old=""
if [ -n "$old_m" ]; then
v_old=$(effective_package_version "$old_m" "$workspace_old")
fi
v_new=$(effective_package_version "$new_m" "$workspace_new")

if [ "$v_old" = "$v_new" ]; then
continue
fi

if excluded_crate "$name"; then
continue
fi
emit["$name"]=1
done < <(echo "$METADATA" | jq -r '.packages[] | "\(.name)\t\(.manifest_path)"')

if [ "${#emit[@]}" -eq 0 ]; then
echo "crates-to-package.sh: no packages with a changed [package] version between refs (nothing to emit)." >&2
# Build a name -> {version, publishable} map for workspace members.
#
# publishable:
# - publish unset (null) => publishable
# - publish array (registries) => publishable if non-empty
# - publish = false => NOT publishable
map_filter='
. as $m
| ($m.packages | map({id, name, version, publish})) as $pkgs
| ($m.workspace_members) as $members
| [ $members[]
| . as $id
| ($pkgs[] | select(.id == $id))
| {
name,
version,
publishable: (
if (.publish == null) then true
elif (.publish | type) == "array" then ((.publish | length) > 0)
else false
end
)
}
]
| map({key: .name, value: {version: .version, publishable: .publishable}})
| from_entries
'

BASE_MAP="$(echo "${BASE_METADATA}" | jq -c "${map_filter}")"
HEAD_MAP="$(echo "${HEAD_METADATA}" | jq -c "${map_filter}")"

# Emit publishable crates whose version changed between base and head.
#
# - If a crate is new in head (missing in base), include it.
# - If a crate disappeared in head, ignore it.
TO_PACKAGE="$(jq -nr --argjson base "${BASE_MAP}" --argjson head "${HEAD_MAP}" '
($head | keys_unsorted) as $names
| [ $names[]
| . as $n
| ($head[$n]) as $h
| select($h.publishable == true)
| ($base[$n].version // null) as $bv
| ($h.version) as $hv
| select($bv != $hv)
| $n
]
| sort
| .[]
')"

if [ -z "${TO_PACKAGE}" ]; then
echo "crates-to-package.sh: no publishable workspace crates had a version change between refs (nothing to emit)." >&2
exit 0
fi

for n in "${!emit[@]}"; do
echo "$n"
done | sort -u
echo "${TO_PACKAGE}"
2 changes: 2 additions & 0 deletions scripts/publication-order.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
# Usage: ./publication-order.sh [OPTIONS] [CRATE...]
#
# If crate names are provided, shows only those crates and their dependencies in publication order.
#
# Important: this script is used by libddprof-build gitlab pipeline

set -euo pipefail

Expand Down
Loading