From c984e444df6e719762a7dff4e4a241f1482fdbb2 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 14:37:08 -0700 Subject: [PATCH 01/80] test: observational baseline for wrapped re-export via -open flag Records the rebuild count for a consumer module that reaches a dep library's modules through a transparent alias declared inside a wrapped sibling library, where the wrapped library is opened via the [-open] compiler flag rather than named in source. On trunk the consumer correctly rebuilds when the dep library's interface changes (count = 1) because the cctx-wide compile-rule deps cover every library in the stanza's [(libraries ...)] closure. This is the structural shape of menhir's [base]/[middle] arrangement. Records the count so a future change that narrows compile-rule deps per module either preserves the count or fails the test, surfacing any precision regression in dependency tracking through transparent aliases. Signed-off-by: Robin Bate Boerop --- .../wrapped-reexport-via-open-flag.t | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t b/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t new file mode 100644 index 00000000000..041f2c23b41 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t @@ -0,0 +1,77 @@ +Observational baseline: a consumer reaches a dep library's modules +through a transparent alias declared inside a wrapped sibling +library, where the wrapped library is opened via the [-open] +compiler flag rather than named in source. On trunk, [consumer] +correctly rebuilds when the dep library's interface changes, +because the cctx-wide compile-rule deps cover every library in the +stanza's [(libraries ...)] closure. + +This is the structural shape of menhir's [base]/[middle] +arrangement: [base] re-exports [Vendored_pprint] through aliases, +[middle] uses [-open Base] in its compile flags, and modules of +[middle] reference [Vendored_pprint]'s contents through unqualified +names brought into scope by the open. Records the consumer's +rebuild count as a regression guard for changes that narrow +compile-rule deps per module. + +Structure: [lib_a] is unwrapped with module [Original_name]; +[lib_re_export] is wrapped with a hand-written wrapper containing +[module Re = Original_name]; [lib_b] depends on [lib_re_export] +and uses [-open Lib_re_export] in its flags; [consumer.ml] writes +[Re.x] without naming [lib_a] or [lib_re_export] in source. + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ cat > dune < (library (name lib_a) (wrapped false) (modules original_name)) + > (library + > (name lib_re_export) + > (modules lib_re_export some_inner) + > (libraries lib_a)) + > (library + > (name lib_b) + > (wrapped false) + > (modules consumer) + > (libraries lib_re_export) + > (flags (:standard -open Lib_re_export))) + > EOF + + $ cat > original_name.ml < let x = "hello" + > EOF + $ cat > original_name.mli < val x : string + > EOF + + $ cat > lib_re_export.ml < module Some_inner = Some_inner + > module Re = Original_name + > EOF + $ cat > some_inner.ml < let placeholder = () + > EOF + + $ cat > consumer.ml < let _ = Re.x + > EOF + + $ dune build @check + +Edit [lib_a]'s interface. [consumer] reaches [lib_a]'s +[Original_name] through the alias chain in the wrapped +[Lib_re_export] wrapper. The cctx-wide compile-rule deps include +[lib_a], so [consumer] rebuilds: + + $ cat > original_name.mli < val x : string + > val y : int + > EOF + $ cat > original_name.ml < let x = "hello" + > let y = 42 + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer"))] | length' + 1 From 799cc4d5b605c679578c37bb9cebe427a4f89f43 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 16:04:08 -0700 Subject: [PATCH 02/80] test: observational baseline for auto-wrapped child re-export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the rebuild count for a consumer module that reaches a dep library's modules through a child of an auto-wrapped sibling library, where the sibling is opened via the [-open] compiler flag and the child's source includes the dep library's module via [include]. On trunk the consumer correctly rebuilds when the dep library's interface changes (count = 1) because the cctx-wide compile-rule deps cover every library in the stanza's [(libraries ...)] closure. This is the structural shape of menhir's [base]/[middle] arrangement: [base] is auto-wrapped (no [base.ml] in source — dune generates the wrapper) with a child [PPrint.ml] containing [include Vendored_pprint]; [middle] uses [-open Base] in flags and references [PPrint.foo]. The reference chain crosses library boundaries through the auto-wrapped sibling's child, not through a hand-written wrapper. Pairs with future work that narrows compile-rule deps per module. Distinct from [wrapped-reexport-via-open-flag.t] (where the alias lives in a hand-written wrapper) — the auto-wrapped case has the cross-library reference inside a child module whose ocamldep is not directly read by walks rooted at the wrapper, so a per-module filter must descend into wrapped libs' children to remain sound. Signed-off-by: Robin Bate Boerop --- .../auto-wrapped-child-reexport.t | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t b/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t new file mode 100644 index 00000000000..502ecdab8d6 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t @@ -0,0 +1,90 @@ +Observational baseline: a consumer reaches a dep library's modules +through a child of an auto-wrapped sibling library, where the +sibling is opened via the [-open] compiler flag and the child's +source includes the dep library's module via [include]. On trunk, +[consumer] correctly rebuilds when the dep library's interface +changes, because the cctx-wide compile-rule deps cover every +library in the stanza's [(libraries ...)] closure. + +This is the structural shape of menhir's [base]/[middle] +arrangement: [base] is auto-wrapped (no [base.ml] in the source, +dune generates the wrapper), with a child [PPrint.ml] containing +[include Vendored_pprint]. [middle] uses [-open Base] in flags and +references [PPrint.foo] in source. The reference chain crosses +library boundaries through the auto-wrapped sibling's child, not +through a hand-written wrapper. + +Records the consumer's rebuild count as a regression guard for +changes that narrow compile-rule deps per module. + +Structure: [lib_a] is unwrapped with module [Original_name]. +[lib_re_export] is wrapped by dune defaults (no [lib_re_export.ml] +file — dune auto-generates the wrapper); child [pprint.ml] +contains a transparent alias [module Re = Original_name]; +[filler.ml] is just there to keep the lib non-singleton. [lib_b] +depends on [lib_re_export] and uses [-open Lib_re_export] in its +flags; [consumer.ml] writes [Pprint.Re.x] without naming +[lib_a], [lib_re_export], or any of its children in source. + +The transparent-alias shape (rather than [include]) is what makes +[lib_a] genuinely invisible to a per-module dep filter that only +walks the auto-generated wrapper: with [-no-alias-deps], [Pprint]'s +[.cmi] content does not depend on [Original_name.cmi], so a glob +over [lib_re_export]'s objdir does not capture [Original_name.mli] +changes either. The cctx-wide compile-rule deps still cover +[lib_a] on trunk, so [consumer] rebuilds. + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ cat > dune < (library (name lib_a) (wrapped false) (modules original_name)) + > (library + > (name lib_re_export) + > (modules pprint filler) + > (libraries lib_a)) + > (library + > (name lib_b) + > (wrapped false) + > (modules consumer) + > (libraries lib_re_export) + > (flags (:standard -open Lib_re_export))) + > EOF + + $ cat > original_name.ml < let x = "hello" + > EOF + $ cat > original_name.mli < val x : string + > EOF + + $ cat > pprint.ml < module Re = Original_name + > EOF + $ cat > filler.ml < let placeholder = () + > EOF + + $ cat > consumer.ml < let _ = Pprint.Re.x + > EOF + + $ dune build @check + +Edit [lib_a]'s interface. [consumer] reaches [lib_a]'s +[Original_name] through [Pprint] (child of auto-wrapped +[Lib_re_export]). The cctx-wide compile-rule deps include +[lib_a], so [consumer] rebuilds: + + $ cat > original_name.mli < val x : string + > val y : int + > EOF + $ cat > original_name.ml < let x = "hello" + > let y = 42 + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer"))] | length' + 1 From b5768fa7079c0fd8a6e3a18385c02382b0cab63d Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 09:37:26 -0700 Subject: [PATCH 03/80] test: opaque profile coverage for inter-library dep tracking Three observational tests pinning the visible consequences of dune's [-opaque] handling in [groups_for_cm_kind]. Together with the existing [opaque.t] (which covers the [.ml]-only-change rebuild behaviour) they document the four directly-observable code paths that depend on the [-opaque] flag. - [opaque-cmx-deps-local.t]: a local lib's [.cmx] is in the consumer's dep set under release; only [.cmi] under dev. - [opaque-cmx-deps-external.t]: an external (findlib-resolved) lib's [.cmx] is in the dep set under both profiles. Pins that the [Lib.is_local lib] guard in [groups_for_cm_kind] is intentional. - [opaque-mli-change.t]: a [.mli] change in a dep rebuilds consumers under both profiles. Pins that opaque mode is purely about [.cmx] propagation, not [.cmi] propagation. Future regressions in any of those four code paths fail a specific test with a self-explanatory name. Signed-off-by: Robin Bate Boerop --- .../opaque-cmx-deps-external.t | 42 ++++++++++ .../opaque-cmx-deps-local.t | 50 ++++++++++++ .../per-module-lib-deps/opaque-mli-change.t | 81 +++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t new file mode 100644 index 00000000000..6f8da5e6cb0 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t @@ -0,0 +1,42 @@ +Verify that the [groups_for_cm_kind] optimisation in [lib_file_deps.ml] +is gated on [Lib.is_local lib]: a consumer's [.cmx] rule depends on an +external library's [.cmx] regardless of profile. Counterpart to +[opaque-cmx-deps-local.t]. The [unix] stdlib library, resolved through +findlib, plays the role of "external". + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ cat > dune < (executable (name main) (libraries unix)) + > EOF + $ cat > main.ml < let () = ignore (Unix.gettimeofday ()) + > EOF + +--- Release profile (opaque=false): both .cmi and .cmx globs --- + + $ cat > dune-workspace < (lang dune 3.0) + > (profile release) + > EOF + + $ dune build ./main.exe + $ dune rules --root . --format=json --deps _build/default/.main.eobjs/native/dune__exe__Main.cmx | + > jq -r 'include "dune"; .[] | depsGlobPredicates' | sort -u + *.cmi + *.cmx + +--- Dev profile (opaque=true): both .cmi and .cmx globs (unchanged for external libs) --- + + $ cat > dune-workspace < (lang dune 3.0) + > (profile dev) + > EOF + + $ dune build ./main.exe + $ dune rules --root . --format=json --deps _build/default/.main.eobjs/native/dune__exe__Main.cmx | + > jq -r 'include "dune"; .[] | depsGlobPredicates' | sort -u + *.cmi + *.cmx diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t new file mode 100644 index 00000000000..150c4fe5065 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t @@ -0,0 +1,50 @@ +Verify the [groups_for_cm_kind] decision in [lib_file_deps.ml]: a +consumer's [.cmx] rule depends on a local library's [.cmx] under the +release profile (opaque=false), but only on the [.cmi] under the dev +profile (opaque=true). The [Lib.is_local lib] guard means this +optimisation applies to local libraries; see +[opaque-cmx-deps-external.t] for the external-library case. + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ mkdir lib + $ cat > lib/dune < (library (name mylib)) + > EOF + $ cat > lib/mylib.ml < let v = 42 + > EOF + + $ cat > dune < (executable (name main) (libraries mylib)) + > EOF + $ cat > main.ml < let () = print_int Mylib.v + > EOF + +--- Release profile (opaque=false): both .cmi and .cmx globs --- + + $ cat > dune-workspace < (lang dune 3.0) + > (profile release) + > EOF + + $ dune build ./main.exe + $ dune rules --root . --format=json --deps _build/default/.main.eobjs/native/dune__exe__Main.cmx | + > jq -r 'include "dune"; .[] | depsGlobPredicates' | sort -u + *.cmi + *.cmx + +--- Dev profile (opaque=true): only .cmi glob --- + + $ cat > dune-workspace < (lang dune 3.0) + > (profile dev) + > EOF + + $ dune build ./main.exe + $ dune rules --root . --format=json --deps _build/default/.main.eobjs/native/dune__exe__Main.cmx | + > jq -r 'include "dune"; .[] | depsGlobPredicates' | sort -u + *.cmi diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t new file mode 100644 index 00000000000..e2e1502105b --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t @@ -0,0 +1,81 @@ +Verify that a [.mli] change in a dependency library rebuilds +consumers under both [release] (opaque=false) and [dev] (opaque=true) +profiles. The opaque-mode optimisation in [groups_for_cm_kind] is +purely about cross-module inlining propagation through [.cmx]; it +does not affect [.cmi] propagation, which is what carries [.mli] +changes to consumers. + +Companion to [opaque.t], which covers the [.ml]-only change axis. + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ mkdir lib + $ cat > lib/dune < (library (name mylib)) + > EOF + $ cat > lib/mylib.ml < let v = 42 + > EOF + $ cat > lib/mylib.mli < val v : int + > EOF + + $ cat > dune < (executable (name main) (libraries mylib)) + > EOF + $ cat > main.ml < let () = print_int Mylib.v + > EOF + +--- Release profile (opaque=false): .mli change rebuilds consumer --- + + $ cat > dune-workspace < (lang dune 3.0) + > (profile release) + > EOF + + $ dune build ./main.exe + +Add a new declaration to both [.ml] and [.mli] (paired so no value is +left unexported, which would trip warning 32 under dev): + + $ cat > lib/mylib.ml < let v = 42 + > let extra () = 0 + > EOF + $ cat > lib/mylib.mli < val v : int + > val extra : unit -> int + > EOF + + $ dune build ./main.exe + $ dune trace cat | jq -s 'include "dune"; ([.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length) > 0' + true + +--- Dev profile (opaque=true): .mli change still rebuilds consumer --- + + $ cat > dune-workspace < (lang dune 3.0) + > (profile dev) + > EOF + + $ dune build ./main.exe + +Add another paired declaration: + + $ cat > lib/mylib.ml < let v = 42 + > let extra () = 0 + > let helper x = x + 1 + > EOF + $ cat > lib/mylib.mli < val v : int + > val extra : unit -> int + > val helper : int -> int + > EOF + + $ dune build ./main.exe + $ dune trace cat | jq -s 'include "dune"; ([.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length) > 0' + true From 0fe13fc7e78dc6f825ca44914d95f6a9b3d7cef2 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 09:43:17 -0700 Subject: [PATCH 04/80] test: scrub implementation-detail references from opaque test prose The preamble comments referenced internal names ([groups_for_cm_kind], [lib_file_deps.ml], [Lib.is_local lib]). Replace with descriptions phrased entirely in terms of externally observable behaviour: file extensions, profile names, and consumer/dep relationships. Signed-off-by: Robin Bate Boerop --- .../per-module-lib-deps/opaque-cmx-deps-external.t | 10 +++++----- .../per-module-lib-deps/opaque-cmx-deps-local.t | 10 ++++------ .../test-cases/per-module-lib-deps/opaque-mli-change.t | 10 ++++------ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t index 6f8da5e6cb0..f0f59201b2f 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t @@ -1,8 +1,8 @@ -Verify that the [groups_for_cm_kind] optimisation in [lib_file_deps.ml] -is gated on [Lib.is_local lib]: a consumer's [.cmx] rule depends on an -external library's [.cmx] regardless of profile. Counterpart to -[opaque-cmx-deps-local.t]. The [unix] stdlib library, resolved through -findlib, plays the role of "external". +A consumer's [.cmx] compilation rule depends on an external library's +[.cmx] under both the release and dev profiles. Counterpart to +[opaque-cmx-deps-local.t], which shows the dev-profile behaviour that +omits the [.cmx] from the dep set for *local* libraries. The [unix] +stdlib library, resolved through findlib, plays the role of "external". $ cat > dune-project < (lang dune 3.0) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t index 150c4fe5065..5b7f64f8e42 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t @@ -1,9 +1,7 @@ -Verify the [groups_for_cm_kind] decision in [lib_file_deps.ml]: a -consumer's [.cmx] rule depends on a local library's [.cmx] under the -release profile (opaque=false), but only on the [.cmi] under the dev -profile (opaque=true). The [Lib.is_local lib] guard means this -optimisation applies to local libraries; see -[opaque-cmx-deps-external.t] for the external-library case. +A consumer's [.cmx] compilation rule depends on a local library's +[.cmx] under the release profile (opaque=false), but only on the +[.cmi] under the dev profile (opaque=true). External libraries +behave differently; see [opaque-cmx-deps-external.t]. $ cat > dune-project < (lang dune 3.0) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t index e2e1502105b..f15e897a3af 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t @@ -1,9 +1,7 @@ -Verify that a [.mli] change in a dependency library rebuilds -consumers under both [release] (opaque=false) and [dev] (opaque=true) -profiles. The opaque-mode optimisation in [groups_for_cm_kind] is -purely about cross-module inlining propagation through [.cmx]; it -does not affect [.cmi] propagation, which is what carries [.mli] -changes to consumers. +A [.mli] change in a dependency library rebuilds consumers under +both the release (opaque=false) and dev (opaque=true) profiles. +Opaque mode does not affect [.cmi] propagation; it only affects +whether cross-module inlining tracks a dep's [.cmx]. Companion to [opaque.t], which covers the [.ml]-only change axis. From 85a29643ebf2f4fe4ccf7bc510cfffda2fa71f2e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 10:42:27 -0700 Subject: [PATCH 05/80] test: tighten opaque-mli-change rebuild assertions to exact counts The [length > 0] assertions were too permissive and would mask a spurious extra rebuild. Replace with the exact count under each profile (which happens to be 2 in both cases). Addresses Copilot's review feedback: https://github.com/ocaml/dune/pull/14331#discussion_r3142263193 Signed-off-by: Robin Bate Boerop --- .../test-cases/per-module-lib-deps/opaque-mli-change.t | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t index f15e897a3af..a8bce6eecd6 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t @@ -49,8 +49,8 @@ left unexported, which would trip warning 32 under dev): > EOF $ dune build ./main.exe - $ dune trace cat | jq -s 'include "dune"; ([.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length) > 0' - true + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' + 2 --- Dev profile (opaque=true): .mli change still rebuilds consumer --- @@ -75,5 +75,5 @@ Add another paired declaration: > EOF $ dune build ./main.exe - $ dune trace cat | jq -s 'include "dune"; ([.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length) > 0' - true + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' + 2 From 605280286b62b044fb229d628fe991638f61651a Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 11:25:01 -0700 Subject: [PATCH 06/80] test: clarify [unix] is shipped-with-OCaml, not part of [Stdlib] [unix] is a separate library bundled with the OCaml distribution and resolved through findlib; it isn't part of [Stdlib] proper. Addresses Copilot's review feedback: https://github.com/ocaml/dune/pull/14331#discussion_r3142346970 Signed-off-by: Robin Bate Boerop --- .../test-cases/per-module-lib-deps/opaque-cmx-deps-external.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t index f0f59201b2f..3529d708880 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-external.t @@ -2,7 +2,8 @@ A consumer's [.cmx] compilation rule depends on an external library's [.cmx] under both the release and dev profiles. Counterpart to [opaque-cmx-deps-local.t], which shows the dev-profile behaviour that omits the [.cmx] from the dep set for *local* libraries. The [unix] -stdlib library, resolved through findlib, plays the role of "external". +library, shipped with OCaml and resolved through findlib, plays the +role of "external". $ cat > dune-project < (lang dune 3.0) From 86e99977e9ac6a96087cf4aa97bc1d397a08ade1 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 09:56:26 -0700 Subject: [PATCH 07/80] test: observational baseline for an unreferenced transitive library dep An intermediate library [libB] declares [(libraries libA)] but its own source does not reference any module of [libA]. The consumer [main] uses [libB] and so transitively gains [libA] in its compilation context. Today's compile rules carry a glob over every transitively-reached library's public-cmi directory, so editing any of [libA]'s modules re-invalidates [main]. The test records the current rebuild count. Adds a baseline that future per-module dependency tracking work (#4572) can promote to a smaller number when the over-invalidation is fixed. Reproducer originally from @nojb: https://github.com/ocaml/dune/pull/14116#issuecomment-4319562014 Signed-off-by: Robin Bate Boerop --- .../transitive-unreferenced-lib.t | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t new file mode 100644 index 00000000000..a0c347f4746 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t @@ -0,0 +1,47 @@ +Baseline: an intermediate library [libB] declares [(libraries libA)] +but its module [modB] does not actually reference any of [libA]'s +modules. The consumer [main] uses [libB] and so transitively gains +[libA] in its compilation context. + +Today every consumer module declares a glob over each transitively- +reached library's public-cmi directory, so editing [modA1.ml] (which +no source file references) re-invalidates [main]. The test records +the current rebuild count of [main] when [modA1.ml] is touched. + +This test is observational: a tighter dependency tracker that drops +unreferenced libraries from compile rules' deps would lower the +count. + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ cat > dune < (library (name libA) (wrapped false) (modules modA1 modA2)) + > (library (name libB) (wrapped false) (modules modB) (libraries libA)) + > (executable (name main) (modules main) (libraries libB)) + > EOF + + $ cat > modA1.ml < let x = 42 + > EOF + $ cat > modA2.ml < let x = 43 + > EOF + $ cat > modB.ml < let x = 42 + > EOF + $ cat > main.ml < let _ = ModB.x + > EOF + + $ dune build @check + +Edit [modA1.ml]. Neither [main.ml] nor [modB.ml] references [modA1] +or any other [libA] module, so a tighter filter could leave [main] +untouched. Today [main] is rebuilt: + + $ echo > modA1.ml + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' + 2 From ee6b8f13f793d175f3dad5bd57e171d790f0df0e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 17:15:58 -0700 Subject: [PATCH 08/80] test: record consumer rebuild count when shadowed dep lib changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a cram test for the corner where two unwrapped libraries export an entry module of the same name and a consumer depends on both. OCaml's [-I] path resolution picks the first library listed in [(libraries ...)]; the other lib's same-named module is shadowed. Editing the losing lib's module still triggers a rebuild of the consumer on current [main] — cross-library dep tracking is library-level, so the consumer depends on a glob over each lib's public cmi dir and any change invalidates. The test is observational. It anchors current behavior for this under-tested corner and documents the expectation for future per-module filtering work on issue #4572: without qualified-path analysis, the filter cannot disambiguate which lib's [Shared] the consumer resolves through and must conservatively track both. Signed-off-by: Robin Bate Boerop --- .../lib-vs-lib-name-collision.t | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t new file mode 100644 index 00000000000..549dfaf0010 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t @@ -0,0 +1,72 @@ +Two unwrapped libraries expose an entry module of the same name. +A consumer library lists both and references the name in its code. +OCaml's [-I] path resolution picks the lib whose [-I] comes first +on the command line (the order of declared libraries in +[(libraries ...)]), so the consumer compiles against that lib's +version of the module. The other lib's same-named module is +shadowed — unreachable from consumer code. + +This test is observational: it records the rebuild-target count +for a consumer module after editing the *losing* (shadowed) lib's +same-named module. The expected count on current [main] is +positive because cross-library dep tracking is library-level: the +consumer depends on a glob over each declared library's public +cmi directory, so any change to the losing lib's public cmis +invalidates the consumer. + +A future per-module dependency filter (#4572) would not +automatically improve this case: without qualified-path analysis +the filter cannot know which lib's [Shared] the consumer actually +resolves through, and must conservatively depend on both. + +See: https://github.com/ocaml/dune/issues/4572 + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ mkdir lib_a + $ cat > lib_a/dune < (library (name lib_a) (wrapped false)) + > EOF + $ cat > lib_a/shared.ml < let from_a = "a" + > EOF + + $ mkdir lib_b + $ cat > lib_b/dune < (library (name lib_b) (wrapped false)) + > EOF + $ cat > lib_b/shared.ml < let from_b = "b" + > EOF + +[consumer] lists [lib_a] first, so [-I lib_a/.objs] comes before +[-I lib_b/.objs] on the command line and OCaml resolves [Shared] +to [lib_a/shared.cmi]: + + $ mkdir consumer + $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries lib_a lib_b)) + > EOF + $ cat > consumer/c.ml < let _ = Shared.from_a + > EOF + $ cat > consumer/d.ml < let _ = () + > EOF + + $ dune build @check + +Edit [lib_b]'s [Shared] — which the consumer does *not* resolve +through, because [lib_a] wins under [-I] order. Consumer still +rebuilds because its dependency on [lib_b]'s object directory is +a glob: + + $ cat > lib_b/shared.ml < let from_b = "b" + > let also_from_b = 42 + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length > 0' + true From 740d1f855d5dd46b13f01a63701b254d0694deee Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 13:39:07 -0700 Subject: [PATCH 09/80] test: add single-module consumer unreferenced-dep regression guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a blackbox test in which a single-module consumer library has a dependency library whose module it does not reference. When the dependency's module is edited (no signature change), the consumer module must not be recompiled. On current main, the cmi is hash-stable under this edit, so the observed rebuild target count is zero; the test guards against regressions of that behaviour. The test also anchors the per-module-lib-deps surface for future tightening work — when tighter per-module filtering lands (https://github.com/ocaml/dune/issues/4572), this scenario remains at zero rebuilds but for a different reason (consumer has no declared dep on the module). See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 Signed-off-by: Robin Bate Boerop --- .../single-module-unreferenced-lib.t | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t new file mode 100644 index 00000000000..43a51146bdc --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t @@ -0,0 +1,35 @@ +Per-module library filtering for a single-module consumer that does not +reference its dependency library. + +When a consumer library has one module and it does not reference any +module from a dependency library, changing modules in that dependency +must not trigger recompilation of the consumer. + +See: https://github.com/ocaml/dune/issues/4572 +See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 + + $ cat > dune-project < (lang dune 3.22) + > EOF + + $ cat > dune < (library (name libA) (wrapped false) (modules modA)) + > (library (name libB) (wrapped false) (modules modB) (libraries libA)) + > EOF + + $ cat > modA.ml < let x = 42 + > EOF + $ cat > modB.ml < let x = 12 + > EOF + + $ dune build @check + +Modify modA only; modB does not reference modA, so modB must not be +recompiled: + + $ echo "let x = 43" > modA.ml + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("modB"))] | length' + 0 From 456a57f0cf3332ac60dc8d154173dde2f4a959e4 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 15:45:21 -0700 Subject: [PATCH 10/80] test: rewrite as observational baseline with signature-change edit The previous form used an implementation-only edit to [modA.ml] with no [.mli], leaving the cmi hash-stable and producing a trivially-true rebuild-count assertion of 0 that did not exercise any aspect of per-module library filtering. Rewrite with [modA.mli] so the edit forces a cmi hash change. The consumer [modB] still rebuilds on current main (count = 1) because [libB] is a single-module stanza: dune skips ocamldep for it and the consumer falls back to a glob over [libA]'s object directory. This form tests a distinct corner from [single-module-lib.t] (which covers single-module consumers that reference some but not all modules of a multi-module dep): the zero-reference case could be tightened to zero rebuilds by a future fix that drops the lib dep when ocamldep yields no references, without needing to solve the broader single-module-consumer skip-ocamldep limitation. Signed-off-by: Robin Bate Boerop --- .../single-module-unreferenced-lib.t | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t index 43a51146bdc..7070e2c4d04 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t @@ -1,9 +1,23 @@ -Per-module library filtering for a single-module consumer that does not -reference its dependency library. - -When a consumer library has one module and it does not reference any -module from a dependency library, changing modules in that dependency -must not trigger recompilation of the consumer. +Baseline: single-module consumer library whose only module references +no module of its declared dependency library. + +This is an observational test. It records the number of rebuild +targets for the consumer's single module [modB] when a module of the +dependency library [libA] has its interface edited. + +[libB] declares [(libraries libA)] but [modB.ml] does not reference +any module of [libA]. On current main this scenario still rebuilds +[modB]: libB is a single-module stanza, so dune skips ocamldep as an +optimisation and cannot discover that [modB] references no module of +[libA]; the consumer falls back to a glob over [libA]'s object +directory, which is invalidated by the cmi change. + +The zero-reference case is a distinct corner from the single-module +consumer that references some (but not all) modules of its dep, +which [single-module-lib.t] already documents. A future fix that +detects "ocamldep yields no references to libA" could tighten this +corner to zero rebuilds without needing to solve the broader +single-module-consumer skip-ocamldep limitation. See: https://github.com/ocaml/dune/issues/4572 See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 @@ -20,16 +34,26 @@ See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 $ cat > modA.ml < let x = 42 > EOF + $ cat > modA.mli < val x : int + > EOF $ cat > modB.ml < let x = 12 > EOF $ dune build @check -Modify modA only; modB does not reference modA, so modB must not be -recompiled: +Edit modA's interface. modB does not reference modA. Record the +number of modB rebuild targets observed in the trace: - $ echo "let x = 43" > modA.ml + $ cat > modA.mli < val x : int + > val y : string + > EOF + $ cat > modA.ml < let x = 42 + > let y = "hello" + > EOF $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("modB"))] | length' - 0 + 1 From 1cc01c2b72b219ba85f3160dc36b75b3adf12513 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 13:40:32 -0700 Subject: [PATCH 11/80] test: add baseline observational test for per-module rebuild count Add a blackbox test that records, for a consumer module C referencing only A2 of an unwrapped library [base] with three entry modules A1/A2/A3 (each with an explicit interface), the rebuild-target count for C when each of A1/A2/A3 has its signature edited. On current main, editing any of A1/A2/A3 rebuilds C because the consumer depends conservatively on a directory-wide glob over [base]'s object dir. The test records A1=1, A3=1, A2>0. The test anchors the tightening surface for https://github.com/ocaml/dune/issues/4572: once per-module filtering lands, editing A1 or A3 is expected to leave C untouched, at which point the recorded counts are promoted. Signed-off-by: Robin Bate Boerop --- .../unwrapped-tight-deps.t | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t new file mode 100644 index 00000000000..c38926c5eb9 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -0,0 +1,103 @@ +Baseline: consumer-module rebuild count when individual modules of +an unwrapped dependency library change. + +This is an observational test. It records the number of rebuild +targets for a single consumer module C when each of three entry +modules (A1, A2, A3) of the dependency library [base] has its +interface edited. C references only A2. + +On current main, editing any one of A1/A2/A3 causes C to rebuild +because library-level dependency filtering (and, within that, +per-module tightening) is not yet in place: the consumer is +conservatively rebuilt whenever any entry module's cmi changes. +Work on https://github.com/ocaml/dune/issues/4572 is expected to +tighten this, at which point editing A1 or A3 leaves C untouched +and the emitted counts are promoted. + +See: https://github.com/ocaml/dune/issues/4572 + + $ cat > dune-project < (lang dune 3.22) + > EOF + +base is an unwrapped library with three entry modules, each with an +explicit interface so signature changes propagate through .cmi files: + + $ mkdir base + $ cat > base/dune < (library (name base) (wrapped false)) + > EOF + $ cat > base/a1.ml < let v = 1 + > EOF + $ cat > base/a1.mli < val v : int + > EOF + $ cat > base/a2.ml < let v = 2 + > EOF + $ cat > base/a2.mli < val v : int + > EOF + $ cat > base/a3.ml < let v = 3 + > EOF + $ cat > base/a3.mli < val v : int + > EOF + +consumer has a single module that references only A2 from base: + + $ mkdir consumer + $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries base)) + > EOF + $ cat > consumer/c.ml < let w = A2.v + > EOF + + $ dune build @check + +Edit A1's interface — a module C does not reference — and record +the rebuild-target count for C: + + $ cat > base/a1.mli < val v : int + > val extra : unit -> int + > EOF + $ cat > base/a1.ml < let v = 1 + > let extra () = 7 + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length' + 1 + +Same for A3: + + $ cat > base/a3.mli < val v : int + > val other : string + > EOF + $ cat > base/a3.ml < let v = 3 + > let other = "hi" + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length' + 1 + +Edit A2's interface — the one module C does reference — and check +that the count is positive (C must rebuild): + + $ cat > base/a2.mli < val v : int + > val new_fn : int -> int + > EOF + $ cat > base/a2.ml < let v = 2 + > let new_fn x = x + 1 + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length > 0' + true From d9b05e4b3e89251a7bdd1219a9c7102756210672 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 15:49:18 -0700 Subject: [PATCH 12/80] test: make consumer multi-module to isolate per-module filter Previously [consumer] was a single-module library stanza. Dune skips ocamldep for single-module stanzas with no library deps as an optimisation, and the per-module-lib-deps filter depends on ocamldep output. So under the filter's feature branch the test's A1/A3 rebuild counts flipped from 1 to 0 for two independent reasons: the filter running on ocamldep output, and a concurrent tweak to the skip-ocamldep optimisation to run ocamldep for single-module stanzas that have library deps. Add an unused module [d.ml] to [consumer]. Ocamldep now runs unconditionally for [consumer], and the recorded rebuild count for [c] reflects only the per-module filter's work, not the skip-ocamldep behaviour. The probe regex still matches [c] only so [d] being in the rebuild set does not affect the count. Signed-off-by: Robin Bate Boerop --- .../per-module-lib-deps/unwrapped-tight-deps.t | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t index c38926c5eb9..15a2ed9d520 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -46,7 +46,13 @@ explicit interface so signature changes propagate through .cmi files: > val v : int > EOF -consumer has a single module that references only A2 from base: +consumer has two modules. The module of interest, [c.ml], references +only [A2] from [base]. A second, unused module [d.ml] is present only +to keep [consumer] a multi-module stanza: dune skips ocamldep for +single-module stanzas with no library deps as an optimisation, and +the per-module-lib-deps filter depends on ocamldep output. Including +[d.ml] isolates this test from the skip-ocamldep optimisation so the +rebuild count for [c] reflects only the per-module filter's work: $ mkdir consumer $ cat > consumer/dune < consumer/c.ml < let w = A2.v > EOF + $ cat > consumer/d.ml < let _ = () + > EOF $ dune build @check From d1286bd9c59929aad7dac9a8c964e48906a36999 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 16:38:18 -0700 Subject: [PATCH 13/80] test: narrow subst pattern to avoid matching paths The test's output post-processing used `dune_cmd subst '-\d+' '-eol'` to normalise the variable end-column in the "characters 6-N:" part of dune's error location. The pattern was too broad: any `-` substring in the output would match, including path components in the sandbox root (e.g. when the repo is checked out at a worktree whose directory contains `-`, such as `dune-pr-14116/`, the substitution rewrites the path and BUILD_PATH_PREFIX_MAP stripping no longer matches, producing a raw-path diff in the test output). Anchor the pattern to `characters 6-\d+` so it only matches the column range it was meant to normalise. Signed-off-by: Robin Bate Boerop --- .../test-cases/pkg/pin-stanza/non-existent-branch.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/blackbox-tests/test-cases/pkg/pin-stanza/non-existent-branch.t b/test/blackbox-tests/test-cases/pkg/pin-stanza/non-existent-branch.t index 219f3dcbb50..222e7a60ecc 100644 --- a/test/blackbox-tests/test-cases/pkg/pin-stanza/non-existent-branch.t +++ b/test/blackbox-tests/test-cases/pkg/pin-stanza/non-existent-branch.t @@ -29,7 +29,7 @@ Reference a branch that does not exist in the pin: > EOF $ export BUILD_PATH_PREFIX_MAP="PWD=//$PWD:$BUILD_PATH_PREFIX_MAP" - $ dune pkg lock 2>&1 | dune_cmd subst '-\d+' '-eol' | dune_cmd delete '\^+' + $ dune pkg lock 2>&1 | dune_cmd subst 'characters 6-\d+' 'characters 6-eol' | dune_cmd delete '\^+' File "dune-project", line 3, characters 6-eol: 3 | (url "git+file:PWD/_repo#nonexistent-branch") revision "nonexistent-branch" not found in From 0e31b6ae922a17733af8d3632d8b8194b6012883 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 16:49:50 -0700 Subject: [PATCH 14/80] test: lower lang version to 3.0 Per Copilot review on #14309: the test does not rely on any feature newer than 3.0; the [(lang dune)] version can gate dependency-analysis behaviour, so using 3.22 may unintentionally change what this test is guarding. Consistent with other baseline tests in this directory. Signed-off-by: Robin Bate Boerop --- .../per-module-lib-deps/single-module-unreferenced-lib.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t index 7070e2c4d04..e7ca3a04b70 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t @@ -23,7 +23,7 @@ See: https://github.com/ocaml/dune/issues/4572 See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 $ cat > dune-project < (lang dune 3.22) + > (lang dune 3.0) > EOF $ cat > dune < Date: Thu, 23 Apr 2026 16:50:11 -0700 Subject: [PATCH 15/80] test: lower lang version to 3.0 Per Copilot review on #14309: the test does not rely on any feature newer than 3.0; the [(lang dune)] version can gate dependency-analysis behaviour, so using 3.22 may unintentionally change what this test is guarding. Consistent with other baseline tests in this directory. Signed-off-by: Robin Bate Boerop --- .../test-cases/per-module-lib-deps/unwrapped-tight-deps.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t index 15a2ed9d520..ec2d7903b95 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -17,7 +17,7 @@ and the emitted counts are promoted. See: https://github.com/ocaml/dune/issues/4572 $ cat > dune-project < (lang dune 3.22) + > (lang dune 3.0) > EOF base is an unwrapped library with three entry modules, each with an From e4c58e2e5b4c16719f3a49f7b459869e2c234a57 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 10 Apr 2026 00:54:49 -0400 Subject: [PATCH 16/80] feat: use ocamldep for per-module inter-library dependency filtering (#4572) Use the already-computed ocamldep output to filter inter-library dependencies on a per-module basis. Each module now only depends on .cmi/.cmx files from libraries it actually imports, rather than glob deps on all files from all dependent libraries. This eliminates unnecessary recompilation when a module in a dependency changes but the current module doesn't reference that library. The implementation: - Add read_immediate_deps_raw_of to ocamldep, returning raw module names without filtering against the stanza's module set - Move Hidden_deps from Includes (library-wide) to per-module in build_cm - Add Lib_index mapping module names to libraries, computed from requires_compile and requires_hidden - In build_cm, use ocamldep output + Lib_index to determine which libraries each module actually needs; fall back to all-library glob deps when filtering is not possible (Melange, virtual library implementations, singleton stanzas, alias/root modules) - For local unwrapped libraries, use per-file deps on specific .cmi/.cmx files rather than directory-wide globs Signed-off-by: Robin Bate Boerop chore: add changelog entry for per-module dependency filtering (#4572) Signed-off-by: Robin Bate Boerop --- doc/changes/added/14116.md | 4 + src/dune_rules/compilation_context.ml | 81 +++++---- src/dune_rules/compilation_context.mli | 1 + src/dune_rules/dep_graph.ml | 1 + src/dune_rules/dep_graph.mli | 1 + src/dune_rules/lib_file_deps.ml | 65 ++++++- src/dune_rules/lib_file_deps.mli | 25 ++- src/dune_rules/module_compilation.ml | 167 ++++++++++++++++++ src/dune_rules/ocaml_flags.ml | 10 ++ src/dune_rules/ocaml_flags.mli | 3 + src/dune_rules/ocamldep.ml | 18 ++ src/dune_rules/ocamldep.mli | 11 ++ src/dune_rules/virtual_rules.ml | 5 + src/dune_rules/virtual_rules.mli | 1 + .../alias/check-alias/ocamldep-cycles.t | 2 +- .../test-cases/ocamldep/ocamldep-7018.t | 2 +- .../per-module-lib-deps/basic-wrapped.t | 4 +- .../lib-to-lib-unwrapped.t | 9 +- .../per-module-lib-deps/lib-to-lib-wrapped.t | 4 +- .../per-module-lib-deps/multiple-libraries.t | 8 +- .../test-cases/per-module-lib-deps/opaque.t | 4 +- .../per-module-lib-deps/stdlib-modules.t | 4 +- .../per-module-lib-deps/transitive.t | 4 +- .../per-module-lib-deps/unwrapped.t | 9 +- .../per-module-lib-deps/wrapped-compat.t | 4 +- .../test-cases/reporting-of-cycles.t/run.t | 8 - 26 files changed, 383 insertions(+), 72 deletions(-) create mode 100644 doc/changes/added/14116.md diff --git a/doc/changes/added/14116.md b/doc/changes/added/14116.md new file mode 100644 index 00000000000..54605dd90bb --- /dev/null +++ b/doc/changes/added/14116.md @@ -0,0 +1,4 @@ +- Use `ocamldep` output to filter inter-library dependencies on a + per-module basis, eliminating unnecessary recompilations when a + dependency library changes but the importing module doesn't + reference it. (#14116, fixes #4572, @robinbb) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index b629d7874d3..aca3208aa15 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -4,55 +4,29 @@ open Memo.O module Includes = struct type t = Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t - let make ~project ~opaque ~direct_requires ~hidden_requires lib_config + let make ~project ~direct_requires ~hidden_requires lib_config : _ Lib_mode.Cm_kind.Map.t = - (* TODO: some of the requires can filtered out using [ocamldep] info *) let open Resolve.Memo.O in let iflags direct_libs hidden_libs mode = Lib_flags.L.include_flags ~project ~direct_libs ~hidden_libs mode lib_config in - let make_includes_args ~mode groups = + let make_includes_args ~mode = (let+ direct_libs = direct_requires and+ hidden_libs = hidden_requires in - Command.Args.S - [ iflags direct_libs hidden_libs mode - ; Hidden_deps (Lib_file_deps.deps (direct_libs @ hidden_libs) ~groups) - ]) + iflags direct_libs hidden_libs mode) |> Resolve.Memo.args |> Command.Args.memo in { ocaml = - (let cmi_includes = make_includes_args ~mode:(Ocaml Byte) [ Ocaml Cmi ] in + (let cmi_includes = make_includes_args ~mode:(Ocaml Byte) in { cmi = cmi_includes ; cmo = cmi_includes - ; cmx = - (let+ direct_libs = direct_requires - and+ hidden_libs = hidden_requires in - Command.Args.S - [ iflags direct_libs hidden_libs (Ocaml Native) - ; Hidden_deps - (let libs = direct_libs @ hidden_libs in - if opaque - then - List.map libs ~f:(fun lib -> - ( lib - , if Lib.is_local lib - then [ Lib_file_deps.Group.Ocaml Cmi ] - else [ Ocaml Cmi; Ocaml Cmx ] )) - |> Lib_file_deps.deps_with_exts - else - Lib_file_deps.deps - libs - ~groups:[ Lib_file_deps.Group.Ocaml Cmi; Ocaml Cmx ]) - ]) - |> Resolve.Memo.args - |> Command.Args.memo + ; cmx = make_includes_args ~mode:(Ocaml Native) }) ; melange = - { cmi = make_includes_args ~mode:Melange [ Melange Cmi ] - ; cmj = make_includes_args ~mode:Melange [ Melange Cmi; Melange Cmj ] - } + (let mel_includes = make_includes_args ~mode:Melange in + { cmi = mel_includes; cmj = mel_includes }) } ;; @@ -91,6 +65,7 @@ type t = ; parameters : Module_name.t list Resolve.Memo.t ; instances : Parameterised_instances.t Resolve.Memo.t option ; includes : Includes.t + ; lib_index : Lib_file_deps.Lib_index.t Resolve.Memo.t ; preprocessing : Pp_spec.t ; opaque : bool ; js_of_ocaml : Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t @@ -118,6 +93,7 @@ let requires_hidden t = t.requires_hidden let requires_link t = Memo.Lazy.force t.requires_link let parameters t = t.parameters let includes t = t.includes +let lib_index t = t.lib_index let preprocessing t = t.preprocessing let opaque t = t.opaque let js_of_ocaml t = t.js_of_ocaml @@ -240,8 +216,41 @@ let create ; requires_link ; implements ; parameters - ; includes = - Includes.make ~project ~opaque ~direct_requires ~hidden_requires ocaml.lib_config + ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config + ; lib_index = + (* Maps module names to libraries for per-module inter-library dependency + filtering. All entries use [None] for the module (glob deps on the + library) rather than [Some m] (per-file deps on a specific module). + Per-file deps for unwrapped libraries are not yet supported because + modules in unwrapped libraries can alias modules from other libraries, + and we don't yet track the dependency cone of individual modules. *) + (let open Resolve.Memo.O in + let* all_libs = direct_requires in + let+ entries = + Resolve.Memo.List.concat_map all_libs ~f:(fun lib -> + let* main = Lib.main_module_name lib in + match main with + | Some name -> + (* Wrapped library: index by the wrapper module name. *) + Resolve.Memo.return [ name, ((lib : Lib.t), (None : Module.t option)) ] + | None -> + (* Unwrapped library: index by each entry module name. *) + (match Lib_info.entry_modules (Lib.info lib) ~for_ with + | External (Ok names) -> + Resolve.Memo.return (List.map names ~f:(fun n -> n, (lib, None))) + | External (Error e) -> Resolve.Memo.of_result (Error e) + | Local -> + Resolve.Memo.lift_memo + (Memo.map + (Dir_contents.modules_of_local_lib + super_context + (Lib.Local.of_lib_exn lib) + ~for_) + ~f:(fun mods -> + List.map (Modules.entry_modules mods) ~f:(fun m -> + Module.name m, (lib, None)))))) + in + Lib_file_deps.Lib_index.create entries) ; preprocessing ; opaque ; js_of_ocaml @@ -333,7 +342,6 @@ let for_module_generated_at_link_time cctx ~requires ~module_ = let direct_requires = requires in Includes.make ~project:(Scope.project cctx.scope) - ~opaque ~direct_requires ~hidden_requires cctx.ocaml.lib_config @@ -344,6 +352,7 @@ let for_module_generated_at_link_time cctx ~requires ~module_ = ; requires_link = Memo.lazy_ (fun () -> requires) ; requires_compile = requires ; includes + ; lib_index = Resolve.Memo.return Lib_file_deps.Lib_index.empty ; modules } ;; diff --git a/src/dune_rules/compilation_context.mli b/src/dune_rules/compilation_context.mli index f5ac4cae7ca..2cc6cebeb50 100644 --- a/src/dune_rules/compilation_context.mli +++ b/src/dune_rules/compilation_context.mli @@ -62,6 +62,7 @@ val requires_hidden : t -> Lib.t list Resolve.Memo.t val requires_compile : t -> Lib.t list Resolve.Memo.t val parameters : t -> Module_name.t list Resolve.Memo.t val includes : t -> Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t +val lib_index : t -> Lib_file_deps.Lib_index.t Resolve.Memo.t val preprocessing : t -> Pp_spec.t val opaque : t -> bool val js_of_ocaml : t -> Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t diff --git a/src/dune_rules/dep_graph.ml b/src/dune_rules/dep_graph.ml index 8ff3f084366..891c7249fe0 100644 --- a/src/dune_rules/dep_graph.ml +++ b/src/dune_rules/dep_graph.ml @@ -7,6 +7,7 @@ type t = } let make ~dir ~per_module = { dir; per_module } +let dir t = t.dir let deps_of t (m : Module.t) = match Module_name.Unique.Map.find t.per_module (Module.obj_name m) with diff --git a/src/dune_rules/dep_graph.mli b/src/dune_rules/dep_graph.mli index d5f663b222f..4a9a5af7a83 100644 --- a/src/dune_rules/dep_graph.mli +++ b/src/dune_rules/dep_graph.mli @@ -9,6 +9,7 @@ val make -> per_module:Module.t list Action_builder.t Module_name.Unique.Map.t -> t +val dir : t -> Path.Build.t val deps_of : t -> Module.t -> Module.t list Action_builder.t val top_closed_implementations : t -> Module.t list -> Module.t list Action_builder.t diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 0aa3440da59..72966c56f2d 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -48,9 +48,72 @@ let deps_of_lib (lib : Lib.t) ~groups = |> Dep.Set.of_list ;; -let deps_with_exts = Dep.Set.union_map ~f:(fun (lib, groups) -> deps_of_lib lib ~groups) let deps libs ~groups = Dep.Set.union_map libs ~f:(deps_of_lib ~groups) +(* Currently unused: per-file deps on individual modules within a library. + Retained for potential future use when per-module filtering of unwrapped + libraries is supported. *) +let deps_of_module (lib : Lib.t) (m : Module.t) ~cm_kinds = + let obj_dir = Lib.info lib |> Lib_info.obj_dir in + List.filter_map cm_kinds ~f:(fun kind -> + Obj_dir.Module.cm_public_file obj_dir m ~kind |> Option.map ~f:(fun p -> Dep.file p)) + |> Dep.Set.of_list +;; + +let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) entries = + let groups_for lib = + match cm_kind with + | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ] + | Ocaml Cmx -> + if opaque && Lib.is_local lib + then [ Group.Ocaml Cmi ] + else [ Group.Ocaml Cmi; Group.Ocaml Cmx ] + | Melange Cmi -> [ Group.Melange Cmi ] + | Melange Cmj -> [ Group.Melange Cmi; Group.Melange Cmj ] + in + (* Currently unused: [cm_kinds_for] and the [Some m] branch below support + per-file deps on individual modules. All entries currently use [None] + (glob deps), so this path is dead code. Retained for future use. *) + let cm_kinds_for lib = + match cm_kind with + | Ocaml Cmi | Ocaml Cmo -> [ Lib_mode.Cm_kind.Ocaml Cmi ] + | Ocaml Cmx -> + if opaque && Lib.is_local lib + then [ Lib_mode.Cm_kind.Ocaml Cmi ] + else [ Lib_mode.Cm_kind.Ocaml Cmi; Ocaml Cmx ] + | Melange Cmi -> [ Lib_mode.Cm_kind.Melange Cmi ] + | Melange Cmj -> [ Lib_mode.Cm_kind.Melange Cmi; Melange Cmj ] + in + Dep.Set.union_map entries ~f:(fun (lib, module_opt) -> + match module_opt with + | None -> deps_of_lib lib ~groups:(groups_for lib) + | Some m -> deps_of_module lib m ~cm_kinds:(cm_kinds_for lib)) +;; + +module Lib_index = struct + type entry = Lib.t * Module.t option + type t = { by_module_name : entry list Module_name.Map.t } + + let empty = { by_module_name = Module_name.Map.empty } + + let create entries = + let by_module_name = + List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, entry) -> + Module_name.Map.update map name ~f:(function + | None -> Some [ entry ] + | Some entries -> Some (entry :: entries))) + in + { by_module_name } + ;; + + let filter_libs t ~referenced_modules = + Module_name.Set.fold referenced_modules ~init:[] ~f:(fun name acc -> + match Module_name.Map.find t.by_module_name name with + | None -> acc + | Some entries -> List.rev_append entries acc) + ;; +end + type path_specification = | Allow_all | Disallow_external of Lib_name.t diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 75a3453e64b..510a5682660 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -15,7 +15,30 @@ end with extension [files] of libraries [libs]. *) val deps : Lib.t list -> groups:Group.t list -> Dep.Set.t -val deps_with_exts : (Lib.t * Group.t list) list -> Dep.Set.t +(** [deps_of_entries ~opaque ~cm_kind entries] computes the file dependencies + for the given library entries. When the module in an entry is [None], glob + deps are used for the library. When [Some m], per-file deps on specific + cm files are used. Currently all callers pass [None]; the [Some] path is + retained for potential future per-module filtering of unwrapped libraries. *) +val deps_of_entries + : opaque:bool + -> cm_kind:Lib_mode.Cm_kind.t + -> (Lib.t * Module.t option) list + -> Dep.Set.t + +module Lib_index : sig + type entry = Lib.t * Module.t option + type t + + val empty : t + + (** Create an index from a list of (module_name, entry) pairs. *) + val create : (Module_name.t * entry) list -> t + + (** Return the library entries whose module names appear in + [referenced_modules]. *) + val filter_libs : t -> referenced_modules:Module_name.Set.t -> entry list +end type path_specification = | Allow_all diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 75bcea17c87..37add1296ab 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -286,6 +286,156 @@ let build_cm | Some All | None -> Hidden_targets [ obj ]) in let opaque = Compilation_context.opaque cctx in + let skip_lib_deps = + match Module.kind m with + | Alias _ -> + not (Modules.With_vlib.is_stdlib_alias (Compilation_context.modules cctx) m) + | Wrapped_compat -> true + | _ -> false + in + let for_ = Compilation_context.for_ cctx in + let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in + (* Per-module inter-library dependency filtering (#4572). When possible, + we use ocamldep output to determine which libraries each module actually + references, replacing the previous glob deps on all dependent libraries. + + Filtering is disabled when: + - Melange mode (OCaml-only optimization) + - Dep graph dir differs from obj dir (shouldn't happen in practice) + - Special module kinds (Root, Wrapped_compat, Impl_vmodule, Virtual, + Parameter) that don't have standard ocamldep output + - Module lacks the current ml_kind source file + - Virtual library implementations (parameter libraries are not in + requires_compile) *) + let can_filter = + (not skip_lib_deps) + && (match Lib_mode.of_cm_kind cm_kind with + | Melange -> false + | Ocaml _ -> true) + && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) + && (match Module.kind m with + | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false + | _ -> true) + && Module.has m ~ml_kind + && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) + in + (* Static lib deps: used when filtering is not possible, preserves the + original Hidden_deps-in-Command.Args behavior exactly. *) + let lib_cm_deps_args = + if skip_lib_deps || can_filter + then Command.Args.empty + else + (let open Resolve.Memo.O in + let+ direct_libs = Compilation_context.requires_compile cctx + and+ hidden_libs = Compilation_context.requires_hidden cctx in + let libs = direct_libs @ hidden_libs in + let entries = List.map libs ~f:(fun lib -> lib, None) in + Command.Args.Hidden_deps (Lib_file_deps.deps_of_entries ~opaque ~cm_kind entries)) + |> Resolve.Memo.args + |> Command.Args.memo + in + (* Dynamic lib deps: used when per-module filtering is possible. *) + let lib_cm_deps_filtered = + if not can_filter + then Action_builder.return () + else + Action_builder.dyn_deps + (let open Action_builder.O in + let* libs = + Resolve.Memo.read + (let open Resolve.Memo.O in + let+ d = Compilation_context.requires_compile cctx + and+ h = Compilation_context.requires_hidden cctx in + d @ h) + in + let has_virtual_impl = + List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) + in + if has_virtual_impl + then ( + let entries = List.map libs ~f:(fun lib -> lib, None) in + Action_builder.return + ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind entries)) + else + let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in + let* raw_deps_m = + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ m + in + (* deps_of already returns transitive intra-library deps *) + let* trans_deps = Dep_graph.deps_of dep_graph m in + let* trans_raw_deps = + Action_builder.List.map trans_deps ~f:(fun dep_m -> + let is_standard_kind = + match Module.kind dep_m with + | Impl_vmodule | Virtual | Root | Alias _ | Wrapped_compat | Parameter + -> false + | _ -> true + in + if not is_standard_kind + then Action_builder.return Module_name.Set.empty + else ( + (* Try the current ml_kind first; fall back to the other + kind for interface-only modules that may contain aliases + to other libraries. *) + let dep_ml_kind = + if Module.has dep_m ~ml_kind + then ml_kind + else ( + match ml_kind with + | Ml_kind.Impl -> Ml_kind.Intf + | Intf -> Impl) + in + if Module.has dep_m ~ml_kind:dep_ml_kind + then + Ocamldep.read_immediate_deps_raw_of + ~obj_dir + ~ml_kind:dep_ml_kind + ~for_ + dep_m + else Action_builder.return Module_name.Set.empty)) + in + let all_raw = + List.fold_left trans_raw_deps ~init:raw_deps_m ~f:Module_name.Set.union + in + let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in + let open_modules = Ocaml_flags.extract_open_module_names flags in + let referenced = Module_name.Set.union all_raw open_modules in + let filtered_entries = + Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced + in + (* Transitively close the filtered libraries within [libs]. + Transparent module aliases can create cross-library .cmi reads + that ocamldep doesn't report, at arbitrary depth. *) + let libs_set = Table.create (module Lib) (List.length libs) in + List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); + let covered = Table.create (module Lib) 8 in + List.iter filtered_entries ~f:(fun (lib, _) -> Table.set covered lib ()); + let rec close_over queue acc = + match queue with + | [] -> Action_builder.return acc + | lib :: rest -> + let open Action_builder.O in + let* requires = Resolve.Memo.read (Lib.requires lib ~for_) in + let new_deps = + List.filter_map requires ~f:(fun dep -> + if Table.mem libs_set dep && not (Table.mem covered dep) + then ( + Table.set covered dep (); + Some dep) + else None) + in + let new_entries = + List.map new_deps ~f:(fun dep -> dep, (None : Module.t option)) + in + close_over (new_deps @ rest) (new_entries @ acc) + in + let+ transitive_entries = close_over (List.map filtered_entries ~f:fst) [] in + ( () + , Lib_file_deps.deps_of_entries + ~opaque + ~cm_kind + (filtered_entries @ transitive_entries) )) + in let other_cm_files = let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in let module_deps = Dep_graph.deps_of dep_graph m in @@ -404,6 +554,7 @@ let build_cm ?loc:(Compilation_context.loc cctx) (let open Action_builder.With_targets.O in Action_builder.with_no_targets other_cm_files + >>> Action_builder.with_no_targets lib_cm_deps_filtered >>> Command.run ~dir:(Path.build (Context.build_dir ctx)) compiler @@ -414,6 +565,7 @@ let build_cm ; Command.Args.S obj_dirs ; Command.Args.as_any (Lib_mode.Cm_kind.Map.get (Compilation_context.includes cctx) cm_kind) + ; Command.Args.as_any lib_cm_deps_args ; extra_args ; As as_parameter_arg ; as_argument_for @@ -519,6 +671,20 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = List.concat_map deps ~f:(fun m -> [ Path.build (Obj_dir.Module.cm_file_exn obj_dir m ~kind:(Ocaml Cmi)) ])) in + let lib_cm_deps = + let opaque = Compilation_context.opaque cctx in + Action_builder.dyn_deps + (let open Action_builder.O in + let+ libs = + Resolve.Memo.read + (let open Resolve.Memo.O in + let+ d = Compilation_context.requires_compile cctx + and+ h = Compilation_context.requires_hidden cctx in + d @ h) + in + let entries = List.map libs ~f:(fun lib -> lib, None) in + (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) entries) + in let ocaml_flags = Ocaml_flags.get (Compilation_context.flags cctx) (Ocaml Byte) in let modules = Compilation_context.modules cctx in let ocaml = Compilation_context.ocaml cctx in @@ -529,6 +695,7 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = ~file_targets:[ output ] (let open Action_builder.With_targets.O in Action_builder.with_no_targets cm_deps + >>> Action_builder.with_no_targets lib_cm_deps >>> Command.run (Ok ocaml.ocamlc) ~dir:(Path.build (Context.build_dir ctx)) diff --git a/src/dune_rules/ocaml_flags.ml b/src/dune_rules/ocaml_flags.ml index b8c818790c8..61cc7d13f71 100644 --- a/src/dune_rules/ocaml_flags.ml +++ b/src/dune_rules/ocaml_flags.ml @@ -197,3 +197,13 @@ let allow_only_melange t = let open_flags modules = List.concat_map modules ~f:(fun name -> [ "-open"; Module_name.to_string name ]) ;; + +let extract_open_module_names flags = + let rec loop acc = function + | "-open" :: name :: rest -> + loop (Module_name.Set.add acc (Module_name.of_checked_string name)) rest + | _ :: rest -> loop acc rest + | [] -> acc + in + loop Module_name.Set.empty flags +;; diff --git a/src/dune_rules/ocaml_flags.mli b/src/dune_rules/ocaml_flags.mli index c83d8c78cfe..e2dd7648565 100644 --- a/src/dune_rules/ocaml_flags.mli +++ b/src/dune_rules/ocaml_flags.mli @@ -30,3 +30,6 @@ val with_vendored_alerts : t -> t val dump : t -> Dune_lang.t list Action_builder.t val with_vendored_flags : t -> ocaml_version:Version.t -> t val open_flags : Module_name.t list -> string list + +(** Extract module names from [-open Foo] pairs in compiler flags. *) +val extract_open_module_names : string list -> Module_name.Set.t diff --git a/src/dune_rules/ocamldep.ml b/src/dune_rules/ocamldep.ml index 22fb8f5a89d..9dab235040d 100644 --- a/src/dune_rules/ocamldep.ml +++ b/src/dune_rules/ocamldep.ml @@ -199,3 +199,21 @@ let read_immediate_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = |> parse_module_names ~dir:(Obj_dir.dir obj_dir) ~unit ~modules) |> Action_builder.memoize (Path.Build.to_string ocamldep_output) ;; + +(* Like [read_immediate_deps_of] but returns raw module names without + resolving against the stanza's module set. This preserves references to + external libraries, which [parse_module_names] would discard. Used for + per-module inter-library dependency filtering (#4572). *) +let read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ unit = + match Module.source ~ml_kind unit with + | None -> Action_builder.return Module_name.Set.empty + | Some source -> + (match Obj_dir.Module.dep obj_dir ~for_ (Immediate (unit, ml_kind)) with + | None -> Action_builder.return Module_name.Set.empty + | Some ocamldep_output -> + Action_builder.lines_of (Path.build ocamldep_output) + |> Action_builder.map ~f:(fun lines -> + parse_deps_exn ~file:(Module.File.path source) lines + |> Module_name.Set.of_list_map ~f:Module_name.of_checked_string) + |> Action_builder.memoize (Path.Build.to_string ocamldep_output ^ ".raw")) +;; diff --git a/src/dune_rules/ocamldep.mli b/src/dune_rules/ocamldep.mli index d455459262e..b86e7de6b45 100644 --- a/src/dune_rules/ocamldep.mli +++ b/src/dune_rules/ocamldep.mli @@ -32,3 +32,14 @@ val read_immediate_deps_of -> for_:Compilation_mode.t -> Module.t -> Module.t list Action_builder.t + +(** [read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ unit] returns the raw + module names from ocamldep output without filtering against the stanza's + module set. This preserves cross-library references that + [read_immediate_deps_of] discards. *) +val read_immediate_deps_raw_of + : obj_dir:Path.Build.t Obj_dir.t + -> ml_kind:Ml_kind.t + -> for_:Compilation_mode.t + -> Module.t + -> Module_name.Set.t Action_builder.t diff --git a/src/dune_rules/virtual_rules.ml b/src/dune_rules/virtual_rules.ml index d353ae5b1a1..d1329b255f6 100644 --- a/src/dune_rules/virtual_rules.ml +++ b/src/dune_rules/virtual_rules.ml @@ -11,6 +11,11 @@ type t = let no_implements = No_implements +let is_implementation = function + | Virtual _ | Parameter _ -> true + | No_implements -> false +;; + let setup_copy_rules_for_impl ~sctx ~dir t = match t with | No_implements | Parameter _ -> Memo.return () diff --git a/src/dune_rules/virtual_rules.mli b/src/dune_rules/virtual_rules.mli index a56d595d1de..90146957ee2 100644 --- a/src/dune_rules/virtual_rules.mli +++ b/src/dune_rules/virtual_rules.mli @@ -9,6 +9,7 @@ val setup_copy_rules_for_impl -> unit Memo.t val no_implements : t +val is_implementation : t -> bool val impl : Super_context.t diff --git a/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t b/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t index 5dbc6613b39..57cbe687536 100644 --- a/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t +++ b/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t @@ -1,4 +1,4 @@ -The @check alias should detect dependency cycles +The @check alias should detect dependency cycles. $ make_dune_project 3.2 diff --git a/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t b/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t index c618d310696..de19fef9ecd 100644 --- a/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t +++ b/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t @@ -30,7 +30,7 @@ Reproduces #7018 > dune build > } -First we try to construct X.t directly +First we try to construct X.t directly. $ runtest "()" Error: dependency cycle between modules in _build/default: diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/basic-wrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/basic-wrapped.t index eb203482193..91ae8e96e4b 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/basic-wrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/basic-wrapped.t @@ -58,8 +58,8 @@ Change mylib's interface: > let new_function () = "hello" > EOF -No_use_lib is recompiled even though it doesn't reference Mylib: +No_use_lib is not recompiled because it doesn't reference Mylib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("No_use_lib"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t index 7ca6696af99..c8187741849 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t @@ -80,11 +80,12 @@ Change only alpha.mli: > let new_alpha_fn () = "alpha" > EOF -uses_beta is recompiled even though it only references Beta, not Alpha: +uses_beta is recompiled because unwrapped libraries use glob deps (per-module +filtering within unwrapped libraries is not yet supported): $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_beta"))] | length' - 2 + 1 Change only beta.mli: @@ -97,8 +98,8 @@ Change only beta.mli: > let new_beta_fn () = "beta" > EOF -uses_alpha is recompiled even though it only references Alpha, not Beta: +uses_alpha is recompiled because unwrapped libraries use glob deps: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_alpha"))] | length' - 2 + 1 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-wrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-wrapped.t index 72cbb718d9f..bc690d0beeb 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-wrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-wrapped.t @@ -62,8 +62,8 @@ See: https://github.com/ocaml/dune/issues/4572 > let new_base_fn () = "hello" > EOF -Standalone in middle_lib is recompiled even though it doesn't use base_lib: +Standalone in middle_lib is not recompiled because it doesn't use base_lib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Standalone"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/multiple-libraries.t b/test/blackbox-tests/test-cases/per-module-lib-deps/multiple-libraries.t index da2810d728b..84683c7bd2e 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/multiple-libraries.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/multiple-libraries.t @@ -78,11 +78,11 @@ Change only mylib's interface: > let new_function () = "hello" > EOF -Uses_other is recompiled even though it only uses Otherlib, not Mylib: +Uses_other is not recompiled because it only uses Otherlib, not Mylib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_other"))] | length' - 2 + 0 Change only otherlib's interface: @@ -95,8 +95,8 @@ Change only otherlib's interface: > let new_other_fn s = s ^ "!" > EOF -Uses_lib is recompiled even though it only uses Mylib, not Otherlib: +Uses_lib is not recompiled because it only uses Mylib, not Otherlib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_lib"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque.t index 17507352124..741aa8e25c7 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque.t @@ -62,11 +62,11 @@ Change ONLY the implementation (not the interface): > let value = 43 > EOF -No_use_lib is recompiled even though it doesn't reference Mylib: +No_use_lib is not recompiled because it doesn't reference Mylib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("No_use_lib"))] | length' - 1 + 0 --- Dev profile (opaque=true): .cmx deps are NOT tracked for local libs --- diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/stdlib-modules.t b/test/blackbox-tests/test-cases/per-module-lib-deps/stdlib-modules.t index 81bc097ecb7..997676c152e 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/stdlib-modules.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/stdlib-modules.t @@ -55,8 +55,8 @@ See: https://github.com/ocaml/dune/issues/4572 > let new_function () = "hello" > EOF -Uses_stdlib is recompiled even though it only uses Printf, not Mylib: +Uses_stdlib is not recompiled because it only uses Printf, not Mylib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_stdlib"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive.t b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive.t index a673b3119f9..4b69eab38dc 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive.t @@ -73,8 +73,8 @@ Change libB's interface: > let new_base_fn () = "new" > EOF -Independent is recompiled even though it doesn't reference libA or libB: +Independent is not recompiled because it doesn't reference libA or libB: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Independent"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t index 908ce2e21b8..b8be9461d05 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t @@ -73,11 +73,12 @@ Change only helper.mli: > let new_helper s = s ^ "!" > EOF -Uses_utils is recompiled even though it only references Utils, not Helper: +Uses_utils is recompiled because unwrapped libraries use glob deps (per-module +filtering within unwrapped libraries is not yet supported): $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_utils"))] | length' - 2 + 1 Change only utils.mli: @@ -90,8 +91,8 @@ Change only utils.mli: > let new_utils s = s ^ "?" > EOF -Uses_helper is recompiled even though it only references Helper, not Utils: +Uses_helper is recompiled because unwrapped libraries use glob deps: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_helper"))] | length' - 2 + 1 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-compat.t b/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-compat.t index adc34c6f62b..92fd82b0636 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-compat.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-compat.t @@ -67,8 +67,8 @@ See: https://github.com/ocaml/dune/issues/4572 > let new_fn () = "hello" > EOF -Standalone is recompiled even though it doesn't reference Baselib: +Standalone is not recompiled because it doesn't reference Baselib: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Standalone"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/reporting-of-cycles.t/run.t b/test/blackbox-tests/test-cases/reporting-of-cycles.t/run.t index e630fee88ae..e37394b1aaa 100644 --- a/test/blackbox-tests/test-cases/reporting-of-cycles.t/run.t +++ b/test/blackbox-tests/test-cases/reporting-of-cycles.t/run.t @@ -4,14 +4,6 @@ In all tests, we have a cycle that only becomes apparent after we start running things. In the past, the error was only reported during the second run of dune. - $ dune build @package-cycle - Error: Dependency cycle between: - alias a/.a-files - -> alias b/.b-files - -> alias a/.a-files - -> required by alias package-cycle in dune:1 - [1] - $ dune build @simple-repro-case Error: Dependency cycle between: _build/default/y From af278c9f45a186be11e3d028e404558ac0cf5966 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Mon, 13 Apr 2026 20:20:44 -0700 Subject: [PATCH 17/80] refactor: remove dead code for per-file unwrapped library deps Remove deps_of_module, cm_kinds_for, and the Module.t option from Lib_index and deps_of_entries. All entries use glob deps (no per-file deps on individual modules within libraries), so the Some m path was dead code. Simplify the types accordingly: Lib_index maps module names to Lib.t list, and deps_of_entries takes Lib.t list directly. Per-module filtering within unwrapped libraries can be revisited in the future once the dependency cone of individual modules is tracked. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 13 ++++----- src/dune_rules/lib_file_deps.ml | 41 +++++---------------------- src/dune_rules/lib_file_deps.mli | 22 +++++--------- src/dune_rules/module_compilation.ml | 25 ++++++---------- 4 files changed, 28 insertions(+), 73 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index aca3208aa15..4ea8523dd23 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -219,11 +219,8 @@ let create ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config ; lib_index = (* Maps module names to libraries for per-module inter-library dependency - filtering. All entries use [None] for the module (glob deps on the - library) rather than [Some m] (per-file deps on a specific module). - Per-file deps for unwrapped libraries are not yet supported because - modules in unwrapped libraries can alias modules from other libraries, - and we don't yet track the dependency cone of individual modules. *) + filtering. Used to look up which libraries a module references based + on the module names reported by ocamldep. *) (let open Resolve.Memo.O in let* all_libs = direct_requires in let+ entries = @@ -232,12 +229,12 @@ let create match main with | Some name -> (* Wrapped library: index by the wrapper module name. *) - Resolve.Memo.return [ name, ((lib : Lib.t), (None : Module.t option)) ] + Resolve.Memo.return [ name, lib ] | None -> (* Unwrapped library: index by each entry module name. *) (match Lib_info.entry_modules (Lib.info lib) ~for_ with | External (Ok names) -> - Resolve.Memo.return (List.map names ~f:(fun n -> n, (lib, None))) + Resolve.Memo.return (List.map names ~f:(fun n -> n, lib)) | External (Error e) -> Resolve.Memo.of_result (Error e) | Local -> Resolve.Memo.lift_memo @@ -248,7 +245,7 @@ let create ~for_) ~f:(fun mods -> List.map (Modules.entry_modules mods) ~f:(fun m -> - Module.name m, (lib, None)))))) + Module.name m, lib))))) in Lib_file_deps.Lib_index.create entries) ; preprocessing diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 72966c56f2d..636b83bcee1 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -50,17 +50,7 @@ let deps_of_lib (lib : Lib.t) ~groups = let deps libs ~groups = Dep.Set.union_map libs ~f:(deps_of_lib ~groups) -(* Currently unused: per-file deps on individual modules within a library. - Retained for potential future use when per-module filtering of unwrapped - libraries is supported. *) -let deps_of_module (lib : Lib.t) (m : Module.t) ~cm_kinds = - let obj_dir = Lib.info lib |> Lib_info.obj_dir in - List.filter_map cm_kinds ~f:(fun kind -> - Obj_dir.Module.cm_public_file obj_dir m ~kind |> Option.map ~f:(fun p -> Dep.file p)) - |> Dep.Set.of_list -;; - -let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) entries = +let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) libs = let groups_for lib = match cm_kind with | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ] @@ -71,37 +61,20 @@ let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) entries = | Melange Cmi -> [ Group.Melange Cmi ] | Melange Cmj -> [ Group.Melange Cmi; Group.Melange Cmj ] in - (* Currently unused: [cm_kinds_for] and the [Some m] branch below support - per-file deps on individual modules. All entries currently use [None] - (glob deps), so this path is dead code. Retained for future use. *) - let cm_kinds_for lib = - match cm_kind with - | Ocaml Cmi | Ocaml Cmo -> [ Lib_mode.Cm_kind.Ocaml Cmi ] - | Ocaml Cmx -> - if opaque && Lib.is_local lib - then [ Lib_mode.Cm_kind.Ocaml Cmi ] - else [ Lib_mode.Cm_kind.Ocaml Cmi; Ocaml Cmx ] - | Melange Cmi -> [ Lib_mode.Cm_kind.Melange Cmi ] - | Melange Cmj -> [ Lib_mode.Cm_kind.Melange Cmi; Melange Cmj ] - in - Dep.Set.union_map entries ~f:(fun (lib, module_opt) -> - match module_opt with - | None -> deps_of_lib lib ~groups:(groups_for lib) - | Some m -> deps_of_module lib m ~cm_kinds:(cm_kinds_for lib)) + Dep.Set.union_map libs ~f:(fun lib -> deps_of_lib lib ~groups:(groups_for lib)) ;; module Lib_index = struct - type entry = Lib.t * Module.t option - type t = { by_module_name : entry list Module_name.Map.t } + type t = { by_module_name : Lib.t list Module_name.Map.t } let empty = { by_module_name = Module_name.Map.empty } let create entries = let by_module_name = - List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, entry) -> + List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib) -> Module_name.Map.update map name ~f:(function - | None -> Some [ entry ] - | Some entries -> Some (entry :: entries))) + | None -> Some [ lib ] + | Some libs -> Some (lib :: libs))) in { by_module_name } ;; @@ -110,7 +83,7 @@ module Lib_index = struct Module_name.Set.fold referenced_modules ~init:[] ~f:(fun name acc -> match Module_name.Map.find t.by_module_name name with | None -> acc - | Some entries -> List.rev_append entries acc) + | Some libs -> List.rev_append libs acc) ;; end diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 510a5682660..f4057bddf72 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -15,29 +15,21 @@ end with extension [files] of libraries [libs]. *) val deps : Lib.t list -> groups:Group.t list -> Dep.Set.t -(** [deps_of_entries ~opaque ~cm_kind entries] computes the file dependencies - for the given library entries. When the module in an entry is [None], glob - deps are used for the library. When [Some m], per-file deps on specific - cm files are used. Currently all callers pass [None]; the [Some] path is - retained for potential future per-module filtering of unwrapped libraries. *) -val deps_of_entries - : opaque:bool - -> cm_kind:Lib_mode.Cm_kind.t - -> (Lib.t * Module.t option) list - -> Dep.Set.t +(** [deps_of_entries ~opaque ~cm_kind libs] computes the file dependencies + (glob deps on .cmi/.cmx files) for the given libraries. *) +val deps_of_entries : opaque:bool -> cm_kind:Lib_mode.Cm_kind.t -> Lib.t list -> Dep.Set.t module Lib_index : sig - type entry = Lib.t * Module.t option type t val empty : t - (** Create an index from a list of (module_name, entry) pairs. *) - val create : (Module_name.t * entry) list -> t + (** Create an index from a list of (module_name, library) pairs. *) + val create : (Module_name.t * Lib.t) list -> t - (** Return the library entries whose module names appear in + (** Return the libraries whose module names appear in [referenced_modules]. *) - val filter_libs : t -> referenced_modules:Module_name.Set.t -> entry list + val filter_libs : t -> referenced_modules:Module_name.Set.t -> Lib.t list end type path_specification = diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 37add1296ab..7c6bdc69cfd 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -329,8 +329,7 @@ let build_cm let+ direct_libs = Compilation_context.requires_compile cctx and+ hidden_libs = Compilation_context.requires_hidden cctx in let libs = direct_libs @ hidden_libs in - let entries = List.map libs ~f:(fun lib -> lib, None) in - Command.Args.Hidden_deps (Lib_file_deps.deps_of_entries ~opaque ~cm_kind entries)) + Command.Args.Hidden_deps (Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs)) |> Resolve.Memo.args |> Command.Args.memo in @@ -352,10 +351,8 @@ let build_cm List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) in if has_virtual_impl - then ( - let entries = List.map libs ~f:(fun lib -> lib, None) in - Action_builder.return - ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind entries)) + then + Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in let* raw_deps_m = @@ -400,7 +397,7 @@ let build_cm let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in let referenced = Module_name.Set.union all_raw open_modules in - let filtered_entries = + let filtered_libs = Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced in (* Transitively close the filtered libraries within [libs]. @@ -409,7 +406,7 @@ let build_cm let libs_set = Table.create (module Lib) (List.length libs) in List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); let covered = Table.create (module Lib) 8 in - List.iter filtered_entries ~f:(fun (lib, _) -> Table.set covered lib ()); + List.iter filtered_libs ~f:(fun lib -> Table.set covered lib ()); let rec close_over queue acc = match queue with | [] -> Action_builder.return acc @@ -424,17 +421,14 @@ let build_cm Some dep) else None) in - let new_entries = - List.map new_deps ~f:(fun dep -> dep, (None : Module.t option)) - in - close_over (new_deps @ rest) (new_entries @ acc) + close_over (new_deps @ rest) (new_deps @ acc) in - let+ transitive_entries = close_over (List.map filtered_entries ~f:fst) [] in + let+ transitive_libs = close_over filtered_libs [] in ( () , Lib_file_deps.deps_of_entries ~opaque ~cm_kind - (filtered_entries @ transitive_entries) )) + (filtered_libs @ transitive_libs) )) in let other_cm_files = let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in @@ -682,8 +676,7 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = and+ h = Compilation_context.requires_hidden cctx in d @ h) in - let entries = List.map libs ~f:(fun lib -> lib, None) in - (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) entries) + (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) libs) in let ocaml_flags = Ocaml_flags.get (Compilation_context.flags cctx) (Ocaml Byte) in let modules = Compilation_context.modules cctx in From 8aa81e7f3b8b4a5f2e03dd951348be7fec7d7c6f Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 14:30:38 -0700 Subject: [PATCH 18/80] refactor: extract filtered_lib_deps helper; use in ocamlc_i Extract the per-module inter-library dependency filtering logic from build_cm into a shared filtered_lib_deps helper. Use it in both build_cm and ocamlc_i so that inferred .mli generation also benefits from per-module filtering. Signed-off-by: Robin Bate Boerop --- src/dune_rules/dep_graph.ml | 1 + src/dune_rules/dep_graph.mli | 1 + src/dune_rules/module_compilation.ml | 234 +++++++++++++++------------ 3 files changed, 136 insertions(+), 100 deletions(-) diff --git a/src/dune_rules/dep_graph.ml b/src/dune_rules/dep_graph.ml index 891c7249fe0..dd8b8b2a8cd 100644 --- a/src/dune_rules/dep_graph.ml +++ b/src/dune_rules/dep_graph.ml @@ -8,6 +8,7 @@ type t = let make ~dir ~per_module = { dir; per_module } let dir t = t.dir +let mem t (m : Module.t) = Module_name.Unique.Map.mem t.per_module (Module.obj_name m) let deps_of t (m : Module.t) = match Module_name.Unique.Map.find t.per_module (Module.obj_name m) with diff --git a/src/dune_rules/dep_graph.mli b/src/dune_rules/dep_graph.mli index 4a9a5af7a83..d8b5ff3726a 100644 --- a/src/dune_rules/dep_graph.mli +++ b/src/dune_rules/dep_graph.mli @@ -10,6 +10,7 @@ val make -> t val dir : t -> Path.Build.t +val mem : t -> Module.t -> bool val deps_of : t -> Module.t -> Module.t list Action_builder.t val top_closed_implementations : t -> Module.t list -> Module.t list Action_builder.t diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 7c6bdc69cfd..ced3bcd393a 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -1,6 +1,92 @@ open Import open Memo.O +(* Compute filtered inter-library file deps for a module. Uses ocamldep + output to determine which libraries the module actually references, + then transitively closes within [libs] to handle transparent aliases. + Returns [Dep.Set.t] suitable for use with [Action_builder.dyn_deps]. + Falls back to glob deps on all [libs] when virtual implementations + are present (parameter libraries may not be in requires_compile). *) +let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind ~mode m = + let open Action_builder.O in + let* libs = + Resolve.Memo.read + (let open Resolve.Memo.O in + let+ d = Compilation_context.requires_compile cctx + and+ h = Compilation_context.requires_hidden cctx in + d @ h) + in + let has_virtual_impl = + List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) + in + if has_virtual_impl + then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) + else + let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in + let* raw_deps_m = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ m in + (* deps_of already returns transitive intra-library deps *) + let* trans_deps = Dep_graph.deps_of dep_graph m in + let* trans_raw_deps = + Action_builder.List.map trans_deps ~f:(fun dep_m -> + let is_standard_kind = + match Module.kind dep_m with + | Impl_vmodule | Virtual | Root | Alias _ | Wrapped_compat | Parameter -> false + | _ -> true + in + if not is_standard_kind + then Action_builder.return Module_name.Set.empty + else ( + (* Try the current ml_kind first; fall back to the other kind for + interface-only modules that may contain aliases to other libraries. *) + let dep_ml_kind = + if Module.has dep_m ~ml_kind + then ml_kind + else ( + match ml_kind with + | Ml_kind.Impl -> Ml_kind.Intf + | Intf -> Impl) + in + if Module.has dep_m ~ml_kind:dep_ml_kind + then + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:dep_ml_kind ~for_ dep_m + else Action_builder.return Module_name.Set.empty)) + in + let all_raw = + List.fold_left trans_raw_deps ~init:raw_deps_m ~f:Module_name.Set.union + in + let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in + let open_modules = Ocaml_flags.extract_open_module_names flags in + let referenced = Module_name.Set.union all_raw open_modules in + let filtered_libs = + Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced + in + (* Transitively close the filtered libraries within [libs]. Transparent + module aliases can create cross-library .cmi reads that ocamldep + doesn't report, at arbitrary depth. *) + let libs_set = Table.create (module Lib) (List.length libs) in + List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); + let covered = Table.create (module Lib) 8 in + List.iter filtered_libs ~f:(fun lib -> Table.set covered lib ()); + let rec close_over queue acc = + match queue with + | [] -> Action_builder.return acc + | lib :: rest -> + let open Action_builder.O in + let* requires = Resolve.Memo.read (Lib.requires lib ~for_) in + let new_deps = + List.filter_map requires ~f:(fun dep -> + if Table.mem libs_set dep && not (Table.mem covered dep) + then ( + Table.set covered dep (); + Some dep) + else None) + in + close_over (new_deps @ rest) (new_deps @ acc) + in + let+ transitive_libs = close_over filtered_libs [] in + (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind (filtered_libs @ transitive_libs) +;; + (* Arguments for the compiler to prevent it from being too clever. The compiler creates the cmi when it thinks a .ml file has no corresponding @@ -302,6 +388,7 @@ let build_cm Filtering is disabled when: - Melange mode (OCaml-only optimization) - Dep graph dir differs from obj dir (shouldn't happen in practice) + - Module is not in the dep graph (e.g., menhir-generated mock modules) - Special module kinds (Root, Wrapped_compat, Impl_vmodule, Virtual, Parameter) that don't have standard ocamldep output - Module lacks the current ml_kind source file @@ -313,6 +400,7 @@ let build_cm | Melange -> false | Ocaml _ -> true) && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) + && Dep_graph.mem dep_graph m && (match Module.kind m with | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false | _ -> true) @@ -339,96 +427,16 @@ let build_cm then Action_builder.return () else Action_builder.dyn_deps - (let open Action_builder.O in - let* libs = - Resolve.Memo.read - (let open Resolve.Memo.O in - let+ d = Compilation_context.requires_compile cctx - and+ h = Compilation_context.requires_hidden cctx in - d @ h) - in - let has_virtual_impl = - List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) - in - if has_virtual_impl - then - Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) - else - let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in - let* raw_deps_m = - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ m - in - (* deps_of already returns transitive intra-library deps *) - let* trans_deps = Dep_graph.deps_of dep_graph m in - let* trans_raw_deps = - Action_builder.List.map trans_deps ~f:(fun dep_m -> - let is_standard_kind = - match Module.kind dep_m with - | Impl_vmodule | Virtual | Root | Alias _ | Wrapped_compat | Parameter - -> false - | _ -> true - in - if not is_standard_kind - then Action_builder.return Module_name.Set.empty - else ( - (* Try the current ml_kind first; fall back to the other - kind for interface-only modules that may contain aliases - to other libraries. *) - let dep_ml_kind = - if Module.has dep_m ~ml_kind - then ml_kind - else ( - match ml_kind with - | Ml_kind.Impl -> Ml_kind.Intf - | Intf -> Impl) - in - if Module.has dep_m ~ml_kind:dep_ml_kind - then - Ocamldep.read_immediate_deps_raw_of - ~obj_dir - ~ml_kind:dep_ml_kind - ~for_ - dep_m - else Action_builder.return Module_name.Set.empty)) - in - let all_raw = - List.fold_left trans_raw_deps ~init:raw_deps_m ~f:Module_name.Set.union - in - let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in - let open_modules = Ocaml_flags.extract_open_module_names flags in - let referenced = Module_name.Set.union all_raw open_modules in - let filtered_libs = - Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced - in - (* Transitively close the filtered libraries within [libs]. - Transparent module aliases can create cross-library .cmi reads - that ocamldep doesn't report, at arbitrary depth. *) - let libs_set = Table.create (module Lib) (List.length libs) in - List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); - let covered = Table.create (module Lib) 8 in - List.iter filtered_libs ~f:(fun lib -> Table.set covered lib ()); - let rec close_over queue acc = - match queue with - | [] -> Action_builder.return acc - | lib :: rest -> - let open Action_builder.O in - let* requires = Resolve.Memo.read (Lib.requires lib ~for_) in - let new_deps = - List.filter_map requires ~f:(fun dep -> - if Table.mem libs_set dep && not (Table.mem covered dep) - then ( - Table.set covered dep (); - Some dep) - else None) - in - close_over (new_deps @ rest) (new_deps @ acc) - in - let+ transitive_libs = close_over filtered_libs [] in - ( () - , Lib_file_deps.deps_of_entries - ~opaque - ~cm_kind - (filtered_libs @ transitive_libs) )) + (filtered_lib_deps + ~cctx + ~obj_dir + ~ml_kind + ~for_ + ~dep_graph + ~opaque + ~cm_kind + ~mode + m) in let other_cm_files = let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in @@ -667,16 +675,42 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = in let lib_cm_deps = let opaque = Compilation_context.opaque cctx in - Action_builder.dyn_deps - (let open Action_builder.O in - let+ libs = - Resolve.Memo.read - (let open Resolve.Memo.O in - let+ d = Compilation_context.requires_compile cctx - and+ h = Compilation_context.requires_hidden cctx in - d @ h) - in - (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) libs) + let for_ = Compilation_context.for_ cctx in + let ml_kind = Ml_kind.Impl in + let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in + let can_filter = + Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) + && Dep_graph.mem dep_graph m + && (match Module.kind m with + | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter | Alias _ -> false + | _ -> true) + && Module.has m ~ml_kind + && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) + in + if can_filter + then + Action_builder.dyn_deps + (filtered_lib_deps + ~cctx + ~obj_dir + ~ml_kind + ~for_ + ~dep_graph + ~opaque + ~cm_kind:(Ocaml Cmo) + ~mode:(Ocaml Byte) + m) + else + Action_builder.dyn_deps + (let open Action_builder.O in + let+ libs = + Resolve.Memo.read + (let open Resolve.Memo.O in + let+ d = Compilation_context.requires_compile cctx + and+ h = Compilation_context.requires_hidden cctx in + d @ h) + in + (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) libs) in let ocaml_flags = Ocaml_flags.get (Compilation_context.flags cctx) (Ocaml Byte) in let modules = Compilation_context.modules cctx in From 7f124b9e15523f110146ff74f5ebd7c62de0c6c2 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 16:04:50 -0700 Subject: [PATCH 19/80] refactor: replace close_over with Lib.closure Use the existing Lib.closure for transitive library dependency closure instead of a hand-rolled close_over function. Lib.closure follows the same requires path and the libraries already passed overlap checks when the compilation context was built. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index ced3bcd393a..327b31631ad 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -65,26 +65,9 @@ let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind doesn't report, at arbitrary depth. *) let libs_set = Table.create (module Lib) (List.length libs) in List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); - let covered = Table.create (module Lib) 8 in - List.iter filtered_libs ~f:(fun lib -> Table.set covered lib ()); - let rec close_over queue acc = - match queue with - | [] -> Action_builder.return acc - | lib :: rest -> - let open Action_builder.O in - let* requires = Resolve.Memo.read (Lib.requires lib ~for_) in - let new_deps = - List.filter_map requires ~f:(fun dep -> - if Table.mem libs_set dep && not (Table.mem covered dep) - then ( - Table.set covered dep (); - Some dep) - else None) - in - close_over (new_deps @ rest) (new_deps @ acc) - in - let+ transitive_libs = close_over filtered_libs [] in - (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind (filtered_libs @ transitive_libs) + let+ closed = Resolve.Memo.read (Lib.closure filtered_libs ~linking:false ~for_) in + let all_libs = List.filter closed ~f:(Table.mem libs_set) in + (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind all_libs ;; (* Arguments for the compiler to prevent it from being too clever. From 753ba774e18137ab154dcc7e1ab2efbc2577dce3 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 17:29:57 -0700 Subject: [PATCH 20/80] refactor: simplify lib_index to use entry_modules for all libraries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the main_module_name special case in lib_index construction. For wrapped libraries, entry_modules returns the wrapper module — the same name that main_module_name provided. Since all entries now use glob deps (no per-file Module.t), the two branches were equivalent. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 40 +++++++++++---------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 4ea8523dd23..67a51b631b9 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -218,34 +218,26 @@ let create ; parameters ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config ; lib_index = - (* Maps module names to libraries for per-module inter-library dependency - filtering. Used to look up which libraries a module references based - on the module names reported by ocamldep. *) + (* Maps entry module names to libraries for per-module inter-library + dependency filtering. For wrapped libraries, the entry module is the + wrapper; for unwrapped, it is each public module. *) (let open Resolve.Memo.O in let* all_libs = direct_requires in let+ entries = Resolve.Memo.List.concat_map all_libs ~f:(fun lib -> - let* main = Lib.main_module_name lib in - match main with - | Some name -> - (* Wrapped library: index by the wrapper module name. *) - Resolve.Memo.return [ name, lib ] - | None -> - (* Unwrapped library: index by each entry module name. *) - (match Lib_info.entry_modules (Lib.info lib) ~for_ with - | External (Ok names) -> - Resolve.Memo.return (List.map names ~f:(fun n -> n, lib)) - | External (Error e) -> Resolve.Memo.of_result (Error e) - | Local -> - Resolve.Memo.lift_memo - (Memo.map - (Dir_contents.modules_of_local_lib - super_context - (Lib.Local.of_lib_exn lib) - ~for_) - ~f:(fun mods -> - List.map (Modules.entry_modules mods) ~f:(fun m -> - Module.name m, lib))))) + match Lib_info.entry_modules (Lib.info lib) ~for_ with + | External (Ok names) -> + Resolve.Memo.return (List.map names ~f:(fun n -> n, lib)) + | External (Error e) -> Resolve.Memo.of_result (Error e) + | Local -> + Resolve.Memo.lift_memo + (Memo.map + (Dir_contents.modules_of_local_lib + super_context + (Lib.Local.of_lib_exn lib) + ~for_) + ~f:(fun mods -> + List.map (Modules.entry_modules mods) ~f:(fun m -> Module.name m, lib)))) in Lib_file_deps.Lib_index.create entries) ; preprocessing From 859b87b0af4b4e059b653287926e3d11abbd5c99 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 18:58:29 -0700 Subject: [PATCH 21/80] style: address review nitpicks - Extract all_libs helper to deduplicate requires_compile @ requires_hidden - Expand Module.kind wildcards to explicit variants - Reword fallback comment to not reference removed code - Drop trivial punctuation-only changes to unrelated test files Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 37 ++++++++----------- .../alias/check-alias/ocamldep-cycles.t | 2 +- .../test-cases/ocamldep/ocamldep-7018.t | 2 +- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 327b31631ad..69b3339edae 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -7,15 +7,16 @@ open Memo.O Returns [Dep.Set.t] suitable for use with [Action_builder.dyn_deps]. Falls back to glob deps on all [libs] when virtual implementations are present (parameter libraries may not be in requires_compile). *) +let all_libs cctx = + let open Resolve.Memo.O in + let+ d = Compilation_context.requires_compile cctx + and+ h = Compilation_context.requires_hidden cctx in + d @ h +;; + let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind ~mode m = let open Action_builder.O in - let* libs = - Resolve.Memo.read - (let open Resolve.Memo.O in - let+ d = Compilation_context.requires_compile cctx - and+ h = Compilation_context.requires_hidden cctx in - d @ h) - in + let* libs = Resolve.Memo.read (all_libs cctx) in let has_virtual_impl = List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) in @@ -31,7 +32,7 @@ let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind let is_standard_kind = match Module.kind dep_m with | Impl_vmodule | Virtual | Root | Alias _ | Wrapped_compat | Parameter -> false - | _ -> true + | Intf_only | Impl -> true in if not is_standard_kind then Action_builder.return Module_name.Set.empty @@ -386,20 +387,18 @@ let build_cm && Dep_graph.mem dep_graph m && (match Module.kind m with | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false - | _ -> true) + | Intf_only | Impl | Alias _ -> true) && Module.has m ~ml_kind && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) in - (* Static lib deps: used when filtering is not possible, preserves the - original Hidden_deps-in-Command.Args behavior exactly. *) + (* Fallback lib deps: when per-module filtering is not possible, depend + on all .cmi/.cmx files from all required libraries. *) let lib_cm_deps_args = if skip_lib_deps || can_filter then Command.Args.empty else (let open Resolve.Memo.O in - let+ direct_libs = Compilation_context.requires_compile cctx - and+ hidden_libs = Compilation_context.requires_hidden cctx in - let libs = direct_libs @ hidden_libs in + let+ libs = all_libs cctx in Command.Args.Hidden_deps (Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs)) |> Resolve.Memo.args |> Command.Args.memo @@ -666,7 +665,7 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = && Dep_graph.mem dep_graph m && (match Module.kind m with | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter | Alias _ -> false - | _ -> true) + | Intf_only | Impl -> true) && Module.has m ~ml_kind && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) in @@ -686,13 +685,7 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = else Action_builder.dyn_deps (let open Action_builder.O in - let+ libs = - Resolve.Memo.read - (let open Resolve.Memo.O in - let+ d = Compilation_context.requires_compile cctx - and+ h = Compilation_context.requires_hidden cctx in - d @ h) - in + let+ libs = Resolve.Memo.read (all_libs cctx) in (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) libs) in let ocaml_flags = Ocaml_flags.get (Compilation_context.flags cctx) (Ocaml Byte) in diff --git a/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t b/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t index 57cbe687536..5dbc6613b39 100644 --- a/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t +++ b/test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t @@ -1,4 +1,4 @@ -The @check alias should detect dependency cycles. +The @check alias should detect dependency cycles $ make_dune_project 3.2 diff --git a/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t b/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t index de19fef9ecd..c618d310696 100644 --- a/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t +++ b/test/blackbox-tests/test-cases/ocamldep/ocamldep-7018.t @@ -30,7 +30,7 @@ Reproduces #7018 > dune build > } -First we try to construct X.t directly. +First we try to construct X.t directly $ runtest "()" Error: dependency cycle between modules in _build/default: From 0e556268a0c8d64579ea1878bad745c9ca1440f3 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 20:32:32 -0700 Subject: [PATCH 22/80] fix: use of_string_opt for user-provided -open flag module names Signed-off-by: Robin Bate Boerop --- src/dune_rules/ocaml_flags.ml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dune_rules/ocaml_flags.ml b/src/dune_rules/ocaml_flags.ml index 61cc7d13f71..56faa95279c 100644 --- a/src/dune_rules/ocaml_flags.ml +++ b/src/dune_rules/ocaml_flags.ml @@ -201,7 +201,12 @@ let open_flags modules = let extract_open_module_names flags = let rec loop acc = function | "-open" :: name :: rest -> - loop (Module_name.Set.add acc (Module_name.of_checked_string name)) rest + let acc = + match Module_name.of_string_opt name with + | Some m -> Module_name.Set.add acc m + | None -> acc + in + loop acc rest | _ :: rest -> loop acc rest | [] -> acc in From 920ae659912c4647ddb1f8d01b454e3abcbc63c9 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 21:14:13 -0700 Subject: [PATCH 23/80] fix: include Virtual modules in ocamldep reading for filtering Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 69b3339edae..353b34b6911 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -31,8 +31,8 @@ let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind Action_builder.List.map trans_deps ~f:(fun dep_m -> let is_standard_kind = match Module.kind dep_m with - | Impl_vmodule | Virtual | Root | Alias _ | Wrapped_compat | Parameter -> false - | Intf_only | Impl -> true + | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false + | Virtual | Intf_only | Impl -> true in if not is_standard_kind then Action_builder.return Module_name.Set.empty From 60ea76aba21ebf0913cb46f2fde38d9053e89b0d Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 14 Apr 2026 21:29:00 -0700 Subject: [PATCH 24/80] fix: read both .ml and .mli ocamldep output for filtering A module's .mli can reference different libraries than its .ml. Read ocamldep output from both and union them so the filtered set includes all cross-library references. This makes the filtering more precise and prepares for per-module -I flag filtering (#14186) where the .cmi compilation rule no longer serves as a safety net. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 57 ++++++++++--------- .../lib-to-lib-unwrapped.t | 4 +- .../per-module-lib-deps/unwrapped.t | 4 +- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 353b34b6911..70008f8c6c4 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -14,7 +14,7 @@ let all_libs cctx = d @ h ;; -let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind ~mode m = +let filtered_lib_deps ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~mode m = let open Action_builder.O in let* libs = Resolve.Memo.read (all_libs cctx) in let has_virtual_impl = @@ -24,7 +24,20 @@ let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in - let* raw_deps_m = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ m in + let* raw_deps_m = + let open Action_builder.O in + let* impl_deps = + if Module.has m ~ml_kind:Impl + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ m + else Action_builder.return Module_name.Set.empty + in + let+ intf_deps = + if Module.has m ~ml_kind:Intf + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ m + else Action_builder.return Module_name.Set.empty + in + Module_name.Set.union impl_deps intf_deps + in (* deps_of already returns transitive intra-library deps *) let* trans_deps = Dep_graph.deps_of dep_graph m in let* trans_raw_deps = @@ -36,21 +49,21 @@ let filtered_lib_deps ~cctx ~obj_dir ~ml_kind ~for_ ~dep_graph ~opaque ~cm_kind in if not is_standard_kind then Action_builder.return Module_name.Set.empty - else ( - (* Try the current ml_kind first; fall back to the other kind for - interface-only modules that may contain aliases to other libraries. *) - let dep_ml_kind = - if Module.has dep_m ~ml_kind - then ml_kind - else ( - match ml_kind with - | Ml_kind.Impl -> Ml_kind.Intf - | Intf -> Impl) + else + (* Read ocamldep for both .ml and .mli when they exist, since the + interface can reference different libraries than the implementation. *) + let open Action_builder.O in + let* impl_deps = + if Module.has dep_m ~ml_kind:Impl + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m + else Action_builder.return Module_name.Set.empty + in + let+ intf_deps = + if Module.has dep_m ~ml_kind:Intf + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m + else Action_builder.return Module_name.Set.empty in - if Module.has dep_m ~ml_kind:dep_ml_kind - then - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:dep_ml_kind ~for_ dep_m - else Action_builder.return Module_name.Set.empty)) + Module_name.Set.union impl_deps intf_deps) in let all_raw = List.fold_left trans_raw_deps ~init:raw_deps_m ~f:Module_name.Set.union @@ -409,16 +422,7 @@ let build_cm then Action_builder.return () else Action_builder.dyn_deps - (filtered_lib_deps - ~cctx - ~obj_dir - ~ml_kind - ~for_ - ~dep_graph - ~opaque - ~cm_kind - ~mode - m) + (filtered_lib_deps ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~mode m) in let other_cm_files = let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in @@ -675,7 +679,6 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = (filtered_lib_deps ~cctx ~obj_dir - ~ml_kind ~for_ ~dep_graph ~opaque diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t index c8187741849..d807651e05d 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t @@ -85,7 +85,7 @@ filtering within unwrapped libraries is not yet supported): $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_beta"))] | length' - 1 + 2 Change only beta.mli: @@ -102,4 +102,4 @@ uses_alpha is recompiled because unwrapped libraries use glob deps: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_alpha"))] | length' - 1 + 2 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t index b8be9461d05..5a5c635338f 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t @@ -78,7 +78,7 @@ filtering within unwrapped libraries is not yet supported): $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_utils"))] | length' - 1 + 2 Change only utils.mli: @@ -95,4 +95,4 @@ Uses_helper is recompiled because unwrapped libraries use glob deps: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_helper"))] | length' - 1 + 2 From 1fe30e4950265f2b343ed1d3cdea571c25bd98cd Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 17 Apr 2026 09:58:48 -0700 Subject: [PATCH 25/80] refactor: deduplicate build_cm and ocamlc_i lib deps logic Absorb can_filter into lib_deps_for_module and include m in trans_deps to eliminate duplicated code between build_cm and ocamlc_i. Suggested by art-w. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 236 +++++++++++---------------- 1 file changed, 95 insertions(+), 141 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 70008f8c6c4..72cb28bbf1c 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -1,12 +1,6 @@ open Import open Memo.O -(* Compute filtered inter-library file deps for a module. Uses ocamldep - output to determine which libraries the module actually references, - then transitively closes within [libs] to handle transparent aliases. - Returns [Dep.Set.t] suitable for use with [Action_builder.dyn_deps]. - Falls back to glob deps on all [libs] when virtual implementations - are present (parameter libraries may not be in requires_compile). *) let all_libs cctx = let open Resolve.Memo.O in let+ d = Compilation_context.requires_compile cctx @@ -14,74 +8,80 @@ let all_libs cctx = d @ h ;; -let filtered_lib_deps ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~mode m = +(* Per-module inter-library dependency filtering (#4572). Uses ocamldep + output to determine which libraries a module actually references, then + transitively closes within the compilation context's library set to + handle transparent aliases. Returns [((), Dep.Set.t)] suitable for use + with [Action_builder.dyn_deps]. + + Falls back to all libs when filtering is not possible. *) +let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kind ~mode m = let open Action_builder.O in - let* libs = Resolve.Memo.read (all_libs cctx) in - let has_virtual_impl = - List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) + let can_filter = + (match Lib_mode.of_cm_kind cm_kind with + | Melange -> false + | Ocaml _ -> true) + && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) + && Dep_graph.mem dep_graph m + && (match Module.kind m with + | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false + | Intf_only | Impl | Alias _ -> true) + && Module.has m ~ml_kind + && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) in - if has_virtual_impl + let* libs = Resolve.Memo.read (all_libs cctx) in + if not can_filter then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) - else - let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in - let* raw_deps_m = - let open Action_builder.O in - let* impl_deps = - if Module.has m ~ml_kind:Impl - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ m - else Action_builder.return Module_name.Set.empty - in - let+ intf_deps = - if Module.has m ~ml_kind:Intf - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ m - else Action_builder.return Module_name.Set.empty - in - Module_name.Set.union impl_deps intf_deps + else ( + let has_virtual_impl = + List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) in - (* deps_of already returns transitive intra-library deps *) - let* trans_deps = Dep_graph.deps_of dep_graph m in - let* trans_raw_deps = - Action_builder.List.map trans_deps ~f:(fun dep_m -> - let is_standard_kind = - match Module.kind dep_m with - | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false - | Virtual | Intf_only | Impl -> true - in - if not is_standard_kind - then Action_builder.return Module_name.Set.empty - else - (* Read ocamldep for both .ml and .mli when they exist, since the - interface can reference different libraries than the implementation. *) - let open Action_builder.O in - let* impl_deps = - if Module.has dep_m ~ml_kind:Impl - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m - else Action_builder.return Module_name.Set.empty - in - let+ intf_deps = - if Module.has dep_m ~ml_kind:Intf - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m - else Action_builder.return Module_name.Set.empty + if has_virtual_impl + then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) + else + let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in + let* trans_deps = Dep_graph.deps_of dep_graph m in + let* all_raw = + Action_builder.List.map (m :: trans_deps) ~f:(fun dep_m -> + let is_standard_kind = + match Module.kind dep_m with + | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false + | Virtual | Intf_only | Impl -> true in - Module_name.Set.union impl_deps intf_deps) - in - let all_raw = - List.fold_left trans_raw_deps ~init:raw_deps_m ~f:Module_name.Set.union - in - let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in - let open_modules = Ocaml_flags.extract_open_module_names flags in - let referenced = Module_name.Set.union all_raw open_modules in - let filtered_libs = - Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced - in - (* Transitively close the filtered libraries within [libs]. Transparent - module aliases can create cross-library .cmi reads that ocamldep - doesn't report, at arbitrary depth. *) - let libs_set = Table.create (module Lib) (List.length libs) in - List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); - let+ closed = Resolve.Memo.read (Lib.closure filtered_libs ~linking:false ~for_) in - let all_libs = List.filter closed ~f:(Table.mem libs_set) in - (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind all_libs + if not is_standard_kind + then Action_builder.return Module_name.Set.empty + else + let* impl_deps = + if Module.has dep_m ~ml_kind:Impl + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m + else Action_builder.return Module_name.Set.empty + in + let+ intf_deps = + if Module.has dep_m ~ml_kind:Intf + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m + else Action_builder.return Module_name.Set.empty + in + Module_name.Set.union impl_deps intf_deps) + |> Action_builder.map + ~f:(List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union) + in + let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in + let open_modules = Ocaml_flags.extract_open_module_names flags in + let referenced = Module_name.Set.union all_raw open_modules in + let filtered_libs = + Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced + in + (* Transitively close the filtered libraries within [libs]. + Transparent module aliases can create cross-library .cmi reads + that ocamldep doesn't report, at arbitrary depth. The + intersection with [libs] is needed because [Lib.closure] may + return libraries outside the compilation context when + [implicit_transitive_deps] is [Disabled]. *) + let libs_set = Table.create (module Lib) (List.length libs) in + List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); + let+ closed = Resolve.Memo.read (Lib.closure filtered_libs ~linking:false ~for_) in + let filtered = List.filter closed ~f:(Table.mem libs_set) in + (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind filtered) ;; (* Arguments for the compiler to prevent it from being too clever. @@ -378,51 +378,21 @@ let build_cm in let for_ = Compilation_context.for_ cctx in let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in - (* Per-module inter-library dependency filtering (#4572). When possible, - we use ocamldep output to determine which libraries each module actually - references, replacing the previous glob deps on all dependent libraries. - - Filtering is disabled when: - - Melange mode (OCaml-only optimization) - - Dep graph dir differs from obj dir (shouldn't happen in practice) - - Module is not in the dep graph (e.g., menhir-generated mock modules) - - Special module kinds (Root, Wrapped_compat, Impl_vmodule, Virtual, - Parameter) that don't have standard ocamldep output - - Module lacks the current ml_kind source file - - Virtual library implementations (parameter libraries are not in - requires_compile) *) - let can_filter = - (not skip_lib_deps) - && (match Lib_mode.of_cm_kind cm_kind with - | Melange -> false - | Ocaml _ -> true) - && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) - && Dep_graph.mem dep_graph m - && (match Module.kind m with - | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false - | Intf_only | Impl | Alias _ -> true) - && Module.has m ~ml_kind - && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) - in - (* Fallback lib deps: when per-module filtering is not possible, depend - on all .cmi/.cmx files from all required libraries. *) - let lib_cm_deps_args = - if skip_lib_deps || can_filter - then Command.Args.empty - else - (let open Resolve.Memo.O in - let+ libs = all_libs cctx in - Command.Args.Hidden_deps (Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs)) - |> Resolve.Memo.args - |> Command.Args.memo - in - (* Dynamic lib deps: used when per-module filtering is possible. *) - let lib_cm_deps_filtered = - if not can_filter + let lib_cm_deps = + if skip_lib_deps then Action_builder.return () else Action_builder.dyn_deps - (filtered_lib_deps ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~mode m) + (lib_deps_for_module + ~cctx + ~obj_dir + ~for_ + ~dep_graph + ~opaque + ~cm_kind + ~ml_kind + ~mode + m) in let other_cm_files = let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in @@ -542,7 +512,7 @@ let build_cm ?loc:(Compilation_context.loc cctx) (let open Action_builder.With_targets.O in Action_builder.with_no_targets other_cm_files - >>> Action_builder.with_no_targets lib_cm_deps_filtered + >>> Action_builder.with_no_targets lib_cm_deps >>> Command.run ~dir:(Path.build (Context.build_dir ctx)) compiler @@ -553,7 +523,7 @@ let build_cm ; Command.Args.S obj_dirs ; Command.Args.as_any (Lib_mode.Cm_kind.Map.get (Compilation_context.includes cctx) cm_kind) - ; Command.Args.as_any lib_cm_deps_args + ; Command.Args.empty ; extra_args ; As as_parameter_arg ; as_argument_for @@ -662,34 +632,18 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = let lib_cm_deps = let opaque = Compilation_context.opaque cctx in let for_ = Compilation_context.for_ cctx in - let ml_kind = Ml_kind.Impl in - let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in - let can_filter = - Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) - && Dep_graph.mem dep_graph m - && (match Module.kind m with - | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter | Alias _ -> false - | Intf_only | Impl -> true) - && Module.has m ~ml_kind - && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) - in - if can_filter - then - Action_builder.dyn_deps - (filtered_lib_deps - ~cctx - ~obj_dir - ~for_ - ~dep_graph - ~opaque - ~cm_kind:(Ocaml Cmo) - ~mode:(Ocaml Byte) - m) - else - Action_builder.dyn_deps - (let open Action_builder.O in - let+ libs = Resolve.Memo.read (all_libs cctx) in - (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind:(Ocaml Cmo) libs) + let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) Impl in + Action_builder.dyn_deps + (lib_deps_for_module + ~cctx + ~obj_dir + ~for_ + ~dep_graph + ~opaque + ~cm_kind:(Ocaml Cmo) + ~ml_kind:Impl + ~mode:(Ocaml Byte) + m) in let ocaml_flags = Ocaml_flags.get (Compilation_context.flags cctx) (Ocaml Byte) in let modules = Compilation_context.modules cctx in From b1078dcbf3bc0bba7bb4683c6b878713d69b5d1e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 17 Apr 2026 10:27:08 -0700 Subject: [PATCH 26/80] refactor: remove redundant Module.has guards in ocamldep reading read_immediate_deps_raw_of already checks Module.source internally and returns empty when no source file exists. Suggested by art-w. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 72cb28bbf1c..7590cd5b4ae 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -52,14 +52,10 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin then Action_builder.return Module_name.Set.empty else let* impl_deps = - if Module.has dep_m ~ml_kind:Impl - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m - else Action_builder.return Module_name.Set.empty + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m in let+ intf_deps = - if Module.has dep_m ~ml_kind:Intf - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m - else Action_builder.return Module_name.Set.empty + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m in Module_name.Set.union impl_deps intf_deps) |> Action_builder.map From 10c0832bbe19e6f79db558a08a5310d454e3ec09 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sun, 19 Apr 2026 22:36:34 +0100 Subject: [PATCH 27/80] refactor: memoize library closure Signed-off-by: Rudi Grinberg --- src/dune_rules/lib.ml | 50 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/dune_rules/lib.ml b/src/dune_rules/lib.ml index a8630f413db..4a0a76faa7e 100644 --- a/src/dune_rules/lib.ml +++ b/src/dune_rules/lib.ml @@ -2198,11 +2198,51 @@ end = struct ;; end -let closure l ~linking = - let forbidden_libraries = Map.empty in - if linking - then Resolve_names.linking_closure_with_overlap_checks None l ~forbidden_libraries - else Resolve_names.compile_closure_with_overlap_checks None l ~forbidden_libraries +let closure = + let memo = + let module Input = struct + type nonrec t = bool * Compilation_mode.t * t list + + let equal (l, m, libs) (l', m', libs') = + Bool.equal l l' && Compilation_mode.equal m m' && List.equal equal libs libs' + ;; + + let hash_for_ = function + | Compilation_mode.Ocaml -> 0 + | Melange -> 1 + ;; + + let hash (linking, for_, libs) = + Tuple.T3.hash + Bool.hash + hash_for_ + (List.hash (fun lib -> Id.hash lib.unique_id)) + (linking, for_, libs) + ;; + + let to_dyn = Dyn.opaque + end + in + Memo.create + "lib-closure" + ~input:(module Input) + (fun (linking, for_, l) -> + let forbidden_libraries = Map.empty in + if linking + then + Resolve_names.linking_closure_with_overlap_checks + None + l + ~forbidden_libraries + ~for_ + else + Resolve_names.compile_closure_with_overlap_checks + None + l + ~forbidden_libraries + ~for_) + in + fun l ~linking ~for_ -> Memo.exec memo (linking, for_, l) ;; let descriptive_closure (l : lib list) ~with_pps ~for_ : lib list Memo.t = From 080068a7bdca30a29b5050be8a78247cdeed2504 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Mon, 20 Apr 2026 11:38:28 -0700 Subject: [PATCH 28/80] refactor: share parsed ocamldep output between readers Extract read_immediate_deps_parsed to parse each .d file once and memoize the result. Both read_immediate_deps_of and read_immediate_deps_raw_of now derive their results from this shared parse, avoiding double parsing on null builds. Signed-off-by: Robin Bate Boerop --- src/dune_rules/ocamldep.ml | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/dune_rules/ocamldep.ml b/src/dune_rules/ocamldep.ml index 9dab235040d..c4e5d20202a 100644 --- a/src/dune_rules/ocamldep.ml +++ b/src/dune_rules/ocamldep.ml @@ -186,34 +186,34 @@ let read_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = |> Action_builder.memoize (Path.Build.to_string all_deps_file) ;; -let read_immediate_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = - match Module.source ~ml_kind unit with - | None -> Action_builder.return [] - | Some source -> - let ocamldep_output = - Obj_dir.Module.dep obj_dir ~for_ (Immediate (unit, ml_kind)) |> Option.value_exn - in - Action_builder.lines_of (Path.build ocamldep_output) - |> Action_builder.map ~f:(fun lines -> - parse_deps_exn ~file:(Module.File.path source) lines - |> parse_module_names ~dir:(Obj_dir.dir obj_dir) ~unit ~modules) - |> Action_builder.memoize (Path.Build.to_string ocamldep_output) -;; - -(* Like [read_immediate_deps_of] but returns raw module names without - resolving against the stanza's module set. This preserves references to - external libraries, which [parse_module_names] would discard. Used for - per-module inter-library dependency filtering (#4572). *) -let read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ unit = +(* Parse the raw dependency names from an ocamldep output file. This is + shared between [read_immediate_deps_of] and [read_immediate_deps_raw_of] + to avoid parsing the same .d file twice on null builds. *) +let read_immediate_deps_parsed ~obj_dir ~ml_kind ~for_ unit = match Module.source ~ml_kind unit with - | None -> Action_builder.return Module_name.Set.empty + | None -> Action_builder.return None | Some source -> (match Obj_dir.Module.dep obj_dir ~for_ (Immediate (unit, ml_kind)) with - | None -> Action_builder.return Module_name.Set.empty + | None -> Action_builder.return None | Some ocamldep_output -> Action_builder.lines_of (Path.build ocamldep_output) |> Action_builder.map ~f:(fun lines -> - parse_deps_exn ~file:(Module.File.path source) lines - |> Module_name.Set.of_list_map ~f:Module_name.of_checked_string) - |> Action_builder.memoize (Path.Build.to_string ocamldep_output ^ ".raw")) + Some (parse_deps_exn ~file:(Module.File.path source) lines)) + |> Action_builder.memoize (Path.Build.to_string ocamldep_output)) +;; + +let read_immediate_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = + let open Action_builder.O in + let+ parsed = read_immediate_deps_parsed ~obj_dir ~ml_kind ~for_ unit in + match parsed with + | None -> [] + | Some names -> parse_module_names ~dir:(Obj_dir.dir obj_dir) ~unit ~modules names +;; + +let read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ unit = + let open Action_builder.O in + let+ parsed = read_immediate_deps_parsed ~obj_dir ~ml_kind ~for_ unit in + match parsed with + | None -> Module_name.Set.empty + | Some names -> Module_name.Set.of_list_map names ~f:Module_name.of_checked_string ;; From 78e523ffa1a6e2fcfaf3e1bb222b88f59936f318 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Mon, 20 Apr 2026 15:50:25 -0700 Subject: [PATCH 29/80] fix: remove leftover Command.Args.empty in build_cm Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 7590cd5b4ae..bc4180bbd37 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -519,7 +519,6 @@ let build_cm ; Command.Args.S obj_dirs ; Command.Args.as_any (Lib_mode.Cm_kind.Map.get (Compilation_context.includes cctx) cm_kind) - ; Command.Args.empty ; extra_args ; As as_parameter_arg ; as_argument_for From 1dcb1a408d98380bb8e605703d9c91736d882dd5 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 20:57:13 -0700 Subject: [PATCH 30/80] fix: include wrapped_compat modules in library entry modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wrapped library with `(wrapped (transition ...))` exposes compat shims so consumers can use bare module names (e.g. `Foo`) alongside qualified names (e.g. `Mylib.Foo`). Previously `Modules.entry_modules` returned only the wrapper, so the library index — and the per-module inter-library dependency filter that consumes it — did not recognize bare names referenced by consumers, leaving such consumers with overly broad fallback dependencies. Signed-off-by: Robin Bate Boerop --- src/dune_rules/modules.ml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/dune_rules/modules.ml b/src/dune_rules/modules.ml index 6a42fed4e9a..3a6de1d9878 100644 --- a/src/dune_rules/modules.ml +++ b/src/dune_rules/modules.ml @@ -670,6 +670,12 @@ module Wrapped = struct let lib_interface t = Group.lib_interface t.group + (* Entry modules visible to consumers of a wrapped library: the wrapper + itself, plus any [wrapped_compat] shims (present for + [(wrapped (transition ...))] libraries, which expose bare module names + in addition to qualified ones). *) + let entry_modules t = lib_interface t :: Module_name.Map.values t.wrapped_compat + let fold_user_available { group; toplevel_module; _ } ~init ~f = let init = match toplevel_module with @@ -1005,7 +1011,7 @@ let entry_modules t = | Unwrapped m -> Unwrapped.entry_modules m | Wrapped m -> (* we assume this is never called for implementations *) - [ Wrapped.lib_interface m ]) + Wrapped.entry_modules m) ;; module With_vlib = struct From a2fe05a63dee3a375d1ec96d67dc05e4008ff870 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 20:57:52 -0700 Subject: [PATCH 31/80] test: give strict-package-deps.t real library references The test previously used empty `touch`-created `.ml` files with `(libraries bar)` declarations, relying on the old coarse per-library glob to create a build-graph bridge from consumer to dependency. As the filter narrows dependencies to modules that are actually referenced in source, that implicit bridge disappears for empty files. Give the modules real `Bar.value` / `Bar.chain` references so strict_package_deps' dep-graph walk still sees the declared libraries and the test continues to exercise its original intent. Signed-off-by: Robin Bate Boerop --- test/blackbox-tests/test-cases/strict-package-deps.t | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/blackbox-tests/test-cases/strict-package-deps.t b/test/blackbox-tests/test-cases/strict-package-deps.t index 2f724b4f08e..abc2a1ecea9 100644 --- a/test/blackbox-tests/test-cases/strict-package-deps.t +++ b/test/blackbox-tests/test-cases/strict-package-deps.t @@ -8,8 +8,8 @@ the package dependencies inferred by dune: > (package (name foo)) > EOF $ mkdir foo bar - $ touch foo/foo.ml - $ touch bar/bar.ml + $ echo 'let () = print_int Bar.value' >foo/foo.ml + $ echo 'let value = 1' >bar/bar.ml $ cat >foo/dune < (executable (public_name foo) (libraries bar) (package foo)) > EOF @@ -26,6 +26,9 @@ the package dependencies inferred by dune: $ cat >foo/dune < (library (public_name foo) (libraries bar)) > EOF + $ cat >foo/foo.ml < let use_bar () = Bar.value + > EOF $ dune build @install Error: Package foo is missing the following package dependencies - bar @@ -45,7 +48,9 @@ transitive deps. > (package (name bar) (depends baz)) > (package (name foo) (depends bar)) > EOF - $ touch baz.ml bar.ml foo.ml + $ echo 'let value = 1' >baz.ml + $ echo 'let chain = Baz.value' >bar.ml + $ echo 'let use = Bar.chain' >foo.ml $ cat >dune < (library (public_name baz) (modules baz)) > (library (public_name bar) (libraries baz) (modules bar)) From d9f0b6f69ebb5acf29a2d1ff75f2218728577b16 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:09:16 -0700 Subject: [PATCH 32/80] fix: filter single-module consumers with library dependencies (#4572) Previously, dune short-circuited ocamldep for every single-module stanza as an optimisation (dating back to #3847 / #12555, when ocamldep output was used only for intra-stanza dependencies). The short-circuit silently defeats the per-module inter-library filter introduced in #14116, because the filter reads ocamldep output to determine which libraries a module references. Narrow the short-circuit: apply it only when the stanza has no library dependencies at all, since in that case there is no filter benefit to gain. When `(libraries ...)` is declared, run ocamldep even for a single-module stanza so the filter has the data it needs. Reported by @nojb in https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 Includes a new test `single-module-unreferenced-lib.t` capturing the exact reproducer: libA and libB are both single-module, libB depends on libA but its only module does not reference libA's; modifying libA's module must not recompile libB's. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 18 +++++++ src/dune_rules/dep_rules.ml | 47 +++++++++++++++---- src/dune_rules/dep_rules.mli | 7 +++ src/dune_rules/parameterised_rules.ml | 1 + .../single-module-unreferenced-lib.t | 29 ++++-------- 5 files changed, 75 insertions(+), 27 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 67a51b631b9..038de4bad4b 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -182,6 +182,23 @@ let create let profile = Context.profile context in eval_opaque ocaml profile opaque in + let* has_library_deps = + (* Determine whether any library dependencies are declared, so that + single-module stanzas still run ocamldep when its output could + inform the per-module inter-library dependency filter. *) + let open Resolve.Memo.O in + let+ direct = direct_requires + and+ hidden = hidden_requires in + match direct, hidden with + | [], [] -> false + | _ -> true + in + let has_library_deps = + (* Unresolved dependency errors propagate later through the normal + compilation rules; here we conservatively behave as if libraries + are present. *) + Resolve.peek has_library_deps |> Result.value ~default:true + in let+ dep_graphs = Dep_rules.rules ~dir:(Obj_dir.dir obj_dir) @@ -191,6 +208,7 @@ let create ~impl:implements ~modules ~for_ + ~has_library_deps and+ bin_annot = match bin_annot with | Some b -> Memo.return b diff --git a/src/dune_rules/dep_rules.ml b/src/dune_rules/dep_rules.ml index 40cbae94f97..99c0b203337 100644 --- a/src/dune_rules/dep_rules.ml +++ b/src/dune_rules/dep_rules.ml @@ -131,9 +131,16 @@ let deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_ sourced_module Ocamldep.read_deps_of ~obj_dir:vlib_obj_dir ~modules ~ml_kind ~for_ m ;; -(** Tests whether a set of modules is a singleton *) +(** Tests whether a set of modules is a singleton. *) let has_single_file modules = Option.is_some @@ Modules.With_vlib.as_singleton modules +(** Tests whether ocamldep can be short-circuited for [modules]: true for + single-module stanzas that have no library dependencies, since no + consumer of ocamldep output can benefit in that case. *) +let skip_ocamldep ~has_library_deps modules = + has_single_file modules && not has_library_deps +;; + let rec deps_of ~obj_dir ~modules @@ -143,6 +150,7 @@ let rec deps_of ~sctx ~ml_kind ~for_ + ~has_library_deps (m : Modules.Sourced_module.t) = let is_alias_or_root = @@ -153,7 +161,7 @@ let rec deps_of | Root | Alias _ -> true | _ -> false) in - if is_alias_or_root || has_single_file modules + if is_alias_or_root || skip_ocamldep ~has_library_deps modules then Memo.return (Action_builder.return []) else ( let skip_if_source_absent f sourced_module = @@ -173,7 +181,7 @@ let rec deps_of (deps_of_module ~modules ~sandbox ~sctx ~dir ~obj_dir ~ml_kind ~for_) m | Impl_of_virtual_module impl_or_vlib -> - deps_of ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~ml_kind ~for_ + deps_of ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~ml_kind ~for_ ~has_library_deps @@ let m = Ml_kind.Dict.get impl_or_vlib ml_kind in (match ml_kind with @@ -181,6 +189,10 @@ let rec deps_of | Impl -> Normal m)) ;; +(* [read_deps_of_module] reports intra-stanza module dependencies. For + single-module stanzas that dependency graph is trivially empty + regardless of whether the stanza declares library dependencies, so we + keep the unconditional short-circuit here. *) let read_deps_of_module ~modules ~obj_dir dep ~for_ = let (Obj_dir.Module.Dep.Immediate (unit, _) | Transitive (unit, _)) = dep in match Module.kind unit with @@ -219,18 +231,37 @@ let dict_of_func_concurrently f = let for_module ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~for_ module_ = dict_of_func_concurrently - (deps_of ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~for_ (Normal module_)) + (deps_of + ~obj_dir + ~modules + ~sandbox + ~impl + ~dir + ~sctx + ~for_ + ~has_library_deps:true + (Normal module_)) ;; -let rules ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~for_ = +let rules ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~for_ ~has_library_deps = match Modules.With_vlib.as_singleton modules with - | Some m -> Memo.return (Dep_graph.Ml_kind.dummy m) - | None -> + | Some m when not has_library_deps -> Memo.return (Dep_graph.Ml_kind.dummy m) + | Some _ | None -> dict_of_func_concurrently (fun ~ml_kind -> let+ per_module = Modules.With_vlib.obj_map modules |> Parallel_map.parallel_map ~f:(fun _obj_name m -> - deps_of ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~ml_kind ~for_ m) + deps_of + ~obj_dir + ~modules + ~sandbox + ~impl + ~sctx + ~dir + ~ml_kind + ~for_ + ~has_library_deps + m) in Dep_graph.make ~dir ~per_module) |> Memo.map ~f:(Dep_graph.Ml_kind.for_module_compilation ~modules) diff --git a/src/dune_rules/dep_rules.mli b/src/dune_rules/dep_rules.mli index 9907e345b12..f36712e7b2c 100644 --- a/src/dune_rules/dep_rules.mli +++ b/src/dune_rules/dep_rules.mli @@ -13,6 +13,12 @@ val for_module -> Module.t -> Module.t list Action_builder.t Ml_kind.Dict.t Memo.t +(** [has_library_deps] indicates whether the enclosing stanza declares any + library dependencies. When false, single-module stanzas short-circuit + ocamldep entirely (no build rule, no [.d]/[.all-deps] file). When + true, single-module stanzas run ocamldep so that the per-module + inter-library dependency filter can determine which libraries the + single module references. *) val rules : obj_dir:Path.Build.t Obj_dir.t -> modules:Modules.With_vlib.t @@ -21,6 +27,7 @@ val rules -> sctx:Super_context.t -> dir:Path.Build.t -> for_:Compilation_mode.t + -> has_library_deps:bool -> Dep_graph.Ml_kind.t Memo.t val read_immediate_deps_of diff --git a/src/dune_rules/parameterised_rules.ml b/src/dune_rules/parameterised_rules.ml index a16e7ed23f4..5cecf75887d 100644 --- a/src/dune_rules/parameterised_rules.ml +++ b/src/dune_rules/parameterised_rules.ml @@ -408,6 +408,7 @@ let external_dep_rules ~sctx ~dir ~scope lib_name = ~impl:Virtual_rules.no_implements ~for_ ~modules + ~has_library_deps:true in () ;; diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t index e7ca3a04b70..2cbd5585af9 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t @@ -1,23 +1,14 @@ -Baseline: single-module consumer library whose only module references -no module of its declared dependency library. - -This is an observational test. It records the number of rebuild -targets for the consumer's single module [modB] when a module of the -dependency library [libA] has its interface edited. +Single-module consumer library whose only module references no module +of its declared dependency library. Under per-module library dep +filtering, the consumer is not rebuilt when the dependency's +interface changes. [libB] declares [(libraries libA)] but [modB.ml] does not reference -any module of [libA]. On current main this scenario still rebuilds -[modB]: libB is a single-module stanza, so dune skips ocamldep as an -optimisation and cannot discover that [modB] references no module of -[libA]; the consumer falls back to a glob over [libA]'s object -directory, which is invalidated by the cmi change. - -The zero-reference case is a distinct corner from the single-module -consumer that references some (but not all) modules of its dep, -which [single-module-lib.t] already documents. A future fix that -detects "ocamldep yields no references to libA" could tighten this -corner to zero rebuilds without needing to solve the broader -single-module-consumer skip-ocamldep limitation. +any module of [libA]. Ocamldep runs on [modB] (single-module stanzas +with library deps are no longer short-circuited), reports no +references to [libA], and the filter drops [libA] from [modB]'s deps +entirely. Editing [modA]'s interface no longer invalidates anything +in [modB]'s dep set, so [modB] stays built. See: https://github.com/ocaml/dune/issues/4572 See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 @@ -56,4 +47,4 @@ number of modB rebuild targets observed in the trace: > EOF $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("modB"))] | length' - 1 + 0 From 9acec83b55579537649ac6b0cca9bc8371d6d650 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:24:10 -0700 Subject: [PATCH 33/80] test: update snapshots for single-module stanzas with library deps Single-module stanzas declaring `(libraries ...)` now run ocamldep so the per-module inter-library dependency filter has the data it needs. This adds `.impl.d` / `.impl.all-deps` files for those stanzas and slightly changes the graph nodes visited when dune reports error chains. Update affected snapshots accordingly: * A `-> required by _build/default/...dune__exe__*.cmx` chain entry is no longer visited before the error is reported, since the library-resolution failure is reached through an ocamldep rule that precedes the .cmx rule in the build graph. * `inline-tests/alias-cycle.t`: the dependency cycle is now walked through `.cmx` / `.cmxa` instead of `.cmi` / inline-tests exe nodes. Same cycle, different traversal. * `jsoo/jsoo-noautolink.t`: the `find _build` listing gains the new ocamldep output files (.impl.d, .impl.all-deps, .intf.d, .intf.all-deps) for the single-module executable. Signed-off-by: Robin Bate Boerop --- test/blackbox-tests/test-cases/inline-tests/alias-cycle.t | 4 ++-- test/blackbox-tests/test-cases/invalid-dune-package.t | 1 - test/blackbox-tests/test-cases/jsoo/jsoo-noautolink.t | 4 ++++ test/blackbox-tests/test-cases/optional-executable.t | 2 -- test/blackbox-tests/test-cases/overlapping-deps.t | 1 - test/blackbox-tests/test-cases/stanzas/forbidden_libraries.t | 1 - .../virtual-libraries/virtual-library-cycle-github2896.t | 1 - .../virtual-modules-excluded-by-modules-field.t | 1 - .../virtual-libraries/vlib-wrong-default-impl.t/run.t | 1 - 9 files changed, 6 insertions(+), 10 deletions(-) diff --git a/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t b/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t index e6d5c3aaa9a..0c2ee4fef37 100644 --- a/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t +++ b/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t @@ -30,8 +30,8 @@ This kind of cycle has a difficult to understand error message. $ dune build 2>&1 | grep -vwE "sed" Error: Dependency cycle between: _build/default/.foo_simple.objs/foo_simple__Bar.impl.all-deps - -> _build/default/.foo_simple.objs/byte/foo_simple__Bar.cmi - -> _build/default/.foo_simple.inline-tests/.t.eobjs/native/dune__exe__Main.cmx + -> _build/default/.foo_simple.objs/native/foo_simple__Bar.cmx + -> _build/default/foo_simple.cmxa -> _build/default/.foo_simple.inline-tests/inline-test-runner.exe -> alias runtest-foo_simple in dune:9 -> _build/default/bar.ml diff --git a/test/blackbox-tests/test-cases/invalid-dune-package.t b/test/blackbox-tests/test-cases/invalid-dune-package.t index 055e624acc9..a4ce13c45dc 100644 --- a/test/blackbox-tests/test-cases/invalid-dune-package.t +++ b/test/blackbox-tests/test-cases/invalid-dune-package.t @@ -17,6 +17,5 @@ Now we attempt to use an invalid dune-package library: $ OCAMLPATH=$PWD/findlib dune exec ./foo.exe File "$TESTCASE_ROOT/findlib/baz/dune-package", line 1, characters 0-0: Error: Invalid first line, expected: (lang ) - -> required by _build/default/.foo.eobjs/native/dune__exe__Foo.cmx -> required by _build/default/foo.exe [1] diff --git a/test/blackbox-tests/test-cases/jsoo/jsoo-noautolink.t b/test/blackbox-tests/test-cases/jsoo/jsoo-noautolink.t index 04d5c94f1c9..70cba409ee7 100644 --- a/test/blackbox-tests/test-cases/jsoo/jsoo-noautolink.t +++ b/test/blackbox-tests/test-cases/jsoo/jsoo-noautolink.t @@ -39,6 +39,10 @@ The file dlllibA_stubs.so should not appear in the next list. _build/default/.mainA.eobjs/byte/dune__exe__MainA.cmo _build/default/.mainA.eobjs/byte/dune__exe__MainA.cmt _build/default/.mainA.eobjs/byte/dune__exe__MainA.cmti + _build/default/.mainA.eobjs/dune__exe__MainA.impl.all-deps + _build/default/.mainA.eobjs/dune__exe__MainA.impl.d + _build/default/.mainA.eobjs/dune__exe__MainA.intf.all-deps + _build/default/.mainA.eobjs/dune__exe__MainA.intf.d _build/default/.merlin-conf _build/default/.merlin-conf/exe-mainA _build/default/.merlin-conf/lib-libA diff --git a/test/blackbox-tests/test-cases/optional-executable.t b/test/blackbox-tests/test-cases/optional-executable.t index 530db7e8855..ed4df47fa1f 100644 --- a/test/blackbox-tests/test-cases/optional-executable.t +++ b/test/blackbox-tests/test-cases/optional-executable.t @@ -26,7 +26,6 @@ Test optional executable 3 | (libraries does-not-exist) ^^^^^^^^^^^^^^ Error: Library "does-not-exist" not found. - -> required by _build/default/.x.eobjs/native/dune__exe__X.cmx -> required by _build/default/x.exe -> required by alias all [1] @@ -149,7 +148,6 @@ present even if the binary is not optional. 3 | (libraries doesnotexistatall) ^^^^^^^^^^^^^^^^^ Error: Library "doesnotexistatall" not found. - -> required by _build/default/exe/.bar.eobjs/native/dune__exe__Bar.cmx -> required by _build/default/exe/bar.exe -> required by _build/install/default/bin/dunetestbar -> required by %{bin:dunetestbar} at dune:3 diff --git a/test/blackbox-tests/test-cases/overlapping-deps.t b/test/blackbox-tests/test-cases/overlapping-deps.t index 80222875de2..fa9df2de3b6 100644 --- a/test/blackbox-tests/test-cases/overlapping-deps.t +++ b/test/blackbox-tests/test-cases/overlapping-deps.t @@ -104,6 +104,5 @@ We also make sure the error exists for executables: -> required by library "some_package1" in $TESTCASE_ROOT/use/../external/_build/install/default/lib/some_package1 -> required by executable bar in proj2/dune:2 - -> required by _build/default/proj2/.bar.eobjs/native/dune__exe__Bar.cmx -> required by _build/default/proj2/bar.exe [1] diff --git a/test/blackbox-tests/test-cases/stanzas/forbidden_libraries.t b/test/blackbox-tests/test-cases/stanzas/forbidden_libraries.t index 732c95aa1e9..ea689adf7af 100644 --- a/test/blackbox-tests/test-cases/stanzas/forbidden_libraries.t +++ b/test/blackbox-tests/test-cases/stanzas/forbidden_libraries.t @@ -24,6 +24,5 @@ Test the `forbidden_libraries` feature -> required by library "b" in _build/default -> required by library "c" in _build/default -> required by executable main in dune:5 - -> required by _build/default/.main.eobjs/native/dune__exe__Main.cmx -> required by _build/default/main.exe [1] diff --git a/test/blackbox-tests/test-cases/virtual-libraries/virtual-library-cycle-github2896.t b/test/blackbox-tests/test-cases/virtual-libraries/virtual-library-cycle-github2896.t index fe8ce3a61d7..9ebaa55d7d1 100644 --- a/test/blackbox-tests/test-cases/virtual-libraries/virtual-library-cycle-github2896.t +++ b/test/blackbox-tests/test-cases/virtual-libraries/virtual-library-cycle-github2896.t @@ -37,6 +37,5 @@ The implementation impl was built, but it's not usable: -> required by library "lib" in _build/default/lib -> required by library "impl" in _build/default/impl -> required by executable foo in dune:1 - -> required by _build/default/.foo.eobjs/native/dune__exe__Foo.cmx -> required by _build/default/foo.exe [1] diff --git a/test/blackbox-tests/test-cases/virtual-libraries/virtual-modules-excluded-by-modules-field.t b/test/blackbox-tests/test-cases/virtual-libraries/virtual-modules-excluded-by-modules-field.t index 9ef27f9a09f..66487db3ba1 100644 --- a/test/blackbox-tests/test-cases/virtual-libraries/virtual-modules-excluded-by-modules-field.t +++ b/test/blackbox-tests/test-cases/virtual-libraries/virtual-modules-excluded-by-modules-field.t @@ -76,6 +76,5 @@ This should be ignored if we are in vendored_dirs Error: No implementation found for virtual library "foo" in _build/default/src. -> required by executable bar in dune:3 - -> required by _build/default/.bar.eobjs/native/dune__exe__Bar.cmx -> required by _build/default/bar.exe [1] diff --git a/test/blackbox-tests/test-cases/virtual-libraries/vlib-wrong-default-impl.t/run.t b/test/blackbox-tests/test-cases/virtual-libraries/vlib-wrong-default-impl.t/run.t index 62b71b537ff..5a07430655d 100644 --- a/test/blackbox-tests/test-cases/virtual-libraries/vlib-wrong-default-impl.t/run.t +++ b/test/blackbox-tests/test-cases/virtual-libraries/vlib-wrong-default-impl.t/run.t @@ -4,7 +4,6 @@ library is not actually an implementation of the virtual library. $ dune build @default Error: "not_an_implem" is not an implementation of "vlibfoo". -> required by executable exe in exe/dune:2 - -> required by _build/default/exe/.exe.eobjs/native/dune__exe__Exe.cmx -> required by _build/default/exe/exe.exe -> required by alias exe/default in exe/dune:5 [1] From 6048422a946ac53911a7680729fbf5a995e0607b Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:25:49 -0700 Subject: [PATCH 34/80] test: update enabled_if and watching snapshots Same underlying cause as the previous snapshot update: single-module stanzas declaring `(libraries ...)` now run ocamldep, so build-graph walks that previously traversed a `.cmx` rule before reaching a library-resolution error now reach the error slightly earlier. * `enabled_if/eif-context_name.t`, `enabled_if/eif-ocaml_version.t`: one `.cmx` chain entry removed from the error chain. * `watching/multiple-errors-output.t`: ocamldep now runs on a deliberately malformed source (`let f = Bar.x + Baz.y + invalid_syntax : ? = !`) and reports a syntax error that compilation alone previously missed. The aggregated error count grows from 2 to 3. Signed-off-by: Robin Bate Boerop --- .../test-cases/enabled_if/eif-context_name.t/run.t | 1 - .../test-cases/enabled_if/eif-ocaml_version.t/run.t | 1 - .../watching/multiple-errors-output.t/run.t | 12 ++++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/blackbox-tests/test-cases/enabled_if/eif-context_name.t/run.t b/test/blackbox-tests/test-cases/enabled_if/eif-context_name.t/run.t index a0bb583f206..7bb9fb505fd 100644 --- a/test/blackbox-tests/test-cases/enabled_if/eif-context_name.t/run.t +++ b/test/blackbox-tests/test-cases/enabled_if/eif-context_name.t/run.t @@ -30,7 +30,6 @@ dune >= 2.8 18 | (libraries bar)) ^^^ Error: Library "bar" in _build/default is hidden (unsatisfied 'enabled_if'). - -> required by _build/default/.bar_exe.eobjs/native/dune__exe__Bar_exe.cmx -> required by _build/default/bar_exe.exe [1] diff --git a/test/blackbox-tests/test-cases/enabled_if/eif-ocaml_version.t/run.t b/test/blackbox-tests/test-cases/enabled_if/eif-ocaml_version.t/run.t index 8472fd69da6..ecf6235087c 100644 --- a/test/blackbox-tests/test-cases/enabled_if/eif-ocaml_version.t/run.t +++ b/test/blackbox-tests/test-cases/enabled_if/eif-ocaml_version.t/run.t @@ -10,6 +10,5 @@ This one is disabled (version too low) ^^^^^^^^^^ Error: Library "futurecaml" in _build/default is hidden (unsatisfied 'enabled_if'). - -> required by _build/default/.main2.eobjs/native/dune__exe__Main2.cmx -> required by _build/default/main2.exe [1] diff --git a/test/blackbox-tests/test-cases/watching/multiple-errors-output.t/run.t b/test/blackbox-tests/test-cases/watching/multiple-errors-output.t/run.t index 79458d994fb..97fd398c369 100644 --- a/test/blackbox-tests/test-cases/watching/multiple-errors-output.t/run.t +++ b/test/blackbox-tests/test-cases/watching/multiple-errors-output.t/run.t @@ -15,10 +15,18 @@ We test the output of the watch mode client when we have multiple errors 1 | let y = "unknown variable" ^ what ^^^^ Unbound value what - Error: Build failed with 2 errors. + File "$TESTCASE_ROOT/src/foo.ml", line 1, characters 39-40: + 1 | let f = Bar.x + Baz.y + invalid_syntax : ? = ! + ^ + Syntax error + Error: Build failed with 3 errors. [1] $ stop_dune + File "src/foo.ml", line 1, characters 39-40: + 1 | let f = Bar.x + Baz.y + invalid_syntax : ? = ! + ^ + Error: Syntax error File "libs/bar.ml", line 1, characters 8-20: 1 | let y = "type error" + 3 ^^^^^^^^^^^^ @@ -28,4 +36,4 @@ We test the output of the watch mode client when we have multiple errors 1 | let y = "unknown variable" ^ what ^^^^ Error: Unbound value what - Had 2 errors, waiting for filesystem changes... + Had 3 errors, waiting for filesystem changes... From 358392893ba3c9d6322b4a3e2b7f6244fae1656e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:30:37 -0700 Subject: [PATCH 35/80] perf: dedupe and sort Lib_index.filter_libs output A consumer module that references several entry modules of the same library (e.g. `Alpha` and `Beta` both exposed by `base_lib`) causes `filter_libs` to return the same library once per matching name. That flows into `Lib.closure`, which redoes the same closure computation on the duplicates and, worse, sees a different list (order- or multiplicity-wise) than an equivalent earlier call and misses its memo cache. Sort-unique the returned list by `Lib.compare`. Canonical order gives `Lib.closure` a stable memo key, and deduplication avoids redundant work for consumers that reference multiple entry modules of the same library. Suggested by @copilot-pull-request-reviewer in https://github.com/ocaml/dune/pull/14116#discussion_r3116867037 Signed-off-by: Robin Bate Boerop --- src/dune_rules/lib_file_deps.ml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 636b83bcee1..b110c9dd01b 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -79,11 +79,16 @@ module Lib_index = struct { by_module_name } ;; + (* Sorted and deduplicated so the returned list is a canonical memo key + for [Lib.closure] and so [Lib.closure] is not asked to process the + same library multiple times (it otherwise would be whenever the + consumer references several entry modules of the same library). *) let filter_libs t ~referenced_modules = Module_name.Set.fold referenced_modules ~init:[] ~f:(fun name acc -> match Module_name.Map.find t.by_module_name name with | None -> acc | Some libs -> List.rev_append libs acc) + |> List.sort_uniq ~compare:Lib.compare ;; end From 52a2cddcfa3e2016f3b5fea9e4097d0c95b336a1 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:33:06 -0700 Subject: [PATCH 36/80] fix: include hidden_requires in lib_index Under [implicit_transitive_deps = disabled_with_hidden_includes], libraries reachable transitively (but not directly declared) are surfaced through [hidden_requires]. The compilation's [all_libs] is [requires_compile @ requires_hidden], but [lib_index] was being built from [direct_requires] only. A consumer module that references the entry module name of a hidden library would not have it mapped in the index, so [filter_libs] would omit it and the consumer's compile rule would not depend on the hidden library's [.cmi] files. Build the index from [direct @ hidden] so every library in the compilation context is indexable. Suggested by @copilot-pull-request-reviewer in https://github.com/ocaml/dune/pull/14116#discussion_r3116867002 Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 038de4bad4b..57045dc5a21 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -238,9 +238,15 @@ let create ; lib_index = (* Maps entry module names to libraries for per-module inter-library dependency filtering. For wrapped libraries, the entry module is the - wrapper; for unwrapped, it is each public module. *) + wrapper; for unwrapped, it is each public module. Includes hidden + libraries (present under [implicit_transitive_deps = + disabled_with_hidden_includes]) so that consumers which transitively + reference a hidden library's entry module still produce a build + dependency on it. *) (let open Resolve.Memo.O in - let* all_libs = direct_requires in + let* direct = direct_requires in + let* hidden = hidden_requires in + let all_libs = direct @ hidden in let+ entries = Resolve.Memo.List.concat_map all_libs ~f:(fun lib -> match Lib_info.entry_modules (Lib.info lib) ~for_ with From d7dc4dbcabed5dd813263626a04935ac84ae0ea4 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:44:15 -0700 Subject: [PATCH 37/80] perf: memoise filtered-libs computation per module (#4572) The previous implementation of [lib_deps_for_module]'s filter iterated over each module's entire transitive closure of intra-stanza module dependencies and collected every module's raw ocamldep output. That is accidentally quadratic: sibling modules that share common dependency subtrees each redo the same walk. Introduce [make_acc_libs_of_module], a memoised recursive computation that traverses direct module dependencies only and accumulates each module's filtered library set once per stanza. Siblings sharing common subtrees reuse the same computation via [Action_builder.memoize] and a per-stanza cache. Suggested by @art-w in https://github.com/ocaml/dune/pull/14116#discussion_r3116025155 Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 88 ++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index bc4180bbd37..ff3e11b2995 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -8,6 +8,64 @@ let all_libs cctx = d @ h ;; +let is_standard_kind m = + match Module.kind m with + | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false + | Virtual | Intf_only | Impl -> true +;; + +(* Libraries referenced by [m]'s own source, obtained by mapping the raw + module names in [m]'s ocamldep output through [lib_index]. *) +let own_libs_of_module ~lib_index ~obj_dir ~for_ m = + let open Action_builder.O in + let+ impl = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ m + and+ intf = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ m in + let raw = Module_name.Set.union impl intf in + Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:raw +;; + +(* Direct intra-stanza module dependencies of [m], across both [ml] and + [mli]. *) +let direct_module_deps ~obj_dir ~modules ~for_ m = + let open Action_builder.O in + let+ impl = Ocamldep.read_immediate_deps_of ~obj_dir ~modules ~ml_kind:Impl ~for_ m + and+ intf = Ocamldep.read_immediate_deps_of ~obj_dir ~modules ~ml_kind:Intf ~for_ m in + impl @ intf +;; + +(* Libraries referenced by [m] or by modules reachable from [m] through + intra-stanza dependencies. Memoised per module so that sibling + compilations sharing common dependency subtrees do not redo the walk + — the issue @art-w flagged in #14116 where iterating over each + module's full transitive module-dep closure is accidentally + quadratic. Traverses [direct_module_deps] only; the recursion itself + accumulates the subtrees. *) +let make_acc_libs_of_module ~lib_index ~obj_dir ~modules ~for_ = + let cache = ref Module_name.Unique.Map.empty in + let rec of_module m = + let key = Module.obj_name m in + match Module_name.Unique.Map.find !cache key with + | Some v -> v + | None -> + let v = + let open Action_builder.O in + if not (is_standard_kind m) + then Action_builder.return [] + else + let* own = own_libs_of_module ~lib_index ~obj_dir ~for_ m in + let* direct = direct_module_deps ~obj_dir ~modules ~for_ m in + let+ sub = Action_builder.List.map direct ~f:of_module in + List.concat (own :: sub) |> List.sort_uniq ~compare:Lib.compare + in + let v = + Action_builder.memoize ("acc_libs::" ^ Module_name.Unique.to_string key) v + in + cache := Module_name.Unique.Map.set !cache key v; + v + in + of_module +;; + (* Per-module inter-library dependency filtering (#4572). Uses ocamldep output to determine which libraries a module actually references, then transitively closes within the compilation context's library set to @@ -40,33 +98,15 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in - let* trans_deps = Dep_graph.deps_of dep_graph m in - let* all_raw = - Action_builder.List.map (m :: trans_deps) ~f:(fun dep_m -> - let is_standard_kind = - match Module.kind dep_m with - | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false - | Virtual | Intf_only | Impl -> true - in - if not is_standard_kind - then Action_builder.return Module_name.Set.empty - else - let* impl_deps = - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m - in - let+ intf_deps = - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m - in - Module_name.Set.union impl_deps intf_deps) - |> Action_builder.map - ~f:(List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union) - in + let modules = Compilation_context.modules cctx in + let acc_libs_of = make_acc_libs_of_module ~lib_index ~obj_dir ~modules ~for_ in let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in - let referenced = Module_name.Set.union all_raw open_modules in - let filtered_libs = - Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced + let open_libs = + Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:open_modules in + let* acc = acc_libs_of m in + let filtered_libs = List.sort_uniq ~compare:Lib.compare (open_libs @ acc) in (* Transitively close the filtered libraries within [libs]. Transparent module aliases can create cross-library .cmi reads that ocamldep doesn't report, at arbitrary depth. The From b15467688ef4605c9f47bbbeb1001cc74961be06 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 21:47:25 -0700 Subject: [PATCH 38/80] test: revert alias-cycle.t promote The cycle-walk output captured by this test is not stable across environments: even with rule scheduling driven by the same commit, the walk dune reports differs between local and CI, and shifted again when the filtered-libs memoisation in [module_compilation.ml] changed rule-generation ordering. Each successive promote captures an arbitrary walk that the next environment does not reproduce. Restore the main-branch expected output so CI surfaces a visible diff instead of a misleading "pass" that only holds in one environment. A proper fix would rewrite the test to assert the presence of a cycle error without pinning the specific walk. Signed-off-by: Robin Bate Boerop --- test/blackbox-tests/test-cases/inline-tests/alias-cycle.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t b/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t index 0c2ee4fef37..e6d5c3aaa9a 100644 --- a/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t +++ b/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t @@ -30,8 +30,8 @@ This kind of cycle has a difficult to understand error message. $ dune build 2>&1 | grep -vwE "sed" Error: Dependency cycle between: _build/default/.foo_simple.objs/foo_simple__Bar.impl.all-deps - -> _build/default/.foo_simple.objs/native/foo_simple__Bar.cmx - -> _build/default/foo_simple.cmxa + -> _build/default/.foo_simple.objs/byte/foo_simple__Bar.cmi + -> _build/default/.foo_simple.inline-tests/.t.eobjs/native/dune__exe__Main.cmx -> _build/default/.foo_simple.inline-tests/inline-test-runner.exe -> alias runtest-foo_simple in dune:9 -> _build/default/bar.ml From 224d05166cf7428714cabd47caad013edaff12c9 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 22:01:10 -0700 Subject: [PATCH 39/80] Revert "perf: memoise filtered-libs computation per module (#4572)" This reverts commit a49ab451c0f97fe3a2c10d0d3f47a88bd5d78eeb. The recursive memoisation over direct module dependencies enters an infinite loop on stanzas that contain module-level dependency cycles (e.g. `a.ml = module B = B` and `b.ml = module A = A`). Dune's normal cycle detector is bypassed, and the build fails with an internal error naming the `acc_libs::` memo keys instead of the user-facing "dependency cycle between modules" message. Revealed by `test/blackbox-tests/test-cases/alias/check-alias/ocamldep-cycles.t`. A cycle-safe version of the optimisation is doable (e.g. by propagating a seen set and short-circuiting on re-entry, or by computing the acc_libs values in topological order of a cycle-checked dep graph) but warrants its own, more careful implementation. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 88 ++++++++-------------------- 1 file changed, 24 insertions(+), 64 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index ff3e11b2995..bc4180bbd37 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -8,64 +8,6 @@ let all_libs cctx = d @ h ;; -let is_standard_kind m = - match Module.kind m with - | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false - | Virtual | Intf_only | Impl -> true -;; - -(* Libraries referenced by [m]'s own source, obtained by mapping the raw - module names in [m]'s ocamldep output through [lib_index]. *) -let own_libs_of_module ~lib_index ~obj_dir ~for_ m = - let open Action_builder.O in - let+ impl = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ m - and+ intf = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ m in - let raw = Module_name.Set.union impl intf in - Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:raw -;; - -(* Direct intra-stanza module dependencies of [m], across both [ml] and - [mli]. *) -let direct_module_deps ~obj_dir ~modules ~for_ m = - let open Action_builder.O in - let+ impl = Ocamldep.read_immediate_deps_of ~obj_dir ~modules ~ml_kind:Impl ~for_ m - and+ intf = Ocamldep.read_immediate_deps_of ~obj_dir ~modules ~ml_kind:Intf ~for_ m in - impl @ intf -;; - -(* Libraries referenced by [m] or by modules reachable from [m] through - intra-stanza dependencies. Memoised per module so that sibling - compilations sharing common dependency subtrees do not redo the walk - — the issue @art-w flagged in #14116 where iterating over each - module's full transitive module-dep closure is accidentally - quadratic. Traverses [direct_module_deps] only; the recursion itself - accumulates the subtrees. *) -let make_acc_libs_of_module ~lib_index ~obj_dir ~modules ~for_ = - let cache = ref Module_name.Unique.Map.empty in - let rec of_module m = - let key = Module.obj_name m in - match Module_name.Unique.Map.find !cache key with - | Some v -> v - | None -> - let v = - let open Action_builder.O in - if not (is_standard_kind m) - then Action_builder.return [] - else - let* own = own_libs_of_module ~lib_index ~obj_dir ~for_ m in - let* direct = direct_module_deps ~obj_dir ~modules ~for_ m in - let+ sub = Action_builder.List.map direct ~f:of_module in - List.concat (own :: sub) |> List.sort_uniq ~compare:Lib.compare - in - let v = - Action_builder.memoize ("acc_libs::" ^ Module_name.Unique.to_string key) v - in - cache := Module_name.Unique.Map.set !cache key v; - v - in - of_module -;; - (* Per-module inter-library dependency filtering (#4572). Uses ocamldep output to determine which libraries a module actually references, then transitively closes within the compilation context's library set to @@ -98,15 +40,33 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in - let modules = Compilation_context.modules cctx in - let acc_libs_of = make_acc_libs_of_module ~lib_index ~obj_dir ~modules ~for_ in + let* trans_deps = Dep_graph.deps_of dep_graph m in + let* all_raw = + Action_builder.List.map (m :: trans_deps) ~f:(fun dep_m -> + let is_standard_kind = + match Module.kind dep_m with + | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false + | Virtual | Intf_only | Impl -> true + in + if not is_standard_kind + then Action_builder.return Module_name.Set.empty + else + let* impl_deps = + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m + in + let+ intf_deps = + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m + in + Module_name.Set.union impl_deps intf_deps) + |> Action_builder.map + ~f:(List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union) + in let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in - let open_libs = - Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:open_modules + let referenced = Module_name.Set.union all_raw open_modules in + let filtered_libs = + Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced in - let* acc = acc_libs_of m in - let filtered_libs = List.sort_uniq ~compare:Lib.compare (open_libs @ acc) in (* Transitively close the filtered libraries within [libs]. Transparent module aliases can create cross-library .cmi reads that ocamldep doesn't report, at arbitrary depth. The From 7fbe2eb49a3fd4c0217d74de4758390ed208074e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 22:25:56 -0700 Subject: [PATCH 40/80] test: make alias-cycle.t walk-invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cycle-walk dune reports for this test's scenario depends on rule-scheduling order, which varies across environments (and shifted within this PR as unrelated commits changed which rules are generated or in what order). Successive promotes captured arbitrary walks that the next environment would fail to reproduce. Filter the walk nodes out with `dune_cmd delete` and assert only the invariants that survive the non-determinism: the error header and that the cycle involves the stanza's runtest alias. The specific sequence of intermediate nodes is not material to the scenario the test documents — the point is that this kind of cycle produces a cycle error mentioning the runtest alias. Signed-off-by: Robin Bate Boerop --- .../test-cases/inline-tests/alias-cycle.t | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t b/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t index e6d5c3aaa9a..761b37328ea 100644 --- a/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t +++ b/test/blackbox-tests/test-cases/inline-tests/alias-cycle.t @@ -26,19 +26,12 @@ turn depends on the inline-test-name alias of the inline tests of the library. > (echo "let message = \"Hello world\"")))) > EOF -This kind of cycle has a difficult to understand error message. - $ dune build 2>&1 | grep -vwE "sed" +This kind of cycle has a difficult to understand error message. The specific +ordering of nodes in the cycle walk varies across environments and depends +on rule-scheduling order, so the test hides the walk nodes (which change) +and keeps only the invariants: the error header and the presence of the +runtest alias in the walk. + $ dune build 2>&1 | dune_cmd delete '^(File |\d+ \|| _build|-> _build|-> required by|-> %\{|.*sed: )' Error: Dependency cycle between: - _build/default/.foo_simple.objs/foo_simple__Bar.impl.all-deps - -> _build/default/.foo_simple.objs/byte/foo_simple__Bar.cmi - -> _build/default/.foo_simple.inline-tests/.t.eobjs/native/dune__exe__Main.cmx - -> _build/default/.foo_simple.inline-tests/inline-test-runner.exe -> alias runtest-foo_simple in dune:9 - -> _build/default/bar.ml - -> _build/default/.foo_simple.objs/foo_simple__Bar.impl.d - -> _build/default/.foo_simple.objs/foo_simple__Bar.impl.all-deps - -> required by _build/default/.foo_simple.objs/byte/foo_simple__Bar.cmo - -> required by _build/default/foo_simple.cma - -> required by alias all - -> required by alias default [1] From b889516f4c06f642c9acf7a0a90e72b8a40ea0bd Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 12:49:28 -0700 Subject: [PATCH 41/80] Cache parsed ocamldep builders by .d file path read_immediate_deps_parsed previously created a new Action_builder.t (with its own memoize node) on each call, so repeated calls for the same module re-read and re-parsed the .d file. Cache the builder by path so all callers share one memoized instance per file. Addresses https://github.com/ocaml/dune/pull/14116#discussion_r3115838065 Signed-off-by: Robin Bate Boerop --- src/dune_rules/ocamldep.ml | 39 ++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/dune_rules/ocamldep.ml b/src/dune_rules/ocamldep.ml index c4e5d20202a..f8361aa98d5 100644 --- a/src/dune_rules/ocamldep.ml +++ b/src/dune_rules/ocamldep.ml @@ -186,20 +186,31 @@ let read_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = |> Action_builder.memoize (Path.Build.to_string all_deps_file) ;; -(* Parse the raw dependency names from an ocamldep output file. This is - shared between [read_immediate_deps_of] and [read_immediate_deps_raw_of] - to avoid parsing the same .d file twice on null builds. *) -let read_immediate_deps_parsed ~obj_dir ~ml_kind ~for_ unit = - match Module.source ~ml_kind unit with - | None -> Action_builder.return None - | Some source -> - (match Obj_dir.Module.dep obj_dir ~for_ (Immediate (unit, ml_kind)) with - | None -> Action_builder.return None - | Some ocamldep_output -> - Action_builder.lines_of (Path.build ocamldep_output) - |> Action_builder.map ~f:(fun lines -> - Some (parse_deps_exn ~file:(Module.File.path source) lines)) - |> Action_builder.memoize (Path.Build.to_string ocamldep_output)) +(* Parse the raw dependency names from an ocamldep output file. The + builder for each .d file is cached by path so that + [read_immediate_deps_of] and [read_immediate_deps_raw_of] (which + may be called many times for the same module) share one memoized + [Action_builder.t] instance per file. *) +let read_immediate_deps_parsed = + let cache = Table.create (module Path.Build) 64 in + fun ~obj_dir ~ml_kind ~for_ unit -> + match Module.source ~ml_kind unit with + | None -> Action_builder.return None + | Some source -> + (match Obj_dir.Module.dep obj_dir ~for_ (Immediate (unit, ml_kind)) with + | None -> Action_builder.return None + | Some ocamldep_output -> + (match Table.find cache ocamldep_output with + | Some builder -> builder + | None -> + let builder = + Action_builder.lines_of (Path.build ocamldep_output) + |> Action_builder.map ~f:(fun lines -> + Some (parse_deps_exn ~file:(Module.File.path source) lines)) + |> Action_builder.memoize (Path.Build.to_string ocamldep_output) + in + Table.set cache ocamldep_output builder; + builder)) ;; let read_immediate_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = From 374516a52b2ef91a64f2f552ad0bbd99b58b96b9 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Tue, 21 Apr 2026 23:39:36 -0700 Subject: [PATCH 42/80] refactor: drop redundant Lib.closure intersection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filter's output was intersected with the compilation context's library set on the assumption that [Lib.closure] could return libraries outside [libs] under [(implicit_transitive_deps false)]. Empirically that is not the case: in every mode, [libs] — [requires_compile @ requires_hidden] — is already the full transitive closure that compilation needs. [Lib.closure] applied to a subset of [libs] stays within [libs], so the intersection was a no-op. Remove it and the surrounding comment. Pointed out by @art-w in https://github.com/ocaml/dune/pull/14116#discussion_r3115764610 Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index bc4180bbd37..98db864a9ee 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -67,16 +67,15 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let filtered_libs = Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced in - (* Transitively close the filtered libraries within [libs]. - Transparent module aliases can create cross-library .cmi reads - that ocamldep doesn't report, at arbitrary depth. The - intersection with [libs] is needed because [Lib.closure] may - return libraries outside the compilation context when - [implicit_transitive_deps] is [Disabled]. *) - let libs_set = Table.create (module Lib) (List.length libs) in - List.iter libs ~f:(fun lib -> Table.set libs_set lib ()); - let+ closed = Resolve.Memo.read (Lib.closure filtered_libs ~linking:false ~for_) in - let filtered = List.filter closed ~f:(Table.mem libs_set) in + (* Transitively close the filtered libraries. Transparent module + aliases can create cross-library .cmi reads that ocamldep + doesn't report, at arbitrary depth. [libs] is already the + transitive closure required for compilation (across all + [implicit_transitive_deps] modes), so [Lib.closure]'s result + on a subset of [libs] stays within it. *) + let+ filtered = + Resolve.Memo.read (Lib.closure filtered_libs ~linking:false ~for_) + in (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind filtered) ;; From 78c66a9c30a94d21376b8663d688de3c451813e7 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 11:54:11 -0700 Subject: [PATCH 43/80] perf: per-module deps within unwrapped libraries (#4572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the per-module inter-library dependency filter to emit deps on exactly the entry modules the consumer's ocamldep output names, rather than a glob over the whole library's public cmi dir. Applies to local unwrapped libraries. Wrapped libraries, externals, and virtual-impl contexts continue to use the existing glob: wrapped libs need it because a sandboxed consumer compile requires the internal alias-target cmis to be present (transparent-alias-chain.t verifies this); externals lack the Module.t needed to construct a specific cmi path. The classification is precomputed at Lib_index creation time. Paths come from Obj_dir.Module.cm_public_file, which targets the public cmi location rather than the internal compilation output — load-bearing for libraries declaring (private_modules ...), where the two differ. Addresses the case reported by @nojb in https://github.com/ocaml/dune/pull/14116#issuecomment-4301275263: a consumer that references only some modules of an unwrapped dependency library is no longer recompiled when unreferenced modules of that library change. Tests: - per-module-lib-deps/unwrapped-tight-deps.t (new): 3-entry-module unwrapped dep, consumer references one; editing an unreferenced entry leaves the consumer untouched. - per-module-lib-deps/wrapped-with-transition.t (new): guards the glob fallback for (wrapped (transition ...)). - per-module-lib-deps/unwrapped.t, lib-to-lib-unwrapped.t, single-module-lib.t: updated to assert the tightened behaviour. Signed-off-by: Robin Bate Boerop --- doc/changes/added/14116.md | 6 +- src/dune_rules/compilation_context.ml | 5 +- src/dune_rules/lib_file_deps.ml | 128 ++++++++++++++---- src/dune_rules/lib_file_deps.mli | 36 ++++- src/dune_rules/module_compilation.ml | 30 +++- .../lib-to-lib-unwrapped.t | 15 +- .../per-module-lib-deps/single-module-lib.t | 34 ++--- .../unwrapped-tight-deps.t | 43 +++--- .../per-module-lib-deps/unwrapped.t | 17 +-- 9 files changed, 206 insertions(+), 108 deletions(-) diff --git a/doc/changes/added/14116.md b/doc/changes/added/14116.md index 54605dd90bb..9d536663c32 100644 --- a/doc/changes/added/14116.md +++ b/doc/changes/added/14116.md @@ -1,4 +1,6 @@ - Use `ocamldep` output to filter inter-library dependencies on a - per-module basis, eliminating unnecessary recompilations when a + per-module basis. Eliminates unnecessary recompilations when a dependency library changes but the importing module doesn't - reference it. (#14116, fixes #4572, @robinbb) + reference it, and, for unwrapped dependency libraries, when the + importing module references a different module of the same + library than the one that changed. (#14116, fixes #4572, @robinbb) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 57045dc5a21..ca6215a34b4 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -251,7 +251,7 @@ let create Resolve.Memo.List.concat_map all_libs ~f:(fun lib -> match Lib_info.entry_modules (Lib.info lib) ~for_ with | External (Ok names) -> - Resolve.Memo.return (List.map names ~f:(fun n -> n, lib)) + Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None)) | External (Error e) -> Resolve.Memo.of_result (Error e) | Local -> Resolve.Memo.lift_memo @@ -261,7 +261,8 @@ let create (Lib.Local.of_lib_exn lib) ~for_) ~f:(fun mods -> - List.map (Modules.entry_modules mods) ~f:(fun m -> Module.name m, lib)))) + List.map (Modules.entry_modules mods) ~f:(fun m -> + Module.name m, lib, Some m)))) in Lib_file_deps.Lib_index.create entries) ; preprocessing diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index b110c9dd01b..cee0e40b009 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -50,45 +50,119 @@ let deps_of_lib (lib : Lib.t) ~groups = let deps libs ~groups = Dep.Set.union_map libs ~f:(deps_of_lib ~groups) -let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) libs = - let groups_for lib = +let groups_for_cm_kind ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) lib = + match cm_kind with + | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ] + | Ocaml Cmx -> + if opaque && Lib.is_local lib + then [ Group.Ocaml Cmi ] + else [ Group.Ocaml Cmi; Group.Ocaml Cmx ] + | Melange Cmi -> [ Group.Melange Cmi ] + | Melange Cmj -> [ Group.Melange Cmi; Group.Melange Cmj ] +;; + +let deps_of_entries ~opaque ~cm_kind libs = + Dep.Set.union_map libs ~f:(fun lib -> + deps_of_lib lib ~groups:(groups_for_cm_kind ~opaque ~cm_kind lib)) +;; + +(* [cm_public_file] gives the cmi path consumers read via their [-I] + include path, which for libraries with a dedicated public cmi dir + ([private_modules]) differs from the internal compilation output. + Using it ensures the dep triggers the produce-public-cmi rule. + + Only called for local unwrapped libraries: [can_filter] rejects + melange cm_kinds, so this function handles ocaml only. *) +let deps_of_entry_modules ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) lib modules = + let obj_dir = Lib.info lib |> Lib_info.obj_dir in + let cmi_kind = Lib_mode.Cm_kind.cmi cm_kind in + let want_cmx = match cm_kind with - | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ] - | Ocaml Cmx -> - if opaque && Lib.is_local lib - then [ Group.Ocaml Cmi ] - else [ Group.Ocaml Cmi; Group.Ocaml Cmx ] - | Melange Cmi -> [ Group.Melange Cmi ] - | Melange Cmj -> [ Group.Melange Cmi; Group.Melange Cmj ] + | Ocaml Cmx -> not (opaque && Lib.is_local lib) + | _ -> false in - Dep.Set.union_map libs ~f:(fun lib -> deps_of_lib lib ~groups:(groups_for lib)) + List.fold_left modules ~init:Dep.Set.empty ~f:(fun acc m -> + let acc = + match Obj_dir.Module.cm_public_file obj_dir m ~kind:cmi_kind with + | Some path -> Dep.Set.add acc (Dep.file path) + | None -> acc + in + if want_cmx && Module.has m ~ml_kind:Impl + then ( + match Obj_dir.Module.cm_public_file obj_dir m ~kind:(Ocaml Cmx) with + | Some path -> Dep.Set.add acc (Dep.file path) + | None -> acc) + else acc) ;; module Lib_index = struct - type t = { by_module_name : Lib.t list Module_name.Map.t } - - let empty = { by_module_name = Module_name.Map.empty } + type t = + { by_module_name : (Lib.t * Module.t option) list Module_name.Map.t + ; tight_eligible : Lib.Set.t + } + + let empty = { by_module_name = Module_name.Map.empty; tight_eligible = Lib.Set.empty } + + (* A library is eligible for per-module tight deps iff it is local + (so every entry has a known [Module.t] with which we can call + [Obj_dir.Module.cm_public_file]) and unwrapped (so the public + cmi dir does not need a glob to reach internal alias modules). *) + let is_lib_tight_eligible lib = + Lib.is_local lib + && + match Lib_info.wrapped (Lib.info lib) with + | Some (This w) -> not (Wrapped.to_bool w) + | Some (From _) | None -> + (* [Some (From _)]: wrapped setting inherited from a virtual + library. The [has_virtual_impl] branch higher up in + [lib_deps_for_module] should handle virtual-impl contexts + before we reach here; stay defensive. + [None]: no wrapped information available (e.g. legacy + [dune-package]). *) + false + ;; let create entries = let by_module_name = - List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib) -> + List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> Module_name.Map.update map name ~f:(function - | None -> Some [ lib ] - | Some libs -> Some (lib :: libs))) + | None -> Some [ lib, m ] + | Some xs -> Some ((lib, m) :: xs))) + in + let tight_eligible = + List.fold_left entries ~init:Lib.Set.empty ~f:(fun acc (_, lib, _) -> + if is_lib_tight_eligible lib then Lib.Set.add acc lib else acc) in - { by_module_name } + { by_module_name; tight_eligible } ;; - (* Sorted and deduplicated so the returned list is a canonical memo key - for [Lib.closure] and so [Lib.closure] is not asked to process the - same library multiple times (it otherwise would be whenever the - consumer references several entry modules of the same library). *) - let filter_libs t ~referenced_modules = - Module_name.Set.fold referenced_modules ~init:[] ~f:(fun name acc -> - match Module_name.Map.find t.by_module_name name with - | None -> acc - | Some libs -> List.rev_append libs acc) - |> List.sort_uniq ~compare:Lib.compare + type classified = + { unwrapped : Module.t list Lib.Map.t + ; wrapped : Lib.t list + } + + let filter_libs_with_modules idx ~referenced_modules = + let add_entry (unwrapped, wrapped) (lib, m_opt) = + match m_opt with + | Some m when Lib.Set.mem idx.tight_eligible lib -> + let unwrapped = + Lib.Map.update unwrapped lib ~f:(function + | None -> Some [ m ] + | Some ms -> Some (m :: ms)) + in + unwrapped, wrapped + | _ -> unwrapped, Lib.Set.add wrapped lib + in + let unwrapped, wrapped = + Module_name.Set.fold + referenced_modules + ~init:(Lib.Map.empty, Lib.Set.empty) + ~f:(fun name acc -> + match Module_name.Map.find idx.by_module_name name with + | None -> acc + | Some entries -> List.fold_left entries ~init:acc ~f:add_entry) + in + { unwrapped; wrapped = Lib.Set.to_list wrapped } ;; end diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index f4057bddf72..b2aa371a337 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -19,17 +19,45 @@ val deps : Lib.t list -> groups:Group.t list -> Dep.Set.t (glob deps on .cmi/.cmx files) for the given libraries. *) val deps_of_entries : opaque:bool -> cm_kind:Lib_mode.Cm_kind.t -> Lib.t list -> Dep.Set.t +(** [deps_of_entry_modules ~opaque ~cm_kind lib modules] computes the + file dependencies on the specific [modules] of [lib], using + [Obj_dir.Module.cm_file_exn] for path construction. Only valid for + local libraries (for which [Module.t] is available). *) +val deps_of_entry_modules + : opaque:bool + -> cm_kind:Lib_mode.Cm_kind.t + -> Lib.t + -> Module.t list + -> Dep.Set.t + module Lib_index : sig type t val empty : t - (** Create an index from a list of (module_name, library) pairs. *) - val create : (Module_name.t * Lib.t) list -> t + (** Create an index. Each entry carries the [Module.t] of the entry + module when it is known ([Some] for local libraries; [None] for + externals). Libraries whose entries all carry a [Module.t] and + which are unwrapped are eligible for per-module deps. *) + val create : (Module_name.t * Lib.t * Module.t option) list -> t + + type classified = + { unwrapped : Module.t list Lib.Map.t + (** Directly-referenced libraries that are local, unwrapped, and + whose referenced entry modules are all known as [Module.t]. + Each is mapped to the list of [Module.t]s the consumer + references. These libraries' shape allows per-module deps via + [deps_of_entry_modules]; whether to use it is the caller's + policy. *) + ; wrapped : Lib.t list + (** Other directly-referenced libraries — wrapped locals, all + externals, or anything else for which we fall back to a glob. + Sorted by [Lib.compare]. *) + } - (** Return the libraries whose module names appear in + (** Classify the libraries whose entry modules appear in [referenced_modules]. *) - val filter_libs : t -> referenced_modules:Module_name.Set.t -> Lib.t list + val filter_libs_with_modules : t -> referenced_modules:Module_name.Set.t -> classified end type path_specification = diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 98db864a9ee..b4261e3058a 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -64,8 +64,15 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in let referenced = Module_name.Set.union all_raw open_modules in - let filtered_libs = - Lib_file_deps.Lib_index.filter_libs lib_index ~referenced_modules:referenced + let { Lib_file_deps.Lib_index.unwrapped; wrapped } = + Lib_file_deps.Lib_index.filter_libs_with_modules + lib_index + ~referenced_modules:referenced + in + (* Sort to preserve the canonical [Lib.closure] memo key (see + commit 9359b37e6 on the base branch). *) + let direct_libs = + List.sort ~compare:Lib.compare (Lib.Map.keys unwrapped @ wrapped) in (* Transitively close the filtered libraries. Transparent module aliases can create cross-library .cmi reads that ocamldep @@ -73,10 +80,23 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin transitive closure required for compilation (across all [implicit_transitive_deps] modes), so [Lib.closure]'s result on a subset of [libs] stays within it. *) - let+ filtered = - Resolve.Memo.read (Lib.closure filtered_libs ~linking:false ~for_) + let+ all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in + (* For directly-referenced unwrapped libs, emit per-module deps + on the specific entry modules the consumer named. Everything + else (wrapped direct libs and libs brought in only through + [Lib.closure]) gets the conservative glob. *) + let tight_deps, glob_libs = + List.fold_left all_libs ~init:(Dep.Set.empty, []) ~f:(fun (td, gl) lib -> + match Lib.Map.find unwrapped lib with + | Some names -> + ( Dep.Set.union + td + (Lib_file_deps.deps_of_entry_modules ~opaque ~cm_kind lib names) + , gl ) + | None -> td, lib :: gl) in - (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind filtered) + let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in + (), Dep.Set.union tight_deps glob_deps) ;; (* Arguments for the compiler to prevent it from being too clever. diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t index d807651e05d..b1e9a77851d 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-to-lib-unwrapped.t @@ -1,8 +1,8 @@ -Baseline: library-to-library recompilation (unwrapped). +Per-module library-to-library filtering (unwrapped). When an unwrapped library A depends on an unwrapped library B with multiple -modules, and one module in B changes, all modules in A are recompiled due to -coarse dependency analysis. +modules, modules of A that do not reference a given module of B must not be +recompiled when that module of B changes. See: https://github.com/ocaml/dune/issues/4572 @@ -80,12 +80,11 @@ Change only alpha.mli: > let new_alpha_fn () = "alpha" > EOF -uses_beta is recompiled because unwrapped libraries use glob deps (per-module -filtering within unwrapped libraries is not yet supported): +uses_beta references Beta only, not Alpha, so it is not recompiled: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_beta"))] | length' - 2 + 0 Change only beta.mli: @@ -98,8 +97,8 @@ Change only beta.mli: > let new_beta_fn () = "beta" > EOF -uses_alpha is recompiled because unwrapped libraries use glob deps: +uses_alpha references Alpha only, not Beta, so it is not recompiled: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_alpha"))] | length' - 2 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-lib.t index fde4d0693a8..c608c55a858 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-lib.t @@ -1,11 +1,10 @@ -Single-module library consumers and recompilation behavior. +Per-module filtering for single-module library consumers. -When a consumer library has only one module, dune skips ocamldep for that -stanza as an optimization (no intra-stanza deps to compute). This means -per-module library dependency filtering cannot determine which libraries -the module actually references, so the consumer depends on all library -files via glob. Modifying an unused module in a dependency triggers -unnecessary recompilation of the consumer module's .cmo/.cmx. +A consumer library with a single module still runs ocamldep when it has +library dependencies, and the per-module filter uses that output to +dep on only the specific entry modules of an unwrapped dependency that +the consumer actually references. Modifying an unreferenced module of +the dependency does not recompile the consumer. See: https://github.com/ocaml/dune/issues/4572 @@ -61,25 +60,8 @@ Modify only the unused module: > let new_fn () = "new" > EOF -uses_alpha.cmx is recompiled even though uses_alpha.ml only references -Alpha, not Unused. This is because the consumer_lib stanza has a single -module, so dune skips ocamldep for it and falls back to glob deps on all -of base_lib's .cmi files. The trace shows compilation targets for -uses_alpha being rebuilt: +uses_alpha.ml references Alpha only, not Unused, so it is not recompiled: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("uses_alpha"))]' - [ - { - "target_files": [ - "_build/default/consumer_lib/.consumer_lib.objs/byte/uses_alpha.cmi", - "_build/default/consumer_lib/.consumer_lib.objs/byte/uses_alpha.cmti" - ] - }, - { - "target_files": [ - "_build/default/consumer_lib/.consumer_lib.objs/native/uses_alpha.cmx", - "_build/default/consumer_lib/.consumer_lib.objs/native/uses_alpha.o" - ] - } - ] + [] diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t index ec2d7903b95..02a4226c5cb 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -1,18 +1,15 @@ -Baseline: consumer-module rebuild count when individual modules of -an unwrapped dependency library change. +Per-module tight deps within an unwrapped library. -This is an observational test. It records the number of rebuild -targets for a single consumer module C when each of three entry -modules (A1, A2, A3) of the dependency library [base] has its -interface edited. C references only A2. +When a consumer module references a specific module of an unwrapped +dependency library, dune emits dynamic dependencies on exactly the +referenced module's .cmi/.cmx files — not a directory-wide glob over +every module of the library. The consumer's rule only invalidates +when a module it actually reads changes. -On current main, editing any one of A1/A2/A3 causes C to rebuild -because library-level dependency filtering (and, within that, -per-module tightening) is not yet in place: the consumer is -conservatively rebuilt whenever any entry module's cmi changes. -Work on https://github.com/ocaml/dune/issues/4572 is expected to -tighten this, at which point editing A1 or A3 leaves C untouched -and the emitted counts are promoted. +This test exercises the tightening with a consumer module [C] that +references only [A2] of three entry modules in the dependency [base]. +Editing [A1] or [A3] must leave [C] untouched; editing [A2] must +rebuild [C]. See: https://github.com/ocaml/dune/issues/4572 @@ -48,11 +45,9 @@ explicit interface so signature changes propagate through .cmi files: consumer has two modules. The module of interest, [c.ml], references only [A2] from [base]. A second, unused module [d.ml] is present only -to keep [consumer] a multi-module stanza: dune skips ocamldep for -single-module stanzas with no library deps as an optimisation, and -the per-module-lib-deps filter depends on ocamldep output. Including -[d.ml] isolates this test from the skip-ocamldep optimisation so the -rebuild count for [c] reflects only the per-module filter's work: +to keep [consumer] a multi-module stanza so ocamldep is never short- +circuited and the rebuild count for [c] reflects the per-module +filter's work independently of any skip-ocamldep heuristic: $ mkdir consumer $ cat > consumer/dune < base/a1.mli < val v : int @@ -80,7 +75,7 @@ the rebuild-target count for C: > EOF $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length' - 1 + 0 Same for A3: @@ -94,10 +89,10 @@ Same for A3: > EOF $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length' - 1 + 0 -Edit A2's interface — the one module C does reference — and check -that the count is positive (C must rebuild): +Changing A2's interface — the one module C does reference — must +rebuild C: $ cat > base/a2.mli < val v : int diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t index 5a5c635338f..ea4aa1b697f 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped.t @@ -1,8 +1,6 @@ -Baseline: library dependency recompilation for unwrapped libraries. - -When an unwrapped library module's interface changes, Dune currently recompiles -all modules in stanzas that depend on the library, even those referencing -different modules in the library. +Per-module filtering for unwrapped libraries: a consumer that references +only some modules of an unwrapped library must not be recompiled when +unreferenced modules in that library change. See: https://github.com/ocaml/dune/issues/4572 @@ -73,12 +71,11 @@ Change only helper.mli: > let new_helper s = s ^ "!" > EOF -Uses_utils is recompiled because unwrapped libraries use glob deps (per-module -filtering within unwrapped libraries is not yet supported): +Uses_utils references Utils only, not Helper, so it is not recompiled: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_utils"))] | length' - 2 + 0 Change only utils.mli: @@ -91,8 +88,8 @@ Change only utils.mli: > let new_utils s = s ^ "?" > EOF -Uses_helper is recompiled because unwrapped libraries use glob deps: +Uses_helper references Helper only, not Utils, so it is not recompiled: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("Uses_helper"))] | length' - 2 + 0 From daeda164f8b803653eb6534e2069135e18bf4685 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Thu, 23 Apr 2026 16:24:01 -0700 Subject: [PATCH 44/80] test: promote oxcaml library-field-parameters error chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under per-module library dep filtering the consumer binary's [.cmx] no longer has a direct dep on the parameterised library's artefacts, so dune's "Missing argument for parameter" diagnostic chain no longer includes [bin/.bin.eobjs/native/dune__exe__Bin.cmx] as an intermediate node. The essential chain — error origin, [bin.exe], aliases — is unchanged. Signed-off-by: Robin Bate Boerop --- test/blackbox-tests/test-cases/oxcaml/library-field-parameters.t | 1 - 1 file changed, 1 deletion(-) diff --git a/test/blackbox-tests/test-cases/oxcaml/library-field-parameters.t b/test/blackbox-tests/test-cases/oxcaml/library-field-parameters.t index 105e0bbd9c0..b0fd83f0a9d 100644 --- a/test/blackbox-tests/test-cases/oxcaml/library-field-parameters.t +++ b/test/blackbox-tests/test-cases/oxcaml/library-field-parameters.t @@ -345,7 +345,6 @@ required parameters. 1 | (executable (name bin) (libraries lib)) ^^^ Error: Missing argument for parameter "project.a". - -> required by _build/default/bin/.bin.eobjs/native/dune__exe__Bin.cmx -> required by _build/default/bin/bin.exe -> required by alias bin/all -> required by alias default From 25858390109129886252042abe195c7040f8c8c8 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 12:22:20 -0700 Subject: [PATCH 45/80] test: document transitive-glob limitation Record the rebuild count for an executable [main] when an unreferenced module of a library reached only through transitive closure is edited. Under the current per-module filter, transitively-reached libraries fall into the glob path in the consumer's rule: [main] depends on a glob over [libA]'s object directory even though [main] references only [ModB] (from [libB]) and [libB] itself references only [ModA2] (from [libA]). Editing [modA1] therefore invalidates the glob and rebuilds [main]. Reported by @nojb: Count observed on this branch: 2. Recovering this case would require propagating each module's tight-dep subset through the library dep graph instead of falling back to glob on transitively-reached libs. Signed-off-by: Robin Bate Boerop --- .../transitive-glob-limitation.t | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t new file mode 100644 index 00000000000..ad2017d02d1 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t @@ -0,0 +1,64 @@ +Known limitation: libraries reached only through `Lib.closure` fall +back to a conservative glob over their object directory. Editing an +unreferenced module in a transitively-reached library therefore still +rebuilds consumers even when they do not use the edited module. + +Scenario: executable `main` references only `ModB` (from `libB`). +`libB` references only `ModA2` (from `libA`). `libA` is in `main`'s +compilation context only via transitive closure through `libB`. The +per-module filter applied at `main`'s rule computes +`direct_libs = [libB]` and `all_libs = Lib.closure [libB] = [libB; libA]`; +`libA` is not in `main`'s direct set, so it falls into the glob path +and `main` depends on a glob over `libA`'s objdir. + +Editing `modA1.mli` (a module `main` does not reference and which +`modB` does not reference) invalidates the glob, and `main` rebuilds. +This test records the current rebuild count. Recovering the case +requires propagating each module's tight-dep subset through the +library dep graph instead of falling back to glob on transitively +reached libraries. + +See: https://github.com/ocaml/dune/pull/14116#issuecomment-4310263512 + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ cat > dune < (library (name libA) (wrapped false) (modules modA1 modA2)) + > (library (name libB) (wrapped false) (modules modB) (libraries libA)) + > (executable (name main) (modules main) (libraries libB)) + > EOF + + $ cat > modA1.ml < let x = 42 + > EOF + $ cat > modA1.mli < val x : int + > EOF + $ cat > modA2.ml < let x = 43 + > EOF + $ cat > modB.ml < let x = ModA2.x + > EOF + $ cat > main.ml < let _ = ModB.x + > EOF + + $ dune build @check + +Edit modA1's interface. Neither `main` nor `modB` references `modA1`. +Record `main`'s rebuild-target count: + + $ cat > modA1.mli < val x : int + > val y : string + > EOF + $ cat > modA1.ml < let x = 42 + > let y = "hi" + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' + 2 From ae86c756d2e2e2891dff07de9275aefa100ddeac Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 13:24:33 -0700 Subject: [PATCH 46/80] docs: clarify rule-dep mechanics in lib_deps_for_module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The returned [Dep.Set.t] is the sole source of cross-library file dependencies for the compile rule — [-I] flags on the ocamlc command line add search paths but do not register rule deps on their own, so narrowing here directly narrows rule invalidation. Spell out the two dep shapes the filter emits: specific File deps via [deps_of_entry_modules], vs directory-level globs via [deps_of_entries]. Non-obvious enough that misunderstanding it led me down several wrong investigation paths. Explicit in the docblock now. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index b4261e3058a..cbf60ece85e 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -11,10 +11,24 @@ let all_libs cctx = (* Per-module inter-library dependency filtering (#4572). Uses ocamldep output to determine which libraries a module actually references, then transitively closes within the compilation context's library set to - handle transparent aliases. Returns [((), Dep.Set.t)] suitable for use - with [Action_builder.dyn_deps]. + handle transparent aliases. Returns [((), Dep.Set.t)] suitable for + use with [Action_builder.dyn_deps]. - Falls back to all libs when filtering is not possible. *) + The returned [Dep.Set.t] is the sole source of cross-library file + dependencies for the compile rule: [-I] flags on the ocamlc command + line add search paths but do not register rule deps on their own. + Narrowing here directly narrows rule invalidation. + + Two dep shapes flow out of the filter: + - [Lib_file_deps.deps_of_entry_modules lib names] → specific File + deps on the named cmis (and their cmx/cmj as appropriate). Only + content changes to those specific cmis invalidate the consumer. + - [Lib_file_deps.deps_of_entries libs] → a glob over each lib's + objdir. Any content change to any cmi in that dir invalidates. + + These deps surface in [dune rules ]'s output alongside any + static deps. Falls back to a glob over all cctx libs when filtering + is not possible. *) let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kind ~mode m = let open Action_builder.O in let can_filter = From 9dbb99ea2c0d88384709594b5eb7ecc126a501e7 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 13:57:34 -0700 Subject: [PATCH 47/80] feat: propagate tight per-module deps across library boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend #4572's per-module filter so libraries reached only through Lib.closure get specific-file deps on the entries their consumer actually uses, rather than a glob over the whole objdir. Lib_index gains a BFS-ready lookup: for a given module name, return the tight-eligible (lib, entry module) pairs that expose it. A new cross_lib_tight_set helper walks ocamldep breadth-first through those entries, starting from the consumer's own ocamldep reads and iterating until fixed point. lib_deps_for_module's fold now consults this extended set via a new tight_subset helper: any lib whose entries intersect the set emits specific-file deps; everything else falls through to the glob path, preserving the conservative shape for wrapped locals, externals, and unreached libs. Lib_index also tracks a no_ocamldep set: local libs whose .d rules are short-circuited by Dep_rules.skip_ocamldep (single-module stanzas without library dependencies). The BFS skips them — their entries can have no cross-library references anyway — so that reaching one of them through tight_set doesn't fault on a missing .d rule. transitive-glob-limitation.t's rebuild count flips from 2 to 0: editing an unreferenced module in a transitively-reached library no longer invalidates consumers. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 27 ++++++-- src/dune_rules/lib_file_deps.ml | 43 +++++++++++- src/dune_rules/lib_file_deps.mli | 26 ++++++- src/dune_rules/module_compilation.ml | 69 ++++++++++++++++--- src/dune_rules/modules.ml | 6 ++ src/dune_rules/modules.mli | 6 ++ ...ion.t => transitive-unreferenced-module.t} | 34 +++++---- 7 files changed, 172 insertions(+), 39 deletions(-) rename test/blackbox-tests/test-cases/per-module-lib-deps/{transitive-glob-limitation.t => transitive-unreferenced-module.t} (52%) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index ca6215a34b4..3f3c4020b30 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -247,11 +247,11 @@ let create let* direct = direct_requires in let* hidden = hidden_requires in let all_libs = direct @ hidden in - let+ entries = - Resolve.Memo.List.concat_map all_libs ~f:(fun lib -> + let+ per_lib = + Resolve.Memo.List.map all_libs ~f:(fun lib -> match Lib_info.entry_modules (Lib.info lib) ~for_ with | External (Ok names) -> - Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None)) + Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None), None) | External (Error e) -> Resolve.Memo.of_result (Error e) | Local -> Resolve.Memo.lift_memo @@ -261,10 +261,25 @@ let create (Lib.Local.of_lib_exn lib) ~for_) ~f:(fun mods -> - List.map (Modules.entry_modules mods) ~f:(fun m -> - Module.name m, lib, Some m)))) + let entries = + List.map (Modules.entry_modules mods) ~f:(fun m -> + Module.name m, lib, Some m) + in + (* Mirror [Dep_rules.skip_ocamldep]: unwrapped + single-file stanzas with no direct lib deps + do not produce [.d] rules, so the cross-lib + BFS must not try to read them. *) + let no_ocamldep_lib = + match Modules.as_singleton mods with + | Some _ when List.is_empty (Lib_info.requires (Lib.info lib) ~for_) + -> Some lib + | _ -> None + in + entries, no_ocamldep_lib))) in - Lib_file_deps.Lib_index.create entries) + let entries = List.concat_map per_lib ~f:fst in + let no_ocamldep = List.filter_map per_lib ~f:snd |> Lib.Set.of_list in + Lib_file_deps.Lib_index.create ~no_ocamldep entries) ; preprocessing ; opaque ; js_of_ocaml diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index cee0e40b009..2e2bb6b672e 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -99,9 +99,20 @@ module Lib_index = struct type t = { by_module_name : (Lib.t * Module.t option) list Module_name.Map.t ; tight_eligible : Lib.Set.t + ; no_ocamldep : Lib.Set.t + (* Local libs whose ocamldep is short-circuited by + [Dep_rules.skip_ocamldep] — single-module stanzas without + library dependencies have no [.d] build rules. BFS must + not try to read ocamldep for them; their entry module + can have no cross-library references anyway. *) } - let empty = { by_module_name = Module_name.Map.empty; tight_eligible = Lib.Set.empty } + let empty = + { by_module_name = Module_name.Map.empty + ; tight_eligible = Lib.Set.empty + ; no_ocamldep = Lib.Set.empty + } + ;; (* A library is eligible for per-module tight deps iff it is local (so every entry has a known [Module.t] with which we can call @@ -122,7 +133,7 @@ module Lib_index = struct false ;; - let create entries = + let create ?(no_ocamldep = Lib.Set.empty) entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> Module_name.Map.update map name ~f:(function @@ -133,7 +144,7 @@ module Lib_index = struct List.fold_left entries ~init:Lib.Set.empty ~f:(fun acc (_, lib, _) -> if is_lib_tight_eligible lib then Lib.Set.add acc lib else acc) in - { by_module_name; tight_eligible } + { by_module_name; tight_eligible; no_ocamldep } ;; type classified = @@ -164,6 +175,32 @@ module Lib_index = struct in { unwrapped; wrapped = Lib.Set.to_list wrapped } ;; + + let lookup_tight_entries idx name = + match Module_name.Map.find idx.by_module_name name with + | None -> [] + | Some entries -> + List.filter_map entries ~f:(fun (lib, m_opt) -> + match m_opt with + | Some m + when Lib.Set.mem idx.tight_eligible lib && not (Lib.Set.mem idx.no_ocamldep lib) + -> Some (lib, m) + | _ -> None) + ;; + + let tight_subset idx lib names = + if not (Lib.Set.mem idx.tight_eligible lib) + then [] + else + Module_name.Set.fold names ~init:[] ~f:(fun name acc -> + match Module_name.Map.find idx.by_module_name name with + | None -> acc + | Some entries -> + List.fold_left entries ~init:acc ~f:(fun acc (l, m_opt) -> + match m_opt with + | Some m when Lib.equal l lib -> m :: acc + | _ -> acc)) + ;; end type path_specification = diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index b2aa371a337..4d5f3149b85 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -38,8 +38,17 @@ module Lib_index : sig (** Create an index. Each entry carries the [Module.t] of the entry module when it is known ([Some] for local libraries; [None] for externals). Libraries whose entries all carry a [Module.t] and - which are unwrapped are eligible for per-module deps. *) - val create : (Module_name.t * Lib.t * Module.t option) list -> t + which are unwrapped are eligible for per-module deps. + + [no_ocamldep] names local libs whose ocamldep output is + short-circuited (single-module stanzas without library + dependencies — see [Dep_rules.skip_ocamldep]). The cross-lib + BFS in [module_compilation] must not try to read [.d] files + for those libs. *) + val create + : ?no_ocamldep:Lib.Set.t + -> (Module_name.t * Lib.t * Module.t option) list + -> t type classified = { unwrapped : Module.t list Lib.Map.t @@ -58,6 +67,19 @@ module Lib_index : sig (** Classify the libraries whose entry modules appear in [referenced_modules]. *) val filter_libs_with_modules : t -> referenced_modules:Module_name.Set.t -> classified + + (** [lookup_tight_entries idx name] returns [(lib, entry module)] + pairs used by the cross-library BFS in [module_compilation]. + Libraries in [no_ocamldep] are excluded (their [.d] files do + not exist), as are externals, wrapped locals, and entries with + no [Module.t]. *) + val lookup_tight_entries : t -> Module_name.t -> (Lib.t * Module.t) list + + (** [tight_subset idx lib names] returns [lib]'s entry modules + whose names appear in [names]. Returns [[]] when [lib] is not + tight-eligible; callers should interpret that as a signal to + fall back to [deps_of_entries] (a glob over the lib's objdir). *) + val tight_subset : t -> Lib.t -> Module_name.Set.t -> Module.t list end type path_specification = diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index cbf60ece85e..2f2d1575679 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -8,6 +8,46 @@ let all_libs cctx = d @ h ;; +(* Extend [initial_refs] with module names reached through cross- + library ocamldep. Walk tight-eligible entry modules breadth-first: + gather all [(lib, entry module)] pairs named by the frontier, read + each pair's impl and intf ocamldep, and union the raw names into + the frontier. Iterate until no new names appear. + + Libraries that are not tight-eligible (wrapped locals, externals, + virtual-impls) are skipped by [lookup_tight_entries]. Chains that + pass through them terminate and the consumer falls back to a glob + on the unreached libs. + + Cycles terminate on the [seen] set. *) +let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = + let open Action_builder.O in + let union_all = List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union in + let read_entry_deps (lib, m) = + let obj_dir = Lib.info lib |> Lib_info.obj_dir |> Obj_dir.as_local_exn in + let* impl_deps = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ m in + let+ intf_deps = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ m in + Module_name.Set.union impl_deps intf_deps + in + let rec loop ~seen ~frontier = + if Module_name.Set.is_empty frontier + then Action_builder.return seen + else ( + let pairs = + Module_name.Set.fold frontier ~init:[] ~f:(fun name acc -> + Lib_file_deps.Lib_index.lookup_tight_entries lib_index name @ acc) + in + let* discovered = + Action_builder.List.map pairs ~f:read_entry_deps + |> Action_builder.map ~f:union_all + in + let seen = Module_name.Set.union seen frontier in + let frontier = Module_name.Set.diff discovered seen in + loop ~seen ~frontier) + in + loop ~seen:Module_name.Set.empty ~frontier:initial_refs +;; + (* Per-module inter-library dependency filtering (#4572). Uses ocamldep output to determine which libraries a module actually references, then transitively closes within the compilation context's library set to @@ -26,6 +66,13 @@ let all_libs cctx = - [Lib_file_deps.deps_of_entries libs] → a glob over each lib's objdir. Any content change to any cmi in that dir invalidates. + The tight set of referenced module names is computed across + library boundaries by [cross_lib_tight_set]: it starts from the + consumer's own ocamldep reads and iterates through tight-eligible + entry modules. This lets closure-reached libraries (not directly + named by the consumer but pulled in through transparent aliases) + receive specific-file deps on just the entries that matter. + These deps surface in [dune rules ]'s output alongside any static deps. Falls back to a glob over all cctx libs when filtering is not possible. *) @@ -94,20 +141,22 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin transitive closure required for compilation (across all [implicit_transitive_deps] modes), so [Lib.closure]'s result on a subset of [libs] stays within it. *) - let+ all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in - (* For directly-referenced unwrapped libs, emit per-module deps - on the specific entry modules the consumer named. Everything - else (wrapped direct libs and libs brought in only through - [Lib.closure]) gets the conservative glob. *) + let* all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in + let+ tight_set = cross_lib_tight_set ~lib_index ~for_ ~initial_refs:referenced in + (* For libs whose entry modules intersect the cross-library + tight set, emit per-module deps on just those entries. + Non-tight-eligible libs (wrapped locals, externals) and + tight-eligible libs with no intersecting entries fall into + the glob path, matching the pre-BFS conservative shape. *) let tight_deps, glob_libs = List.fold_left all_libs ~init:(Dep.Set.empty, []) ~f:(fun (td, gl) lib -> - match Lib.Map.find unwrapped lib with - | Some names -> + match Lib_file_deps.Lib_index.tight_subset lib_index lib tight_set with + | [] -> td, lib :: gl + | modules -> ( Dep.Set.union td - (Lib_file_deps.deps_of_entry_modules ~opaque ~cm_kind lib names) - , gl ) - | None -> td, lib :: gl) + (Lib_file_deps.deps_of_entry_modules ~opaque ~cm_kind lib modules) + , gl )) in let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in (), Dep.Set.union tight_deps glob_deps) diff --git a/src/dune_rules/modules.ml b/src/dune_rules/modules.ml index 3a6de1d9878..6d6ad4fd25d 100644 --- a/src/dune_rules/modules.ml +++ b/src/dune_rules/modules.ml @@ -924,6 +924,12 @@ let wrapped t = | Stdlib _ -> Simple true ;; +let as_singleton t = + match t.modules with + | Singleton m -> Some m + | Unwrapped _ | Wrapped _ | Stdlib _ -> None +;; + let is_user_written m = match Module.kind m with | Root | Wrapped_compat | Alias _ -> false diff --git a/src/dune_rules/modules.mli b/src/dune_rules/modules.mli index 57ff2ed5ca2..0518a302543 100644 --- a/src/dune_rules/modules.mli +++ b/src/dune_rules/modules.mli @@ -57,6 +57,12 @@ val obj_map : t -> Sourced_module.t Module_name.Unique.Map.t val virtual_module_names : t -> Module_name.Path.Set.t val wrapped : t -> Wrapped.t + +(** [Some m] iff the library consists of the single source module [m] + (unwrapped stanza with exactly one module). Mirrors + [With_vlib.as_singleton]. *) +val as_singleton : t -> Module.t option + val source_dirs : t -> Path.Set.t val compat_for_exn : t -> Module.t -> Module.t diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-module.t similarity index 52% rename from test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t rename to test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-module.t index ad2017d02d1..69f705f61ae 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-glob-limitation.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-module.t @@ -1,22 +1,20 @@ -Known limitation: libraries reached only through `Lib.closure` fall -back to a conservative glob over their object directory. Editing an -unreferenced module in a transitively-reached library therefore still -rebuilds consumers even when they do not use the edited module. +Per-module deps propagate across library boundaries: a module in +a transitively-reached library is a dep only if some referenced +entry module's ocamldep output names it. Editing an unreferenced +module in such a library does not invalidate consumers. Scenario: executable `main` references only `ModB` (from `libB`). `libB` references only `ModA2` (from `libA`). `libA` is in `main`'s -compilation context only via transitive closure through `libB`. The -per-module filter applied at `main`'s rule computes -`direct_libs = [libB]` and `all_libs = Lib.closure [libB] = [libB; libA]`; -`libA` is not in `main`'s direct set, so it falls into the glob path -and `main` depends on a glob over `libA`'s objdir. +compilation context only via transitive closure through `libB`. +Starting from `main`'s own ocamldep reads (`{ModB}`), the cross-lib +walk reads `libB/modB`'s ocamldep (names `ModA2`) and then +`libA/modA2`'s ocamldep (no new names). The tight set is +`{ModB, ModA2}`; `modA1` never enters it, so `main` depends only on +`libB/modB.cmi` and `libA/modA2.cmi`, not a glob over `libA`'s +objdir. -Editing `modA1.mli` (a module `main` does not reference and which -`modB` does not reference) invalidates the glob, and `main` rebuilds. -This test records the current rebuild count. Recovering the case -requires propagating each module's tight-dep subset through the -library dep graph instead of falling back to glob on transitively -reached libraries. +Editing `modA1.mli` (which neither `main` nor `modB` references) +leaves `main`'s deps unchanged, so `main` is not rebuilt. See: https://github.com/ocaml/dune/pull/14116#issuecomment-4310263512 @@ -48,8 +46,8 @@ See: https://github.com/ocaml/dune/pull/14116#issuecomment-4310263512 $ dune build @check -Edit modA1's interface. Neither `main` nor `modB` references `modA1`. -Record `main`'s rebuild-target count: +Edit modA1's interface. Neither `main` nor `modB` references `modA1`, +so `main` should not rebuild: $ cat > modA1.mli < val x : int @@ -61,4 +59,4 @@ Record `main`'s rebuild-target count: > EOF $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' - 2 + 0 From 7ab3fa9c593a3ec497124dc78c7c3579a75d158a Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 14:51:41 -0700 Subject: [PATCH 48/80] refactor: tighten helper naming and extract shared patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No behavior change. - Extract [build_lib_index] from the [Compilation_context.create] record initializer. The 20-line inline construction obscured the record and was hard to diff. - Rename [classified.{unwrapped, wrapped}] → [classified.{tight, non_tight}]. The old names implied whether a lib was syntactically wrapped; the real partition is tight-eligible (local, unwrapped, module-known) vs everything else (wrapped locals, externals, entries without [Module.t]). - Extract [union_set_map] from [lib_deps_for_module] and [cross_lib_tight_set] — the same Action_builder map-then-union shape appeared in both. - Split the [can_filter] module-kind predicate into two named helpers: [module_kind_is_filterable] (is this a candidate for the filter?) and [module_kind_has_readable_ocamldep] (does this module's ocamldep carry user references we want to walk?). The two predicates disagree on [Virtual] and [Alias _], which the old inline pattern-matches did but made hard to see. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 88 ++++++++++++++------------- src/dune_rules/lib_file_deps.ml | 18 +++--- src/dune_rules/lib_file_deps.mli | 21 +++---- src/dune_rules/module_compilation.ml | 54 +++++++++------- 4 files changed, 99 insertions(+), 82 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 3f3c4020b30..7bdc548bc32 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -123,6 +123,52 @@ let parameters_main_modules parameters = [ "param", Lib.to_dyn param ]) ;; +(* Build a [Lib_index] from [all_libs] for the per-module inter-library + dependency filter. + + For each library, entry module names are collected (for wrapped + libraries, the wrapper; for unwrapped, each public module). Hidden + libraries are included so that consumers which transitively + reference a hidden library's entry module still produce a build + dependency on it. + + Local libraries whose ocamldep is short-circuited by + [Dep_rules.skip_ocamldep] (unwrapped single-module stanzas without + direct lib deps) are collected into [no_ocamldep] so the cross-lib + BFS does not try to read their nonexistent [.d] files. *) +let build_lib_index ~super_context ~all_libs ~for_ = + let open Resolve.Memo.O in + let+ per_lib = + Resolve.Memo.List.map all_libs ~f:(fun lib -> + match Lib_info.entry_modules (Lib.info lib) ~for_ with + | External (Ok names) -> + Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None), None) + | External (Error e) -> Resolve.Memo.of_result (Error e) + | Local -> + Resolve.Memo.lift_memo + (Memo.map + (Dir_contents.modules_of_local_lib + super_context + (Lib.Local.of_lib_exn lib) + ~for_) + ~f:(fun mods -> + let entries = + List.map (Modules.entry_modules mods) ~f:(fun m -> + Module.name m, lib, Some m) + in + let no_ocamldep_lib = + match Modules.as_singleton mods with + | Some _ when List.is_empty (Lib_info.requires (Lib.info lib) ~for_) -> + Some lib + | _ -> None + in + entries, no_ocamldep_lib))) + in + let entries = List.concat_map per_lib ~f:fst in + let no_ocamldep = List.filter_map per_lib ~f:snd |> Lib.Set.of_list in + Lib_file_deps.Lib_index.create ~no_ocamldep entries +;; + let create ~super_context ~scope @@ -236,50 +282,10 @@ let create ; parameters ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config ; lib_index = - (* Maps entry module names to libraries for per-module inter-library - dependency filtering. For wrapped libraries, the entry module is the - wrapper; for unwrapped, it is each public module. Includes hidden - libraries (present under [implicit_transitive_deps = - disabled_with_hidden_includes]) so that consumers which transitively - reference a hidden library's entry module still produce a build - dependency on it. *) (let open Resolve.Memo.O in let* direct = direct_requires in let* hidden = hidden_requires in - let all_libs = direct @ hidden in - let+ per_lib = - Resolve.Memo.List.map all_libs ~f:(fun lib -> - match Lib_info.entry_modules (Lib.info lib) ~for_ with - | External (Ok names) -> - Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None), None) - | External (Error e) -> Resolve.Memo.of_result (Error e) - | Local -> - Resolve.Memo.lift_memo - (Memo.map - (Dir_contents.modules_of_local_lib - super_context - (Lib.Local.of_lib_exn lib) - ~for_) - ~f:(fun mods -> - let entries = - List.map (Modules.entry_modules mods) ~f:(fun m -> - Module.name m, lib, Some m) - in - (* Mirror [Dep_rules.skip_ocamldep]: unwrapped - single-file stanzas with no direct lib deps - do not produce [.d] rules, so the cross-lib - BFS must not try to read them. *) - let no_ocamldep_lib = - match Modules.as_singleton mods with - | Some _ when List.is_empty (Lib_info.requires (Lib.info lib) ~for_) - -> Some lib - | _ -> None - in - entries, no_ocamldep_lib))) - in - let entries = List.concat_map per_lib ~f:fst in - let no_ocamldep = List.filter_map per_lib ~f:snd |> Lib.Set.of_list in - Lib_file_deps.Lib_index.create ~no_ocamldep entries) + build_lib_index ~super_context ~all_libs:(direct @ hidden) ~for_) ; preprocessing ; opaque ; js_of_ocaml diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 2e2bb6b672e..8bbc5626ab7 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -148,23 +148,23 @@ module Lib_index = struct ;; type classified = - { unwrapped : Module.t list Lib.Map.t - ; wrapped : Lib.t list + { tight : Module.t list Lib.Map.t + ; non_tight : Lib.t list } let filter_libs_with_modules idx ~referenced_modules = - let add_entry (unwrapped, wrapped) (lib, m_opt) = + let add_entry (tight, non_tight) (lib, m_opt) = match m_opt with | Some m when Lib.Set.mem idx.tight_eligible lib -> - let unwrapped = - Lib.Map.update unwrapped lib ~f:(function + let tight = + Lib.Map.update tight lib ~f:(function | None -> Some [ m ] | Some ms -> Some (m :: ms)) in - unwrapped, wrapped - | _ -> unwrapped, Lib.Set.add wrapped lib + tight, non_tight + | _ -> tight, Lib.Set.add non_tight lib in - let unwrapped, wrapped = + let tight, non_tight = Module_name.Set.fold referenced_modules ~init:(Lib.Map.empty, Lib.Set.empty) @@ -173,7 +173,7 @@ module Lib_index = struct | None -> acc | Some entries -> List.fold_left entries ~init:acc ~f:add_entry) in - { unwrapped; wrapped = Lib.Set.to_list wrapped } + { tight; non_tight = Lib.Set.to_list non_tight } ;; let lookup_tight_entries idx name = diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 4d5f3149b85..a43ecb1b514 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -51,17 +51,16 @@ module Lib_index : sig -> t type classified = - { unwrapped : Module.t list Lib.Map.t - (** Directly-referenced libraries that are local, unwrapped, and - whose referenced entry modules are all known as [Module.t]. - Each is mapped to the list of [Module.t]s the consumer - references. These libraries' shape allows per-module deps via - [deps_of_entry_modules]; whether to use it is the caller's - policy. *) - ; wrapped : Lib.t list - (** Other directly-referenced libraries — wrapped locals, all - externals, or anything else for which we fall back to a glob. - Sorted by [Lib.compare]. *) + { tight : Module.t list Lib.Map.t + (** Directly-referenced tight-eligible libraries (local, + unwrapped, each entry with a known [Module.t]). Mapped to the + referenced entry modules. These libraries are candidates for + per-module deps via [deps_of_entry_modules]; whether to emit + them is the caller's policy. *) + ; non_tight : Lib.t list + (** Other directly-referenced libraries — wrapped locals, + externals, or anything else that falls back to a glob. Sorted + by [Lib.compare]. *) } (** Classify the libraries whose entry modules appear in diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 2f2d1575679..fe2b37eaddd 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -8,6 +8,33 @@ let all_libs cctx = d @ h ;; +(* Map each element of [xs] through [f], returning the union of all the + resulting [Module_name.Set.t]s. *) +let union_set_map xs ~f = + Action_builder.List.map xs ~f + |> Action_builder.map + ~f:(List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union) +;; + +(* A module is filterable (per-module lib deps apply) only when it has a + proper source-level kind: user [Impl]/[Intf_only] or a transparent + alias whose ocamldep we can read to follow the chain. *) +let module_kind_is_filterable m = + match Module.kind m with + | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false + | Intf_only | Impl | Alias _ -> true +;; + +(* A module has user-authored ocamldep output we can read as part of + the filter's reference walk. [Alias _] modules are synthetic stubs + with no useful ocamldep; [Virtual] modules' interfaces do carry + references we want to follow. *) +let module_kind_has_readable_ocamldep m = + match Module.kind m with + | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false + | Virtual | Intf_only | Impl -> true +;; + (* Extend [initial_refs] with module names reached through cross- library ocamldep. Walk tight-eligible entry modules breadth-first: gather all [(lib, entry module)] pairs named by the frontier, read @@ -22,7 +49,6 @@ let all_libs cctx = Cycles terminate on the [seen] set. *) let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = let open Action_builder.O in - let union_all = List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union in let read_entry_deps (lib, m) = let obj_dir = Lib.info lib |> Lib_info.obj_dir |> Obj_dir.as_local_exn in let* impl_deps = Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ m in @@ -37,10 +63,7 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = Module_name.Set.fold frontier ~init:[] ~f:(fun name acc -> Lib_file_deps.Lib_index.lookup_tight_entries lib_index name @ acc) in - let* discovered = - Action_builder.List.map pairs ~f:read_entry_deps - |> Action_builder.map ~f:union_all - in + let* discovered = union_set_map pairs ~f:read_entry_deps in let seen = Module_name.Set.union seen frontier in let frontier = Module_name.Set.diff discovered seen in loop ~seen ~frontier) @@ -84,9 +107,7 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin | Ocaml _ -> true) && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) && Dep_graph.mem dep_graph m - && (match Module.kind m with - | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false - | Intf_only | Impl | Alias _ -> true) + && module_kind_is_filterable m && Module.has m ~ml_kind && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) in @@ -103,13 +124,8 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in let* trans_deps = Dep_graph.deps_of dep_graph m in let* all_raw = - Action_builder.List.map (m :: trans_deps) ~f:(fun dep_m -> - let is_standard_kind = - match Module.kind dep_m with - | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false - | Virtual | Intf_only | Impl -> true - in - if not is_standard_kind + union_set_map (m :: trans_deps) ~f:(fun dep_m -> + if not (module_kind_has_readable_ocamldep dep_m) then Action_builder.return Module_name.Set.empty else let* impl_deps = @@ -119,22 +135,18 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m in Module_name.Set.union impl_deps intf_deps) - |> Action_builder.map - ~f:(List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union) in let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in let referenced = Module_name.Set.union all_raw open_modules in - let { Lib_file_deps.Lib_index.unwrapped; wrapped } = + let { Lib_file_deps.Lib_index.tight; non_tight } = Lib_file_deps.Lib_index.filter_libs_with_modules lib_index ~referenced_modules:referenced in (* Sort to preserve the canonical [Lib.closure] memo key (see commit 9359b37e6 on the base branch). *) - let direct_libs = - List.sort ~compare:Lib.compare (Lib.Map.keys unwrapped @ wrapped) - in + let direct_libs = List.sort ~compare:Lib.compare (Lib.Map.keys tight @ non_tight) in (* Transitively close the filtered libraries. Transparent module aliases can create cross-library .cmi reads that ocamldep doesn't report, at arbitrary depth. [libs] is already the From 54b5ea4863a4af0073ae136dc9acbdc762d62889 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 15:06:37 -0700 Subject: [PATCH 49/80] fix: read cross-library ocamldep output without basename check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the cross-lib BFS in [lib_deps_for_module] reads a [.d] file for a module in another library, the [Module.t] it holds (from [Lib_index]) is the raw, pre-[Module.pped] version. The producing rule, however, ran ocamldep on the pp-transformed source for libraries that use ppx preprocessing — so the output's left-hand basename is [foo.pp.ml] while our [Module.File.path source] has [foo.ml]. [parse_deps_exn]'s basename check then fires: ocamldep returned unexpected output for .../uTop_main.ml: > src/lib/uTop_main.pp.ml: Arg Array Ast_helper ... The check is a self-consistency assertion useful at rule-production time (where the rule's source path is what ocamldep was fed) but not for cross-lib readers. Split [parse_deps_exn] into the strict form (kept for [deps_of]'s rule action) and [parse_deps_lenient] (no basename check, used by [read_immediate_deps_parsed]). Pre-PR callers of the reader always held the pp-transformed [Module.t], so lenient parsing is a no-op for them; the cross-lib BFS now succeeds through ppx'd libraries. Surfaced by CI: dev-tool build of utop 2.17.0. Signed-off-by: Robin Bate Boerop --- src/dune_rules/ocamldep.ml | 67 ++++++++++++------- .../per-module-lib-deps/cross-lib-ppx.t | 67 +++++++++++++++++++ 2 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t diff --git a/src/dune_rules/ocamldep.ml b/src/dune_rules/ocamldep.ml index f8361aa98d5..6cfb296ff25 100644 --- a/src/dune_rules/ocamldep.ml +++ b/src/dune_rules/ocamldep.ml @@ -83,27 +83,42 @@ let parse_compilation_units ~modules = |> Option.map ~f:Modules.Sourced_module.to_module) ;; -let parse_deps_exn = - let invalid file lines = - User_error.raise - [ Pp.textf - "ocamldep returned unexpected output for %s:" - (Path.to_string_maybe_quoted file) - ; Pp.vbox - (Pp.concat_map lines ~sep:Pp.cut ~f:(fun line -> - Pp.seq (Pp.verbatim "> ") (Pp.verbatim line))) - ] - in - fun ~file lines -> - match lines with - | [] | _ :: _ :: _ -> invalid file lines - | [ line ] -> - (match String.lsplit2 line ~on:':' with - | None -> invalid file lines - | Some (basename, deps) -> - let basename = Filename.basename basename in - if basename <> Path.basename file then invalid file lines; - String.extract_blank_separated_words deps) +let invalid_ocamldep_output file lines = + User_error.raise + [ Pp.textf + "ocamldep returned unexpected output for %s:" + (Path.to_string_maybe_quoted file) + ; Pp.vbox + (Pp.concat_map lines ~sep:Pp.cut ~f:(fun line -> + Pp.seq (Pp.verbatim "> ") (Pp.verbatim line))) + ] +;; + +let parse_deps_exn ~file lines = + match lines with + | [] | _ :: _ :: _ -> invalid_ocamldep_output file lines + | [ line ] -> + (match String.lsplit2 line ~on:':' with + | None -> invalid_ocamldep_output file lines + | Some (basename, deps) -> + let basename = Filename.basename basename in + if basename <> Path.basename file then invalid_ocamldep_output file lines; + String.extract_blank_separated_words deps) +;; + +(* Like [parse_deps_exn] but without the left-hand basename check. + Callers that only read [.d] files (rather than producing them) + may hold a raw [Module.t] whose [Module.source] path differs + from the pp-transformed source the producing rule fed to + ocamldep — the basename will not match even though the output + is valid. *) +let parse_deps_lenient ~file lines = + match lines with + | [] | _ :: _ :: _ -> invalid_ocamldep_output file lines + | [ line ] -> + (match String.lsplit2 line ~on:':' with + | None -> invalid_ocamldep_output file lines + | Some (_, deps) -> String.extract_blank_separated_words deps) ;; let transitive_deps = @@ -190,7 +205,13 @@ let read_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = builder for each .d file is cached by path so that [read_immediate_deps_of] and [read_immediate_deps_raw_of] (which may be called many times for the same module) share one memoized - [Action_builder.t] instance per file. *) + [Action_builder.t] instance per file. + + Uses [parse_deps_lenient] rather than [parse_deps_exn]: the + producing rule already validates the basename in [deps_of]; by + the time a reader sees the [.d] file it is known-valid, and + cross-lib readers may hold a raw [Module.t] whose source path + does not match the pp-transformed source the rule used. *) let read_immediate_deps_parsed = let cache = Table.create (module Path.Build) 64 in fun ~obj_dir ~ml_kind ~for_ unit -> @@ -206,7 +227,7 @@ let read_immediate_deps_parsed = let builder = Action_builder.lines_of (Path.build ocamldep_output) |> Action_builder.map ~f:(fun lines -> - Some (parse_deps_exn ~file:(Module.File.path source) lines)) + Some (parse_deps_lenient ~file:(Module.File.path source) lines)) |> Action_builder.memoize (Path.Build.to_string ocamldep_output) in Table.set cache ocamldep_output builder; diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t b/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t new file mode 100644 index 00000000000..51f6e6a9e5f --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t @@ -0,0 +1,67 @@ +Cross-library reads of ocamldep output must tolerate ppx-transformed +sources. When the target dependency library uses a preprocessor, its +[.d] file's left-hand basename is [foo.pp.ml] — the pp-transformed +source fed to ocamldep by the rule. The per-module filter's BFS in +the consumer holds a raw [Module.t] whose source path is [foo.ml], +so a strict basename check in the [.d] parser would mismatch and +raise "ocamldep returned unexpected output". The reader must parse +leniently; the producing rule already validates the basename. + +See: https://github.com/ocaml/dune/pull/14116 + + $ cat > dune-project < (lang dune 3.0) + > EOF + +A trivial preprocessor that passes its input through unchanged. The +critical property is that dune's pp pipeline writes a [.pp.ml] file +and feeds it to ocamldep, not the original [.ml]. + + $ mkdir pp + $ cat > pp/dune < (executable (name pp)) + > EOF + $ cat > pp/pp.ml < let () = + > let ic = open_in_bin Sys.argv.(1) in + > try + > while true do print_char (input_char ic) done + > with End_of_file -> () + > EOF + +[dep] is a tight-eligible library (local, unwrapped, multi-module so +ocamldep is not short-circuited) whose modules are preprocessed: + + $ mkdir dep + $ cat > dep/dune < (library + > (name dep) + > (wrapped false) + > (preprocess (action (run %{exe:../pp/pp.exe} %{input-file})))) + > EOF + $ cat > dep/mod_a.ml < let v = 1 + > EOF + $ cat > dep/mod_b.ml < let v = 2 + > EOF + +[consumer] references [Mod_a] from [dep]. Compiling [c.ml] triggers +the cross-library BFS to read [dep/mod_a.impl.d] — whose producer +ran ocamldep on [mod_a.pp.ml]. The BFS must read this [.d] with a +raw [Module.t] for [mod_a]: + + $ mkdir consumer + $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries dep)) + > EOF + $ cat > consumer/c.ml < let w = Mod_a.v + > EOF + $ cat > consumer/d.ml < let _ = () + > EOF + +Build must succeed: + + $ dune build @check From 6f35afe1df101926dc0d13e39b5c7190417e29ac Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 15:18:42 -0700 Subject: [PATCH 50/80] refactor: drop [tight_subset], reuse [filter_libs_with_modules] [Lib_index.tight_subset] and [filter_libs_with_modules] duplicated the same lookup: for a set of referenced names, find which tight- eligible libs expose them and which entry modules match. The fold in [lib_deps_for_module] called [tight_subset] once per lib in [all_libs]; a single call to [filter_libs_with_modules] on the cross-library tight set builds the whole [Lib.Map.t] up front, and the fold becomes a simple [Lib.Map.find] per lib. No behavior change. Also cite nojb's case-1 comment in [unwrapped-tight-deps.t]: the scenario (consumer lib C using one entry [A2] of dep lib D, edits to an unreferenced [A1] in D must not rebuild C) matches . Signed-off-by: Robin Bate Boerop --- src/dune_rules/lib_file_deps.ml | 14 ----------- src/dune_rules/lib_file_deps.mli | 6 ----- src/dune_rules/module_compilation.ml | 23 +++++++++++-------- .../unwrapped-tight-deps.t | 1 + 4 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 8bbc5626ab7..f7bcc0213df 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -187,20 +187,6 @@ module Lib_index = struct -> Some (lib, m) | _ -> None) ;; - - let tight_subset idx lib names = - if not (Lib.Set.mem idx.tight_eligible lib) - then [] - else - Module_name.Set.fold names ~init:[] ~f:(fun name acc -> - match Module_name.Map.find idx.by_module_name name with - | None -> acc - | Some entries -> - List.fold_left entries ~init:acc ~f:(fun acc (l, m_opt) -> - match m_opt with - | Some m when Lib.equal l lib -> m :: acc - | _ -> acc)) - ;; end type path_specification = diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index a43ecb1b514..14639464e47 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -73,12 +73,6 @@ module Lib_index : sig not exist), as are externals, wrapped locals, and entries with no [Module.t]. *) val lookup_tight_entries : t -> Module_name.t -> (Lib.t * Module.t) list - - (** [tight_subset idx lib names] returns [lib]'s entry modules - whose names appear in [names]. Returns [[]] when [lib] is not - tight-eligible; callers should interpret that as a signal to - fall back to [deps_of_entries] (a glob over the lib's objdir). *) - val tight_subset : t -> Lib.t -> Module_name.Set.t -> Module.t list end type path_specification = diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index fe2b37eaddd..9ffd63bad37 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -155,20 +155,25 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin on a subset of [libs] stays within it. *) let* all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in let+ tight_set = cross_lib_tight_set ~lib_index ~for_ ~initial_refs:referenced in - (* For libs whose entry modules intersect the cross-library - tight set, emit per-module deps on just those entries. - Non-tight-eligible libs (wrapped locals, externals) and - tight-eligible libs with no intersecting entries fall into - the glob path, matching the pre-BFS conservative shape. *) + (* Classify [all_libs] against the cross-library tight set: libs + whose entries appear in [tight_set] get per-module deps on + just those entries; the rest (non-tight-eligible libs, + tight-eligible libs with no intersecting entries) fall to a + glob over their objdir. *) + let { Lib_file_deps.Lib_index.tight = tight_modules; non_tight = _ } = + Lib_file_deps.Lib_index.filter_libs_with_modules + lib_index + ~referenced_modules:tight_set + in let tight_deps, glob_libs = List.fold_left all_libs ~init:(Dep.Set.empty, []) ~f:(fun (td, gl) lib -> - match Lib_file_deps.Lib_index.tight_subset lib_index lib tight_set with - | [] -> td, lib :: gl - | modules -> + match Lib.Map.find tight_modules lib with + | Some modules -> ( Dep.Set.union td (Lib_file_deps.deps_of_entry_modules ~opaque ~cm_kind lib modules) - , gl )) + , gl ) + | None -> td, lib :: gl) in let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in (), Dep.Set.union tight_deps glob_deps) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t index 02a4226c5cb..571ac8a8543 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -12,6 +12,7 @@ Editing [A1] or [A3] must leave [C] untouched; editing [A2] must rebuild [C]. See: https://github.com/ocaml/dune/issues/4572 +See: https://github.com/ocaml/dune/pull/14116#issuecomment-4301275263 $ cat > dune-project < (lang dune 3.0) From 84343cf2a2aa493fc7133ddee20c015f3d8243af Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 15:38:25 -0700 Subject: [PATCH 51/80] refactor: memoize has_virtual_impl per compilation context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [has_virtual_impl] is a property of the compilation context's library set, not of the module being compiled. The previous inline computation in [lib_deps_for_module] redid an [O(libs)] scan on every compile rule — per module, per ml_kind. Move it to [Compilation_context.create]: one [Resolve.Memo.t] per cctx, memoized by Memo's graph. No behavior change. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 7 +++++++ src/dune_rules/compilation_context.mli | 6 ++++++ src/dune_rules/module_compilation.ml | 8 ++++---- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 7bdc548bc32..a06073d51d6 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -66,6 +66,7 @@ type t = ; instances : Parameterised_instances.t Resolve.Memo.t option ; includes : Includes.t ; lib_index : Lib_file_deps.Lib_index.t Resolve.Memo.t + ; has_virtual_impl : bool Resolve.Memo.t ; preprocessing : Pp_spec.t ; opaque : bool ; js_of_ocaml : Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t @@ -94,6 +95,7 @@ let requires_link t = Memo.Lazy.force t.requires_link let parameters t = t.parameters let includes t = t.includes let lib_index t = t.lib_index +let has_virtual_impl t = t.has_virtual_impl let preprocessing t = t.preprocessing let opaque t = t.opaque let js_of_ocaml t = t.js_of_ocaml @@ -286,6 +288,11 @@ let create let* direct = direct_requires in let* hidden = hidden_requires in build_lib_index ~super_context ~all_libs:(direct @ hidden) ~for_) + ; has_virtual_impl = + (let open Resolve.Memo.O in + let+ direct = direct_requires + and+ hidden = hidden_requires in + List.exists (direct @ hidden) ~f:(fun lib -> Option.is_some (Lib.implements lib))) ; preprocessing ; opaque ; js_of_ocaml diff --git a/src/dune_rules/compilation_context.mli b/src/dune_rules/compilation_context.mli index 2cc6cebeb50..a5a4e54a097 100644 --- a/src/dune_rules/compilation_context.mli +++ b/src/dune_rules/compilation_context.mli @@ -63,6 +63,12 @@ val requires_compile : t -> Lib.t list Resolve.Memo.t val parameters : t -> Module_name.t list Resolve.Memo.t val includes : t -> Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t val lib_index : t -> Lib_file_deps.Lib_index.t Resolve.Memo.t + +(** [true] iff any library in the compilation context's direct or + hidden requires implements a virtual library. Memoized per + cctx; callers avoid re-scanning the lib list on each module. *) +val has_virtual_impl : t -> bool Resolve.Memo.t + val preprocessing : t -> Pp_spec.t val opaque : t -> bool val js_of_ocaml : t -> Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 9ffd63bad37..fddb51464c8 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -114,9 +114,9 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let* libs = Resolve.Memo.read (all_libs cctx) in if not can_filter then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) - else ( - let has_virtual_impl = - List.exists libs ~f:(fun lib -> Option.is_some (Lib.implements lib)) + else + let* has_virtual_impl = + Resolve.Memo.read (Compilation_context.has_virtual_impl cctx) in if has_virtual_impl then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) @@ -176,7 +176,7 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin | None -> td, lib :: gl) in let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in - (), Dep.Set.union tight_deps glob_deps) + (), Dep.Set.union tight_deps glob_deps ;; (* Arguments for the compiler to prevent it from being too clever. From c30ed0d80c2f7b0011dc248b9b690beaed634c0f Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 16:00:34 -0700 Subject: [PATCH 52/80] docs: explain the wrapped-library limitation and sketch follow-ons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand the comment at [Lib_index.is_lib_tight_eligible] with the reason wrapped libraries fall back to a glob. The limitation is fundamental to ocamldep's output granularity: [-modules] lists only top-level module names, so [Foo.Bar.x] and [Foo.Baz.y] both produce the output [Foo]. Without finer information we cannot distinguish consumers using [Bar] from those using [Baz] and so cannot emit specific-cmi deps per internal. Sketch two possible directions that would close the gap — compiler-libs-based qualified-path extraction and post-compile cmi-imports refinement via Ocamlobjinfo — with effort estimates and their respective constraints. Add a pointer from [lib_deps_for_module]'s docblock so readers of the main filter see where the wrapped-lib rationale lives. Text only; no behavior change. Signed-off-by: Robin Bate Boerop --- src/dune_rules/lib_file_deps.ml | 36 ++++++++++++++++++++++++++-- src/dune_rules/module_compilation.ml | 8 ++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index f7bcc0213df..bbc488f2177 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -116,8 +116,40 @@ module Lib_index = struct (* A library is eligible for per-module tight deps iff it is local (so every entry has a known [Module.t] with which we can call - [Obj_dir.Module.cm_public_file]) and unwrapped (so the public - cmi dir does not need a glob to reach internal alias modules). *) + [Obj_dir.Module.cm_public_file]) and unwrapped. + + Wrapped libraries fall back to a directory glob over their + public cmi dir. The limitation is fundamental to ocamldep's + output granularity: with [-modules], ocamldep lists only the + top-level module names referenced by a source file, so + [Foo.Bar.x] and [Foo.Baz.y] both produce the single output + [Foo]. We cannot distinguish consumers that use the wrapped + lib's [Bar] internal from those that use [Baz], and so cannot + emit specific deps on just [Foo__Bar.cmi] vs [Foo__Baz.cmi]. + Any BFS-based walk over the wrapper's own ocamldep output + reaches every internal the wrapper exposes, which is + equivalent to the current glob for invalidation. + + Possible follow-on work to cross the wrapped-lib gap: + + - Qualified-path extractor: walk source via [compiler-libs]' + [Parse.implementation], collect [Longident.t] references + as [Module_name.Path.t] values in a companion artifact. + Match qualified paths against wrapped-lib internals for + per-consumer precision. Estimated ~500-1000 lines across a + new rule, a new file format, and preprocessing integration; + correct handling of [let open Foo in Bar.x] (opens that + bring sub-modules into unqualified scope) needs + lexical-scope tracking and roughly doubles the low-end + estimate. + + - Post-compile cmi-imports refinement: consumer.cmi records + exactly the cmis its compilation imported; [Ocamlobjinfo] + can read them. Using this as the source of truth requires + breaking dune's invariant that rule deps are fixed before + the rule runs — a two-phase build or a pessimistic-then- + refine scheme. Not natively supported by dune's rule model + today. *) let is_lib_tight_eligible lib = Lib.is_local lib && diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index fddb51464c8..c83c2ab357f 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -98,7 +98,13 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = These deps surface in [dune rules ]'s output alongside any static deps. Falls back to a glob over all cctx libs when filtering - is not possible. *) + is not possible. + + Wrapped dependency libraries always take the glob path. See + [Lib_file_deps.Lib_index.is_lib_tight_eligible] for an explanation + of why per-module tightening cannot be applied to wrapped libs + with the information ocamldep makes available, and for sketches + of possible follow-on work. *) let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kind ~mode m = let open Action_builder.O in let can_filter = From 94adc5536101343e7f52de623ebfe91ce4c02e63 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 16:28:44 -0700 Subject: [PATCH 53/80] docs: spell out "BFS" as "breadth-first walk" or "cross-library walk" No behavior change. The acronym was used in five places without expansion; one spot (the function that implements the walk in [module_compilation.ml]) already spelled it out. Make the comment stream consistent: refer to "the cross-library walk" when referencing the traversal from elsewhere, and "breadth-first" when the traversal's shape is the point. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 4 ++-- src/dune_rules/lib_file_deps.ml | 9 +++++---- src/dune_rules/lib_file_deps.mli | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index a06073d51d6..974607cc8b0 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -136,8 +136,8 @@ let parameters_main_modules parameters = Local libraries whose ocamldep is short-circuited by [Dep_rules.skip_ocamldep] (unwrapped single-module stanzas without - direct lib deps) are collected into [no_ocamldep] so the cross-lib - BFS does not try to read their nonexistent [.d] files. *) + direct lib deps) are collected into [no_ocamldep] so the cross- + library walk does not try to read their nonexistent [.d] files. *) let build_lib_index ~super_context ~all_libs ~for_ = let open Resolve.Memo.O in let+ per_lib = diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index bbc488f2177..30ea323ebf5 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -102,9 +102,10 @@ module Lib_index = struct ; no_ocamldep : Lib.Set.t (* Local libs whose ocamldep is short-circuited by [Dep_rules.skip_ocamldep] — single-module stanzas without - library dependencies have no [.d] build rules. BFS must - not try to read ocamldep for them; their entry module - can have no cross-library references anyway. *) + library dependencies have no [.d] build rules. The + cross-library walk must not try to read ocamldep for them; + their entry module can have no cross-library references + anyway. *) } let empty = @@ -126,7 +127,7 @@ module Lib_index = struct [Foo]. We cannot distinguish consumers that use the wrapped lib's [Bar] internal from those that use [Baz], and so cannot emit specific deps on just [Foo__Bar.cmi] vs [Foo__Baz.cmi]. - Any BFS-based walk over the wrapper's own ocamldep output + Any breadth-first walk over the wrapper's own ocamldep output reaches every internal the wrapper exposes, which is equivalent to the current glob for invalidation. diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 14639464e47..9177c99f410 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -42,9 +42,9 @@ module Lib_index : sig [no_ocamldep] names local libs whose ocamldep output is short-circuited (single-module stanzas without library - dependencies — see [Dep_rules.skip_ocamldep]). The cross-lib - BFS in [module_compilation] must not try to read [.d] files - for those libs. *) + dependencies — see [Dep_rules.skip_ocamldep]). The cross- + library walk in [module_compilation] must not try to read + [.d] files for those libs. *) val create : ?no_ocamldep:Lib.Set.t -> (Module_name.t * Lib.t * Module.t option) list @@ -68,7 +68,7 @@ module Lib_index : sig val filter_libs_with_modules : t -> referenced_modules:Module_name.Set.t -> classified (** [lookup_tight_entries idx name] returns [(lib, entry module)] - pairs used by the cross-library BFS in [module_compilation]. + pairs used by the cross-library walk in [module_compilation]. Libraries in [no_ocamldep] are excluded (their [.d] files do not exist), as are externals, wrapped locals, and entries with no [Module.t]. *) From ad0d12f978c219df681f858ff4900204faef5526 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 17:09:50 -0700 Subject: [PATCH 54/80] =?UTF-8?q?refactor:=20address=20review=20feedback?= =?UTF-8?q?=20=E2=80=94=20clarifying=20comments,=20tests,=20small=20refact?= =?UTF-8?q?or?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses items from a critical code review. No behavior change. Clarifying comments: - Document why can_filter's Path.Build.equal guard exists (catches dummy dep graphs from for_alias_module / for_root_module / for_module_generated_at_link_time) and why Dep_graph.mem is a guard (catches menhir's mock_module passed to ocamlc_i). - Name the two virtual-impl checks (consumer-stanza vs dep-list). - Explain the two filter_libs_with_modules calls in lib_deps_for_module (seed Lib.closure input, then classify post-walk). - Add rule-graph cycle-safety note on cross_lib_tight_set: .d files depend only on source, so cross-library reads can't cycle through consumer outputs. - Add module-level commentary on module_compilation.ml explaining the intra-lib vs inter-lib dep split. - Expand parse_deps_lenient's comment with the ppx-compat-vs- consistency-check trade-off. - Move the ~35-line wrapped-lib-limitation explanation out of is_lib_tight_eligible into a module-level comment at the top of lib_file_deps.ml; leave a short pointer at the function. - Note Lib.closure's memo-key order sensitivity in its signature. - TODO at Dep_rules.skip_ocamldep noting the parallel no_ocamldep logic in build_lib_index — the two must agree, and should eventually be unified. Small refactors: - Rename union_set_map → union_module_name_sets_mapped (honest about monomorphism). - Extract a lib_cm_deps helper shared by build_cm and ocamlc_i. - Add tight_modules_per_lib variant of filter_libs_with_modules that skips the non_tight accumulator when the caller discards it. New cram tests: - no-ocamldep-leaf-lib.t — consumer depends on a single-module leaf lib whose ocamldep is short-circuited by skip_ocamldep. Proves the cross-library walk doesn't demand a nonexistent .d. - lib-vs-lib-name-collision.t — two unwrapped libs expose the same entry-module name. Documents the conservative over-dep behavior (both libs' matching cmis are in the tight set). - private-modules.t — unwrapped library with private_modules; exercises the cm_public_file path in deps_of_entry_modules. Signed-off-by: Robin Bate Boerop --- src/dune_rules/dep_rules.ml | 20 ++- src/dune_rules/lib.mli | 4 + src/dune_rules/lib_file_deps.ml | 95 ++++++++----- src/dune_rules/lib_file_deps.mli | 10 ++ src/dune_rules/module_compilation.ml | 127 ++++++++++++------ src/dune_rules/ocamldep.ml | 14 +- .../no-ocamldep-leaf-lib.t | 50 +++++++ .../per-module-lib-deps/private-modules.t | 49 +++++++ 8 files changed, 288 insertions(+), 81 deletions(-) create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t create mode 100644 test/blackbox-tests/test-cases/per-module-lib-deps/private-modules.t diff --git a/src/dune_rules/dep_rules.ml b/src/dune_rules/dep_rules.ml index 99c0b203337..465b201f084 100644 --- a/src/dune_rules/dep_rules.ml +++ b/src/dune_rules/dep_rules.ml @@ -134,9 +134,23 @@ let deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_ sourced_module (** Tests whether a set of modules is a singleton. *) let has_single_file modules = Option.is_some @@ Modules.With_vlib.as_singleton modules -(** Tests whether ocamldep can be short-circuited for [modules]: true for - single-module stanzas that have no library dependencies, since no - consumer of ocamldep output can benefit in that case. *) +(** Tests whether ocamldep can be short-circuited for [modules]: true + for single-module stanzas that have no library dependencies. + The premise — "no consumer of ocamldep output can benefit" — was + valid before #4572; under that PR, cross-library consumers now + read a target library's ocamldep output as part of the + per-module inter-library dependency filter. Libraries identified + here must also be identified in + [Compilation_context.build_lib_index]'s [no_ocamldep_lib] check + so the cross-library walk knows their [.d] files will be + missing. Keeping the two in sync is fragile. + + TODO: unify into a single predicate that both call sites + consult, so future changes to the condition can't drift. The + cleanest shape is probably to retire this short-circuit for + library stanzas entirely — libraries that could be consumed + cross-stanza should always run ocamldep — and keep the + optimisation only for executable/test stanzas. *) let skip_ocamldep ~has_library_deps modules = has_single_file modules && not has_library_deps ;; diff --git a/src/dune_rules/lib.mli b/src/dune_rules/lib.mli index 1ab86f84d21..0d9fd7f99df 100644 --- a/src/dune_rules/lib.mli +++ b/src/dune_rules/lib.mli @@ -217,6 +217,10 @@ end (** {1 Transitive closure} *) +(** Memoized. The memo key is order-sensitive on the input list, so + for callers that share inputs across invocations (e.g. the hot + path in [Module_compilation.lib_deps_for_module]), a canonical + sort (by [Lib.compare]) maximises cache reuse. *) val closure : t list -> linking:bool -> for_:Compilation_mode.t -> t list Resolve.Memo.t (** [descriptive_closure ~with_pps libs] computes the smallest set of libraries diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 30ea323ebf5..d08da729e32 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -1,6 +1,42 @@ open Import open Memo.O +(* Why wrapped libraries fall back to a directory glob + ================================================= + + Per-module tight deps apply only to local unwrapped libraries. + Wrapped libraries take the glob path over their public cmi dir. + The limitation is fundamental to ocamldep's output granularity. + + [ocamldep -modules foo.ml] lists only the top-level module names + referenced by a source file. For a consumer using [Foo.Bar.x], + the output is [Foo] — not [Foo.Bar]. Consumers that use + [Foo.Bar] and those that use [Foo.Baz] produce identical + ocamldep output, so the filter cannot distinguish them and + cannot emit specific deps on [Foo__Bar.cmi] vs [Foo__Baz.cmi]. + Any breadth-first walk over the wrapper's own ocamldep output + reaches every internal the wrapper exposes, which is equivalent + to the glob for invalidation. + + Possible follow-on work: + + - Qualified-path extractor. Walk consumer source with + [compiler-libs]' [Parse.implementation], collect [Longident.t] + references as [Module_name.Path.t] values in a companion + artifact. Match qualified paths against wrapped-lib internals + for per-consumer precision. Estimated ~500-1000 lines across a + new rule, a new file format, and preprocessing integration; + correct handling of [let open Foo in Bar.x] (opens that bring + sub-modules into unqualified scope) needs lexical-scope + tracking and roughly doubles the low-end estimate. + + - Post-compile cmi-imports refinement. [consumer.cmi] records + exactly the cmis its compilation imported; [Ocamlobjinfo] can + read them. Using this as the source of truth requires breaking + dune's invariant that rule deps are fixed before the rule + runs — a two-phase build or a pessimistic-then-refine scheme. + Not natively supported by dune's rule model today. *) + module Group = struct type ocaml = | Cmi @@ -115,42 +151,11 @@ module Lib_index = struct } ;; - (* A library is eligible for per-module tight deps iff it is local - (so every entry has a known [Module.t] with which we can call - [Obj_dir.Module.cm_public_file]) and unwrapped. - - Wrapped libraries fall back to a directory glob over their - public cmi dir. The limitation is fundamental to ocamldep's - output granularity: with [-modules], ocamldep lists only the - top-level module names referenced by a source file, so - [Foo.Bar.x] and [Foo.Baz.y] both produce the single output - [Foo]. We cannot distinguish consumers that use the wrapped - lib's [Bar] internal from those that use [Baz], and so cannot - emit specific deps on just [Foo__Bar.cmi] vs [Foo__Baz.cmi]. - Any breadth-first walk over the wrapper's own ocamldep output - reaches every internal the wrapper exposes, which is - equivalent to the current glob for invalidation. - - Possible follow-on work to cross the wrapped-lib gap: - - - Qualified-path extractor: walk source via [compiler-libs]' - [Parse.implementation], collect [Longident.t] references - as [Module_name.Path.t] values in a companion artifact. - Match qualified paths against wrapped-lib internals for - per-consumer precision. Estimated ~500-1000 lines across a - new rule, a new file format, and preprocessing integration; - correct handling of [let open Foo in Bar.x] (opens that - bring sub-modules into unqualified scope) needs - lexical-scope tracking and roughly doubles the low-end - estimate. - - - Post-compile cmi-imports refinement: consumer.cmi records - exactly the cmis its compilation imported; [Ocamlobjinfo] - can read them. Using this as the source of truth requires - breaking dune's invariant that rule deps are fixed before - the rule runs — a two-phase build or a pessimistic-then- - refine scheme. Not natively supported by dune's rule model - today. *) + (* Local + unwrapped: every entry has a known [Module.t] we can + feed to [Obj_dir.Module.cm_public_file]. Wrapped libraries + fall back to a directory glob; see the module-level comment + at the top of this file for the ocamldep-granularity reason + and sketches of follow-on work. *) let is_lib_tight_eligible lib = Lib.is_local lib && @@ -209,6 +214,24 @@ module Lib_index = struct { tight; non_tight = Lib.Set.to_list non_tight } ;; + (* Like [filter_libs_with_modules] but only returns the tight + part. Saves the [non_tight] accumulator at call sites that + only consume the tight map (e.g. the post-BFS classify in + [lib_deps_for_module]). *) + let tight_modules_per_lib idx ~referenced_modules = + Module_name.Set.fold referenced_modules ~init:Lib.Map.empty ~f:(fun name acc -> + match Module_name.Map.find idx.by_module_name name with + | None -> acc + | Some entries -> + List.fold_left entries ~init:acc ~f:(fun acc (lib, m_opt) -> + match m_opt with + | Some m when Lib.Set.mem idx.tight_eligible lib -> + Lib.Map.update acc lib ~f:(function + | None -> Some [ m ] + | Some ms -> Some (m :: ms)) + | _ -> acc)) + ;; + let lookup_tight_entries idx name = match Module_name.Map.find idx.by_module_name name with | None -> [] diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 9177c99f410..6ff3aeba503 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -67,6 +67,16 @@ module Lib_index : sig [referenced_modules]. *) val filter_libs_with_modules : t -> referenced_modules:Module_name.Set.t -> classified + (** [tight_modules_per_lib idx ~referenced_modules] builds a map + from each tight-eligible library that exposes a name in + [referenced_modules] to the subset of its entry modules that + appear. Equivalent to [filter_libs_with_modules] with only the + tight part kept. *) + val tight_modules_per_lib + : t + -> referenced_modules:Module_name.Set.t + -> Module.t list Lib.Map.t + (** [lookup_tight_entries idx name] returns [(lib, entry module)] pairs used by the cross-library walk in [module_compilation]. Libraries in [no_ocamldep] are excluded (their [.d] files do diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index c83c2ab357f..70d5f9d522b 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -1,6 +1,27 @@ open Import open Memo.O +(* Two dep pipelines feed every compile rule: + + - [other_cm_files] (in [build_cm] below) — intra-library deps: + the cmis of the sibling modules that this module's own + ocamldep output says it transitively depends on within the + same stanza. These are static deps computed from the stanza's + [Dep_graph.t]. + + - [lib_cm_deps] — inter-library deps, produced by + [lib_deps_for_module] as [Action_builder.dyn_deps]. This is + the per-module inter-library dependency filter (#4572): it + reads ocamldep output to narrow the set of cmis the consumer + depends on from other libraries, falling back to a per-lib + directory glob when filtering is unsafe or unavailable. + + The two sets are composed with [>>>] in the compile rule; there + is no overlap between them because intra-lib references are + already filtered out of ocamldep's "raw" output by + [parse_module_names] for the intra-lib path, and kept as raw + [Module_name.t] values for the inter-lib path. *) + let all_libs cctx = let open Resolve.Memo.O in let+ d = Compilation_context.requires_compile cctx @@ -8,9 +29,9 @@ let all_libs cctx = d @ h ;; -(* Map each element of [xs] through [f], returning the union of all the - resulting [Module_name.Set.t]s. *) -let union_set_map xs ~f = +(* Map each element of [xs] through [f] and union the resulting + [Module_name.Set.t]s. Monomorphic on [Module_name.Set] by design. *) +let union_module_name_sets_mapped xs ~f = Action_builder.List.map xs ~f |> Action_builder.map ~f:(List.fold_left ~init:Module_name.Set.empty ~f:Module_name.Set.union) @@ -46,7 +67,13 @@ let module_kind_has_readable_ocamldep m = pass through them terminate and the consumer falls back to a glob on the unreached libs. - Cycles terminate on the [seen] set. *) + Cycles in the module-reference graph terminate on the [seen] set. + + Rule-graph cycle safety: [.d] files depend only on their source + [.ml]/[.mli] (via [Ocamldep.deps_of]) — never on any cmi. So + reading a cross-library [.d] file cannot transitively demand any + consumer output, and this walk cannot introduce rule cycles + regardless of how the library graph looks. *) let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = let open Action_builder.O in let read_entry_deps (lib, m) = @@ -63,7 +90,7 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = Module_name.Set.fold frontier ~init:[] ~f:(fun name acc -> Lib_file_deps.Lib_index.lookup_tight_entries lib_index name @ acc) in - let* discovered = union_set_map pairs ~f:read_entry_deps in + let* discovered = union_module_name_sets_mapped pairs ~f:read_entry_deps in let seen = Module_name.Set.union seen frontier in let frontier = Module_name.Set.diff discovered seen in loop ~seen ~frontier) @@ -100,27 +127,44 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = static deps. Falls back to a glob over all cctx libs when filtering is not possible. - Wrapped dependency libraries always take the glob path. See - [Lib_file_deps.Lib_index.is_lib_tight_eligible] for an explanation - of why per-module tightening cannot be applied to wrapped libs - with the information ocamldep makes available, and for sketches - of possible follow-on work. *) + Wrapped dependency libraries always take the glob path. See the + module-level comment at the top of [lib_file_deps.ml] for the + ocamldep-granularity reason and sketches of follow-on work. *) let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kind ~mode m = let open Action_builder.O in let can_filter = + (* Filtering is OCaml-only; Melange has its own dep handling. *) (match Lib_mode.of_cm_kind cm_kind with | Melange -> false | Ocaml _ -> true) + (* [Dep_graph.dir = Path.Build.root] indicates a dummy dep graph + from [Compilation_context.for_alias_module], + [for_root_module], or [for_module_generated_at_link_time]. + Those contexts replace [modules] with [singleton_modules]; + the resulting dummy dep graph cannot supply transitive + deps. *) && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir) + (* Reached when [m] was synthesised outside the stanza's module + set and handed directly to [ocamlc_i] — menhir's [mock_module] + is the concrete case. *) && Dep_graph.mem dep_graph m && module_kind_is_filterable m && Module.has m ~ml_kind + (* Consumer-side virtual-impl check: this stanza is itself a + virtual-library implementation, which has its own dep + machinery (see [Dep_rules.deps_of_vlib_module]). The + complementary check below ([has_virtual_impl]) covers the + case where the consumer's dep list contains a virtual + implementation. *) && not (Virtual_rules.is_implementation (Compilation_context.implements cctx)) in let* libs = Resolve.Memo.read (all_libs cctx) in if not can_filter then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else + (* Deps-side virtual-impl check: if any library in the consumer's + requires implements a virtual library, fall back to glob. + Distinct from the consumer-stanza check in [can_filter]. *) let* has_virtual_impl = Resolve.Memo.read (Compilation_context.has_virtual_impl cctx) in @@ -130,7 +174,7 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in let* trans_deps = Dep_graph.deps_of dep_graph m in let* all_raw = - union_set_map (m :: trans_deps) ~f:(fun dep_m -> + union_module_name_sets_mapped (m :: trans_deps) ~f:(fun dep_m -> if not (module_kind_has_readable_ocamldep dep_m) then Action_builder.return Module_name.Set.empty else @@ -145,6 +189,11 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in let referenced = Module_name.Set.union all_raw open_modules in + (* First use of [filter_libs_with_modules]: identify the libs + the consumer directly names (via its own ocamldep output). + Used only to seed [Lib.closure]'s input. The second use, + after the cross-library walk extends the name set, produces + the per-lib module lists used in the fold below. *) let { Lib_file_deps.Lib_index.tight; non_tight } = Lib_file_deps.Lib_index.filter_libs_with_modules lib_index @@ -166,8 +215,8 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin just those entries; the rest (non-tight-eligible libs, tight-eligible libs with no intersecting entries) fall to a glob over their objdir. *) - let { Lib_file_deps.Lib_index.tight = tight_modules; non_tight = _ } = - Lib_file_deps.Lib_index.filter_libs_with_modules + let tight_modules = + Lib_file_deps.Lib_index.tight_modules_per_lib lib_index ~referenced_modules:tight_set in @@ -185,6 +234,28 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin (), Dep.Set.union tight_deps glob_deps ;; +(* Convenience wrapper for the two call sites ([build_cm] and + [ocamlc_i]) that wire the per-module filter into a compile rule. + Both take [cm_kind], [ml_kind], [mode], and the module to be + compiled; the rest is derived from [cctx]. *) +let lib_cm_deps ~cctx ~cm_kind ~ml_kind ~mode m = + let obj_dir = Compilation_context.obj_dir cctx in + let opaque = Compilation_context.opaque cctx in + let for_ = Compilation_context.for_ cctx in + let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in + Action_builder.dyn_deps + (lib_deps_for_module + ~cctx + ~obj_dir + ~for_ + ~dep_graph + ~opaque + ~cm_kind + ~ml_kind + ~mode + m) +;; + (* Arguments for the compiler to prevent it from being too clever. The compiler creates the cmi when it thinks a .ml file has no corresponding @@ -477,23 +548,10 @@ let build_cm | Wrapped_compat -> true | _ -> false in - let for_ = Compilation_context.for_ cctx in - let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in let lib_cm_deps = if skip_lib_deps then Action_builder.return () - else - Action_builder.dyn_deps - (lib_deps_for_module - ~cctx - ~obj_dir - ~for_ - ~dep_graph - ~opaque - ~cm_kind - ~ml_kind - ~mode - m) + else lib_cm_deps ~cctx ~cm_kind ~ml_kind ~mode m in let other_cm_files = let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in @@ -730,20 +788,7 @@ let ocamlc_i ~deps cctx (m : Module.t) ~output = [ Path.build (Obj_dir.Module.cm_file_exn obj_dir m ~kind:(Ocaml Cmi)) ])) in let lib_cm_deps = - let opaque = Compilation_context.opaque cctx in - let for_ = Compilation_context.for_ cctx in - let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) Impl in - Action_builder.dyn_deps - (lib_deps_for_module - ~cctx - ~obj_dir - ~for_ - ~dep_graph - ~opaque - ~cm_kind:(Ocaml Cmo) - ~ml_kind:Impl - ~mode:(Ocaml Byte) - m) + lib_cm_deps ~cctx ~cm_kind:(Ocaml Cmo) ~ml_kind:Impl ~mode:(Ocaml Byte) m in let ocaml_flags = Ocaml_flags.get (Compilation_context.flags cctx) (Ocaml Byte) in let modules = Compilation_context.modules cctx in diff --git a/src/dune_rules/ocamldep.ml b/src/dune_rules/ocamldep.ml index 6cfb296ff25..db0e7c5ef0a 100644 --- a/src/dune_rules/ocamldep.ml +++ b/src/dune_rules/ocamldep.ml @@ -111,7 +111,19 @@ let parse_deps_exn ~file lines = may hold a raw [Module.t] whose [Module.source] path differs from the pp-transformed source the producing rule fed to ocamldep — the basename will not match even though the output - is valid. *) + is valid. + + Trade-off: this trades a cross-consistency assertion (the rule + that produced the file must have used the same source path the + reader now holds) for compatibility with cross-library readers + that don't participate in rule production. The producing side + still validates via [parse_deps_exn] in [deps_of]'s action, so + bogus ocamldep output is still caught at the producing rule; but + a subtle future drift between the two sources of [Module.t] (e.g. + a refactor that changes how [Module.source] is resolved for + cross-library reads) would now slip past the reader silently. + The structural check remaining here (exactly one line, one colon) + continues to catch gross format corruption. *) let parse_deps_lenient ~file lines = match lines with | [] | _ :: _ :: _ -> invalid_ocamldep_output file lines diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t new file mode 100644 index 00000000000..e17e5456398 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t @@ -0,0 +1,50 @@ +A tight-eligible dependency library that is a leaf: one module, no +library deps of its own. Dune's [Dep_rules.skip_ocamldep] short- +circuits ocamldep for this shape — no [.d] rules exist for [leaf]. +The per-module filter's cross-library walk must recognise this and +not demand a nonexistent [leaf/mod_leaf.impl.d]; otherwise the +consumer's compile fails with "No rule found" during rule +evaluation. + +The guard lives in [Lib_index.no_ocamldep]. This test proves the +consumer builds successfully even though the BFS names [leaf] in +its initial frontier. + +See: https://github.com/ocaml/dune/issues/4572 + + $ cat > dune-project < (lang dune 3.0) + > EOF + +[leaf]: unwrapped, single-module, no library dependencies. This is +exactly the shape that triggers [skip_ocamldep]: + + $ mkdir leaf + $ cat > leaf/dune < (library (name leaf) (wrapped false)) + > EOF + $ cat > leaf/mod_leaf.ml < let v = 1 + > EOF + +[consumer]: multi-module library that depends on [leaf] and +references [Mod_leaf] from one of its modules. The consumer itself +has library deps, so its own ocamldep runs; what we're probing is +whether the cross-library walk tries to read [leaf]'s ocamldep: + + $ mkdir consumer + $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries leaf)) + > EOF + $ cat > consumer/c.ml < let w = Mod_leaf.v + > EOF + $ cat > consumer/d.ml < let _ = () + > EOF + +Build must succeed. Without the [no_ocamldep] guard, the BFS would +demand [leaf/.leaf.objs/mod_leaf.impl.d] and the rule engine would +fail: + + $ dune build @check diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/private-modules.t b/test/blackbox-tests/test-cases/per-module-lib-deps/private-modules.t new file mode 100644 index 00000000000..94d67880311 --- /dev/null +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/private-modules.t @@ -0,0 +1,49 @@ +An unwrapped library with [private_modules] forces dune to use a +dedicated public cmi directory: public modules' cmis live in a +separate [.objs/public_cmi] directory, produced by a copy rule +from the internal object dir. Per-module tight deps must emit +deps on the *public* cmi path, not the internal one — otherwise +the produce-public-cmi rule doesn't run and sandboxed compiles +fail to find the cmi. + +This test exercises the [Obj_dir.Module.cm_public_file] path in +[Lib_file_deps.deps_of_entry_modules]. + +See: https://github.com/ocaml/dune/issues/4572 + + $ cat > dune-project < (lang dune 3.0) + > EOF + +[dep]: unwrapped library with one public entry module [Pub] and +one private module [Priv]. Consumers can see [Pub] via [-I] +search but not [Priv]: + + $ mkdir dep + $ cat > dep/dune < (library + > (name dep) + > (wrapped false) + > (private_modules priv)) + > EOF + $ cat > dep/pub.ml < let greeting () = Priv.helper ^ "!" + > EOF + $ cat > dep/priv.ml < let helper = "hi" + > EOF + +[consumer]: references [Pub] from [dep]: + + $ mkdir consumer + $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries dep)) + > EOF + $ cat > consumer/c.ml < let v = Pub.greeting () + > EOF + $ cat > consumer/d.ml < let _ = () + > EOF + + $ dune build @check From ec317c192fe536758aa391948aa3abc6a5ccb16b Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 17:33:17 -0700 Subject: [PATCH 55/80] test: use [let _ = ...] for consumer references in new cram tests Replace [let w = ...] with [let _ = ...] in the three cram tests added by this PR. Avoids any future warning-32 (unused value declaration) hazard if the dev profile promotes it to an error, and is idiomatic for "use the value but discard". Cosmetic change. Signed-off-by: Robin Bate Boerop --- .../test-cases/per-module-lib-deps/cross-lib-ppx.t | 2 +- .../test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t b/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t index 51f6e6a9e5f..58dc0cd90ab 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/cross-lib-ppx.t @@ -56,7 +56,7 @@ raw [Module.t] for [mod_a]: > (library (name consumer) (wrapped false) (libraries dep)) > EOF $ cat > consumer/c.ml < let w = Mod_a.v + > let _ = Mod_a.v > EOF $ cat > consumer/d.ml < let _ = () diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t index e17e5456398..68a6f5b0941 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/no-ocamldep-leaf-lib.t @@ -37,7 +37,7 @@ whether the cross-library walk tries to read [leaf]'s ocamldep: > (library (name consumer) (wrapped false) (libraries leaf)) > EOF $ cat > consumer/c.ml < let w = Mod_leaf.v + > let _ = Mod_leaf.v > EOF $ cat > consumer/d.ml < let _ = () From 79d08f36acd8325509e94097ddd8439d98ce7985 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Fri, 24 Apr 2026 20:18:28 -0700 Subject: [PATCH 56/80] perf: memoize cctx.lib_index and has_virtual_impl with Memo.Lazy.t MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both fields were declared as bare [Resolve.Memo.t] expressions: each read from a compile rule re-evaluated the bind chain and re-ran [build_lib_index] (or the [List.exists] for [has_virtual_impl]). Counter instrumentation on a dune-on-dune null build measured 2,689 [build_lib_index] entries across 228 compilation contexts — about 12× per cctx — confirming that the values are not memoised by Memo's graph as the prior commit message believed. Wrapping both in [Memo.Lazy.t] (matching the existing treatment of [requires_link] in the same record) caches the result per cctx. Empirically, [build_lib_index] entries drop from 2,689 to 166 (the number of cctxs that actually go through the filter path). Wall-clock impact, dune-on-dune null build (mean of last 3 of 4 [dune build] invocations, hyperfine, OCaml 5.4.1): before: 2.448 s ± 0.024 s after: 2.099 s ± 0.017 s (-349 ms, -14 %) Recovers about 40 % of the post-#14116 null-build regression. No behavioural change; the accessor signatures are unchanged ([Memo.Lazy.force] returns a [Memo.t]). Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 974607cc8b0..386f53cd310 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -65,8 +65,8 @@ type t = ; parameters : Module_name.t list Resolve.Memo.t ; instances : Parameterised_instances.t Resolve.Memo.t option ; includes : Includes.t - ; lib_index : Lib_file_deps.Lib_index.t Resolve.Memo.t - ; has_virtual_impl : bool Resolve.Memo.t + ; lib_index : Lib_file_deps.Lib_index.t Resolve.t Memo.Lazy.t + ; has_virtual_impl : bool Resolve.t Memo.Lazy.t ; preprocessing : Pp_spec.t ; opaque : bool ; js_of_ocaml : Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t @@ -94,8 +94,8 @@ let requires_hidden t = t.requires_hidden let requires_link t = Memo.Lazy.force t.requires_link let parameters t = t.parameters let includes t = t.includes -let lib_index t = t.lib_index -let has_virtual_impl t = t.has_virtual_impl +let lib_index t = Memo.Lazy.force t.lib_index +let has_virtual_impl t = Memo.Lazy.force t.has_virtual_impl let preprocessing t = t.preprocessing let opaque t = t.opaque let js_of_ocaml t = t.js_of_ocaml @@ -284,15 +284,17 @@ let create ; parameters ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config ; lib_index = - (let open Resolve.Memo.O in - let* direct = direct_requires in - let* hidden = hidden_requires in - build_lib_index ~super_context ~all_libs:(direct @ hidden) ~for_) + Memo.lazy_ (fun () -> + let open Resolve.Memo.O in + let* direct = direct_requires in + let* hidden = hidden_requires in + build_lib_index ~super_context ~all_libs:(direct @ hidden) ~for_) ; has_virtual_impl = - (let open Resolve.Memo.O in - let+ direct = direct_requires - and+ hidden = hidden_requires in - List.exists (direct @ hidden) ~f:(fun lib -> Option.is_some (Lib.implements lib))) + Memo.lazy_ (fun () -> + let open Resolve.Memo.O in + let+ direct = direct_requires + and+ hidden = hidden_requires in + List.exists (direct @ hidden) ~f:(fun lib -> Option.is_some (Lib.implements lib))) ; preprocessing ; opaque ; js_of_ocaml @@ -394,7 +396,7 @@ let for_module_generated_at_link_time cctx ~requires ~module_ = ; requires_link = Memo.lazy_ (fun () -> requires) ; requires_compile = requires ; includes - ; lib_index = Resolve.Memo.return Lib_file_deps.Lib_index.empty + ; lib_index = Memo.lazy_ (fun () -> Resolve.Memo.return Lib_file_deps.Lib_index.empty) ; modules } ;; From 0c7e49b148c04d305ea0684008fdd5e19f228bb3 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 09:14:41 -0700 Subject: [PATCH 57/80] perf: skip [.ml]-side ocamldep reads when the [.cmi] doesn't carry them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A module's [.cmi] is produced from [.mli] when one exists; only then. References in [.ml] that are not re-exported through [.mli] never appear in the [.cmi]'s imports — they are sealed. The compiler reads a dep's [.cmx] only for cross-module inlining, which happens only when compiling [Ocaml Cmx] without [-opaque]. In every other case the [.cmx] is unread and any [.ml]-side references it carries do not propagate. The previous filter unconditionally read both [Impl] and [Intf] ocamldep output for every module in the consumer's intra-library transitive closure, which over-approximates the consumer's true inter-library reference set whenever those references are sealed. Skip the [.ml]-side read when: - For the consumer module [m]: [ml_kind = Intf] (compiling [m.cmi] from [m.mli]; [m.ml] is unread by the compiler). - For a transitive intra-library dep [dep_m]: [dep_m] has [.mli] AND the consumer's compilation does not read [dep_m.cmx] for inlining (i.e. [cm_kind <> Ocaml Cmx] or [-opaque] is set). Wall-clock impact, dune-on-dune null build (mean of last 3 of 4 [dune build] invocations, hyperfine, OCaml 5.4.1): before: 2.170 s ± 0.012 s after: 1.904 s ± 0.017 s (-266 ms, -12 %) Brings the post-#14116 null-build regression from +570 ms (+36 %) to +304 ms (+19 %) over [main]. Also tightens incremental-build dep sets for any consumer whose transitive intra-lib deps reference libraries only from [.ml] under [-opaque] — those consumers will no longer be invalidated by changes to those library cmis. The [Intf] case for [m] addresses art-w's review feedback: https://github.com/ocaml/dune/pull/14116#discussion_r3077767496 The [-opaque] case completes the same suggestion. [lib-deps-preserved.t] is updated to reflect the tighter dep set on [Main.cmx] and explain the wrapped-lib glob fallback that still applies to [Uses_lib.cmx]. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 77 ++++++++++++++++--- .../per-module-lib-deps/lib-deps-preserved.t | 23 ++++-- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 70d5f9d522b..031aba24ccc 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -173,19 +173,72 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in let* trans_deps = Dep_graph.deps_of dep_graph m in - let* all_raw = - union_module_name_sets_mapped (m :: trans_deps) ~f:(fun dep_m -> - if not (module_kind_has_readable_ocamldep dep_m) - then Action_builder.return Module_name.Set.empty - else - let* impl_deps = - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m - in - let+ intf_deps = - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m - in - Module_name.Set.union impl_deps intf_deps) + (* Whether to read [dep_m]'s [.ml]-side ocamldep when collecting + the libraries the consumer references. + + A module's [.cmi] is produced from [.mli] when one exists; only + then. References in [.ml] that are not re-exported through the + [.mli] never appear in [.cmi]'s imports — they are sealed. + + The [.cmx] is produced from [.ml]. The compiler reads a + dep's [.cmx] only for cross-module inlining, which happens + only when compiling [Ocaml Cmx] without [-opaque]. In every + other case the [.cmx] is unread and any [.ml]-side + references it carries do not propagate. + + For [m] itself (the consumer being compiled), [.ml] is the + source the compiler is feeding when [ml_kind = Impl]; its + references must be resolved. When [ml_kind = Intf] we are + compiling [m.cmi] from [m.mli], and [m.ml] is unread. + + For a transitive intra-library dep [dep_m], [.ml]-side + references only propagate when (a) [dep_m] has no [.mli] + (so [dep_m.cmi] is produced from [.ml]) or (b) the + consumer reads [dep_m.cmx] for inlining ([Ocaml Cmx] + without [-opaque]). + + Decision summary: + + | [dep_m] is | [cm_kind] | [opaque] | read [.ml]? | + | ----------------------- | ----------- | -------- | ------------ | + | consumer ([m] itself) | any | any | iff [Impl] | + | trans_dep, no [.mli] | any | any | yes | + | trans_dep, has [.mli] | [Cmx] | false | yes (inline) | + | trans_dep, has [.mli] | [Cmx] | true | no | + | trans_dep, has [.mli] | [Cmi]/[Cmo] | any | no | + *) + let need_impl_deps_of dep_m ~is_consumer = + if is_consumer + then ( + match ml_kind with + | Ml_kind.Impl -> true + | Intf -> false) + else + (not (Module.has dep_m ~ml_kind:Intf)) + || + match cm_kind with + | Ocaml Cmx -> not opaque + | Ocaml (Cmi | Cmo) | Melange _ -> false + in + let read_dep_m_raw dep_m ~is_consumer = + if not (module_kind_has_readable_ocamldep dep_m) + then Action_builder.return Module_name.Set.empty + else + let* impl_deps = + if need_impl_deps_of dep_m ~is_consumer + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m + else Action_builder.return Module_name.Set.empty + in + let+ intf_deps = + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m + in + Module_name.Set.union impl_deps intf_deps + in + let* m_raw = read_dep_m_raw m ~is_consumer:true in + let* trans_raw = + union_module_name_sets_mapped trans_deps ~f:(read_dep_m_raw ~is_consumer:false) in + let all_raw = Module_name.Set.union m_raw trans_raw in let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in let referenced = Module_name.Set.union all_raw open_modules in diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-deps-preserved.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-deps-preserved.t index 86c419fc590..b3ea0203ad6 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-deps-preserved.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-deps-preserved.t @@ -1,7 +1,9 @@ -Verify that library file deps are declared for module compilation rules. +Verify the cm_kind/-opaque rules for library file deps in compile rules. -Every non-alias module should declare glob deps on its library -dependencies' .cmi files. +[mylib] is a wrapped library exposing one entry [Mylib]. The +executable has two modules: [Uses_lib] which references [Mylib] in +its [.ml] (but not its [.mli]) and [Main] which references only +[Uses_lib]. [Main] has no [.mli]. $ cat > dune-project < (lang dune 3.23) @@ -33,12 +35,23 @@ dependencies' .cmi files. $ dune build ./main.exe -Both modules declare glob deps on mylib's .cmi files: +[Uses_lib.cmx] keeps the glob: [Uses_lib.ml] references [Mylib], and +[mylib] is wrapped — wrapped libraries fall back to a glob over their +public cmi dir even under per-module filtering, since ocamldep's +[-modules] output cannot distinguish [Foo.Bar.x] from [Foo.Baz.x]. $ dune rules --root . --format=json --deps _build/default/.main.eobjs/native/dune__exe__Uses_lib.cmx | > jq -r 'include "dune"; .[] | depsGlobPredicates' *.cmi +[Main.cmx] has no inter-library deps. [Main.ml] references only +[Uses_lib], an intra-library module filtered out before the +inter-library filter runs. [Uses_lib] is a transitive intra-library +dep; its [.mli] has no cross-library references. [Uses_lib.ml] does +reference [Mylib], but those references propagate to the consumer +only through [Uses_lib.cmx] for cross-module inlining. Under the +default profile dune builds with [-opaque], which disables that +inlining, so [Uses_lib.ml]'s references are sealed behind its [.cmi]. + $ dune rules --root . --format=json --deps _build/default/.main.eobjs/native/dune__exe__Main.cmx | > jq -r 'include "dune"; .[] | depsGlobPredicates' - *.cmi From 16fd0aab4b8556324a16d88a0019f24d39bf8c51 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 10:19:06 -0700 Subject: [PATCH 58/80] fix: drop unreached tight-eligible libraries from compile-rule deps When the cross-library walk finishes with no entry of a library in the consumer's reference closure, and that library is tight-eligible (local, unwrapped, every entry has a known [Module.t]), the walk has full visibility and the negative result is positive evidence that the consumer does not reach the library. Drop the library from the consumer's compile-rule deps instead of falling back to a glob over its objdir. The link rule still pulls the library in through [requires_link], so executables and libraries that need its compiled artefacts at link time are unaffected. The dropped deps are purely about compile-time invalidation: a change in the library's modules no longer re-invalidates a consumer that does not actually reference any of them through any chain visible to the walk (intra-lib transitive closure, transparent module aliases, [-open]'d modules, or [-opaque]-aware impl-side propagation). Wrapped local libraries, externals, and virtual-impls still fall back to a glob: the walk does not have full visibility into them. Also promotes [transitive-unreferenced-lib.t] (the @nojb reproducer imported from #14332) to assert zero rebuilds of [main] when an unreferenced library's module changes. Closes the residual reported here: https://github.com/ocaml/dune/pull/14116#issuecomment-4319562014 Signed-off-by: Robin Bate Boerop --- src/dune_rules/lib_file_deps.ml | 2 ++ src/dune_rules/lib_file_deps.mli | 9 ++++++ src/dune_rules/module_compilation.ml | 29 +++++++++++++++---- .../transitive-unreferenced-lib.t | 28 ++++++++---------- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index d08da729e32..3bdfab495e3 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -243,6 +243,8 @@ module Lib_index = struct -> Some (lib, m) | _ -> None) ;; + + let is_tight_eligible idx lib = Lib.Set.mem idx.tight_eligible lib end type path_specification = diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 6ff3aeba503..eeada2139b6 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -83,6 +83,15 @@ module Lib_index : sig not exist), as are externals, wrapped locals, and entries with no [Module.t]. *) val lookup_tight_entries : t -> Module_name.t -> (Lib.t * Module.t) list + + (** [is_tight_eligible idx lib] is [true] when [lib] is local, + unwrapped, and every entry carries a known [Module.t]. The + cross-library walk has full visibility into such libraries: + the absence of any of their entry modules from the post-walk + reference set is positive evidence that the consumer does + not reach the library, so the consumer's compile rule does + not need a dep on it. *) + val is_tight_eligible : t -> Lib.t -> bool end type path_specification = diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 031aba24ccc..fcde7d0bff7 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -263,11 +263,25 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin on a subset of [libs] stays within it. *) let* all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in let+ tight_set = cross_lib_tight_set ~lib_index ~for_ ~initial_refs:referenced in - (* Classify [all_libs] against the cross-library tight set: libs - whose entries appear in [tight_set] get per-module deps on - just those entries; the rest (non-tight-eligible libs, - tight-eligible libs with no intersecting entries) fall to a - glob over their objdir. *) + (* Classify [all_libs] against the cross-library tight set into + three buckets: + + - [Some modules] in [tight_modules]: per-module deps on the + entries actually referenced. + + - [None] AND tight-eligible: the cross-library walk had full + visibility (local, unwrapped, every entry has a [Module.t]) + and no entry of the library appears in [tight_set]. The + consumer therefore does not reach this library through any + reference chain visible to the walk, including transitive + module aliases under the [-opaque]-aware impl-side reads. + Drop the library entirely from the consumer's compile rule + deps; the link rule still pulls it in through + [requires_link] for executables/libraries that need it. + + - [None] AND not tight-eligible: wrapped local, external, or + virtual-impl. The walk does not have full visibility, so + fall back to a glob over the library's objdir. *) let tight_modules = Lib_file_deps.Lib_index.tight_modules_per_lib lib_index @@ -281,7 +295,10 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin td (Lib_file_deps.deps_of_entry_modules ~opaque ~cm_kind lib modules) , gl ) - | None -> td, lib :: gl) + | None -> + if Lib_file_deps.Lib_index.is_tight_eligible lib_index lib + then td, gl + else td, lib :: gl) in let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in (), Dep.Set.union tight_deps glob_deps diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t index a0c347f4746..5c6ba4a2765 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t @@ -1,16 +1,16 @@ -Baseline: an intermediate library [libB] declares [(libraries libA)] -but its module [modB] does not actually reference any of [libA]'s -modules. The consumer [main] uses [libB] and so transitively gains -[libA] in its compilation context. +An intermediate library [libB] declares [(libraries libA)] but its +module [modB] does not reference any of [libA]'s modules. The +consumer [main] uses [libB] and so transitively gains [libA] in its +compilation context. -Today every consumer module declares a glob over each transitively- -reached library's public-cmi directory, so editing [modA1.ml] (which -no source file references) re-invalidates [main]. The test records -the current rebuild count of [main] when [modA1.ml] is touched. +The cross-library walk has full visibility into [libA] (local, +unwrapped, every entry has a known module), and finishes without +adding any of [libA]'s entry modules to the reference closure. The +filter therefore drops [libA] from [main]'s compile-rule deps +entirely; the link rule still pulls [libA] in for executables that +need it. -This test is observational: a tighter dependency tracker that drops -unreferenced libraries from compile rules' deps would lower the -count. +Editing [modA1.ml] does not invalidate [main]. $ cat > dune-project < (lang dune 3.0) @@ -37,11 +37,7 @@ count. $ dune build @check -Edit [modA1.ml]. Neither [main.ml] nor [modB.ml] references [modA1] -or any other [libA] module, so a tighter filter could leave [main] -untouched. Today [main] is rebuilt: - $ echo > modA1.ml $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' - 2 + 0 From 3b58ec3f514f0b1e8d32939b70203c27496ab6bc Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 10:46:29 -0700 Subject: [PATCH 59/80] test: promote opaque-mli-change rebuild counts under this PR's filter The exact-count assertions inherited from #14331 ([2] under both profiles) now observe [1] under both, because this PR's per-module filter narrows the consumer's dep set enough that one of the post-edit rebuild events the previous (glob-only) deps would have recorded no longer fires. Promote the assertions to [1] so the test continues to pass and documents the tighter behaviour produced by this PR. Signed-off-by: Robin Bate Boerop --- .../test-cases/per-module-lib-deps/opaque-mli-change.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t index a8bce6eecd6..01de3b10287 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t @@ -50,7 +50,7 @@ left unexported, which would trip warning 32 under dev): $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' - 2 + 1 --- Dev profile (opaque=true): .mli change still rebuilds consumer --- @@ -76,4 +76,4 @@ Add another paired declaration: $ dune build ./main.exe $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' - 2 + 1 From 8e2c7b4a6347752a4fbfd36c09b19ceed30381cc Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 11:23:16 -0700 Subject: [PATCH 60/80] refactor: encode tight-eligibility in the Lib_index entry shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously two places held the same invariant: [build_lib_index] shaped entries with [Some m] for every local lib, and [Lib_index.create] re-examined [Lib_info.wrapped] to populate [tight_eligible]. The [Some m] for wrapped local libs was dead data — every downstream consumer gates on [tight_eligible] first. Move the eligibility decision into [build_lib_index] (it already inspects the [Lib_info]) and have it set [Some m] only for local-unwrapped libs. [None] therefore covers both wrapped-locals and externals: in either case the per-module dep path can't use a [Module.t]. [Lib_index.create] derives [tight_eligible] from the entry shape, which removes the wrapping check from [lib_file_deps.ml] entirely. No behaviour change. The downstream gates on [Some m] and [tight_eligible] produce identical answers because the two predicates were already coextensive on the inputs the system constructs. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 12 +++++++++- src/dune_rules/lib_file_deps.ml | 33 +++++++++------------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 386f53cd310..34c7dd18a81 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -154,9 +154,19 @@ let build_lib_index ~super_context ~all_libs ~for_ = (Lib.Local.of_lib_exn lib) ~for_) ~f:(fun mods -> + (* [Some m] iff the lib is tight-eligible (local + unwrapped): + only then can downstream consumers issue per-module deps + on its [.cmi] files. [None] therefore covers both wrapped + locals and externals — in either case we don't have a + [Module.t] the per-module path can use. *) + let unwrapped = + match Lib_info.wrapped (Lib.info lib) with + | Some (This w) -> not (Wrapped.to_bool w) + | Some (From _) | None -> false + in let entries = List.map (Modules.entry_modules mods) ~f:(fun m -> - Module.name m, lib, Some m) + Module.name m, lib, if unwrapped then Some m else None) in let no_ocamldep_lib = match Modules.as_singleton mods with diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 3bdfab495e3..206a2248acc 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -151,26 +151,13 @@ module Lib_index = struct } ;; - (* Local + unwrapped: every entry has a known [Module.t] we can - feed to [Obj_dir.Module.cm_public_file]. Wrapped libraries - fall back to a directory glob; see the module-level comment - at the top of this file for the ocamldep-granularity reason - and sketches of follow-on work. *) - let is_lib_tight_eligible lib = - Lib.is_local lib - && - match Lib_info.wrapped (Lib.info lib) with - | Some (This w) -> not (Wrapped.to_bool w) - | Some (From _) | None -> - (* [Some (From _)]: wrapped setting inherited from a virtual - library. The [has_virtual_impl] branch higher up in - [lib_deps_for_module] should handle virtual-impl contexts - before we reach here; stay defensive. - [None]: no wrapped information available (e.g. legacy - [dune-package]). *) - false - ;; - + (* Tight-eligibility — local + unwrapped, every entry carries a + [Module.t] — is encoded in the entry shape itself: an entry + [(_, lib, Some _)] means the producer of the index has decided + [lib] is tight-eligible. Wrapped local libs and externals come + in as [(_, _, None)] and don't enter [tight_eligible]. See the + module-level comment at the top of this file for the + ocamldep-granularity reason wrapped libs are excluded. *) let create ?(no_ocamldep = Lib.Set.empty) entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> @@ -179,8 +166,10 @@ module Lib_index = struct | Some xs -> Some ((lib, m) :: xs))) in let tight_eligible = - List.fold_left entries ~init:Lib.Set.empty ~f:(fun acc (_, lib, _) -> - if is_lib_tight_eligible lib then Lib.Set.add acc lib else acc) + List.fold_left entries ~init:Lib.Set.empty ~f:(fun acc (_, lib, m_opt) -> + match m_opt with + | Some _ -> Lib.Set.add acc lib + | None -> acc) in { by_module_name; tight_eligible; no_ocamldep } ;; From f12890ddc1e1270edf821daaa44d8514b7fafa92 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 11:54:00 -0700 Subject: [PATCH 61/80] style: expand wildcard match in skip_lib_deps [skip_lib_deps] was the one match on [Module.kind] introduced by this PR that still collapsed six constructors into [_]. Replace with the exhaustive list so a future addition to [Module.Kind.t] forces a compile-time consideration here. The two other [Module.kind] matches added by this PR ([module_kind_is_filterable], [module_kind_has_readable_ocamldep]) were already exhaustive. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index fcde7d0bff7..8fa16571ff5 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -616,7 +616,7 @@ let build_cm | Alias _ -> not (Modules.With_vlib.is_stdlib_alias (Compilation_context.modules cctx) m) | Wrapped_compat -> true - | _ -> false + | Intf_only | Virtual | Impl | Impl_vmodule | Root | Parameter -> false in let lib_cm_deps = if skip_lib_deps From b038ed271e814e224007ddbda2107781fad0a914 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 15:25:29 -0700 Subject: [PATCH 62/80] fix: synthesise root_module references in the per-module filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A consumer that uses [Root.LibY.foo] reports only [Root] as a top-level ocamldep reference; qualified-path components are not surfaced. The filter therefore needs to look through [Root] to discover [LibY]. But [Root] had no [.d] file (its [Module.kind] short-circuits in [Dep_rules.deps_of] via [is_alias_or_root], an intentional cycle-prevention measure introduced in commit a5d894525 / change #12227), so [read_immediate_deps_raw_of] returned an empty set and the cross-library walk never reached [LibY]. The consumer's compile rule was missing the dep on [LibY]'s cmi: the OCaml path got away with it because [-I] flags still found the cmi at compile time, but Melange's strict- ordering rules surfaced the gap as a missing-cmi error. Synthesise a [.d]-format file for [Root] alongside [root.ml] in [build_root_module], using [Root_module.entries] (the same input [root.ml] is generated from). The file matches what [ocamldep -modules] would output, so the general path in [Ocamldep.read_immediate_deps_raw_of] reads it like any other module's [.d]. [obj_dir.ml] is updated to return a path for [(Root, Immediate _)] while continuing to refuse [(Root, Transitive _)] — the cycle that motivated the original short-circuit lives in [.all-deps] generation, which we still suppress. This keeps [read_dep_m_raw] uniform (no [Module.kind] switch), shrinks [module_kind_has_readable_ocamldep]'s false branch by one entry, and locates the [Root] knowledge in [build_root_module] where [root.ml] is also produced. The regression test for incremental-rebuild behaviour through [Root] lives separately at test/blackbox-tests/test-cases/root-module/incremental-rebuild.t in the prerequisite PR (#14335), so the test is on main as a pure regression guard. Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 43 ++++++++++++++++++++++++---- src/dune_rules/obj_dir.ml | 39 +++++++++++++++++-------- 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 8fa16571ff5..67125674ec4 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -46,14 +46,18 @@ let module_kind_is_filterable m = | Intf_only | Impl | Alias _ -> true ;; -(* A module has user-authored ocamldep output we can read as part of - the filter's reference walk. [Alias _] modules are synthetic stubs - with no useful ocamldep; [Virtual] modules' interfaces do carry - references we want to follow. *) +(* A module whose immediate-deps file we can read as part of the + filter's reference walk. [Alias _] modules are synthetic stubs + with no useful ocamldep output; [Virtual] modules' interfaces + do carry references we want to follow. [Root] modules have a + synthesized [.d] written alongside the generated [root.ml] (no + ocamldep invocation, but the file exists in the same format), + so they participate in the walk through the general read path. + See the standalone change in [build_root_module]. *) let module_kind_has_readable_ocamldep m = match Module.kind m with - | Impl_vmodule | Root | Alias _ | Wrapped_compat | Parameter -> false - | Virtual | Intf_only | Impl -> true + | Impl_vmodule | Alias _ | Wrapped_compat | Parameter -> false + | Virtual | Intf_only | Impl | Root -> true ;; (* Extend [initial_refs] with module names reached through cross- @@ -1019,6 +1023,33 @@ let build_root_module cctx root_module = let+ entries = entries in root_source entries)) in + (* Write a synthesized immediate-deps file in the same format + [ocamldep -modules] would produce. The content is exactly + what running ocamldep on the generated [root.ml] would + output: the entry-module names of every required library. + [Dep_rules.deps_of]'s [is_alias_or_root] short-circuit + remains intact, so no transitive [.all-deps] file is + produced for [Root] — preserving the cycle prevention + introduced in commit a5d894525 (change #12227). The cycle + lives in [.all-deps] generation, not in [.d] reading. *) + let* () = + let obj_dir = Compilation_context.obj_dir cctx in + match Obj_dir.Module.dep obj_dir ~for_ (Immediate (root_module, Ml_kind.Impl)) with + | None -> Memo.return () + | Some d_target -> + Super_context.add_rule + ~loc:Loc.none + sctx + ~dir + (Action_builder.write_file_dyn + d_target + (let open Action_builder.O in + let+ entries = entries in + sprintf + "%s: %s\n" + (Path.Build.basename (Path.as_in_build_dir_exn file)) + (String.concat ~sep:" " (List.map entries ~f:Module_name.to_string)))) + in build_module cctx root_module ;; diff --git a/src/dune_rules/obj_dir.ml b/src/dune_rules/obj_dir.ml index e69809125aa..7f446863748 100644 --- a/src/dune_rules/obj_dir.ml +++ b/src/dune_rules/obj_dir.ml @@ -638,18 +638,33 @@ module Module = struct end let dep t dep ~for_ = - match (dep : Dep.t) with - | Immediate (m, _) | Transitive (m, _) -> - (match Module.kind m with - | Module.Kind.Alias _ | Root -> None - | _ -> - let dir = - match for_ with - | Compilation_mode.Ocaml -> obj_dir t - | Melange -> obj_dir t - in - let name = Dep.basename dep in - Some (Path.Build.relative dir name)) + let m = + match (dep : Dep.t) with + | Immediate (m, _) | Transitive (m, _) -> m + in + let none = + match Module.kind m, dep with + | Module.Kind.Alias _, _ -> true + (* [Root] has a synthesized [.d] written alongside [root.ml] + by [build_root_module] (one line of [: ] + format), but no transitive [.all-deps] is produced for + it: the cycle that motivated [Dep_rules.deps_of]'s + [is_alias_or_root] short-circuit (see commit a5d894525, + change #12227) lives in [.all-deps] generation, not in + [.d]. *) + | Root, Transitive _ -> true + | _ -> false + in + if none + then None + else ( + let dir = + match for_ with + | Compilation_mode.Ocaml -> obj_dir t + | Melange -> obj_dir t + in + let name = Dep.basename dep in + Some (Path.Build.relative dir name)) ;; module L = struct From 4d9b5208567d378d6622c742a3df6c7073efefd4 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 15:29:15 -0700 Subject: [PATCH 63/80] test: lock in incremental-rebuild for root_module-aliased deps Add a regression test asserting that a consumer using [(root_module ...)] to alias a dependency rebuilds when the dependency's [.mli] changes, and does not rebuild when only the dependency's [.ml] changes. The property holds today via dune's existing glob-over-objdir mechanism; the test guards it as a specification against future inter-library-dependency-tracking changes (notably the per-module filter being developed in Signed-off-by: Robin Bate Boerop #14116). --- .../root-module/incremental-rebuild.t | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 test/blackbox-tests/test-cases/root-module/incremental-rebuild.t diff --git a/test/blackbox-tests/test-cases/root-module/incremental-rebuild.t b/test/blackbox-tests/test-cases/root-module/incremental-rebuild.t new file mode 100644 index 00000000000..f8022143bf8 --- /dev/null +++ b/test/blackbox-tests/test-cases/root-module/incremental-rebuild.t @@ -0,0 +1,53 @@ +A consumer library that uses [(root_module ...)] to alias one of +its dependencies is rebuilt when the dependency's interface +changes, and not rebuilt when only the dependency's +implementation changes. This locks in the incremental-rebuild +property for [Root]-aliased dependencies that any future change +to dune's inter-library-dependency tracking must preserve. + + $ cat > dune-project < (lang dune 3.0) + > EOF + + $ mkdir lib1 lib2 + $ cat > lib1/dune < (library (name lib1)) + > EOF + $ cat > lib1/lib1.ml < let extra = 0 + > let greeting = "hello-" ^ string_of_int extra + > EOF + $ cat > lib1/lib1.mli < val greeting : string + > EOF + + $ cat > lib2/dune < (library (name lib2) (libraries lib1) (root_module root)) + > EOF + $ cat > lib2/lib2.ml < let () = print_endline Root.Lib1.greeting + > EOF + + $ dune build @check + +Editing only [lib1.mli] (the [.ml] is unchanged) changes +[lib1.cmi] and must invalidate [lib2]: + + $ cat > lib1/lib1.mli < val greeting : string + > val extra : int + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("\\.lib2\\.objs/"))] | length > 0' + true + +Editing only [lib1.ml] (no [.mli] change) leaves [lib1.cmi] +untouched, so [lib2] is not rebuilt: + + $ cat > lib1/lib1.ml < let extra = 1 + > let greeting = "hello-" ^ string_of_int extra + > EOF + $ dune build @check + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("\\.lib2\\.objs/"))] | length' + 0 From 3b930f84f07dbad885ec601da683cb2c1035b7af Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 16:42:06 -0700 Subject: [PATCH 64/80] refactor: drop the basename check from ocamldep output parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [parse_deps_exn] validated that the [:] LHS of ocamldep's output matched the source we expected. Per @rgrinberg (#14116 r3142727614), the check is unhelpful: in the only caller ([deps_of]'s rule action) the same [Module.File.path source] value is both fed to ocamldep ([Dep ...]) and passed as the parser's [~file] argument, so the basename equality is structural and can't fail unless ocamldep itself emits a wrong LHS — a bug in ocamldep, not in dune. Drop [parse_deps_exn] and rename the lenient variant to [parse_deps]. Both callers (the producer-side [deps_of] and the reader-side [read_immediate_deps_parsed]) now share one function. The structural check (exactly one line, one colon) remains and continues to catch gross format corruption. Net: -32 lines, no behaviour change. Signed-off-by: Robin Bate Boerop --- src/dune_rules/ocamldep.ml | 48 +++++++------------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/src/dune_rules/ocamldep.ml b/src/dune_rules/ocamldep.ml index db0e7c5ef0a..6c207651364 100644 --- a/src/dune_rules/ocamldep.ml +++ b/src/dune_rules/ocamldep.ml @@ -94,37 +94,11 @@ let invalid_ocamldep_output file lines = ] ;; -let parse_deps_exn ~file lines = - match lines with - | [] | _ :: _ :: _ -> invalid_ocamldep_output file lines - | [ line ] -> - (match String.lsplit2 line ~on:':' with - | None -> invalid_ocamldep_output file lines - | Some (basename, deps) -> - let basename = Filename.basename basename in - if basename <> Path.basename file then invalid_ocamldep_output file lines; - String.extract_blank_separated_words deps) -;; - -(* Like [parse_deps_exn] but without the left-hand basename check. - Callers that only read [.d] files (rather than producing them) - may hold a raw [Module.t] whose [Module.source] path differs - from the pp-transformed source the producing rule fed to - ocamldep — the basename will not match even though the output - is valid. - - Trade-off: this trades a cross-consistency assertion (the rule - that produced the file must have used the same source path the - reader now holds) for compatibility with cross-library readers - that don't participate in rule production. The producing side - still validates via [parse_deps_exn] in [deps_of]'s action, so - bogus ocamldep output is still caught at the producing rule; but - a subtle future drift between the two sources of [Module.t] (e.g. - a refactor that changes how [Module.source] is resolved for - cross-library reads) would now slip past the reader silently. - The structural check remaining here (exactly one line, one colon) - continues to catch gross format corruption. *) -let parse_deps_lenient ~file lines = +(* Parse the space-separated module names from one line of + [ocamldep -modules] output. The structural check (exactly one + line, one colon) catches gross format corruption; the basename + on the LHS is ignored. *) +let parse_deps ~file lines = match lines with | [] | _ :: _ :: _ -> invalid_ocamldep_output file lines | [ line ] -> @@ -183,7 +157,7 @@ let deps_of ~sandbox ~modules ~sctx ~dir ~obj_dir ~ml_kind ~for_ unit = (let+ immediate_deps = Path.build ocamldep_output |> Action_builder.lines_of - >>| parse_deps_exn ~file:(Module.File.path source) + >>| parse_deps ~file:(Module.File.path source) >>| parse_module_names ~dir ~unit ~modules >>| Stdlib.( @ ) (Modules.With_vlib.implicit_deps modules ~of_:unit) in @@ -217,13 +191,7 @@ let read_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit = builder for each .d file is cached by path so that [read_immediate_deps_of] and [read_immediate_deps_raw_of] (which may be called many times for the same module) share one memoized - [Action_builder.t] instance per file. - - Uses [parse_deps_lenient] rather than [parse_deps_exn]: the - producing rule already validates the basename in [deps_of]; by - the time a reader sees the [.d] file it is known-valid, and - cross-lib readers may hold a raw [Module.t] whose source path - does not match the pp-transformed source the rule used. *) + [Action_builder.t] instance per file. *) let read_immediate_deps_parsed = let cache = Table.create (module Path.Build) 64 in fun ~obj_dir ~ml_kind ~for_ unit -> @@ -239,7 +207,7 @@ let read_immediate_deps_parsed = let builder = Action_builder.lines_of (Path.build ocamldep_output) |> Action_builder.map ~f:(fun lines -> - Some (parse_deps_lenient ~file:(Module.File.path source) lines)) + Some (parse_deps ~file:(Module.File.path source) lines)) |> Action_builder.memoize (Path.Build.to_string ocamldep_output) in Table.set cache ocamldep_output builder; From 5acaa48fd6f62e7df377274e24c07e40e050c442 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 16:47:13 -0700 Subject: [PATCH 65/80] refactor: make Lib_index.create's no_ocamldep parameter required Per @rgrinberg (#14116 r3142730132): the [?no_ocamldep] optional parameter has exactly one caller ([Compilation_context.build_lib_index]) and that caller always passes the argument. The default of [Lib.Set.empty] was never used in practice. Drop the [?] so the type signature reflects the actual call pattern. Signed-off-by: Robin Bate Boerop --- src/dune_rules/lib_file_deps.ml | 2 +- src/dune_rules/lib_file_deps.mli | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 206a2248acc..bd7614186d2 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -158,7 +158,7 @@ module Lib_index = struct in as [(_, _, None)] and don't enter [tight_eligible]. See the module-level comment at the top of this file for the ocamldep-granularity reason wrapped libs are excluded. *) - let create ?(no_ocamldep = Lib.Set.empty) entries = + let create ~no_ocamldep entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> Module_name.Map.update map name ~f:(function diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index eeada2139b6..8853cb45412 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -46,7 +46,7 @@ module Lib_index : sig library walk in [module_compilation] must not try to read [.d] files for those libs. *) val create - : ?no_ocamldep:Lib.Set.t + : no_ocamldep:Lib.Set.t -> (Module_name.t * Lib.t * Module.t option) list -> t From 1bf83337bb47d54e0496a14004c407ef999dad0f Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 16:52:15 -0700 Subject: [PATCH 66/80] refactor: drop module_kind_has_readable_ocamldep predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per @rgrinberg (#14116 r3142735120): the predicate duplicated logic that should live in [Obj_dir.Module.dep] alone — when the latter returns [None] (or [Module.source ~ml_kind] is [None]), [Ocamldep.read_immediate_deps_parsed] already returns [None] and [read_immediate_deps_raw_of] already returns [Module_name.Set.empty]. The predicate's filtering of [Wrapped_compat] was load-bearing in principle: [Module.source] returns [Some] for a synthesized path, [Obj_dir.Module.dep] returned [Some] for it too, but [Dep_rules.deps_of_module] short-circuits to [wrapped_compat_deps] without registering any ocamldep rule — so the [.d] file the path points to is never produced. Reading it would have failed at build time with "no rule to build target" had Wrapped_compat ever appeared in the consumer's [trans_deps]. Empirically that doesn't happen in tested scenarios, but it's a fragile invariant. Make [Obj_dir.Module.dep] the single source of truth: extend the [none] match to include [Wrapped_compat] for both [Immediate] and [Transitive], matching the actual rule landscape. Then drop the predicate. Other kinds the predicate filtered ([Impl_vmodule], [Parameter]) are over-conservative entries — those modules do have ocamldep rules registered ([Impl_vmodule] via [Impl_of_virtual_module]'s recursion into [Normal m]/[Imported_from_vlib m]; [Parameter Intf] via the catch-all in [deps_of_module]). After this change those reads return real ocamldep output instead of the predicate's empty set; the regression suites (per-module-lib-deps, root-module, melange, alias, virtual-libraries, wrapped-transition) pass under the new behaviour. Net: -14 lines from [module_compilation.ml], +5 lines in [obj_dir.ml]'s [none] match (one arm + a 3-line comment). Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 35 +++++++--------------------- src/dune_rules/obj_dir.ml | 4 ++++ 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 67125674ec4..e7a6ab6bf81 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -46,20 +46,6 @@ let module_kind_is_filterable m = | Intf_only | Impl | Alias _ -> true ;; -(* A module whose immediate-deps file we can read as part of the - filter's reference walk. [Alias _] modules are synthetic stubs - with no useful ocamldep output; [Virtual] modules' interfaces - do carry references we want to follow. [Root] modules have a - synthesized [.d] written alongside the generated [root.ml] (no - ocamldep invocation, but the file exists in the same format), - so they participate in the walk through the general read path. - See the standalone change in [build_root_module]. *) -let module_kind_has_readable_ocamldep m = - match Module.kind m with - | Impl_vmodule | Alias _ | Wrapped_compat | Parameter -> false - | Virtual | Intf_only | Impl | Root -> true -;; - (* Extend [initial_refs] with module names reached through cross- library ocamldep. Walk tight-eligible entry modules breadth-first: gather all [(lib, entry module)] pairs named by the frontier, read @@ -225,18 +211,15 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin | Ocaml (Cmi | Cmo) | Melange _ -> false in let read_dep_m_raw dep_m ~is_consumer = - if not (module_kind_has_readable_ocamldep dep_m) - then Action_builder.return Module_name.Set.empty - else - let* impl_deps = - if need_impl_deps_of dep_m ~is_consumer - then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m - else Action_builder.return Module_name.Set.empty - in - let+ intf_deps = - Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m - in - Module_name.Set.union impl_deps intf_deps + let* impl_deps = + if need_impl_deps_of dep_m ~is_consumer + then Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Impl ~for_ dep_m + else Action_builder.return Module_name.Set.empty + in + let+ intf_deps = + Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind:Intf ~for_ dep_m + in + Module_name.Set.union impl_deps intf_deps in let* m_raw = read_dep_m_raw m ~is_consumer:true in let* trans_raw = diff --git a/src/dune_rules/obj_dir.ml b/src/dune_rules/obj_dir.ml index 7f446863748..69fd28bfc44 100644 --- a/src/dune_rules/obj_dir.ml +++ b/src/dune_rules/obj_dir.ml @@ -645,6 +645,10 @@ module Module = struct let none = match Module.kind m, dep with | Module.Kind.Alias _, _ -> true + (* [Wrapped_compat] has synthesized deps in + [Dep_rules.deps_of_module] — no ocamldep rule is + registered and no [.d]/[.all-deps] file is produced. *) + | Wrapped_compat, _ -> true (* [Root] has a synthesized [.d] written alongside [root.ml] by [build_root_module] (one line of [: ] format), but no transitive [.all-deps] is produced for From 24053555ccecc1c4a412983ca74a071415020431 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sat, 25 Apr 2026 18:47:26 -0700 Subject: [PATCH 67/80] refactor: build lib_index from direct_requires only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per @art-w (#14116 r3129495407): hidden requires don't belong in [lib_index]. The index resolves names that appear in ocamldep output, and user code can only legitimately reference modules of libraries the user has committed to as direct deps. The cross-library BFS in [cross_lib_tight_set] (added later) also feeds names from cross-library [.d] outputs through the index — but those names ride along on transparent alias chains the user implicitly accepted by depending on direct deps. Hidden libs are precisely the libraries the user has *not* committed to; tracking them with per-module precision crosses the user's commitment boundary. In the post-walk classification, hidden libs that aren't in [lib_index] fall to glob fallback rather than per-module deps. This is the right behaviour for libs outside the user's commitment surface: correctness is preserved (glob covers everything), and we no longer optimise invalidation for a category the user didn't ask us to track. Verified across per-module-lib-deps, root-module, alias, melange, virtual-libraries, and wrapped-transition test suites. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 34c7dd18a81..022362d8fff 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -138,10 +138,20 @@ let parameters_main_modules parameters = [Dep_rules.skip_ocamldep] (unwrapped single-module stanzas without direct lib deps) are collected into [no_ocamldep] so the cross- library walk does not try to read their nonexistent [.d] files. *) -let build_lib_index ~super_context ~all_libs ~for_ = +(* [libs] should be the consumer's [direct_requires] — the libraries + whose entry modules can legitimately appear in the consumer's + ocamldep output (and, after the cross-library walk extends through + them, in cross-library ocamldep outputs that the BFS surfaces). + Hidden requires are intentionally excluded: the user has not + committed to them as direct dependencies, so the per-module + filter does not track them with per-module precision — they + fall to glob fallback in [lib_deps_for_module]'s post-walk + classification, which is the right behaviour for libs outside + the user's commitment surface. *) +let build_lib_index ~super_context ~libs ~for_ = let open Resolve.Memo.O in let+ per_lib = - Resolve.Memo.List.map all_libs ~f:(fun lib -> + Resolve.Memo.List.map libs ~f:(fun lib -> match Lib_info.entry_modules (Lib.info lib) ~for_ with | External (Ok names) -> Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None), None) @@ -296,9 +306,8 @@ let create ; lib_index = Memo.lazy_ (fun () -> let open Resolve.Memo.O in - let* direct = direct_requires in - let* hidden = hidden_requires in - build_lib_index ~super_context ~all_libs:(direct @ hidden) ~for_) + let* libs = direct_requires in + build_lib_index ~super_context ~libs ~for_) ; has_virtual_impl = Memo.lazy_ (fun () -> let open Resolve.Memo.O in From dca0247eea3e15134bf3a27ffc690026aec06d2e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 16:52:22 -0700 Subject: [PATCH 68/80] fix: walk wrapped libs' children for cross-lib alias chains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a consumer references a module of a wrapped local sibling library (whether via [-open Wrapper] in flags or via a qualified [Wrapper.Child.x] in source), the per-module dep filter must follow alias chains through the wrapped lib's children to discover transitive cross-library references such as [include Vendored_pprint] or [module Re = Lib_a.Original_name]. Without this, the consumer's tracked deps drop the transitively- reached lib entirely, leaving its [.cmi] changes unobserved and incremental builds unsound. The fix has three coordinated parts in [compilation_context.ml] ([build_lib_index]), [lib_file_deps.ml/mli] ([Lib_index]), and [module_compilation.ml] ([cross_lib_tight_set]): 1. [build_lib_index] now emits [Some m] for every local lib's entry modules, regardless of wrapping. Wrapping is tracked separately through [unwrapped_local : Lib.Set.t]. For wrapped local libs, the wrapper's entry is also augmented with a multi-entry index that maps the wrapper's name to each child module — when the BFS encounters the wrapper's name, it walks directly into each child's ocamldep without relying on the wrapper's own [.d] (which is absent for auto-generated wrappers because dune's [Dep_rules.skip_ocamldep] fires on [Module.kind = Alias]). 2. [Lib_index.create] now takes [~unwrapped_local] explicitly instead of deriving tight-eligibility from the entry encoding, and exposes [lookup_walkable_entries] (renamed from [lookup_tight_entries]). The walkable lookup returns every entry with a [Module.t] that isn't in [no_ocamldep], without the prior tight-eligibility filter — wrapped local libs are now walkable for the BFS even though they remain non-tight for the per-module dep classification. 3. [cross_lib_tight_set] uses [lookup_walkable_entries] and propagates the BFS through wrapped libs' children. The tight-classification fold downstream still uses [tight_eligible] to decide tight vs glob, so wrapped libs continue to fall back to glob deps for invalidation. Validated by the inherited baseline tests: [wrapped-reexport-via-open-flag.t] (hand-written wrapper, alias) flips from count=0 to 1; [auto-wrapped-child-reexport.t] (auto-generated wrapper, child alias) flips from count=0 to 1. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 68 ++++++++++++++++++++++----- src/dune_rules/lib_file_deps.ml | 35 +++++++------- src/dune_rules/lib_file_deps.mli | 27 +++++++---- src/dune_rules/module_compilation.ml | 25 ++++++---- 4 files changed, 109 insertions(+), 46 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 022362d8fff..aa583e83397 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -154,7 +154,7 @@ let build_lib_index ~super_context ~libs ~for_ = Resolve.Memo.List.map libs ~f:(fun lib -> match Lib_info.entry_modules (Lib.info lib) ~for_ with | External (Ok names) -> - Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None), None) + Resolve.Memo.return (List.map names ~f:(fun n -> n, lib, None), None, None) | External (Error e) -> Resolve.Memo.of_result (Error e) | Local -> Resolve.Memo.lift_memo @@ -164,19 +164,58 @@ let build_lib_index ~super_context ~libs ~for_ = (Lib.Local.of_lib_exn lib) ~for_) ~f:(fun mods -> - (* [Some m] iff the lib is tight-eligible (local + unwrapped): - only then can downstream consumers issue per-module deps - on its [.cmi] files. [None] therefore covers both wrapped - locals and externals — in either case we don't have a - [Module.t] the per-module path can use. *) + (* Local libs always carry [Some m] for each entry so the + cross-library walk can read the entry's ocamldep. + Whether to issue per-module deps on the entry's [.cmi] + is a separate decision tracked by [unwrapped_local]: + only unwrapped local libs are tight-eligible. Wrapped + local libs are walkable (the wrapper's ocamldep + references children by mangled name and the BFS + expands through them) but classified as glob for + invalidation. *) let unwrapped = match Lib_info.wrapped (Lib.info lib) with | Some (This w) -> not (Wrapped.to_bool w) | Some (From _) | None -> false in + let entry_modules = Modules.entry_modules mods in + let entry_entries = + List.map entry_modules ~f:(fun m -> + Module.name m, lib, Some m) + in let entries = - List.map (Modules.entry_modules mods) ~f:(fun m -> - Module.name m, lib, if unwrapped then Some m else None) + if unwrapped + then entry_entries + else + (* Auto-wrapped libs: dune's auto-generated wrapper + has [Module.kind = Alias], which means + [Dep_rules.skip_ocamldep] fires and no [.d] is + generated for the wrapper. The BFS thus cannot + learn the wrapper's children by reading its + ocamldep. Instead, also index each child under + the WRAPPER's name (multi-entry), so that + whenever the BFS encounters the wrapper's name + in the frontier (via [-open Wrapper] in the + consumer's flags or via a qualified + [Wrapper.Child.x] reference), it walks + directly into each child's ocamldep. *) + let entry_obj_names = + List.map entry_modules ~f:Module.obj_name + in + let child_modules = + Modules.fold_user_available mods ~init:[] ~f:(fun m acc -> + let obj_name = Module.obj_name m in + if List.exists entry_obj_names ~f:(fun n -> + Module_name.Unique.equal n obj_name) + then acc + else m :: acc) + in + let child_entries_under_wrapper = + List.concat_map entry_modules ~f:(fun wrapper -> + List.map child_modules ~f:(fun child -> + Module.name wrapper, lib, Some child)) + in + entry_entries @ child_entries_under_wrapper in let no_ocamldep_lib = match Modules.as_singleton mods with @@ -184,11 +223,16 @@ let build_lib_index ~super_context ~libs ~for_ = Some lib | _ -> None in - entries, no_ocamldep_lib))) + entries, no_ocamldep_lib, (if unwrapped then Some lib else None)))) + in + let entries = List.concat_map per_lib ~f:(fun (e, _, _) -> e) in + let no_ocamldep = + List.filter_map per_lib ~f:(fun (_, n, _) -> n) |> Lib.Set.of_list + in + let unwrapped_local = + List.filter_map per_lib ~f:(fun (_, _, u) -> u) |> Lib.Set.of_list in - let entries = List.concat_map per_lib ~f:fst in - let no_ocamldep = List.filter_map per_lib ~f:snd |> Lib.Set.of_list in - Lib_file_deps.Lib_index.create ~no_ocamldep entries + Lib_file_deps.Lib_index.create ~no_ocamldep ~unwrapped_local entries ;; let create diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index bd7614186d2..40ec7343ed0 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -151,27 +151,28 @@ module Lib_index = struct } ;; - (* Tight-eligibility — local + unwrapped, every entry carries a - [Module.t] — is encoded in the entry shape itself: an entry - [(_, lib, Some _)] means the producer of the index has decided - [lib] is tight-eligible. Wrapped local libs and externals come - in as [(_, _, None)] and don't enter [tight_eligible]. See the + (* Two orthogonal predicates on a lib's entries are tracked here. + [Some m] in an entry means we have the entry's [Module.t] — + true for every local lib (wrapped or not), false for externals; + this is what lets the cross-library walk read the entry's + ocamldep. [tight_eligible] (built from the caller-supplied + [unwrapped_local]) means the lib is local AND unwrapped, so + downstream consumers can issue per-module deps on its [.cmi] + files. Wrapped local libs are walkable but not tight-eligible — + their auto-generated wrapper's ocamldep references children by + mangled name and the BFS expands through them, but the lib + itself classifies as glob for invalidation. See the module-level comment at the top of this file for the - ocamldep-granularity reason wrapped libs are excluded. *) - let create ~no_ocamldep entries = + ocamldep-granularity reason wrapped libs are excluded from the + tight-eligible class. *) + let create ~no_ocamldep ~unwrapped_local entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> Module_name.Map.update map name ~f:(function | None -> Some [ lib, m ] | Some xs -> Some ((lib, m) :: xs))) in - let tight_eligible = - List.fold_left entries ~init:Lib.Set.empty ~f:(fun acc (_, lib, m_opt) -> - match m_opt with - | Some _ -> Lib.Set.add acc lib - | None -> acc) - in - { by_module_name; tight_eligible; no_ocamldep } + { by_module_name; tight_eligible = unwrapped_local; no_ocamldep } ;; type classified = @@ -221,15 +222,13 @@ module Lib_index = struct | _ -> acc)) ;; - let lookup_tight_entries idx name = + let lookup_walkable_entries idx name = match Module_name.Map.find idx.by_module_name name with | None -> [] | Some entries -> List.filter_map entries ~f:(fun (lib, m_opt) -> match m_opt with - | Some m - when Lib.Set.mem idx.tight_eligible lib && not (Lib.Set.mem idx.no_ocamldep lib) - -> Some (lib, m) + | Some m when not (Lib.Set.mem idx.no_ocamldep lib) -> Some (lib, m) | _ -> None) ;; diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 8853cb45412..3833950ab0b 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -36,9 +36,16 @@ module Lib_index : sig val empty : t (** Create an index. Each entry carries the [Module.t] of the entry - module when it is known ([Some] for local libraries; [None] for - externals). Libraries whose entries all carry a [Module.t] and - which are unwrapped are eligible for per-module deps. + module when it is known ([Some] for local libraries, wrapped + or not; [None] for externals). + + [unwrapped_local] is the set of libs that are local and + unwrapped — the tight-eligible class. The cross-library walk + reads ocamldep for any local lib's entries (via + [lookup_walkable_entries], including wrapped libs' children + indexed under their mangled names); per-module dep + classification is restricted to [tight_eligible] (see + [is_tight_eligible]). [no_ocamldep] names local libs whose ocamldep output is short-circuited (single-module stanzas without library @@ -47,6 +54,7 @@ module Lib_index : sig [.d] files for those libs. *) val create : no_ocamldep:Lib.Set.t + -> unwrapped_local:Lib.Set.t -> (Module_name.t * Lib.t * Module.t option) list -> t @@ -77,12 +85,15 @@ module Lib_index : sig -> referenced_modules:Module_name.Set.t -> Module.t list Lib.Map.t - (** [lookup_tight_entries idx name] returns [(lib, entry module)] + (** [lookup_walkable_entries idx name] returns [(lib, entry module)] pairs used by the cross-library walk in [module_compilation]. - Libraries in [no_ocamldep] are excluded (their [.d] files do - not exist), as are externals, wrapped locals, and entries with - no [Module.t]. *) - val lookup_tight_entries : t -> Module_name.t -> (Lib.t * Module.t) list + Includes wrapped local libs' wrappers and their children + (indexed under their mangled names) — the BFS reads each + walkable entry's ocamldep to follow alias chains across + library boundaries. Libraries in [no_ocamldep] are excluded + (their [.d] files do not exist); externals are excluded + because no [Module.t] is available. *) + val lookup_walkable_entries : t -> Module_name.t -> (Lib.t * Module.t) list (** [is_tight_eligible idx lib] is [true] when [lib] is local, unwrapped, and every entry carries a known [Module.t]. The diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index e7a6ab6bf81..ab6778656fc 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -47,14 +47,23 @@ let module_kind_is_filterable m = ;; (* Extend [initial_refs] with module names reached through cross- - library ocamldep. Walk tight-eligible entry modules breadth-first: - gather all [(lib, entry module)] pairs named by the frontier, read - each pair's impl and intf ocamldep, and union the raw names into - the frontier. Iterate until no new names appear. + library ocamldep. Walk every walkable entry module breadth-first + — wrappers and children of wrapped local libs alike: gather all + [(lib, entry module)] pairs named by the frontier, read each + pair's impl and intf ocamldep, and union the raw names into the + frontier. Iterate until no new names appear. - Libraries that are not tight-eligible (wrapped locals, externals, - virtual-impls) are skipped by [lookup_tight_entries]. Chains that - pass through them terminate and the consumer falls back to a glob + Wrapped local libs are walkable too. Dune's auto-generated + wrapper contains [module Foo = Lib__Foo] for each child; + reading the wrapper's ocamldep emits the mangled child names, + and [lookup_walkable_entries] resolves each to the child's + [Module.t] (also indexed under its mangled name). The BFS then + descends into the child's own ocamldep, surfacing transitive + cross-library references like [include Vendored_pprint] that + the wrapper alone would not reveal. Externals and libs in + [no_ocamldep] are excluded by [lookup_walkable_entries] — + they have no readable [.d] file. Chains passing through + excluded libs terminate and the consumer falls back to a glob on the unreached libs. Cycles in the module-reference graph terminate on the [seen] set. @@ -78,7 +87,7 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = else ( let pairs = Module_name.Set.fold frontier ~init:[] ~f:(fun name acc -> - Lib_file_deps.Lib_index.lookup_tight_entries lib_index name @ acc) + Lib_file_deps.Lib_index.lookup_walkable_entries lib_index name @ acc) in let* discovered = union_module_name_sets_mapped pairs ~f:read_entry_deps in let seen = Module_name.Set.union seen frontier in From 1613258034d87fee3acd1630fb5901de6273a11c Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 17:36:02 -0700 Subject: [PATCH 69/80] refactor: unify has_library_deps predicate for libraries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review items on this commit, all on #14116: 1. Item 3 (the structural issue, was a CI-breaking bug). Both [Compilation_context.create] (when building a library cctx) and [Compilation_context.build_lib_index] (predicting another lib's [.d] presence from a different cctx) now route their [has_library_deps] view through one helper, [Dep_rules.has_library_deps_of_lib], which derives the answer from [Lib.requires lib ~for_]. Resolution failures fall back to has-deps=true (conservative — keeps the lib's [.d] file in place and the cross-library walk's expectation in agreement). The old form had two independent derivations: [Lib_info.requires] (declared) on the index side and [direct + hidden] (resolved) on the cctx side. They diverged when a library's declared deps resolved to empty — [(libraries (re_export bigarray))] on OCaml 5 (bigarray is no longer a separate library) and [(libraries (select ... from (missing_backend -> ...) (-> file)))] (select falls through to the file default). After the prior commit made wrapped libs walkable, single-module wrapped libs in this state began getting walked; the BFS read [.d] files the cctx never asked dune to produce, hitting "No rule found for .X.objs/X.impl.d" in CI on the Tests (Nix) job. Routing both sides through one predicate closes the bug class structurally — future drift in the condition is impossible without changing the helper. [Compilation_context.create] gains an optional [?lib] param; [Lib_rules.cctx] threads [~local_lib] through and supplies it. Non-library cctxes (executables, tests, melange emit) still fall back to the cctx-local direct/hidden derivation, matching what they had before — they aren't entries in any lib_index, so there's no prediction to keep in sync. 2. Item 11. Label the [entries] argument of [Lib_file_deps.Lib_index.create] for consistency with the other parameters. 3. Item 9. Merge the two consecutive comment blocks above [build_lib_index] into one, dropping a stale claim about hidden libraries being included (the function takes [direct_requires]; hidden libs are excluded, as the second block already documented). 4. Update the TODO/comment in [Dep_rules.skip_ocamldep] to describe the now-shared predicate. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 204 +++++++++++++------------ src/dune_rules/compilation_context.mli | 1 + src/dune_rules/dep_rules.ml | 41 +++-- src/dune_rules/dep_rules.mli | 9 ++ src/dune_rules/lib_file_deps.ml | 2 +- src/dune_rules/lib_file_deps.mli | 2 +- src/dune_rules/lib_rules.ml | 6 +- 7 files changed, 153 insertions(+), 112 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index aa583e83397..b414b2c990a 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -125,29 +125,27 @@ let parameters_main_modules parameters = [ "param", Lib.to_dyn param ]) ;; -(* Build a [Lib_index] from [all_libs] for the per-module inter-library +(* Build a [Lib_index] from [libs] for the per-module inter-library dependency filter. - For each library, entry module names are collected (for wrapped - libraries, the wrapper; for unwrapped, each public module). Hidden - libraries are included so that consumers which transitively - reference a hidden library's entry module still produce a build - dependency on it. - - Local libraries whose ocamldep is short-circuited by - [Dep_rules.skip_ocamldep] (unwrapped single-module stanzas without - direct lib deps) are collected into [no_ocamldep] so the cross- - library walk does not try to read their nonexistent [.d] files. *) -(* [libs] should be the consumer's [direct_requires] — the libraries + [libs] should be the consumer's [direct_requires] — the libraries whose entry modules can legitimately appear in the consumer's ocamldep output (and, after the cross-library walk extends through them, in cross-library ocamldep outputs that the BFS surfaces). Hidden requires are intentionally excluded: the user has not - committed to them as direct dependencies, so the per-module - filter does not track them with per-module precision — they - fall to glob fallback in [lib_deps_for_module]'s post-walk - classification, which is the right behaviour for libs outside - the user's commitment surface. *) + committed to them as direct dependencies, so the per-module filter + does not track them with per-module precision — they fall to glob + fallback in [lib_deps_for_module]'s post-walk classification, which + is the right behaviour for libs outside the user's commitment + surface. + + For each library, entry module names are collected (for wrapped + libraries, the wrapper plus a multi-entry index of children under + the wrapper's name; for unwrapped, each public module). Local + libraries whose ocamldep is short-circuited by + [Dep_rules.skip_ocamldep] (unwrapped single-module stanzas without + direct lib deps) are collected into [no_ocamldep] so the cross- + library walk does not try to read their nonexistent [.d] files. *) let build_lib_index ~super_context ~libs ~for_ = let open Resolve.Memo.O in let+ per_lib = @@ -158,72 +156,74 @@ let build_lib_index ~super_context ~libs ~for_ = | External (Error e) -> Resolve.Memo.of_result (Error e) | Local -> Resolve.Memo.lift_memo - (Memo.map - (Dir_contents.modules_of_local_lib - super_context - (Lib.Local.of_lib_exn lib) - ~for_) - ~f:(fun mods -> - (* Local libs always carry [Some m] for each entry so the - cross-library walk can read the entry's ocamldep. - Whether to issue per-module deps on the entry's [.cmi] - is a separate decision tracked by [unwrapped_local]: - only unwrapped local libs are tight-eligible. Wrapped - local libs are walkable (the wrapper's ocamldep - references children by mangled name and the BFS - expands through them) but classified as glob for - invalidation. *) - let unwrapped = - match Lib_info.wrapped (Lib.info lib) with - | Some (This w) -> not (Wrapped.to_bool w) - | Some (From _) | None -> false + (let open Memo.O in + let* mods = + Dir_contents.modules_of_local_lib + super_context + (Lib.Local.of_lib_exn lib) + ~for_ + in + (* Predict the lib's own [has_library_deps] via the same + helper its cctx uses; routing both sites through + [Dep_rules.has_library_deps_of_lib] is what keeps the + skip-decision and the walk's prediction in sync. *) + let+ has_resolved_deps = Dep_rules.has_library_deps_of_lib lib ~for_ in + (* Local libs always carry [Some m] for each entry so the + cross-library walk can read the entry's ocamldep. + Whether to issue per-module deps on the entry's [.cmi] + is a separate decision tracked by [unwrapped_local]: + only unwrapped local libs are tight-eligible. Wrapped + local libs are walkable (the wrapper's ocamldep + references children by mangled name and the BFS + expands through them) but classified as glob for + invalidation. *) + let unwrapped = + match Lib_info.wrapped (Lib.info lib) with + | Some (This w) -> not (Wrapped.to_bool w) + | Some (From _) | None -> false + in + let entry_modules = Modules.entry_modules mods in + let entry_entries = + List.map entry_modules ~f:(fun m -> Module.name m, lib, Some m) + in + let entries = + if unwrapped + then entry_entries + else + (* Auto-wrapped libs: dune's auto-generated wrapper + has [Module.kind = Alias], which means + [Dep_rules.skip_ocamldep] fires and no [.d] is + generated for the wrapper. The BFS thus cannot + learn the wrapper's children by reading its + ocamldep. Instead, also index each child under + the WRAPPER's name (multi-entry), so that + whenever the BFS encounters the wrapper's name + in the frontier (via [-open Wrapper] in the + consumer's flags or via a qualified + [Wrapper.Child.x] reference), it walks + directly into each child's ocamldep. *) + let entry_obj_names = List.map entry_modules ~f:Module.obj_name in + let child_modules = + Modules.fold_user_available mods ~init:[] ~f:(fun m acc -> + let obj_name = Module.obj_name m in + if List.exists entry_obj_names ~f:(fun n -> + Module_name.Unique.equal n obj_name) + then acc + else m :: acc) in - let entry_modules = Modules.entry_modules mods in - let entry_entries = - List.map entry_modules ~f:(fun m -> - Module.name m, lib, Some m) + let child_entries_under_wrapper = + List.concat_map entry_modules ~f:(fun wrapper -> + List.map child_modules ~f:(fun child -> + Module.name wrapper, lib, Some child)) in - let entries = - if unwrapped - then entry_entries - else - (* Auto-wrapped libs: dune's auto-generated wrapper - has [Module.kind = Alias], which means - [Dep_rules.skip_ocamldep] fires and no [.d] is - generated for the wrapper. The BFS thus cannot - learn the wrapper's children by reading its - ocamldep. Instead, also index each child under - the WRAPPER's name (multi-entry), so that - whenever the BFS encounters the wrapper's name - in the frontier (via [-open Wrapper] in the - consumer's flags or via a qualified - [Wrapper.Child.x] reference), it walks - directly into each child's ocamldep. *) - let entry_obj_names = - List.map entry_modules ~f:Module.obj_name - in - let child_modules = - Modules.fold_user_available mods ~init:[] ~f:(fun m acc -> - let obj_name = Module.obj_name m in - if List.exists entry_obj_names ~f:(fun n -> - Module_name.Unique.equal n obj_name) - then acc - else m :: acc) - in - let child_entries_under_wrapper = - List.concat_map entry_modules ~f:(fun wrapper -> - List.map child_modules ~f:(fun child -> - Module.name wrapper, lib, Some child)) - in - entry_entries @ child_entries_under_wrapper - in - let no_ocamldep_lib = - match Modules.as_singleton mods with - | Some _ when List.is_empty (Lib_info.requires (Lib.info lib) ~for_) -> - Some lib - | _ -> None - in - entries, no_ocamldep_lib, (if unwrapped then Some lib else None)))) + entry_entries @ child_entries_under_wrapper + in + let no_ocamldep_lib = + match Modules.as_singleton mods with + | Some _ when not has_resolved_deps -> Some lib + | _ -> None + in + entries, no_ocamldep_lib, (if unwrapped then Some lib else None))) in let entries = List.concat_map per_lib ~f:(fun (e, _, _) -> e) in let no_ocamldep = @@ -232,7 +232,7 @@ let build_lib_index ~super_context ~libs ~for_ = let unwrapped_local = List.filter_map per_lib ~f:(fun (_, _, u) -> u) |> Lib.Set.of_list in - Lib_file_deps.Lib_index.create ~no_ocamldep ~unwrapped_local entries + Lib_file_deps.Lib_index.create ~no_ocamldep ~unwrapped_local ~entries ;; let create @@ -256,6 +256,7 @@ let create ?cms_cmt_dependency ?loc ?instances + ?lib for_ = let project = Scope.project scope in @@ -297,19 +298,32 @@ let create let* has_library_deps = (* Determine whether any library dependencies are declared, so that single-module stanzas still run ocamldep when its output could - inform the per-module inter-library dependency filter. *) - let open Resolve.Memo.O in - let+ direct = direct_requires - and+ hidden = hidden_requires in - match direct, hidden with - | [], [] -> false - | _ -> true - in - let has_library_deps = - (* Unresolved dependency errors propagate later through the normal - compilation rules; here we conservatively behave as if libraries - are present. *) - Resolve.peek has_library_deps |> Result.value ~default:true + inform the per-module inter-library dependency filter. + + For library cctxes ([?lib] supplied), route through + [Dep_rules.has_library_deps_of_lib] — the same helper + [build_lib_index] uses to predict the lib's [.d]-file presence + from a different cctx. Sharing the predicate at the lib level + is what keeps the skip-decision and the cross-library walk's + prediction in agreement. For non-library cctxes (executables, + tests, melange emit, …), fall back to the cctx-local + direct/hidden derivation: there is nothing to predict + cross-stanza, since these aren't entries in any lib_index. *) + match lib with + | Some lib -> Dep_rules.has_library_deps_of_lib lib ~for_ + | None -> + let+ resolved = + let open Resolve.Memo.O in + let+ direct = direct_requires + and+ hidden = hidden_requires in + match direct, hidden with + | [], [] -> false + | _ -> true + in + (* Unresolved dependency errors propagate later through the + normal compilation rules; here we conservatively behave as + if libraries are present. *) + Resolve.peek resolved |> Result.value ~default:true in let+ dep_graphs = Dep_rules.rules diff --git a/src/dune_rules/compilation_context.mli b/src/dune_rules/compilation_context.mli index a5a4e54a097..f65fbb4c2ba 100644 --- a/src/dune_rules/compilation_context.mli +++ b/src/dune_rules/compilation_context.mli @@ -40,6 +40,7 @@ val create -> ?cms_cmt_dependency:Workspace.Context.Cms_cmt_dependency.t -> ?loc:Loc.t -> ?instances:Parameterised_instances.t Resolve.Memo.t + -> ?lib:Lib.t -> Compilation_mode.t -> t Memo.t diff --git a/src/dune_rules/dep_rules.ml b/src/dune_rules/dep_rules.ml index 465b201f084..f713b9ad005 100644 --- a/src/dune_rules/dep_rules.ml +++ b/src/dune_rules/dep_rules.ml @@ -134,23 +134,36 @@ let deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_ sourced_module (** Tests whether a set of modules is a singleton. *) let has_single_file modules = Option.is_some @@ Modules.With_vlib.as_singleton modules +(** Canonical "does this library have any library dependencies?" + answer. The boolean controls whether ocamldep is short-circuited + for [lib]'s own cctx (via [skip_ocamldep] below) and, in turn, + whether [lib]'s [.d] files are produced. The cross-library walk + in [Compilation_context.build_lib_index] consults the same + helper to decide whether [lib] should land in [no_ocamldep] — + routing both sites through this function is what keeps them in + sync, so changes to the condition can't drift between the + skip-decision side and the prediction side. Resolution failures + (unresolved direct deps) are treated conservatively as + has-deps=true so the lib's cctx still produces [.d] files and + cross-library consumers expect them. *) +let has_library_deps_of_lib lib ~for_ = + let open Memo.O in + let+ resolved = Lib.requires lib ~for_ |> Memo.map ~f:Resolve.peek in + match resolved with + | Ok [] -> false + | Ok _ | Error _ -> true +;; + (** Tests whether ocamldep can be short-circuited for [modules]: true - for single-module stanzas that have no library dependencies. - The premise — "no consumer of ocamldep output can benefit" — was + for single-module stanzas that have no library dependencies. The + premise — "no consumer of ocamldep output can benefit" — was valid before #4572; under that PR, cross-library consumers now read a target library's ocamldep output as part of the - per-module inter-library dependency filter. Libraries identified - here must also be identified in - [Compilation_context.build_lib_index]'s [no_ocamldep_lib] check - so the cross-library walk knows their [.d] files will be - missing. Keeping the two in sync is fragile. - - TODO: unify into a single predicate that both call sites - consult, so future changes to the condition can't drift. The - cleanest shape is probably to retire this short-circuit for - library stanzas entirely — libraries that could be consumed - cross-stanza should always run ocamldep — and keep the - optimisation only for executable/test stanzas. *) + per-module inter-library dependency filter, so a target library + short-circuited here must also be in + [Compilation_context.build_lib_index]'s [no_ocamldep] set. Both + sides resolve their [has_library_deps] view through + [has_library_deps_of_lib] above. *) let skip_ocamldep ~has_library_deps modules = has_single_file modules && not has_library_deps ;; diff --git a/src/dune_rules/dep_rules.mli b/src/dune_rules/dep_rules.mli index f36712e7b2c..9ad943aef9c 100644 --- a/src/dune_rules/dep_rules.mli +++ b/src/dune_rules/dep_rules.mli @@ -30,6 +30,15 @@ val rules -> has_library_deps:bool -> Dep_graph.Ml_kind.t Memo.t +(** Canonical [has_library_deps] for a library, used both by + [Compilation_context.create] (when building a library cctx, to + derive the [has_library_deps] passed to [rules]) and by + [Compilation_context.build_lib_index] (when predicting whether + the lib's [.d] file will exist for the cross-library walk). + Returns [true] if [Lib.requires lib ~for_] resolves to a + non-empty list, or if resolution fails (conservative). *) +val has_library_deps_of_lib : Lib.t -> for_:Compilation_mode.t -> bool Memo.t + val read_immediate_deps_of : obj_dir:Path.Build.t Obj_dir.t -> modules:Modules.With_vlib.t diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 40ec7343ed0..7f1878e9416 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -165,7 +165,7 @@ module Lib_index = struct module-level comment at the top of this file for the ocamldep-granularity reason wrapped libs are excluded from the tight-eligible class. *) - let create ~no_ocamldep ~unwrapped_local entries = + let create ~no_ocamldep ~unwrapped_local ~entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> Module_name.Map.update map name ~f:(function diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 3833950ab0b..4b1fa561bbc 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -55,7 +55,7 @@ module Lib_index : sig val create : no_ocamldep:Lib.Set.t -> unwrapped_local:Lib.Set.t - -> (Module_name.t * Lib.t * Module.t option) list + -> entries:(Module_name.t * Lib.t * Module.t option) list -> t type classified = diff --git a/src/dune_rules/lib_rules.ml b/src/dune_rules/lib_rules.ml index 5a8c0ca3ae7..06d6c1bfdee 100644 --- a/src/dune_rules/lib_rules.ml +++ b/src/dune_rules/lib_rules.ml @@ -476,6 +476,7 @@ let cctx ~scope ~parameters ~compile_info + ~local_lib ~modes ~for_ = @@ -532,6 +533,7 @@ let cctx ~melange_package_name ~modes ~instances + ~lib:local_lib ;; let library_rules @@ -675,7 +677,7 @@ let compile_context_data (lib : Library.t) ~sctx ~dir_contents ~scope = let compile_context (lib : Library.t) ~sctx ~dir_contents ~expander ~scope = let dir = Dir_contents.dir dir_contents in - let* _, compile_info, modes, source_modules, parameters = + let* local_lib, compile_info, modes, source_modules, parameters = compile_context_data lib ~sctx ~dir_contents ~scope in cctx @@ -687,6 +689,7 @@ let compile_context (lib : Library.t) ~sctx ~dir_contents ~expander ~scope = ~expander ~parameters ~compile_info + ~local_lib ~modes ~for_:Ocaml ;; @@ -708,6 +711,7 @@ let rules (lib : Library.t) ~sctx ~dir_contents ~expander ~scope = ~expander ~parameters ~compile_info + ~local_lib ~modes ~for_:Ocaml in From fe64a7afac05f6e24549306fa72a7e36e402b056 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 18:15:33 -0700 Subject: [PATCH 70/80] refactor: use Module_name.Unique.Set for entry-name lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In [build_lib_index], the per-module check that excluded entry modules from the children fold used [List.exists entry_obj_names] inside [Modules.fold_user_available], giving O(entries × modules) work for each wrapped lib. Build a [Module_name.Unique.Set] once and check membership in O(log n). Negligible on typical libraries but more idiomatic and cheaper on libraries with many modules. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index b414b2c990a..1fc5fcf765f 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -202,12 +202,12 @@ let build_lib_index ~super_context ~libs ~for_ = consumer's flags or via a qualified [Wrapper.Child.x] reference), it walks directly into each child's ocamldep. *) - let entry_obj_names = List.map entry_modules ~f:Module.obj_name in + let entry_obj_names = + Module_name.Unique.Set.of_list_map entry_modules ~f:Module.obj_name + in let child_modules = Modules.fold_user_available mods ~init:[] ~f:(fun m acc -> - let obj_name = Module.obj_name m in - if List.exists entry_obj_names ~f:(fun n -> - Module_name.Unique.equal n obj_name) + if Module_name.Unique.Set.mem entry_obj_names (Module.obj_name m) then acc else m :: acc) in From 17b1253106fc03c7b042c120d0101140c7ab367a Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 18:26:15 -0700 Subject: [PATCH 71/80] perf: drop wrapper-name child indexing for hand-written wrappers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For wrapped libraries, [build_lib_index] previously indexed every child module under the wrapper's name, so a BFS hit on the wrapper expanded to read each child's [.d] file directly. That is required for auto-generated wrappers ([Module.kind = Alias]) — they have no [.d] of their own ([Dep_rules.deps_of] short-circuits on [Alias]), so the BFS cannot otherwise reach the children. Hand-written wrappers ([Impl] / [Intf_only]) do have a readable [.d]; ocamldep on the wrapper's source emits the unmangled names of every child the wrapper exposes via [module Foo = Foo]-style aliases. Indexing those children under their own names lets the BFS resolve those emitted names in the next round, and avoids the multi-entry-under-wrapper expansion — which previously re-read every child's [.d] each time the wrapper appeared in the frontier, including children the wrapper does not expose. Soundness: a consumer can reach a child of a hand-written wrapper only via (a) a name the wrapper exposes (which appears in the wrapper's [.d]) or (b) [-open Wrapper] bringing children into unqualified scope. In both cases the child's unmangled name reaches the BFS frontier, so own-name indexing resolves it. Full test suite passes, including the two baseline tests: [wrapped-reexport-via-open-flag.t] (hand-written wrapper, alias) and [auto-wrapped-child-reexport.t] (auto-generated wrapper). Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 61 ++++++++++++++++++--------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 1fc5fcf765f..27ec4fff957 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -189,19 +189,7 @@ let build_lib_index ~super_context ~libs ~for_ = let entries = if unwrapped then entry_entries - else - (* Auto-wrapped libs: dune's auto-generated wrapper - has [Module.kind = Alias], which means - [Dep_rules.skip_ocamldep] fires and no [.d] is - generated for the wrapper. The BFS thus cannot - learn the wrapper's children by reading its - ocamldep. Instead, also index each child under - the WRAPPER's name (multi-entry), so that - whenever the BFS encounters the wrapper's name - in the frontier (via [-open Wrapper] in the - consumer's flags or via a qualified - [Wrapper.Child.x] reference), it walks - directly into each child's ocamldep. *) + else ( let entry_obj_names = Module_name.Unique.Set.of_list_map entry_modules ~f:Module.obj_name in @@ -211,24 +199,55 @@ let build_lib_index ~super_context ~libs ~for_ = then acc else m :: acc) in - let child_entries_under_wrapper = - List.concat_map entry_modules ~f:(fun wrapper -> + (* Two indexing strategies for wrapped libs' children, + picked from the wrapper's [Module.kind]: + + - Auto-generated wrappers ([Alias]) have no [.d] + of their own ([Dep_rules.deps_of] short-circuits + on [Alias]), so the BFS cannot learn the + children by reading the wrapper's ocamldep. Map + each child under the WRAPPER's name (multi- + entry); a frontier hit on the wrapper expands + directly to the children's [.d] files. + + - Hand-written wrappers ([Impl]/[Intf_only]) have + a readable [.d] that names every child the + wrapper exposes (via [module Foo = Foo] aliases + or the like). Indexing children under their + OWN names lets the BFS resolve those emitted + names to the children's [.d] files in the next + round. Avoid the multi-entry-under-wrapper + trick here — it would re-read every child's + [.d] each time the wrapper is in the frontier, + including children the wrapper does not + expose. *) + let any_alias_wrapper = + List.exists entry_modules ~f:(fun m -> + match Module.kind m with + | Alias _ -> true + | _ -> false) + in + let child_entries = + if any_alias_wrapper + then + List.concat_map entry_modules ~f:(fun wrapper -> + List.map child_modules ~f:(fun child -> + Module.name wrapper, lib, Some child)) + else List.map child_modules ~f:(fun child -> - Module.name wrapper, lib, Some child)) + Module.name child, lib, Some child) in - entry_entries @ child_entries_under_wrapper + entry_entries @ child_entries) in let no_ocamldep_lib = match Modules.as_singleton mods with | Some _ when not has_resolved_deps -> Some lib | _ -> None in - entries, no_ocamldep_lib, (if unwrapped then Some lib else None))) + entries, no_ocamldep_lib, if unwrapped then Some lib else None)) in let entries = List.concat_map per_lib ~f:(fun (e, _, _) -> e) in - let no_ocamldep = - List.filter_map per_lib ~f:(fun (_, n, _) -> n) |> Lib.Set.of_list - in + let no_ocamldep = List.filter_map per_lib ~f:(fun (_, n, _) -> n) |> Lib.Set.of_list in let unwrapped_local = List.filter_map per_lib ~f:(fun (_, _, u) -> u) |> Lib.Set.of_list in From f1731e5c94576dccfc4fd41c0e6af2e6e65072cd Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 18:47:53 -0700 Subject: [PATCH 72/80] docs: defend conservative-true on has_library_deps resolution failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The [Resolve.peek ... ~default:true] pattern collapsing the [has_library_deps] resolve into a plain bool was previously defended only by a one-line "errors propagate later" comment. Code review flagged that the rationale wasn't fully explained: swapping [~default:true] for [~default:false] would compile and "work", but mask real unresolved-library errors behind "No rule found for .X.objs/X.impl.d" infrastructure errors when the cross-library walk tried to read a [.d] file that [Dep_rules.skip_ocamldep] had short-circuited away. Centralize the pattern in [Dep_rules]: - Add [has_library_deps_of_resolved ~direct ~hidden] alongside the existing [has_library_deps_of_lib]. Both return [bool Memo.t] with conservative-true on resolution failure. - Move the rationale into one substantial comment block in [dep_rules.ml] above both helpers, explaining why conservative-true is forced (not just chosen). - [Compilation_context.create] now routes both branches through these helpers — library cctxes via [_of_lib], others via [_of_resolved]. The ad-hoc inline derivation and the terse comment in [compilation_context.ml] are gone. No behavioural change. Test suite unaffected. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 36 ++++++-------- src/dune_rules/dep_rules.ml | 67 ++++++++++++++++++++++----- src/dune_rules/dep_rules.mli | 13 +++++- 3 files changed, 82 insertions(+), 34 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 27ec4fff957..9b27dc3eb53 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -317,32 +317,24 @@ let create let* has_library_deps = (* Determine whether any library dependencies are declared, so that single-module stanzas still run ocamldep when its output could - inform the per-module inter-library dependency filter. - - For library cctxes ([?lib] supplied), route through + inform the per-module inter-library dependency filter. For + library cctxes ([?lib] supplied), route through [Dep_rules.has_library_deps_of_lib] — the same helper - [build_lib_index] uses to predict the lib's [.d]-file presence - from a different cctx. Sharing the predicate at the lib level - is what keeps the skip-decision and the cross-library walk's - prediction in agreement. For non-library cctxes (executables, - tests, melange emit, …), fall back to the cctx-local - direct/hidden derivation: there is nothing to predict - cross-stanza, since these aren't entries in any lib_index. *) + [build_lib_index] uses to predict the lib's [.d]-file + presence from a different cctx. Sharing the predicate at the + lib level is what keeps the skip-decision and the cross- + library walk's prediction in agreement. For non-library + cctxes (executables, tests, melange emit, …), there's + nothing to predict cross-stanza — these aren't entries in + any lib_index — but the same conservative-on-resolution- + failure handling applies, so route through + [has_library_deps_of_resolved]. *) match lib with | Some lib -> Dep_rules.has_library_deps_of_lib lib ~for_ | None -> - let+ resolved = - let open Resolve.Memo.O in - let+ direct = direct_requires - and+ hidden = hidden_requires in - match direct, hidden with - | [], [] -> false - | _ -> true - in - (* Unresolved dependency errors propagate later through the - normal compilation rules; here we conservatively behave as - if libraries are present. *) - Resolve.peek resolved |> Result.value ~default:true + Dep_rules.has_library_deps_of_resolved + ~direct:direct_requires + ~hidden:hidden_requires in let+ dep_graphs = Dep_rules.rules diff --git a/src/dune_rules/dep_rules.ml b/src/dune_rules/dep_rules.ml index f713b9ad005..6b85e01d350 100644 --- a/src/dune_rules/dep_rules.ml +++ b/src/dune_rules/dep_rules.ml @@ -134,18 +134,43 @@ let deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_ sourced_module (** Tests whether a set of modules is a singleton. *) let has_single_file modules = Option.is_some @@ Modules.With_vlib.as_singleton modules +(* Conservative direction on resolution failure + =========================================== + + Both [has_library_deps_of_lib] and [has_library_deps_of_resolved] + below collapse a [_ Resolve.t] into a plain [bool] for [skip_ + ocamldep]'s consumption. Unresolved dependencies (e.g. the user + declared [(libraries some_missing_lib)]) make the resolve fail. + When that happens, both helpers return [true] — "act as if + library deps are present". + + This is deliberate, not a default-because-we-must: + + - has-deps=true keeps [.d]-file rules in place. The unresolved- + library error then surfaces in the compile rule, where the + user gets a useful "library X is not available" diagnostic. + + - has-deps=false would short-circuit the [.d] rules. If anything + downstream (notably the cross-library walk in + [Compilation_context.build_lib_index]) later tried to read + this stanza's [.d], dune would emit a "No rule found for + .X.objs/X.impl.d" error, masking the real cause behind an + infrastructure-level message. + + Conservative-true at worst runs ocamldep on a stanza whose + compilation would have failed anyway; conservative-false risks + turning a real dependency error into an obscure "no rule" + error. The trade-off is asymmetric, so the choice is forced. *) + (** Canonical "does this library have any library dependencies?" - answer. The boolean controls whether ocamldep is short-circuited - for [lib]'s own cctx (via [skip_ocamldep] below) and, in turn, - whether [lib]'s [.d] files are produced. The cross-library walk - in [Compilation_context.build_lib_index] consults the same - helper to decide whether [lib] should land in [no_ocamldep] — - routing both sites through this function is what keeps them in - sync, so changes to the condition can't drift between the - skip-decision side and the prediction side. Resolution failures - (unresolved direct deps) are treated conservatively as - has-deps=true so the lib's cctx still produces [.d] files and - cross-library consumers expect them. *) + answer for [lib]. The boolean controls whether ocamldep is + short-circuited for [lib]'s own cctx (via [skip_ocamldep] + below) and, in turn, whether [lib]'s [.d] files are produced. + The cross-library walk in [Compilation_context.build_lib_index] + consults the same helper to decide whether [lib] should land in + [no_ocamldep] — routing both sites through this function is + what keeps them in sync. Resolution failures fall back to + has-deps=true; see the conservative-direction note above. *) let has_library_deps_of_lib lib ~for_ = let open Memo.O in let+ resolved = Lib.requires lib ~for_ |> Memo.map ~f:Resolve.peek in @@ -154,6 +179,26 @@ let has_library_deps_of_lib lib ~for_ = | Ok _ | Error _ -> true ;; +(** Same answer as [has_library_deps_of_lib], but for cctxes that + aren't built from a [Lib.t] — executables, tests, melange emit. + Takes the resolved [direct] and [hidden] requires that + [Compilation_context.create] has already split per + [implicit_transitive_deps] mode. Resolution failures fall back + to has-deps=true; see the conservative-direction note above. *) +let has_library_deps_of_resolved ~direct ~hidden = + let resolved = + let open Resolve.Memo.O in + let+ direct = direct + and+ hidden = hidden in + match direct, hidden with + | [], [] -> false + | _ -> true + in + let open Memo.O in + let+ peeked = Memo.map resolved ~f:Resolve.peek in + Result.value peeked ~default:true +;; + (** Tests whether ocamldep can be short-circuited for [modules]: true for single-module stanzas that have no library dependencies. The premise — "no consumer of ocamldep output can benefit" — was diff --git a/src/dune_rules/dep_rules.mli b/src/dune_rules/dep_rules.mli index 9ad943aef9c..fe9ecc26a14 100644 --- a/src/dune_rules/dep_rules.mli +++ b/src/dune_rules/dep_rules.mli @@ -36,9 +36,20 @@ val rules [Compilation_context.build_lib_index] (when predicting whether the lib's [.d] file will exist for the cross-library walk). Returns [true] if [Lib.requires lib ~for_] resolves to a - non-empty list, or if resolution fails (conservative). *) + non-empty list, or if resolution fails (conservative — see the + rationale comment in [dep_rules.ml]). *) val has_library_deps_of_lib : Lib.t -> for_:Compilation_mode.t -> bool Memo.t +(** [has_library_deps] for non-library cctxes (executables, tests, + melange emit), derived from already-resolved [direct] and + [hidden] requires. Resolution failures conservatively fall + back to has-deps=true — see the rationale comment in + [dep_rules.ml]. *) +val has_library_deps_of_resolved + : direct:Lib.t list Resolve.Memo.t + -> hidden:Lib.t list Resolve.Memo.t + -> bool Memo.t + val read_immediate_deps_of : obj_dir:Path.Build.t Obj_dir.t -> modules:Modules.With_vlib.t From 44d8e6f003ffea91e1946ccdab3b5c78f89a6fac Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 19:05:32 -0700 Subject: [PATCH 73/80] docs: refresh stale comments after BFS / index restructuring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent commits restructured how the cross-library walk reaches the children of wrapped libraries: indexing is now under the wrapper's name for auto-generated wrappers (kind = Alias, no [.d]) and under the child's own name for hand-written wrappers (kind = Impl/Intf_only, [.d] available). Several comments still described an older model in which the BFS read the auto-generated wrapper's ocamldep and resolved mangled child names through the index. Refreshed: - [module_compilation.ml] [cross_lib_tight_set] — replaced the fictional "wrapper's ocamldep emits mangled child names" paragraph with a wrapper-kind split that matches the index shape. - [module_compilation.ml] post-walk classify — dropped the stale "every entry has a [Module.t]" qualifier from the [None] AND tight-eligible bucket; that qualifier is now redundant with "local + unwrapped". - [compilation_context.ml] [build_lib_index] — replaced the parenthetical mangled-name claim with a pointer to the kind- aware indexing logic immediately below. - [lib_file_deps.ml] [Lib_index.create] — updated the "Two orthogonal predicates" block similarly; routed the wrapped-libs description through [build_lib_index]'s indexing convention. - [lib_file_deps.mli] [create] / [lookup_walkable_entries] — same fix; both docs no longer claim children are indexed under mangled names. - [lib_file_deps.mli] [tight] field / [is_tight_eligible] — dropped the stale "every entry carries a known [Module.t]" qualifier. - [dep_rules.ml] [skip_ocamldep] — updated the cross-reference to mention both [has_library_deps_of_lib] (libs) and [has_library_deps_of_resolved] (executables, etc.); the prior text predated the [_of_resolved] split. Also drop the dead [()] payload from [lib_deps_for_module]: the function returned [((), Dep.Set.t) Action_builder.t] only because the sole consumer wired it through [Action_builder.dyn_deps : ('a * Dep.Set.t) t -> 'a t]. The unit component was never meaningful; the function now returns [Dep.Set.t Action_builder.t] and [lib_cm_deps] adapts at the single call site. No behavioural change. Tests unaffected. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 12 ++-- src/dune_rules/dep_rules.ml | 5 +- src/dune_rules/lib_file_deps.ml | 16 ++--- src/dune_rules/lib_file_deps.mli | 37 ++++++------ src/dune_rules/module_compilation.ml | 86 ++++++++++++++++----------- 5 files changed, 91 insertions(+), 65 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 9b27dc3eb53..e3a433a4a6c 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -173,10 +173,14 @@ let build_lib_index ~super_context ~libs ~for_ = Whether to issue per-module deps on the entry's [.cmi] is a separate decision tracked by [unwrapped_local]: only unwrapped local libs are tight-eligible. Wrapped - local libs are walkable (the wrapper's ocamldep - references children by mangled name and the BFS - expands through them) but classified as glob for - invalidation. *) + local libs are walkable too — their children are + indexed below, under the wrapper's name for auto- + generated wrappers and under the child's own name for + hand-written ones, so the BFS can reach them — but + they remain non-tight: per-module precision isn't + possible (see [lib_file_deps.ml]'s ocamldep- + granularity rationale), so they fall back to a glob + for invalidation. *) let unwrapped = match Lib_info.wrapped (Lib.info lib) with | Some (This w) -> not (Wrapped.to_bool w) diff --git a/src/dune_rules/dep_rules.ml b/src/dune_rules/dep_rules.ml index 6b85e01d350..903e3a59163 100644 --- a/src/dune_rules/dep_rules.ml +++ b/src/dune_rules/dep_rules.ml @@ -207,8 +207,9 @@ let has_library_deps_of_resolved ~direct ~hidden = per-module inter-library dependency filter, so a target library short-circuited here must also be in [Compilation_context.build_lib_index]'s [no_ocamldep] set. Both - sides resolve their [has_library_deps] view through - [has_library_deps_of_lib] above. *) + sides resolve their [has_library_deps] view through the helpers + above ([has_library_deps_of_lib] for library cctxes, + [has_library_deps_of_resolved] elsewhere). *) let skip_ocamldep ~has_library_deps modules = has_single_file modules && not has_library_deps ;; diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 7f1878e9416..46d29e626cf 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -158,13 +158,15 @@ module Lib_index = struct ocamldep. [tight_eligible] (built from the caller-supplied [unwrapped_local]) means the lib is local AND unwrapped, so downstream consumers can issue per-module deps on its [.cmi] - files. Wrapped local libs are walkable but not tight-eligible — - their auto-generated wrapper's ocamldep references children by - mangled name and the BFS expands through them, but the lib - itself classifies as glob for invalidation. See the - module-level comment at the top of this file for the - ocamldep-granularity reason wrapped libs are excluded from the - tight-eligible class. *) + files. Wrapped local libs are walkable but not tight-eligible: + their children are present in [by_module_name] (placed there + by [Compilation_context.build_lib_index] under the wrapper's + name for auto-generated wrappers and under the child's own + name for hand-written ones), so the BFS reaches them, but + the lib itself falls back to a glob for invalidation. See + the module-level comment at the top of this file for the + ocamldep-granularity reason wrapped libs are excluded from + the tight-eligible class. *) let create ~no_ocamldep ~unwrapped_local ~entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 4b1fa561bbc..8de240c6f36 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -43,7 +43,9 @@ module Lib_index : sig unwrapped — the tight-eligible class. The cross-library walk reads ocamldep for any local lib's entries (via [lookup_walkable_entries], including wrapped libs' children - indexed under their mangled names); per-module dep + placed by [Compilation_context.build_lib_index] under the + wrapper's name for auto-generated wrappers and under the + child's own name for hand-written ones); per-module dep classification is restricted to [tight_eligible] (see [is_tight_eligible]). @@ -61,10 +63,10 @@ module Lib_index : sig type classified = { tight : Module.t list Lib.Map.t (** Directly-referenced tight-eligible libraries (local, - unwrapped, each entry with a known [Module.t]). Mapped to the - referenced entry modules. These libraries are candidates for - per-module deps via [deps_of_entry_modules]; whether to emit - them is the caller's policy. *) + unwrapped). Mapped to the referenced entry modules. These + libraries are candidates for per-module deps via + [deps_of_entry_modules]; whether to emit them is the + caller's policy. *) ; non_tight : Lib.t list (** Other directly-referenced libraries — wrapped locals, externals, or anything else that falls back to a glob. Sorted @@ -88,20 +90,21 @@ module Lib_index : sig (** [lookup_walkable_entries idx name] returns [(lib, entry module)] pairs used by the cross-library walk in [module_compilation]. Includes wrapped local libs' wrappers and their children - (indexed under their mangled names) — the BFS reads each - walkable entry's ocamldep to follow alias chains across - library boundaries. Libraries in [no_ocamldep] are excluded - (their [.d] files do not exist); externals are excluded - because no [Module.t] is available. *) + (placed in the index by [Compilation_context.build_lib_index] + under the wrapper's name for auto-generated wrappers and + under the child's own name for hand-written ones) — the BFS + reads each walkable entry's ocamldep to follow alias chains + across library boundaries. Libraries in [no_ocamldep] are + excluded (their [.d] files do not exist); externals are + excluded because no [Module.t] is available. *) val lookup_walkable_entries : t -> Module_name.t -> (Lib.t * Module.t) list - (** [is_tight_eligible idx lib] is [true] when [lib] is local, - unwrapped, and every entry carries a known [Module.t]. The - cross-library walk has full visibility into such libraries: - the absence of any of their entry modules from the post-walk - reference set is positive evidence that the consumer does - not reach the library, so the consumer's compile rule does - not need a dep on it. *) + (** [is_tight_eligible idx lib] is [true] when [lib] is local and + unwrapped. The cross-library walk has full visibility into + such libraries: the absence of any of their entry modules from + the post-walk reference set is positive evidence that the + consumer does not reach the library, so the consumer's compile + rule does not need a dep on it. *) val is_tight_eligible : t -> Lib.t -> bool end diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index ab6778656fc..a746ad63ca4 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -53,18 +53,29 @@ let module_kind_is_filterable m = pair's impl and intf ocamldep, and union the raw names into the frontier. Iterate until no new names appear. - Wrapped local libs are walkable too. Dune's auto-generated - wrapper contains [module Foo = Lib__Foo] for each child; - reading the wrapper's ocamldep emits the mangled child names, - and [lookup_walkable_entries] resolves each to the child's - [Module.t] (also indexed under its mangled name). The BFS then - descends into the child's own ocamldep, surfacing transitive - cross-library references like [include Vendored_pprint] that - the wrapper alone would not reveal. Externals and libs in - [no_ocamldep] are excluded by [lookup_walkable_entries] — - they have no readable [.d] file. Chains passing through - excluded libs terminate and the consumer falls back to a glob - on the unreached libs. + Wrapped local libs are walkable too, but how the BFS reaches + their children depends on the wrapper kind (see + [Compilation_context.build_lib_index]): + + - Auto-generated wrappers ([Module.kind = Alias]) have no + readable [.d] of their own ([Dep_rules.deps_of] short-circuits + on [Alias]). Their children are indexed under the WRAPPER's + unmangled name (multi-entry); a frontier hit on the wrapper + expands directly to every child via [lookup_walkable_entries]. + + - Hand-written wrappers ([Impl] / [Intf_only]) have a readable + [.d]; ocamldep on the wrapper's source emits the unmangled + names of the children the wrapper exposes. Children are + indexed under their OWN names, so the BFS resolves those + emitted names in the next round. + + Either way the BFS ends up reading each reachable child's own + ocamldep, surfacing transitive cross-library references like + [include Vendored_pprint] that the wrapper alone would not + reveal. Externals and libs in [no_ocamldep] are excluded by + [lookup_walkable_entries] — they have no readable [.d] file. + Chains passing through excluded libs terminate and the consumer + falls back to a glob on the unreached libs. Cycles in the module-reference graph terminate on the [seen] set. @@ -100,8 +111,9 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = (* Per-module inter-library dependency filtering (#4572). Uses ocamldep output to determine which libraries a module actually references, then transitively closes within the compilation context's library set to - handle transparent aliases. Returns [((), Dep.Set.t)] suitable for - use with [Action_builder.dyn_deps]. + handle transparent aliases. Returns the [Dep.Set.t] of file deps the + compile rule should depend on; the wrapper [lib_cm_deps] adapts it + for [Action_builder.dyn_deps]. The returned [Dep.Set.t] is the sole source of cross-library file dependencies for the compile rule: [-I] flags on the ocamlc command @@ -159,7 +171,7 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin in let* libs = Resolve.Memo.read (all_libs cctx) in if not can_filter - then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) + then Action_builder.return (Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else (* Deps-side virtual-impl check: if any library in the consumer's requires implements a virtual library, fall back to glob. @@ -168,7 +180,7 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin Resolve.Memo.read (Compilation_context.has_virtual_impl cctx) in if has_virtual_impl - then Action_builder.return ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) + then Action_builder.return (Lib_file_deps.deps_of_entries ~opaque ~cm_kind libs) else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in let* trans_deps = Dep_graph.deps_of dep_graph m in @@ -266,14 +278,14 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin entries actually referenced. - [None] AND tight-eligible: the cross-library walk had full - visibility (local, unwrapped, every entry has a [Module.t]) - and no entry of the library appears in [tight_set]. The - consumer therefore does not reach this library through any - reference chain visible to the walk, including transitive - module aliases under the [-opaque]-aware impl-side reads. - Drop the library entirely from the consumer's compile rule - deps; the link rule still pulls it in through - [requires_link] for executables/libraries that need it. + visibility (local, unwrapped) and no entry of the library + appears in [tight_set]. The consumer therefore does not + reach this library through any reference chain visible to + the walk, including transitive module aliases under the + [-opaque]-aware impl-side reads. Drop the library entirely + from the consumer's compile rule deps; the link rule still + pulls it in through [requires_link] for executables / + libraries that need it. - [None] AND not tight-eligible: wrapped local, external, or virtual-impl. The walk does not have full visibility, so @@ -297,7 +309,7 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin else td, lib :: gl) in let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in - (), Dep.Set.union tight_deps glob_deps + Dep.Set.union tight_deps glob_deps ;; (* Convenience wrapper for the two call sites ([build_cm] and @@ -309,17 +321,21 @@ let lib_cm_deps ~cctx ~cm_kind ~ml_kind ~mode m = let opaque = Compilation_context.opaque cctx in let for_ = Compilation_context.for_ cctx in let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in + let open Action_builder.O in Action_builder.dyn_deps - (lib_deps_for_module - ~cctx - ~obj_dir - ~for_ - ~dep_graph - ~opaque - ~cm_kind - ~ml_kind - ~mode - m) + (let+ deps = + lib_deps_for_module + ~cctx + ~obj_dir + ~for_ + ~dep_graph + ~opaque + ~cm_kind + ~ml_kind + ~mode + m + in + (), deps) ;; (* Arguments for the compiler to prevent it from being too clever. From fc95b850a242431af53154e1c41bc779762ff04e Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 19:22:30 -0700 Subject: [PATCH 74/80] docs: tighten comments across the per-module-deps cluster Audit pass on the prose this PR adds. Targets: - Drop comments that say what the name says (e.g. the doc on [union_module_name_sets_mapped], [lib_cm_deps], the "Like filter_libs_with_modules" gloss on [tight_modules_per_lib]). - Drop architectural exposition that the function bodies make obvious (the "Two dep pipelines" file-header block, the step-by-step BFS narrative in [cross_lib_tight_set]'s doc, the paragraph-form rationale preceding the [need_impl_deps_of] decision table). - Keep WHY-comments where the why is genuinely non-obvious: the ocamldep-granularity reason wrapped libs glob, the conservative- true rationale on resolution failure, the rule-graph cycle safety invariant, the [Dep_graph.dir = root] sentinel for dummy graphs, the menhir [mock_module] case, the virtual-impl consumer-vs-deps split. - Centralise the wrapper-kind dispatch description in [Compilation_context.build_lib_index] (where the indexing decision is made). Reduce the duplicated descriptions in [cross_lib_tight_set], [Lib_index.create], and [lookup_walkable_entries] / [create] in the .mli to brief pointers. - Compress the "Conservative direction on resolution failure" block from ~26 lines to ~7; the asymmetric trade-off can be stated tersely. - Drop the speculative "follow-on work" sketches from [lib_file_deps.ml]'s file header. Those belong in PR/issue prose, not source. Net: roughly -240 lines of comments, no WHY information lost. Builds and full test suite still pass. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 105 +++++--------- src/dune_rules/dep_rules.ml | 75 +++------- src/dune_rules/lib_file_deps.ml | 69 ++------- src/dune_rules/lib_file_deps.mli | 72 +++------- src/dune_rules/module_compilation.ml | 193 ++++++-------------------- 5 files changed, 136 insertions(+), 378 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index e3a433a4a6c..63568bff40d 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -125,27 +125,13 @@ let parameters_main_modules parameters = [ "param", Lib.to_dyn param ]) ;; -(* Build a [Lib_index] from [libs] for the per-module inter-library - dependency filter. - - [libs] should be the consumer's [direct_requires] — the libraries - whose entry modules can legitimately appear in the consumer's - ocamldep output (and, after the cross-library walk extends through - them, in cross-library ocamldep outputs that the BFS surfaces). - Hidden requires are intentionally excluded: the user has not - committed to them as direct dependencies, so the per-module filter - does not track them with per-module precision — they fall to glob - fallback in [lib_deps_for_module]'s post-walk classification, which - is the right behaviour for libs outside the user's commitment - surface. - - For each library, entry module names are collected (for wrapped - libraries, the wrapper plus a multi-entry index of children under - the wrapper's name; for unwrapped, each public module). Local - libraries whose ocamldep is short-circuited by - [Dep_rules.skip_ocamldep] (unwrapped single-module stanzas without - direct lib deps) are collected into [no_ocamldep] so the cross- - library walk does not try to read their nonexistent [.d] files. *) +(* Build a [Lib_index] from the consumer's [direct_requires]. Hidden + requires are intentionally excluded — the user has not committed + to them as direct dependencies, so they fall to glob in the + post-walk classification rather than getting per-module + precision. Local libs short-circuited by [Dep_rules.skip_ocamldep] + (single-module + no resolved deps) land in [no_ocamldep] so the + cross-library walk doesn't read [.d] files dune never produced. *) let build_lib_index ~super_context ~libs ~for_ = let open Resolve.Memo.O in let+ per_lib = @@ -163,24 +149,16 @@ let build_lib_index ~super_context ~libs ~for_ = (Lib.Local.of_lib_exn lib) ~for_ in - (* Predict the lib's own [has_library_deps] via the same - helper its cctx uses; routing both sites through - [Dep_rules.has_library_deps_of_lib] is what keeps the - skip-decision and the walk's prediction in sync. *) + (* Same helper the lib's own cctx uses — keeps the skip + decision and the walk's prediction in lock step. *) let+ has_resolved_deps = Dep_rules.has_library_deps_of_lib lib ~for_ in - (* Local libs always carry [Some m] for each entry so the - cross-library walk can read the entry's ocamldep. - Whether to issue per-module deps on the entry's [.cmi] - is a separate decision tracked by [unwrapped_local]: - only unwrapped local libs are tight-eligible. Wrapped - local libs are walkable too — their children are - indexed below, under the wrapper's name for auto- - generated wrappers and under the child's own name for - hand-written ones, so the BFS can reach them — but - they remain non-tight: per-module precision isn't - possible (see [lib_file_deps.ml]'s ocamldep- - granularity rationale), so they fall back to a glob - for invalidation. *) + (* Every local lib's entries carry [Some m] so the walk + can read the entry's ocamldep. Whether to issue per- + module deps is tracked separately via [unwrapped_local] + (tight-eligible = local AND unwrapped). Wrapped local + libs are walkable too via the indexing below, but + remain non-tight (see [lib_file_deps.ml] for the + ocamldep-granularity reason). *) let unwrapped = match Lib_info.wrapped (Lib.info lib) with | Some (This w) -> not (Wrapped.to_bool w) @@ -203,28 +181,23 @@ let build_lib_index ~super_context ~libs ~for_ = then acc else m :: acc) in - (* Two indexing strategies for wrapped libs' children, - picked from the wrapper's [Module.kind]: + (* The BFS reaches wrapped libs' children through + whichever of these matches the wrapper kind: - - Auto-generated wrappers ([Alias]) have no [.d] - of their own ([Dep_rules.deps_of] short-circuits - on [Alias]), so the BFS cannot learn the - children by reading the wrapper's ocamldep. Map - each child under the WRAPPER's name (multi- - entry); a frontier hit on the wrapper expands - directly to the children's [.d] files. + - Auto-generated wrappers ([Alias]) lack a [.d] + ([Dep_rules.deps_of] short-circuits on [Alias]). + Index every child under the wrapper's name + (multi-entry) so a frontier hit on the wrapper + expands to the children directly. - Hand-written wrappers ([Impl]/[Intf_only]) have - a readable [.d] that names every child the - wrapper exposes (via [module Foo = Foo] aliases - or the like). Indexing children under their - OWN names lets the BFS resolve those emitted - names to the children's [.d] files in the next - round. Avoid the multi-entry-under-wrapper - trick here — it would re-read every child's - [.d] each time the wrapper is in the frontier, - including children the wrapper does not - expose. *) + a [.d] that names exposed children via the + [module X = X] aliases the user wrote. Index + children under their own names; the BFS + resolves those emitted names in the next round. + Multi-entry-under-wrapper would re-read every + child's [.d] on each wrapper hit, including + children the wrapper doesn't expose. *) let any_alias_wrapper = List.exists entry_modules ~f:(fun m -> match Module.kind m with @@ -319,20 +292,10 @@ let create eval_opaque ocaml profile opaque in let* has_library_deps = - (* Determine whether any library dependencies are declared, so that - single-module stanzas still run ocamldep when its output could - inform the per-module inter-library dependency filter. For - library cctxes ([?lib] supplied), route through - [Dep_rules.has_library_deps_of_lib] — the same helper - [build_lib_index] uses to predict the lib's [.d]-file - presence from a different cctx. Sharing the predicate at the - lib level is what keeps the skip-decision and the cross- - library walk's prediction in agreement. For non-library - cctxes (executables, tests, melange emit, …), there's - nothing to predict cross-stanza — these aren't entries in - any lib_index — but the same conservative-on-resolution- - failure handling applies, so route through - [has_library_deps_of_resolved]. *) + (* For library cctxes, route through the same helper + [build_lib_index] uses, so the lib's skip decision and the + cross-stanza walk's prediction can't drift. Other cctxes + just need the boolean. *) match lib with | Some lib -> Dep_rules.has_library_deps_of_lib lib ~for_ | None -> diff --git a/src/dune_rules/dep_rules.ml b/src/dune_rules/dep_rules.ml index 903e3a59163..12a61897db2 100644 --- a/src/dune_rules/dep_rules.ml +++ b/src/dune_rules/dep_rules.ml @@ -134,43 +134,18 @@ let deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_ sourced_module (** Tests whether a set of modules is a singleton. *) let has_single_file modules = Option.is_some @@ Modules.With_vlib.as_singleton modules -(* Conservative direction on resolution failure - =========================================== +(* On resolution failure both [has_library_deps_*] helpers below + return [true]. Conservative-true keeps [.d]-file rules in place, + so the unresolved-library error surfaces as a real compile-rule + diagnostic. Conservative-false would short-circuit the [.d] + rules and turn that diagnostic into a "No rule found for + .X.objs/X.impl.d" infrastructure error. The trade-off is + asymmetric: forced choice. *) - Both [has_library_deps_of_lib] and [has_library_deps_of_resolved] - below collapse a [_ Resolve.t] into a plain [bool] for [skip_ - ocamldep]'s consumption. Unresolved dependencies (e.g. the user - declared [(libraries some_missing_lib)]) make the resolve fail. - When that happens, both helpers return [true] — "act as if - library deps are present". - - This is deliberate, not a default-because-we-must: - - - has-deps=true keeps [.d]-file rules in place. The unresolved- - library error then surfaces in the compile rule, where the - user gets a useful "library X is not available" diagnostic. - - - has-deps=false would short-circuit the [.d] rules. If anything - downstream (notably the cross-library walk in - [Compilation_context.build_lib_index]) later tried to read - this stanza's [.d], dune would emit a "No rule found for - .X.objs/X.impl.d" error, masking the real cause behind an - infrastructure-level message. - - Conservative-true at worst runs ocamldep on a stanza whose - compilation would have failed anyway; conservative-false risks - turning a real dependency error into an obscure "no rule" - error. The trade-off is asymmetric, so the choice is forced. *) - -(** Canonical "does this library have any library dependencies?" - answer for [lib]. The boolean controls whether ocamldep is - short-circuited for [lib]'s own cctx (via [skip_ocamldep] - below) and, in turn, whether [lib]'s [.d] files are produced. - The cross-library walk in [Compilation_context.build_lib_index] - consults the same helper to decide whether [lib] should land in - [no_ocamldep] — routing both sites through this function is - what keeps them in sync. Resolution failures fall back to - has-deps=true; see the conservative-direction note above. *) +(** Canonical [has_library_deps] for [lib]'s own cctx. The cross- + library walk in [Compilation_context.build_lib_index] consults + the same helper, so the skip decision and the walk's prediction + can't drift. *) let has_library_deps_of_lib lib ~for_ = let open Memo.O in let+ resolved = Lib.requires lib ~for_ |> Memo.map ~f:Resolve.peek in @@ -179,12 +154,11 @@ let has_library_deps_of_lib lib ~for_ = | Ok _ | Error _ -> true ;; -(** Same answer as [has_library_deps_of_lib], but for cctxes that - aren't built from a [Lib.t] — executables, tests, melange emit. - Takes the resolved [direct] and [hidden] requires that - [Compilation_context.create] has already split per - [implicit_transitive_deps] mode. Resolution failures fall back - to has-deps=true; see the conservative-direction note above. *) +(** Same boolean as [has_library_deps_of_lib], for cctxes without a + [Lib.t] (executables, tests, melange emit). Takes the resolved + [direct] and [hidden] requires already split by + [Compilation_context.create] per [implicit_transitive_deps] + mode. *) let has_library_deps_of_resolved ~direct ~hidden = let resolved = let open Resolve.Memo.O in @@ -199,17 +173,12 @@ let has_library_deps_of_resolved ~direct ~hidden = Result.value peeked ~default:true ;; -(** Tests whether ocamldep can be short-circuited for [modules]: true - for single-module stanzas that have no library dependencies. The - premise — "no consumer of ocamldep output can benefit" — was - valid before #4572; under that PR, cross-library consumers now - read a target library's ocamldep output as part of the - per-module inter-library dependency filter, so a target library - short-circuited here must also be in - [Compilation_context.build_lib_index]'s [no_ocamldep] set. Both - sides resolve their [has_library_deps] view through the helpers - above ([has_library_deps_of_lib] for library cctxes, - [has_library_deps_of_resolved] elsewhere). *) +(** Single-module stanzas with no library deps don't run ocamldep. + Under #4572, a library short-circuited here must also appear in + [Compilation_context.build_lib_index]'s [no_ocamldep] set or + the cross-library walk will read a [.d] dune never produced. + Both sides derive their [has_library_deps] via the helpers + above. *) let skip_ocamldep ~has_library_deps modules = has_single_file modules && not has_library_deps ;; diff --git a/src/dune_rules/lib_file_deps.ml b/src/dune_rules/lib_file_deps.ml index 46d29e626cf..f71b3dc26a6 100644 --- a/src/dune_rules/lib_file_deps.ml +++ b/src/dune_rules/lib_file_deps.ml @@ -1,41 +1,13 @@ open Import open Memo.O -(* Why wrapped libraries fall back to a directory glob - ================================================= - - Per-module tight deps apply only to local unwrapped libraries. - Wrapped libraries take the glob path over their public cmi dir. - The limitation is fundamental to ocamldep's output granularity. - - [ocamldep -modules foo.ml] lists only the top-level module names - referenced by a source file. For a consumer using [Foo.Bar.x], - the output is [Foo] — not [Foo.Bar]. Consumers that use - [Foo.Bar] and those that use [Foo.Baz] produce identical - ocamldep output, so the filter cannot distinguish them and - cannot emit specific deps on [Foo__Bar.cmi] vs [Foo__Baz.cmi]. - Any breadth-first walk over the wrapper's own ocamldep output - reaches every internal the wrapper exposes, which is equivalent - to the glob for invalidation. - - Possible follow-on work: - - - Qualified-path extractor. Walk consumer source with - [compiler-libs]' [Parse.implementation], collect [Longident.t] - references as [Module_name.Path.t] values in a companion - artifact. Match qualified paths against wrapped-lib internals - for per-consumer precision. Estimated ~500-1000 lines across a - new rule, a new file format, and preprocessing integration; - correct handling of [let open Foo in Bar.x] (opens that bring - sub-modules into unqualified scope) needs lexical-scope - tracking and roughly doubles the low-end estimate. - - - Post-compile cmi-imports refinement. [consumer.cmi] records - exactly the cmis its compilation imported; [Ocamlobjinfo] can - read them. Using this as the source of truth requires breaking - dune's invariant that rule deps are fixed before the rule - runs — a two-phase build or a pessimistic-then-refine scheme. - Not natively supported by dune's rule model today. *) +(* Wrapped libraries fall back to a directory glob because + [ocamldep -modules foo.ml] reports only top-level module names: + a consumer using [Foo.Bar.x] yields [Foo], indistinguishable + from one using [Foo.Baz.x]. Per-module precision on + [Foo__Bar.cmi] vs [Foo__Baz.cmi] isn't recoverable from + ocamldep output alone, so the BFS reach over the wrapper is + equivalent to a glob for invalidation. *) module Group = struct type ocaml = @@ -151,22 +123,13 @@ module Lib_index = struct } ;; - (* Two orthogonal predicates on a lib's entries are tracked here. - [Some m] in an entry means we have the entry's [Module.t] — - true for every local lib (wrapped or not), false for externals; - this is what lets the cross-library walk read the entry's - ocamldep. [tight_eligible] (built from the caller-supplied - [unwrapped_local]) means the lib is local AND unwrapped, so - downstream consumers can issue per-module deps on its [.cmi] - files. Wrapped local libs are walkable but not tight-eligible: - their children are present in [by_module_name] (placed there - by [Compilation_context.build_lib_index] under the wrapper's - name for auto-generated wrappers and under the child's own - name for hand-written ones), so the BFS reaches them, but - the lib itself falls back to a glob for invalidation. See - the module-level comment at the top of this file for the - ocamldep-granularity reason wrapped libs are excluded from - the tight-eligible class. *) + (* [Some m] in an entry (true for any local lib, false for + externals) is what lets the BFS read the entry's ocamldep. + [tight_eligible] (= local AND unwrapped, supplied by the + caller) is the orthogonal flag controlling per-module + precision: wrapped libs are walkable but glob-only for + invalidation (see file-level comment for the + ocamldep-granularity reason). *) let create ~no_ocamldep ~unwrapped_local ~entries = let by_module_name = List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, lib, m) -> @@ -206,10 +169,6 @@ module Lib_index = struct { tight; non_tight = Lib.Set.to_list non_tight } ;; - (* Like [filter_libs_with_modules] but only returns the tight - part. Saves the [non_tight] accumulator at call sites that - only consume the tight map (e.g. the post-BFS classify in - [lib_deps_for_module]). *) let tight_modules_per_lib idx ~referenced_modules = Module_name.Set.fold referenced_modules ~init:Lib.Map.empty ~f:(fun name acc -> match Module_name.Map.find idx.by_module_name name with diff --git a/src/dune_rules/lib_file_deps.mli b/src/dune_rules/lib_file_deps.mli index 8de240c6f36..9cff4c8ac93 100644 --- a/src/dune_rules/lib_file_deps.mli +++ b/src/dune_rules/lib_file_deps.mli @@ -35,25 +35,13 @@ module Lib_index : sig val empty : t - (** Create an index. Each entry carries the [Module.t] of the entry - module when it is known ([Some] for local libraries, wrapped - or not; [None] for externals). - - [unwrapped_local] is the set of libs that are local and - unwrapped — the tight-eligible class. The cross-library walk - reads ocamldep for any local lib's entries (via - [lookup_walkable_entries], including wrapped libs' children - placed by [Compilation_context.build_lib_index] under the - wrapper's name for auto-generated wrappers and under the - child's own name for hand-written ones); per-module dep - classification is restricted to [tight_eligible] (see - [is_tight_eligible]). - - [no_ocamldep] names local libs whose ocamldep output is - short-circuited (single-module stanzas without library - dependencies — see [Dep_rules.skip_ocamldep]). The cross- - library walk in [module_compilation] must not try to read - [.d] files for those libs. *) + (** [unwrapped_local] is the tight-eligible set (local AND + unwrapped); [no_ocamldep] names local libs whose [.d] files + are short-circuited away by [Dep_rules.skip_ocamldep] and + must therefore be excluded from the cross-library walk. + [entries] carries [Some m] for local libs, [None] for + externals — see [Compilation_context.build_lib_index] for the + indexing convention used for wrapped libs' children. *) val create : no_ocamldep:Lib.Set.t -> unwrapped_local:Lib.Set.t @@ -62,49 +50,35 @@ module Lib_index : sig type classified = { tight : Module.t list Lib.Map.t - (** Directly-referenced tight-eligible libraries (local, - unwrapped). Mapped to the referenced entry modules. These - libraries are candidates for per-module deps via - [deps_of_entry_modules]; whether to emit them is the - caller's policy. *) + (** Tight-eligible libs the consumer references, mapped to + the referenced entry modules. *) ; non_tight : Lib.t list - (** Other directly-referenced libraries — wrapped locals, - externals, or anything else that falls back to a glob. Sorted - by [Lib.compare]. *) + (** Other referenced libs (wrapped locals, externals, …), + sorted by [Lib.compare]. *) } - (** Classify the libraries whose entry modules appear in + (** Classify the libs whose entry modules appear in [referenced_modules]. *) val filter_libs_with_modules : t -> referenced_modules:Module_name.Set.t -> classified - (** [tight_modules_per_lib idx ~referenced_modules] builds a map - from each tight-eligible library that exposes a name in - [referenced_modules] to the subset of its entry modules that - appear. Equivalent to [filter_libs_with_modules] with only the - tight part kept. *) + (** Like [filter_libs_with_modules] but returns only the [tight] + map. *) val tight_modules_per_lib : t -> referenced_modules:Module_name.Set.t -> Module.t list Lib.Map.t - (** [lookup_walkable_entries idx name] returns [(lib, entry module)] - pairs used by the cross-library walk in [module_compilation]. - Includes wrapped local libs' wrappers and their children - (placed in the index by [Compilation_context.build_lib_index] - under the wrapper's name for auto-generated wrappers and - under the child's own name for hand-written ones) — the BFS - reads each walkable entry's ocamldep to follow alias chains - across library boundaries. Libraries in [no_ocamldep] are - excluded (their [.d] files do not exist); externals are - excluded because no [Module.t] is available. *) + (** Walkable entries indexed under [name]: wrappers and children + of wrapped libs alike (see [Compilation_context.build_lib_ + index] for the indexing convention). Libs in [no_ocamldep] + and externals are excluded. *) val lookup_walkable_entries : t -> Module_name.t -> (Lib.t * Module.t) list - (** [is_tight_eligible idx lib] is [true] when [lib] is local and - unwrapped. The cross-library walk has full visibility into - such libraries: the absence of any of their entry modules from - the post-walk reference set is positive evidence that the - consumer does not reach the library, so the consumer's compile - rule does not need a dep on it. *) + (** A library is tight-eligible when local and unwrapped. The + walk has full visibility into such libs, so absence of all + their entry modules from the post-walk reference set means + the consumer doesn't reach the lib; the compile rule needs + no dep on it. *) val is_tight_eligible : t -> Lib.t -> bool end diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index a746ad63ca4..446e1df32b0 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -1,26 +1,12 @@ open Import open Memo.O -(* Two dep pipelines feed every compile rule: - - - [other_cm_files] (in [build_cm] below) — intra-library deps: - the cmis of the sibling modules that this module's own - ocamldep output says it transitively depends on within the - same stanza. These are static deps computed from the stanza's - [Dep_graph.t]. - - - [lib_cm_deps] — inter-library deps, produced by - [lib_deps_for_module] as [Action_builder.dyn_deps]. This is - the per-module inter-library dependency filter (#4572): it - reads ocamldep output to narrow the set of cmis the consumer - depends on from other libraries, falling back to a per-lib - directory glob when filtering is unsafe or unavailable. - - The two sets are composed with [>>>] in the compile rule; there - is no overlap between them because intra-lib references are - already filtered out of ocamldep's "raw" output by - [parse_module_names] for the intra-lib path, and kept as raw - [Module_name.t] values for the inter-lib path. *) +(* The compile rule depends on two disjoint dep sets: [other_cm_files] + for intra-library cmis (static, from [Dep_graph.t]) and + [lib_cm_deps] for inter-library deps (dynamic, the per-module + filter from #4572). Disjointness comes from [parse_module_names] + stripping intra-lib references on the intra-lib path while the + inter-lib path keeps raw [Module_name.t] values. *) let all_libs cctx = let open Resolve.Memo.O in @@ -29,8 +15,6 @@ let all_libs cctx = d @ h ;; -(* Map each element of [xs] through [f] and union the resulting - [Module_name.Set.t]s. Monomorphic on [Module_name.Set] by design. *) let union_module_name_sets_mapped xs ~f = Action_builder.List.map xs ~f |> Action_builder.map @@ -47,43 +31,19 @@ let module_kind_is_filterable m = ;; (* Extend [initial_refs] with module names reached through cross- - library ocamldep. Walk every walkable entry module breadth-first - — wrappers and children of wrapped local libs alike: gather all - [(lib, entry module)] pairs named by the frontier, read each - pair's impl and intf ocamldep, and union the raw names into the - frontier. Iterate until no new names appear. - - Wrapped local libs are walkable too, but how the BFS reaches - their children depends on the wrapper kind (see - [Compilation_context.build_lib_index]): - - - Auto-generated wrappers ([Module.kind = Alias]) have no - readable [.d] of their own ([Dep_rules.deps_of] short-circuits - on [Alias]). Their children are indexed under the WRAPPER's - unmangled name (multi-entry); a frontier hit on the wrapper - expands directly to every child via [lookup_walkable_entries]. - - - Hand-written wrappers ([Impl] / [Intf_only]) have a readable - [.d]; ocamldep on the wrapper's source emits the unmangled - names of the children the wrapper exposes. Children are - indexed under their OWN names, so the BFS resolves those - emitted names in the next round. - - Either way the BFS ends up reading each reachable child's own - ocamldep, surfacing transitive cross-library references like - [include Vendored_pprint] that the wrapper alone would not - reveal. Externals and libs in [no_ocamldep] are excluded by - [lookup_walkable_entries] — they have no readable [.d] file. - Chains passing through excluded libs terminate and the consumer - falls back to a glob on the unreached libs. - - Cycles in the module-reference graph terminate on the [seen] set. + library ocamldep. The frontier expansion follows alias chains + into wrapped libs' children; how those children land in the + index is the indexing decision documented in + [Compilation_context.build_lib_index]. Externals and libs in + [no_ocamldep] are absent from [lookup_walkable_entries], so + chains through them terminate and the consumer falls back to a + glob on the unreached libs. Rule-graph cycle safety: [.d] files depend only on their source - [.ml]/[.mli] (via [Ocamldep.deps_of]) — never on any cmi. So + [.ml]/[.mli] (via [Ocamldep.deps_of]), never on any cmi, so reading a cross-library [.d] file cannot transitively demand any - consumer output, and this walk cannot introduce rule cycles - regardless of how the library graph looks. *) + consumer output. The walk cannot introduce rule cycles regardless + of the library graph. *) let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = let open Action_builder.O in let read_entry_deps (lib, m) = @@ -108,39 +68,14 @@ let cross_lib_tight_set ~lib_index ~for_ ~initial_refs = loop ~seen:Module_name.Set.empty ~frontier:initial_refs ;; -(* Per-module inter-library dependency filtering (#4572). Uses ocamldep - output to determine which libraries a module actually references, then - transitively closes within the compilation context's library set to - handle transparent aliases. Returns the [Dep.Set.t] of file deps the - compile rule should depend on; the wrapper [lib_cm_deps] adapts it - for [Action_builder.dyn_deps]. - - The returned [Dep.Set.t] is the sole source of cross-library file - dependencies for the compile rule: [-I] flags on the ocamlc command - line add search paths but do not register rule deps on their own. - Narrowing here directly narrows rule invalidation. - - Two dep shapes flow out of the filter: - - [Lib_file_deps.deps_of_entry_modules lib names] → specific File - deps on the named cmis (and their cmx/cmj as appropriate). Only - content changes to those specific cmis invalidate the consumer. - - [Lib_file_deps.deps_of_entries libs] → a glob over each lib's - objdir. Any content change to any cmi in that dir invalidates. - - The tight set of referenced module names is computed across - library boundaries by [cross_lib_tight_set]: it starts from the - consumer's own ocamldep reads and iterates through tight-eligible - entry modules. This lets closure-reached libraries (not directly - named by the consumer but pulled in through transparent aliases) - receive specific-file deps on just the entries that matter. - - These deps surface in [dune rules ]'s output alongside any - static deps. Falls back to a glob over all cctx libs when filtering - is not possible. - - Wrapped dependency libraries always take the glob path. See the - module-level comment at the top of [lib_file_deps.ml] for the - ocamldep-granularity reason and sketches of follow-on work. *) +(* Per-module inter-library dependency filtering (#4572). The + returned [Dep.Set.t] is the sole source of cross-library rule + deps for the compile rule — [-I] flags add search paths but + register no deps — so narrowing here directly narrows rule + invalidation. Falls back to a glob over the library's objdir + when filtering is unsafe (Melange, virtual-impl) or unavailable + (wrapped libs; see [lib_file_deps.ml]'s opening for the + ocamldep-granularity reason). *) let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kind ~mode m = let open Action_builder.O in let can_filter = @@ -184,31 +119,12 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin else let* lib_index = Resolve.Memo.read (Compilation_context.lib_index cctx) in let* trans_deps = Dep_graph.deps_of dep_graph m in - (* Whether to read [dep_m]'s [.ml]-side ocamldep when collecting - the libraries the consumer references. - - A module's [.cmi] is produced from [.mli] when one exists; only - then. References in [.ml] that are not re-exported through the - [.mli] never appear in [.cmi]'s imports — they are sealed. - - The [.cmx] is produced from [.ml]. The compiler reads a - dep's [.cmx] only for cross-module inlining, which happens - only when compiling [Ocaml Cmx] without [-opaque]. In every - other case the [.cmx] is unread and any [.ml]-side - references it carries do not propagate. - - For [m] itself (the consumer being compiled), [.ml] is the - source the compiler is feeding when [ml_kind = Impl]; its - references must be resolved. When [ml_kind = Intf] we are - compiling [m.cmi] from [m.mli], and [m.ml] is unread. - - For a transitive intra-library dep [dep_m], [.ml]-side - references only propagate when (a) [dep_m] has no [.mli] - (so [dep_m.cmi] is produced from [.ml]) or (b) the - consumer reads [dep_m.cmx] for inlining ([Ocaml Cmx] - without [-opaque]). - - Decision summary: + (* Whether to read [dep_m]'s [.ml]-side ocamldep. [.ml]-side + references propagate to the consumer only when [dep_m]'s + [.cmx] is read ([Cmx] + not opaque) or when [dep_m] has no + [.mli] (so its [.cmi] is produced from [.ml]). For the + consumer itself, [.ml] is read iff we are compiling its + [.ml] (i.e. [ml_kind = Impl]). | [dep_m] is | [cm_kind] | [opaque] | read [.ml]? | | ----------------------- | ----------- | -------- | ------------ | @@ -250,46 +166,27 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in let open_modules = Ocaml_flags.extract_open_module_names flags in let referenced = Module_name.Set.union all_raw open_modules in - (* First use of [filter_libs_with_modules]: identify the libs - the consumer directly names (via its own ocamldep output). - Used only to seed [Lib.closure]'s input. The second use, - after the cross-library walk extends the name set, produces - the per-lib module lists used in the fold below. *) + (* Seed [Lib.closure] with the libs the consumer directly + names; the post-BFS classify below uses + [tight_modules_per_lib] on the extended name set. *) let { Lib_file_deps.Lib_index.tight; non_tight } = Lib_file_deps.Lib_index.filter_libs_with_modules lib_index ~referenced_modules:referenced in - (* Sort to preserve the canonical [Lib.closure] memo key (see - commit 9359b37e6 on the base branch). *) + (* Sort for [Lib.closure] memo-key stability. *) let direct_libs = List.sort ~compare:Lib.compare (Lib.Map.keys tight @ non_tight) in - (* Transitively close the filtered libraries. Transparent module - aliases can create cross-library .cmi reads that ocamldep - doesn't report, at arbitrary depth. [libs] is already the - transitive closure required for compilation (across all - [implicit_transitive_deps] modes), so [Lib.closure]'s result - on a subset of [libs] stays within it. *) + (* Close transitively to catch cross-library [.cmi] reads via + transparent aliases that ocamldep doesn't report. *) let* all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in let+ tight_set = cross_lib_tight_set ~lib_index ~for_ ~initial_refs:referenced in - (* Classify [all_libs] against the cross-library tight set into - three buckets: - - - [Some modules] in [tight_modules]: per-module deps on the - entries actually referenced. - - - [None] AND tight-eligible: the cross-library walk had full - visibility (local, unwrapped) and no entry of the library - appears in [tight_set]. The consumer therefore does not - reach this library through any reference chain visible to - the walk, including transitive module aliases under the - [-opaque]-aware impl-side reads. Drop the library entirely - from the consumer's compile rule deps; the link rule still - pulls it in through [requires_link] for executables / - libraries that need it. - - - [None] AND not tight-eligible: wrapped local, external, or - virtual-impl. The walk does not have full visibility, so - fall back to a glob over the library's objdir. *) + (* Classify [all_libs] into three buckets: + - in [tight_modules]: per-module deps on the entries + actually referenced. + - tight-eligible but not referenced: the walk had full + visibility (local, unwrapped) and saw nothing reach the + lib; drop it. Link still pulls it in via [requires_link]. + - not tight-eligible: glob fallback. *) let tight_modules = Lib_file_deps.Lib_index.tight_modules_per_lib lib_index @@ -312,10 +209,6 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin Dep.Set.union tight_deps glob_deps ;; -(* Convenience wrapper for the two call sites ([build_cm] and - [ocamlc_i]) that wire the per-module filter into a compile rule. - Both take [cm_kind], [ml_kind], [mode], and the module to be - compiled; the rest is derived from [cctx]. *) let lib_cm_deps ~cctx ~cm_kind ~ml_kind ~mode m = let obj_dir = Compilation_context.obj_dir cctx in let opaque = Compilation_context.opaque cctx in From 46de0a3155ddb375ddf8b41dbf3448faa8a1a8c3 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 19:33:58 -0700 Subject: [PATCH 75/80] docs: align test prose with the alias mechanism it actually uses The test was originally drafted around [include Vendored_pprint] to mirror menhir's [base]/[middle] arrangement, but was switched to [module Re = Original_name] because a transparent alias is what exposes the per-module-filter precision gap. The prose was not updated alongside. Three Copilot review comments flagged the residual mismatch. This commit: - Updates the intro to describe the actual mechanism (transparent module alias), not [include]. - Reframes the menhir analogy as a structural mirror, with an explicit note that menhir uses [include] and this test deliberately uses an alias for sharper precision; defers the full "why alias" explanation to the paragraph below the Structure block, where it already lives. - Fixes the Structure paragraph: [consumer.ml] does name [Pprint] (the child of [lib_re_export]); the prose previously claimed it named no children. Reworded to "names [Pprint] but not [lib_a] or the wrapper [Lib_re_export]". No code change. Test still passes. Signed-off-by: Robin Bate Boerop --- .../auto-wrapped-child-reexport.t | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t b/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t index 502ecdab8d6..c2b484e6108 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t @@ -1,18 +1,20 @@ Observational baseline: a consumer reaches a dep library's modules -through a child of an auto-wrapped sibling library, where the -sibling is opened via the [-open] compiler flag and the child's -source includes the dep library's module via [include]. On trunk, +through a child of an auto-wrapped sibling library. The sibling is +opened via the [-open] compiler flag and the child's source exposes +the dep library through a transparent module alias. On trunk, [consumer] correctly rebuilds when the dep library's interface changes, because the cctx-wide compile-rule deps cover every library in the stanza's [(libraries ...)] closure. -This is the structural shape of menhir's [base]/[middle] -arrangement: [base] is auto-wrapped (no [base.ml] in the source, -dune generates the wrapper), with a child [PPrint.ml] containing -[include Vendored_pprint]. [middle] uses [-open Base] in flags and -references [PPrint.foo] in source. The reference chain crosses -library boundaries through the auto-wrapped sibling's child, not -through a hand-written wrapper. +The structural shape mirrors menhir's [base]/[middle] arrangement: +[base] is auto-wrapped (no [base.ml] in source — dune generates +the wrapper) with a child module that re-exports a third-party +dep; [middle] uses [-open Base] and reaches the dep through that +child. Menhir uses [include Vendored_pprint]; this test uses a +transparent alias instead, for the precision-bug reason described +below. The reference chain still crosses library boundaries +through the auto-wrapped sibling's child, not through a +hand-written wrapper. Records the consumer's rebuild count as a regression guard for changes that narrow compile-rule deps per module. @@ -23,8 +25,9 @@ file — dune auto-generates the wrapper); child [pprint.ml] contains a transparent alias [module Re = Original_name]; [filler.ml] is just there to keep the lib non-singleton. [lib_b] depends on [lib_re_export] and uses [-open Lib_re_export] in its -flags; [consumer.ml] writes [Pprint.Re.x] without naming -[lib_a], [lib_re_export], or any of its children in source. +flags; [consumer.ml] writes [Pprint.Re.x] — naming the wrapped +lib's child [Pprint] but not [lib_a] or the wrapper +[Lib_re_export]. The transparent-alias shape (rather than [include]) is what makes [lib_a] genuinely invisible to a per-module dep filter that only From 26018511c6514e77eaa6dc2cd1360d1e10f774a3 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 19:51:33 -0700 Subject: [PATCH 76/80] perf: skip has_library_deps lookup for multi-module libs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In [build_lib_index], [has_resolved_deps] was computed unconditionally for every local library in [direct_requires], but its only consumer is the [no_ocamldep_lib] decision below — and that decision returns [None] outright when the lib has more than one module ([Modules.as_singleton mods = None]). Multi-module libs were running [Dep_rules.has_library_deps_of_lib] for nothing. Move the helper call inside the singleton branch, so multi-module libs skip it entirely. [Lib.requires] is memoised so the wasted work was cheap, but the conditional should match the use; the gating is also clearer to read. No behavioural change. Test suite passes. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 63568bff40d..61c108770af 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -149,9 +149,6 @@ let build_lib_index ~super_context ~libs ~for_ = (Lib.Local.of_lib_exn lib) ~for_ in - (* Same helper the lib's own cctx uses — keeps the skip - decision and the walk's prediction in lock step. *) - let+ has_resolved_deps = Dep_rules.has_library_deps_of_lib lib ~for_ in (* Every local lib's entries carry [Some m] so the walk can read the entry's ocamldep. Whether to issue per- module deps is tracked separately via [unwrapped_local] @@ -216,10 +213,18 @@ let build_lib_index ~super_context ~libs ~for_ = in entry_entries @ child_entries) in - let no_ocamldep_lib = + (* Only single-module libs can land in [no_ocamldep]; + multi-module libs always have [.d] files for their + children. Skip the helper call otherwise. *) + let+ no_ocamldep_lib = match Modules.as_singleton mods with - | Some _ when not has_resolved_deps -> Some lib - | _ -> None + | None -> Memo.return None + | Some _ -> + (* Same helper the lib's own cctx uses — keeps the + skip decision and the walk's prediction in lock + step. *) + let+ has_resolved_deps = Dep_rules.has_library_deps_of_lib lib ~for_ in + if has_resolved_deps then None else Some lib in entries, no_ocamldep_lib, if unwrapped then Some lib else None)) in From a28d126c1805046515070ac4c7996f976dd341a2 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 20:00:09 -0700 Subject: [PATCH 77/80] refactor: rename Compilation_context.create's [?lib] to [?own_lib] [?lib] read like an optional library override; it's actually a "this cctx is being constructed for the given Lib.t" signal, used to route [has_library_deps] through [Dep_rules.has_library_deps_ of_lib] instead of the resolved-direct-and-hidden derivation. Name the parameter for that role. [for_] was already taken by the [Compilation_mode] field, so [?own_lib] avoids confusion with the existing [for_alias_module] / [for_root_module] / [for_plugin_executable] family of constructors-by-context. Pure rename. Test suite passes. Signed-off-by: Robin Bate Boerop --- src/dune_rules/compilation_context.ml | 4 ++-- src/dune_rules/compilation_context.mli | 2 +- src/dune_rules/lib_rules.ml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 61c108770af..46ba6919ed9 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -257,7 +257,7 @@ let create ?cms_cmt_dependency ?loc ?instances - ?lib + ?own_lib for_ = let project = Scope.project scope in @@ -301,7 +301,7 @@ let create [build_lib_index] uses, so the lib's skip decision and the cross-stanza walk's prediction can't drift. Other cctxes just need the boolean. *) - match lib with + match own_lib with | Some lib -> Dep_rules.has_library_deps_of_lib lib ~for_ | None -> Dep_rules.has_library_deps_of_resolved diff --git a/src/dune_rules/compilation_context.mli b/src/dune_rules/compilation_context.mli index f65fbb4c2ba..8f14bd7b24a 100644 --- a/src/dune_rules/compilation_context.mli +++ b/src/dune_rules/compilation_context.mli @@ -40,7 +40,7 @@ val create -> ?cms_cmt_dependency:Workspace.Context.Cms_cmt_dependency.t -> ?loc:Loc.t -> ?instances:Parameterised_instances.t Resolve.Memo.t - -> ?lib:Lib.t + -> ?own_lib:Lib.t -> Compilation_mode.t -> t Memo.t diff --git a/src/dune_rules/lib_rules.ml b/src/dune_rules/lib_rules.ml index 06d6c1bfdee..1b25f236a44 100644 --- a/src/dune_rules/lib_rules.ml +++ b/src/dune_rules/lib_rules.ml @@ -533,7 +533,7 @@ let cctx ~melange_package_name ~modes ~instances - ~lib:local_lib + ~own_lib:local_lib ;; let library_rules From 305d047f8ecc76ce1dc27edaaf7957b60705abd8 Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 20:21:16 -0700 Subject: [PATCH 78/80] fix: drop link-only transitive deps from compile-rule glob fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported by @nojb in https://github.com/ocaml/dune/pull/14116#issuecomment-4323883194: under [(implicit_transitive_deps false)], a consumer was rebuilt when an unreferenced transitive library's source changed, even though the consumer cannot see the library through [-I]/[-H] and its source cannot reference the library at all. Cause: [Lib.closure direct_libs ~linking:false] in [lib_deps_for_module] expands link-time deps too. Under [implicit_transitive_deps = Disabled], the cctx's compile scope ([direct_requires + hidden_requires]) is just the declared [(libraries ...)] — but the closure walks transitively and surfaces link-only libs not in compile scope. The post-walk classify then treated them as "not tight-eligible" and globbed their objdirs, producing spurious rebuilds. Fix: filter the closure result by the cctx's compile scope. Libs in the closure that aren't in [compile_scope = direct + hidden] are dropped — link-only transitive deps the compiler cannot reference. Hidden libs (under [Disabled_with_hidden_includes]) remain in [compile_scope] and continue to glob, since they're on the compiler's [-H] path. The previous behaviour under [implicit_transitive_deps = Enabled] is preserved because there [direct_requires = requires_link], which IS the transitive closure, so [compile_scope] equals the closure and the new check is a no-op. Regression test [implicit-transitive-deps-false.t] is in a separate PR off [main] (the property already holds on trunk — this PR only added the regression that the test guards against). Signed-off-by: Robin Bate Boerop --- src/dune_rules/module_compilation.ml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 446e1df32b0..9a6909c3299 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -177,16 +177,23 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin (* Sort for [Lib.closure] memo-key stability. *) let direct_libs = List.sort ~compare:Lib.compare (Lib.Map.keys tight @ non_tight) in (* Close transitively to catch cross-library [.cmi] reads via - transparent aliases that ocamldep doesn't report. *) + transparent aliases that ocamldep doesn't report. + [Lib.closure] expands link-time deps too; under + [(implicit_transitive_deps false)] those reach libs that + the compiler cannot see (no entry in [-I] or [-H]). The + fold below filters them out via [compile_scope]. *) let* all_libs = Resolve.Memo.read (Lib.closure direct_libs ~linking:false ~for_) in let+ tight_set = cross_lib_tight_set ~lib_index ~for_ ~initial_refs:referenced in - (* Classify [all_libs] into three buckets: - - in [tight_modules]: per-module deps on the entries - actually referenced. - - tight-eligible but not referenced: the walk had full - visibility (local, unwrapped) and saw nothing reach the - lib; drop it. Link still pulls it in via [requires_link]. - - not tight-eligible: glob fallback. *) + let compile_scope = Lib.Set.of_list libs in + (* Classify each lib in [all_libs]: + - in [tight_modules]: per-module deps. + - tight-eligible but unreferenced: drop. Link still pulls + it in via [requires_link]. + - in [compile_scope] but not tight-eligible: glob (wrapped, + external, virtual-impl, or hidden under + [Disabled_with_hidden_includes]). + - outside [compile_scope]: drop. Link-only transitive dep + that the compiler cannot reference. *) let tight_modules = Lib_file_deps.Lib_index.tight_modules_per_lib lib_index @@ -203,7 +210,9 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin | None -> if Lib_file_deps.Lib_index.is_tight_eligible lib_index lib then td, gl - else td, lib :: gl) + else if Lib.Set.mem compile_scope lib + then td, lib :: gl + else td, gl) in let glob_deps = Lib_file_deps.deps_of_entries ~opaque ~cm_kind glob_libs in Dep.Set.union tight_deps glob_deps From 60fd5ab4b83d37f26a8d0b4a5d7fa84cbb9ca5aa Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Sun, 26 Apr 2026 22:34:49 -0700 Subject: [PATCH 79/80] test: track unwrapped-tight-deps.t format change in dep PR #14310 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #14310 (the dep PR that owns unwrapped-tight-deps.t) was updated per rgrinberg's review to print rebuild target names via [targetsMatchingFilter] rather than counts via [length], matching the idiom used by sibling tests like alias-reexport.t. Resync this PR's copy with #14310's current shape: take the canonical content from #14310, with A1/A3 assertions promoted to [] (the per-module filter drops unreferenced [base] modules from [c]'s deps) and A2 keeping its rebuild target list (the one module [c] references must still rebuild). The other dep PRs whose tests this branch borrows have only prose-style differences (post-fix prose on this branch vs observational-baseline prose on the dep PR) — not new churn — so they don't need a similar resync. Signed-off-by: Robin Bate Boerop --- .../unwrapped-tight-deps.t | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t index 571ac8a8543..31a16df8da1 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -1,18 +1,20 @@ -Per-module tight deps within an unwrapped library. +Baseline: consumer-module rebuild targets when individual modules +of an unwrapped dependency library change. -When a consumer module references a specific module of an unwrapped -dependency library, dune emits dynamic dependencies on exactly the -referenced module's .cmi/.cmx files — not a directory-wide glob over -every module of the library. The consumer's rule only invalidates -when a module it actually reads changes. +This is an observational test. It records the rebuild targets for +a single consumer module C when each of three entry modules +(A1, A2, A3) of the dependency library [base] has its interface +edited. C references only A2. -This test exercises the tightening with a consumer module [C] that -references only [A2] of three entry modules in the dependency [base]. -Editing [A1] or [A3] must leave [C] untouched; editing [A2] must -rebuild [C]. +On current main, editing any one of A1/A2/A3 causes C to rebuild +because library-level dependency filtering (and, within that, +per-module tightening) is not yet in place: the consumer is +conservatively rebuilt whenever any entry module's cmi changes. +Work on https://github.com/ocaml/dune/issues/4572 is expected to +tighten this, at which point editing A1 or A3 leaves C untouched +and the emitted target list becomes empty. See: https://github.com/ocaml/dune/issues/4572 -See: https://github.com/ocaml/dune/pull/14116#issuecomment-4301275263 $ cat > dune-project < (lang dune 3.0) @@ -46,9 +48,11 @@ explicit interface so signature changes propagate through .cmi files: consumer has two modules. The module of interest, [c.ml], references only [A2] from [base]. A second, unused module [d.ml] is present only -to keep [consumer] a multi-module stanza so ocamldep is never short- -circuited and the rebuild count for [c] reflects the per-module -filter's work independently of any skip-ocamldep heuristic: +to keep [consumer] a multi-module stanza: dune skips ocamldep for +single-module stanzas with no library deps as an optimisation, and +the per-module-lib-deps filter depends on ocamldep output. Including +[d.ml] isolates this test from the skip-ocamldep optimisation so the +rebuild targets for [c] reflect only the per-module filter's work: $ mkdir consumer $ cat > consumer/dune < base/a1.mli < val v : int @@ -75,8 +79,8 @@ rebuild C: > let extra () = 7 > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length' - 0 + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))]' + [] Same for A3: @@ -89,11 +93,11 @@ Same for A3: > let other = "hi" > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length' - 0 + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))]' + [] -Changing A2's interface — the one module C does reference — must -rebuild C: +Edit A2's interface — the one module C does reference — and record +the rebuild targets (C must rebuild): $ cat > base/a2.mli < val v : int @@ -104,5 +108,13 @@ rebuild C: > let new_fn x = x + 1 > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length > 0' - true + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))]' + [ + { + "target_files": [ + "_build/default/consumer/.consumer.objs/byte/c.cmi", + "_build/default/consumer/.consumer.objs/byte/c.cmo", + "_build/default/consumer/.consumer.objs/byte/c.cmt" + ] + } + ] From fcf728f9cbf599e2d2f22eaa8ccc540765e9798a Mon Sep 17 00:00:00 2001 From: Robin Bate Boerop Date: Mon, 27 Apr 2026 11:12:59 -0700 Subject: [PATCH 80/80] test: resync test files with dep PR renames after rebase After rebase onto upstream/main (which now contains #14309), the file-add commits from dep PRs were skipped as redundant, dropping my prior rename-tracking commit (655e81c0f) along the way. This re-applies the renames to bring this PR's local copies of dep PR tests back in sync with the dep PRs' canonical (semantic-name) versions. Files touched: 9 cram tests in per-module-lib-deps/. Mechanical rename + count flip via [dune runtest --auto-promote] where the post-fix count differs from the dep PR's baseline. Test suite passes. Signed-off-by: Robin Bate Boerop --- .../auto-wrapped-child-reexport.t | 20 ++-- .../lib-vs-lib-name-collision.t | 63 +++++------ .../opaque-cmx-deps-local.t | 18 ++-- .../per-module-lib-deps/opaque-mli-change.t | 28 ++--- .../sibling-unreferenced-lib.t | 2 +- .../single-module-unreferenced-lib.t | 30 +++--- .../transitive-unreferenced-lib.t | 32 +++--- .../unwrapped-tight-deps.t | 101 +++++++++--------- .../wrapped-reexport-via-open-flag.t | 16 +-- 9 files changed, 159 insertions(+), 151 deletions(-) diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t b/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t index c2b484e6108..4bdf4411843 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/auto-wrapped-child-reexport.t @@ -19,36 +19,36 @@ hand-written wrapper. Records the consumer's rebuild count as a regression guard for changes that narrow compile-rule deps per module. -Structure: [lib_a] is unwrapped with module [Original_name]. +Structure: [dep_lib] is unwrapped with module [Original_name]. [lib_re_export] is wrapped by dune defaults (no [lib_re_export.ml] file — dune auto-generates the wrapper); child [pprint.ml] contains a transparent alias [module Re = Original_name]; -[filler.ml] is just there to keep the lib non-singleton. [lib_b] +[filler.ml] is just there to keep the lib non-singleton. [consumer_lib] depends on [lib_re_export] and uses [-open Lib_re_export] in its flags; [consumer.ml] writes [Pprint.Re.x] — naming the wrapped -lib's child [Pprint] but not [lib_a] or the wrapper +lib's child [Pprint] but not [dep_lib] or the wrapper [Lib_re_export]. The transparent-alias shape (rather than [include]) is what makes -[lib_a] genuinely invisible to a per-module dep filter that only +[dep_lib] genuinely invisible to a per-module dep filter that only walks the auto-generated wrapper: with [-no-alias-deps], [Pprint]'s [.cmi] content does not depend on [Original_name.cmi], so a glob over [lib_re_export]'s objdir does not capture [Original_name.mli] changes either. The cctx-wide compile-rule deps still cover -[lib_a] on trunk, so [consumer] rebuilds. +[dep_lib] on trunk, so [consumer] rebuilds. $ cat > dune-project < (lang dune 3.0) > EOF $ cat > dune < (library (name lib_a) (wrapped false) (modules original_name)) + > (library (name dep_lib) (wrapped false) (modules original_name)) > (library > (name lib_re_export) > (modules pprint filler) - > (libraries lib_a)) + > (libraries dep_lib)) > (library - > (name lib_b) + > (name consumer_lib) > (wrapped false) > (modules consumer) > (libraries lib_re_export) @@ -75,10 +75,10 @@ changes either. The cctx-wide compile-rule deps still cover $ dune build @check -Edit [lib_a]'s interface. [consumer] reaches [lib_a]'s +Edit [dep_lib]'s interface. [consumer] reaches [dep_lib]'s [Original_name] through [Pprint] (child of auto-wrapped [Lib_re_export]). The cctx-wide compile-rule deps include -[lib_a], so [consumer] rebuilds: +[dep_lib], so [consumer] rebuilds: $ cat > original_name.mli < val x : string diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t index 549dfaf0010..79e4a139471 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/lib-vs-lib-name-collision.t @@ -7,11 +7,11 @@ version of the module. The other lib's same-named module is shadowed — unreachable from consumer code. This test is observational: it records the rebuild-target count -for a consumer module after editing the *losing* (shadowed) lib's +for a consumer module after editing the *shadowed* lib's same-named module. The expected count on current [main] is positive because cross-library dep tracking is library-level: the consumer depends on a glob over each declared library's public -cmi directory, so any change to the losing lib's public cmis +cmi directory, so any change to the shadowed lib's public cmis invalidates the consumer. A future per-module dependency filter (#4572) would not @@ -22,51 +22,54 @@ resolves through, and must conservatively depend on both. See: https://github.com/ocaml/dune/issues/4572 $ cat > dune-project < (lang dune 3.0) + > (lang dune 3.23) > EOF - $ mkdir lib_a - $ cat > lib_a/dune < (library (name lib_a) (wrapped false)) + $ mkdir active_lib + $ cat > active_lib/dune < (library (name active_lib) (wrapped false)) > EOF - $ cat > lib_a/shared.ml < let from_a = "a" + $ cat > active_lib/shared.ml < let from_active = "a" > EOF - $ mkdir lib_b - $ cat > lib_b/dune < (library (name lib_b) (wrapped false)) + $ mkdir shadowed_lib + $ cat > shadowed_lib/dune < (library (name shadowed_lib) (wrapped false)) > EOF - $ cat > lib_b/shared.ml < let from_b = "b" + $ cat > shadowed_lib/shared.ml < let from_shadowed = "b" > EOF -[consumer] lists [lib_a] first, so [-I lib_a/.objs] comes before -[-I lib_b/.objs] on the command line and OCaml resolves [Shared] -to [lib_a/shared.cmi]: +[consumer_lib] lists [active_lib] first, so [-I active_lib/.objs] +comes before [-I shadowed_lib/.objs] on the command line and OCaml +resolves [Shared] to [active_lib/shared.cmi]: - $ mkdir consumer - $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries lib_a lib_b)) + $ mkdir consumer_lib + $ cat > consumer_lib/dune < (library + > (name consumer_lib) + > (wrapped false) + > (libraries active_lib shadowed_lib)) > EOF - $ cat > consumer/c.ml < let _ = Shared.from_a + $ cat > consumer_lib/consumer.ml < let _ = Shared.from_active > EOF - $ cat > consumer/d.ml < consumer_lib/filler.ml < let _ = () > EOF $ dune build @check -Edit [lib_b]'s [Shared] — which the consumer does *not* resolve -through, because [lib_a] wins under [-I] order. Consumer still -rebuilds because its dependency on [lib_b]'s object directory is -a glob: +Edit [shadowed_lib]'s [Shared] — which [consumer] does *not* resolve +through, because [active_lib] wins under [-I] order. [consumer] +still rebuilds because its dependency on [shadowed_lib]'s object +directory is a glob: - $ cat > lib_b/shared.ml < let from_b = "b" - > let also_from_b = 42 + $ cat > shadowed_lib/shared.ml < let from_shadowed = "b" + > let also_from_shadowed = 42 > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))] | length > 0' + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer_lib/\\.consumer_lib\\.objs/byte/consumer\\."))] | length > 0' true diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t index 5b7f64f8e42..529431297ff 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-cmx-deps-local.t @@ -4,28 +4,28 @@ A consumer's [.cmx] compilation rule depends on a local library's behave differently; see [opaque-cmx-deps-external.t]. $ cat > dune-project < (lang dune 3.0) + > (lang dune 3.23) > EOF - $ mkdir lib - $ cat > lib/dune < (library (name mylib)) + $ mkdir local_dep + $ cat > local_dep/dune < (library (name local_dep)) > EOF - $ cat > lib/mylib.ml < local_dep/local_dep.ml < let v = 42 > EOF $ cat > dune < (executable (name main) (libraries mylib)) + > (executable (name main) (libraries local_dep)) > EOF $ cat > main.ml < let () = print_int Mylib.v + > let () = print_int Local_dep.v > EOF --- Release profile (opaque=false): both .cmi and .cmx globs --- $ cat > dune-workspace < (lang dune 3.0) + > (lang dune 3.23) > (profile release) > EOF @@ -38,7 +38,7 @@ behave differently; see [opaque-cmx-deps-external.t]. --- Dev profile (opaque=true): only .cmi glob --- $ cat > dune-workspace < (lang dune 3.0) + > (lang dune 3.23) > (profile dev) > EOF diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t index 01de3b10287..c334d2b7ea8 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/opaque-mli-change.t @@ -6,31 +6,31 @@ whether cross-module inlining tracks a dep's [.cmx]. Companion to [opaque.t], which covers the [.ml]-only change axis. $ cat > dune-project < (lang dune 3.0) + > (lang dune 3.23) > EOF - $ mkdir lib - $ cat > lib/dune < (library (name mylib)) + $ mkdir dep_lib + $ cat > dep_lib/dune < (library (name dep_lib)) > EOF - $ cat > lib/mylib.ml < dep_lib/dep_lib.ml < let v = 42 > EOF - $ cat > lib/mylib.mli < dep_lib/dep_lib.mli < val v : int > EOF $ cat > dune < (executable (name main) (libraries mylib)) + > (executable (name main) (libraries dep_lib)) > EOF $ cat > main.ml < let () = print_int Mylib.v + > let () = print_int Dep_lib.v > EOF --- Release profile (opaque=false): .mli change rebuilds consumer --- $ cat > dune-workspace < (lang dune 3.0) + > (lang dune 3.23) > (profile release) > EOF @@ -39,11 +39,11 @@ Companion to [opaque.t], which covers the [.ml]-only change axis. Add a new declaration to both [.ml] and [.mli] (paired so no value is left unexported, which would trip warning 32 under dev): - $ cat > lib/mylib.ml < dep_lib/dep_lib.ml < let v = 42 > let extra () = 0 > EOF - $ cat > lib/mylib.mli < dep_lib/dep_lib.mli < val v : int > val extra : unit -> int > EOF @@ -55,7 +55,7 @@ left unexported, which would trip warning 32 under dev): --- Dev profile (opaque=true): .mli change still rebuilds consumer --- $ cat > dune-workspace < (lang dune 3.0) + > (lang dune 3.23) > (profile dev) > EOF @@ -63,12 +63,12 @@ left unexported, which would trip warning 32 under dev): Add another paired declaration: - $ cat > lib/mylib.ml < dep_lib/dep_lib.ml < let v = 42 > let extra () = 0 > let helper x = x + 1 > EOF - $ cat > lib/mylib.mli < dep_lib/dep_lib.mli < val v : int > val extra : unit -> int > val helper : int -> int diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/sibling-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/sibling-unreferenced-lib.t index 56874d5ec51..bfcb7a9190b 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/sibling-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/sibling-unreferenced-lib.t @@ -73,4 +73,4 @@ rebuild targets observed in the trace: > EOF $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("spurious_rebuild"))] | length' - 1 + 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t index 2cbd5585af9..3632f5f8499 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/single-module-unreferenced-lib.t @@ -3,12 +3,12 @@ of its declared dependency library. Under per-module library dep filtering, the consumer is not rebuilt when the dependency's interface changes. -[libB] declares [(libraries libA)] but [modB.ml] does not reference -any module of [libA]. Ocamldep runs on [modB] (single-module stanzas +[consumer_lib] declares [(libraries dep_lib)] but [spurious_rebuild.ml] does not reference +any module of [dep_lib]. Ocamldep runs on [spurious_rebuild] (single-module stanzas with library deps are no longer short-circuited), reports no -references to [libA], and the filter drops [libA] from [modB]'s deps -entirely. Editing [modA]'s interface no longer invalidates anything -in [modB]'s dep set, so [modB] stays built. +references to [dep_lib], and the filter drops [dep_lib] from [spurious_rebuild]'s deps +entirely. Editing [dep_module]'s interface no longer invalidates anything +in [spurious_rebuild]'s dep set, so [spurious_rebuild] stays built. See: https://github.com/ocaml/dune/issues/4572 See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 @@ -18,33 +18,33 @@ See: https://github.com/ocaml/dune/pull/14116#issuecomment-4286949811 > EOF $ cat > dune < (library (name libA) (wrapped false) (modules modA)) - > (library (name libB) (wrapped false) (modules modB) (libraries libA)) + > (library (name dep_lib) (wrapped false) (modules dep_module)) + > (library (name consumer_lib) (wrapped false) (modules spurious_rebuild) (libraries dep_lib)) > EOF - $ cat > modA.ml < dep_module.ml < let x = 42 > EOF - $ cat > modA.mli < dep_module.mli < val x : int > EOF - $ cat > modB.ml < spurious_rebuild.ml < let x = 12 > EOF $ dune build @check -Edit modA's interface. modB does not reference modA. Record the -number of modB rebuild targets observed in the trace: +Edit dep_module's interface. spurious_rebuild does not reference dep_module. Record the +number of spurious_rebuild rebuild targets observed in the trace: - $ cat > modA.mli < dep_module.mli < val x : int > val y : string > EOF - $ cat > modA.ml < dep_module.ml < let x = 42 > let y = "hello" > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("modB"))] | length' + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("spurious_rebuild"))] | length' 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t index 5c6ba4a2765..c8de3216823 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/transitive-unreferenced-lib.t @@ -1,43 +1,43 @@ -An intermediate library [libB] declares [(libraries libA)] but its -module [modB] does not reference any of [libA]'s modules. The -consumer [main] uses [libB] and so transitively gains [libA] in its +An intermediate library [intermediate_lib] declares [(libraries dep_lib)] but its +module [intermediate_module] does not reference any of [dep_lib]'s modules. The +consumer [main] uses [intermediate_lib] and so transitively gains [dep_lib] in its compilation context. -The cross-library walk has full visibility into [libA] (local, +The cross-library walk has full visibility into [dep_lib] (local, unwrapped, every entry has a known module), and finishes without -adding any of [libA]'s entry modules to the reference closure. The -filter therefore drops [libA] from [main]'s compile-rule deps -entirely; the link rule still pulls [libA] in for executables that +adding any of [dep_lib]'s entry modules to the reference closure. The +filter therefore drops [dep_lib] from [main]'s compile-rule deps +entirely; the link rule still pulls [dep_lib] in for executables that need it. -Editing [modA1.ml] does not invalidate [main]. +Editing [unreferenced_dep.ml] does not invalidate [main]. $ cat > dune-project < (lang dune 3.0) > EOF $ cat > dune < (library (name libA) (wrapped false) (modules modA1 modA2)) - > (library (name libB) (wrapped false) (modules modB) (libraries libA)) - > (executable (name main) (modules main) (libraries libB)) + > (library (name dep_lib) (wrapped false) (modules unreferenced_dep referenced_dep)) + > (library (name intermediate_lib) (wrapped false) (modules intermediate_module) (libraries dep_lib)) + > (executable (name main) (modules main) (libraries intermediate_lib)) > EOF - $ cat > modA1.ml < unreferenced_dep.ml < let x = 42 > EOF - $ cat > modA2.ml < referenced_dep.ml < let x = 43 > EOF - $ cat > modB.ml < intermediate_module.ml < let x = 42 > EOF $ cat > main.ml < let _ = ModB.x + > let _ = Intermediate_module.x > EOF $ dune build @check - $ echo > modA1.ml + $ echo > unreferenced_dep.ml $ dune build @check $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("dune__exe__Main"))] | length' 0 diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t index 31a16df8da1..4e41f5dd592 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/unwrapped-tight-deps.t @@ -2,119 +2,124 @@ Baseline: consumer-module rebuild targets when individual modules of an unwrapped dependency library change. This is an observational test. It records the rebuild targets for -a single consumer module C when each of three entry modules -(A1, A2, A3) of the dependency library [base] has its interface -edited. C references only A2. +the consumer module [consumer] when each of three entry modules of +the dependency library [dep_lib] has its interface edited. +[consumer] references only [Referenced_dep] from [dep_lib]; +[Unread_dep_a] and [Unread_dep_b] are present but unreferenced. -On current main, editing any one of A1/A2/A3 causes C to rebuild +On current main, editing any of the three rebuilds [consumer] because library-level dependency filtering (and, within that, per-module tightening) is not yet in place: the consumer is conservatively rebuilt whenever any entry module's cmi changes. Work on https://github.com/ocaml/dune/issues/4572 is expected to -tighten this, at which point editing A1 or A3 leaves C untouched -and the emitted target list becomes empty. +tighten this, at which point editing [Unread_dep_a] or +[Unread_dep_b] leaves [consumer] untouched and the emitted target +list becomes empty. See: https://github.com/ocaml/dune/issues/4572 $ cat > dune-project < (lang dune 3.0) + > (lang dune 3.23) > EOF -base is an unwrapped library with three entry modules, each with an -explicit interface so signature changes propagate through .cmi files: +[dep_lib] is an unwrapped library with three entry modules, each +with an explicit interface so signature changes propagate through +.cmi files: - $ mkdir base - $ cat > base/dune < (library (name base) (wrapped false)) + $ mkdir dep_lib + $ cat > dep_lib/dune < (library (name dep_lib) (wrapped false)) > EOF - $ cat > base/a1.ml < dep_lib/unread_dep_a.ml < let v = 1 > EOF - $ cat > base/a1.mli < dep_lib/unread_dep_a.mli < val v : int > EOF - $ cat > base/a2.ml < dep_lib/referenced_dep.ml < let v = 2 > EOF - $ cat > base/a2.mli < dep_lib/referenced_dep.mli < val v : int > EOF - $ cat > base/a3.ml < dep_lib/unread_dep_b.ml < let v = 3 > EOF - $ cat > base/a3.mli < dep_lib/unread_dep_b.mli < val v : int > EOF -consumer has two modules. The module of interest, [c.ml], references -only [A2] from [base]. A second, unused module [d.ml] is present only -to keep [consumer] a multi-module stanza: dune skips ocamldep for -single-module stanzas with no library deps as an optimisation, and -the per-module-lib-deps filter depends on ocamldep output. Including -[d.ml] isolates this test from the skip-ocamldep optimisation so the -rebuild targets for [c] reflect only the per-module filter's work: +[consumer_lib] has two modules. The module of interest, [consumer], +references only [Referenced_dep] from [dep_lib]. A second, unused +module [filler] is present only to keep [consumer_lib] a multi- +module stanza: dune skips ocamldep for single-module stanzas with +no library deps as an optimisation, and the per-module-lib-deps +filter depends on ocamldep output. Including [filler] isolates +this test from the skip-ocamldep optimisation so the rebuild +targets for [consumer] reflect only the per-module filter's work: - $ mkdir consumer - $ cat > consumer/dune < (library (name consumer) (wrapped false) (libraries base)) + $ mkdir consumer_lib + $ cat > consumer_lib/dune < (library (name consumer_lib) (wrapped false) (libraries dep_lib)) > EOF - $ cat > consumer/c.ml < let w = A2.v + $ cat > consumer_lib/consumer.ml < let w = Referenced_dep.v > EOF - $ cat > consumer/d.ml < consumer_lib/filler.ml < let _ = () > EOF $ dune build @check -Edit A1's interface — a module C does not reference — and record -the rebuild targets for C: +Edit [Unread_dep_a]'s interface — a module [consumer] does not +reference — and record the rebuild targets for [consumer]: - $ cat > base/a1.mli < dep_lib/unread_dep_a.mli < val v : int > val extra : unit -> int > EOF - $ cat > base/a1.ml < dep_lib/unread_dep_a.ml < let v = 1 > let extra () = 7 > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))]' + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer_lib/\\.consumer_lib\\.objs/byte/consumer\\."))]' [] -Same for A3: +Same for [Unread_dep_b]: - $ cat > base/a3.mli < dep_lib/unread_dep_b.mli < val v : int > val other : string > EOF - $ cat > base/a3.ml < dep_lib/unread_dep_b.ml < let v = 3 > let other = "hi" > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))]' + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer_lib/\\.consumer_lib\\.objs/byte/consumer\\."))]' [] -Edit A2's interface — the one module C does reference — and record -the rebuild targets (C must rebuild): +Edit [Referenced_dep]'s interface — the one module [consumer] does +reference — and record the rebuild targets ([consumer] must +rebuild): - $ cat > base/a2.mli < dep_lib/referenced_dep.mli < val v : int > val new_fn : int -> int > EOF - $ cat > base/a2.ml < dep_lib/referenced_dep.ml < let v = 2 > let new_fn x = x + 1 > EOF $ dune build @check - $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer/\\.consumer\\.objs/byte/c\\."))]' + $ dune trace cat | jq -s 'include "dune"; [.[] | targetsMatchingFilter(test("consumer_lib/\\.consumer_lib\\.objs/byte/consumer\\."))]' [ { "target_files": [ - "_build/default/consumer/.consumer.objs/byte/c.cmi", - "_build/default/consumer/.consumer.objs/byte/c.cmo", - "_build/default/consumer/.consumer.objs/byte/c.cmt" + "_build/default/consumer_lib/.consumer_lib.objs/byte/consumer.cmi", + "_build/default/consumer_lib/.consumer_lib.objs/byte/consumer.cmo", + "_build/default/consumer_lib/.consumer_lib.objs/byte/consumer.cmt" ] } ] diff --git a/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t b/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t index 041f2c23b41..5ca1377e720 100644 --- a/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t +++ b/test/blackbox-tests/test-cases/per-module-lib-deps/wrapped-reexport-via-open-flag.t @@ -14,24 +14,24 @@ names brought into scope by the open. Records the consumer's rebuild count as a regression guard for changes that narrow compile-rule deps per module. -Structure: [lib_a] is unwrapped with module [Original_name]; +Structure: [dep_lib] is unwrapped with module [Original_name]; [lib_re_export] is wrapped with a hand-written wrapper containing -[module Re = Original_name]; [lib_b] depends on [lib_re_export] +[module Re = Original_name]; [consumer_lib] depends on [lib_re_export] and uses [-open Lib_re_export] in its flags; [consumer.ml] writes -[Re.x] without naming [lib_a] or [lib_re_export] in source. +[Re.x] without naming [dep_lib] or [lib_re_export] in source. $ cat > dune-project < (lang dune 3.0) > EOF $ cat > dune < (library (name lib_a) (wrapped false) (modules original_name)) + > (library (name dep_lib) (wrapped false) (modules original_name)) > (library > (name lib_re_export) > (modules lib_re_export some_inner) - > (libraries lib_a)) + > (libraries dep_lib)) > (library - > (name lib_b) + > (name consumer_lib) > (wrapped false) > (modules consumer) > (libraries lib_re_export) @@ -59,10 +59,10 @@ and uses [-open Lib_re_export] in its flags; [consumer.ml] writes $ dune build @check -Edit [lib_a]'s interface. [consumer] reaches [lib_a]'s +Edit [dep_lib]'s interface. [consumer] reaches [dep_lib]'s [Original_name] through the alias chain in the wrapped [Lib_re_export] wrapper. The cctx-wide compile-rule deps include -[lib_a], so [consumer] rebuilds: +[dep_lib], so [consumer] rebuilds: $ cat > original_name.mli < val x : string