feat: use ocamldep for per-module inter-library dependency filtering (#4572)

Use the already-computed ocamldep output to filter inter-library
dependencies on a per-module basis. Each module now only depends on
.cmi/.cmx files from libraries it actually imports, rather than glob
deps on all files from all dependent libraries. This eliminates
unnecessary recompilation when a module in a dependency changes but the
current module doesn't reference that library.

The implementation:

- Add read_immediate_deps_raw_of to ocamldep, returning raw module names
  without filtering against the stanza's module set
- Move Hidden_deps from Includes (library-wide) to per-module in build_cm
- Add Lib_index mapping module names to libraries, computed from
  requires_compile and requires_hidden
- In build_cm, use ocamldep output + Lib_index to determine which
  libraries each module actually needs; fall back to all-library glob
  deps when filtering is not possible (Melange, virtual library
  implementations, singleton stanzas, alias/root modules)
- For local unwrapped libraries, use per-file deps on specific .cmi/.cmx
  files rather than directory-wide globs

Signed-off-by: Robin Bate Boerop <me@robinbb.com>

Author: robinbb

src/dune_rules/compilation_context.ml

@@ -4,55 +4,29 @@ open Memo.O
 module Includes = struct
   type t = Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t
 
-  let make ~project ~opaque ~direct_requires ~hidden_requires lib_config
+  let make ~project ~direct_requires ~hidden_requires lib_config
     : _ Lib_mode.Cm_kind.Map.t
     =
-    (* TODO: some of the requires can filtered out using [ocamldep] info *)
     let open Resolve.Memo.O in
     let iflags direct_libs hidden_libs mode =
       Lib_flags.L.include_flags ~project ~direct_libs ~hidden_libs mode lib_config
     in
-    let make_includes_args ~mode groups =
+    let make_includes_args ~mode =
       (let+ direct_libs = direct_requires
        and+ hidden_libs = hidden_requires in
-       Command.Args.S
-         [ iflags direct_libs hidden_libs mode
-         ; Hidden_deps (Lib_file_deps.deps (direct_libs @ hidden_libs) ~groups)
-         ])
+       iflags direct_libs hidden_libs mode)
       |> Resolve.Memo.args
       |> Command.Args.memo
     in
     { ocaml =
-        (let cmi_includes = make_includes_args ~mode:(Ocaml Byte) [ Ocaml Cmi ] in
+        (let cmi_includes = make_includes_args ~mode:(Ocaml Byte) in
          { cmi = cmi_includes
          ; cmo = cmi_includes
-         ; cmx =
-             (let+ direct_libs = direct_requires
-              and+ hidden_libs = hidden_requires in
-              Command.Args.S
-                [ iflags direct_libs hidden_libs (Ocaml Native)
-                ; Hidden_deps
-                    (let libs = direct_libs @ hidden_libs in
-                     if opaque
-                     then
-                       List.map libs ~f:(fun lib ->
-                         ( lib
-                         , if Lib.is_local lib
-                           then [ Lib_file_deps.Group.Ocaml Cmi ]
-                           else [ Ocaml Cmi; Ocaml Cmx ] ))
-                       |> Lib_file_deps.deps_with_exts
-                     else
-                       Lib_file_deps.deps
-                         libs
-                         ~groups:[ Lib_file_deps.Group.Ocaml Cmi; Ocaml Cmx ])
-                ])
-             |> Resolve.Memo.args
-             |> Command.Args.memo
+         ; cmx = make_includes_args ~mode:(Ocaml Native)
          })
     ; melange =
-        { cmi = make_includes_args ~mode:Melange [ Melange Cmi ]
-        ; cmj = make_includes_args ~mode:Melange [ Melange Cmi; Melange Cmj ]
-        }
+        (let mel_includes = make_includes_args ~mode:Melange in
+         { cmi = mel_includes; cmj = mel_includes })
     }
   ;;
 
@@ -91,6 +65,7 @@ type t =
   ; parameters : Module_name.t list Resolve.Memo.t
   ; instances : Parameterised_instances.t Resolve.Memo.t option
   ; includes : Includes.t
+  ; lib_index : Lib_file_deps.Lib_index.t Resolve.Memo.t
   ; preprocessing : Pp_spec.t
   ; opaque : bool
   ; js_of_ocaml : Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t
@@ -118,6 +93,7 @@ let requires_hidden t = t.requires_hidden
 let requires_link t = Memo.Lazy.force t.requires_link
 let parameters t = t.parameters
 let includes t = t.includes
+let lib_index t = t.lib_index
 let preprocessing t = t.preprocessing
 let opaque t = t.opaque
 let js_of_ocaml t = t.js_of_ocaml
@@ -240,8 +216,37 @@ let create
   ; requires_link
   ; implements
   ; parameters
-  ; includes =
-      Includes.make ~project ~opaque ~direct_requires ~hidden_requires ocaml.lib_config
+  ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config
+  ; lib_index =
+      (let open Resolve.Memo.O in
+       let* all_libs =
+         let+ direct = direct_requires
+         and+ hidden = hidden_requires in
+         direct @ hidden
+       in
+       let+ entries =
+         Resolve.Memo.List.concat_map all_libs ~f:(fun lib ->
+           let* main = Lib.main_module_name lib in
+           match main with
+           | Some name ->
+             Resolve.Memo.return [ name, ((lib : Lib.t), (None : Module.t option)) ]
+           | None ->
+             (match Lib_info.entry_modules (Lib.info lib) ~for_ with
+              | External (Ok names) ->
+                Resolve.Memo.return (List.map names ~f:(fun n -> n, (lib, None)))
+              | External (Error e) -> Resolve.Memo.of_result (Error e)
+              | Local ->
+                Resolve.Memo.lift_memo
+                  (Memo.map
+                     (Dir_contents.modules_of_local_lib
+                        super_context
+                        (Lib.Local.of_lib_exn lib)
+                        ~for_)
+                     ~f:(fun mods ->
+                       List.map (Modules.entry_modules mods) ~f:(fun m ->
+                         Module.name m, (lib, Some m))))))
+       in
+       Lib_file_deps.Lib_index.create entries)
   ; preprocessing
   ; opaque
   ; js_of_ocaml
@@ -333,7 +338,6 @@ let for_module_generated_at_link_time cctx ~requires ~module_ =
     let direct_requires = requires in
     Includes.make
       ~project:(Scope.project cctx.scope)
-      ~opaque
       ~direct_requires
       ~hidden_requires
       cctx.ocaml.lib_config
@@ -344,6 +348,7 @@ let for_module_generated_at_link_time cctx ~requires ~module_ =
   ; requires_link = Memo.lazy_ (fun () -> requires)
   ; requires_compile = requires
   ; includes
+  ; lib_index = Resolve.Memo.return Lib_file_deps.Lib_index.empty
   ; modules
   }
 ;;

src/dune_rules/compilation_context.mli

@@ -62,6 +62,7 @@ val requires_hidden : t -> Lib.t list Resolve.Memo.t
 val requires_compile : t -> Lib.t list Resolve.Memo.t
 val parameters : t -> Module_name.t list Resolve.Memo.t
 val includes : t -> Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t
+val lib_index : t -> Lib_file_deps.Lib_index.t Resolve.Memo.t
 val preprocessing : t -> Pp_spec.t
 val opaque : t -> bool
 val js_of_ocaml : t -> Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t

src/dune_rules/dep_graph.ml

@@ -7,6 +7,7 @@ type t =
   }
 
 let make ~dir ~per_module = { dir; per_module }
+let dir t = t.dir
 
 let deps_of t (m : Module.t) =
   match Module_name.Unique.Map.find t.per_module (Module.obj_name m) with

src/dune_rules/dep_graph.mli

@@ -9,6 +9,7 @@ val make
   -> per_module:Module.t list Action_builder.t Module_name.Unique.Map.t
   -> t
 
+val dir : t -> Path.Build.t
 val deps_of : t -> Module.t -> Module.t list Action_builder.t
 val top_closed_implementations : t -> Module.t list -> Module.t list Action_builder.t
 

src/dune_rules/lib_file_deps.ml

@@ -48,9 +48,66 @@ let deps_of_lib (lib : Lib.t) ~groups =
   |> Dep.Set.of_list
 ;;
 
-let deps_with_exts = Dep.Set.union_map ~f:(fun (lib, groups) -> deps_of_lib lib ~groups)
 let deps libs ~groups = Dep.Set.union_map libs ~f:(deps_of_lib ~groups)
 
+let deps_of_module (lib : Lib.t) (m : Module.t) ~cm_kinds =
+  let obj_dir = Lib.info lib |> Lib_info.obj_dir in
+  List.filter_map cm_kinds ~f:(fun kind ->
+    Obj_dir.Module.cm_public_file obj_dir m ~kind |> Option.map ~f:(fun p -> Dep.file p))
+  |> Dep.Set.of_list
+;;
+
+let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) entries =
+  let groups_for lib =
+    match cm_kind with
+    | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ]
+    | Ocaml Cmx ->
+      if opaque && Lib.is_local lib
+      then [ Group.Ocaml Cmi ]
+      else [ Group.Ocaml Cmi; Group.Ocaml Cmx ]
+    | Melange Cmi -> [ Group.Melange Cmi ]
+    | Melange Cmj -> [ Group.Melange Cmi; Group.Melange Cmj ]
+  in
+  let cm_kinds_for lib =
+    match cm_kind with
+    | Ocaml Cmi | Ocaml Cmo -> [ Lib_mode.Cm_kind.Ocaml Cmi ]
+    | Ocaml Cmx ->
+      if opaque && Lib.is_local lib
+      then [ Lib_mode.Cm_kind.Ocaml Cmi ]
+      else [ Lib_mode.Cm_kind.Ocaml Cmi; Ocaml Cmx ]
+    | Melange Cmi -> [ Lib_mode.Cm_kind.Melange Cmi ]
+    | Melange Cmj -> [ Lib_mode.Cm_kind.Melange Cmi; Melange Cmj ]
+  in
+  Dep.Set.union_map entries ~f:(fun (lib, module_opt) ->
+    match module_opt with
+    | None -> deps_of_lib lib ~groups:(groups_for lib)
+    | Some m -> deps_of_module lib m ~cm_kinds:(cm_kinds_for lib))
+;;
+
+module Lib_index = struct
+  type entry = Lib.t * Module.t option
+  type t = { by_module_name : entry list Module_name.Map.t }
+
+  let empty = { by_module_name = Module_name.Map.empty }
+
+  let create entries =
+    let by_module_name =
+      List.fold_left entries ~init:Module_name.Map.empty ~f:(fun map (name, entry) ->
+        Module_name.Map.update map name ~f:(function
+          | None -> Some [ entry ]
+          | Some entries -> Some (entry :: entries)))
+    in
+    { by_module_name }
+  ;;
+
+  let filter_libs t ~referenced_modules =
+    Module_name.Set.fold referenced_modules ~init:[] ~f:(fun name acc ->
+      match Module_name.Map.find t.by_module_name name with
+      | None -> acc
+      | Some entries -> List.rev_append entries acc)
+  ;;
+end
+
 type path_specification =
   | Allow_all
   | Disallow_external of Lib_name.t

src/dune_rules/lib_file_deps.mli

@@ -15,7 +15,29 @@ end
     with extension [files] of libraries [libs]. *)
 val deps : Lib.t list -> groups:Group.t list -> Dep.Set.t
 
-val deps_with_exts : (Lib.t * Group.t list) list -> Dep.Set.t
+(** [deps_of_entries ~opaque ~cm_kind entries] computes the file dependencies
+    for the given library entries. When the module in an entry is [None], glob
+    deps are used for the library. When [Some m], per-file deps on specific
+    cm files are used. *)
+val deps_of_entries
+  :  opaque:bool
+  -> cm_kind:Lib_mode.Cm_kind.t
+  -> (Lib.t * Module.t option) list
+  -> Dep.Set.t
+
+module Lib_index : sig
+  type entry = Lib.t * Module.t option
+  type t
+
+  val empty : t
+
+  (** Create an index from a list of (module_name, entry) pairs. *)
+  val create : (Module_name.t * entry) list -> t
+
+  (** Return the library entries whose module names appear in
+      [referenced_modules]. *)
+  val filter_libs : t -> referenced_modules:Module_name.Set.t -> entry list
+end
 
 type path_specification =
   | Allow_all