Auto merge of #153122 - Zalathar:dep-kind-vtable, r=nnethercote

bors · bors · commit 28b5c1cc084d · 2026-03-01T16:28:20.000Z
Improve the forcing/promotion functions in `DepKindVTable`

This is a bundle of changes to the two function pointers in `DepKindVTable` that are responsible for forcing dep nodes, or promoting disk-cached values from the previous session into memory.

The perf improvements to incr-unchanged and incr-patched are likely from skipping more of the “promotion” plumbing for queries that never cache to disk.
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -121,7 +121,7 @@ impl DepNode {
 
         #[cfg(debug_assertions)]
         {
-            if !tcx.key_fingerprint_style(kind).reconstructible()
+            if !tcx.key_fingerprint_style(kind).is_maybe_recoverable()
                 && (tcx.sess.opts.unstable_opts.incremental_info
                     || tcx.sess.opts.unstable_opts.query_dep_graph)
             {
@@ -225,12 +225,12 @@ pub struct DepKindVTable<'tcx> {
     /// with kind `mir_promoted`, we know that the key fingerprint of the `DepNode`
     /// is actually a `DefPathHash`, and can therefore just look up the corresponding
     /// `DefId` in `tcx.def_path_hash_to_def_id`.
-    pub force_from_dep_node: Option<
+    pub force_from_dep_node_fn: Option<
         fn(tcx: TyCtxt<'tcx>, dep_node: DepNode, prev_index: SerializedDepNodeIndex) -> bool,
     >,
 
     /// Invoke a query to put the on-disk cached value in memory.
-    pub try_load_from_on_disk_cache: Option<fn(TyCtxt<'tcx>, DepNode)>,
+    pub promote_from_disk_fn: Option<fn(TyCtxt<'tcx>, DepNode)>,
 }
 
 /// A "work product" corresponds to a `.o` (or other) file that we
diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs
@@ -1064,7 +1064,11 @@ impl DepGraph {
             match data.colors.get(prev_index) {
                 DepNodeColor::Green(_) => {
                     let dep_node = data.previous.index_to_node(prev_index);
-                    tcx.try_load_from_on_disk_cache(dep_node);
+                    if let Some(promote_fn) =
+                        tcx.dep_kind_vtable(dep_node.kind).promote_from_disk_fn
+                    {
+                        promote_fn(tcx, *dep_node)
+                    };
                 }
                 DepNodeColor::Unknown | DepNodeColor::Red => {
                     // We can skip red nodes because a node can only be marked
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -42,8 +42,12 @@ pub enum KeyFingerprintStyle {
 }
 
 impl KeyFingerprintStyle {
+    /// True if a key can _potentially_ be recovered from a key fingerprint
+    /// with this style.
+    ///
+    /// For some key types, recovery is possible but not guaranteed.
     #[inline]
-    pub const fn reconstructible(self) -> bool {
+    pub const fn is_maybe_recoverable(self) -> bool {
         match self {
             KeyFingerprintStyle::DefPathHash
             | KeyFingerprintStyle::Unit
@@ -103,7 +107,7 @@ impl<'tcx> TyCtxt<'tcx> {
         prev_index: SerializedDepNodeIndex,
         frame: &MarkFrame<'_>,
     ) -> bool {
-        if let Some(force_fn) = self.dep_kind_vtable(dep_node.kind).force_from_dep_node {
+        if let Some(force_fn) = self.dep_kind_vtable(dep_node.kind).force_from_dep_node_fn {
             match panic::catch_unwind(panic::AssertUnwindSafe(|| {
                 force_fn(self, dep_node, prev_index)
             })) {
@@ -119,11 +123,4 @@ impl<'tcx> TyCtxt<'tcx> {
             false
         }
     }
-
-    /// Load data from the on-disk cache.
-    fn try_load_from_on_disk_cache(self, dep_node: &DepNode) {
-        if let Some(try_load_fn) = self.dep_kind_vtable(dep_node.kind).try_load_from_on_disk_cache {
-            try_load_fn(self, *dep_node)
-        }
-    }
 }
diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs
@@ -3,7 +3,7 @@ use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle};
 use rustc_middle::query::QueryCache;
 
 use crate::GetQueryVTable;
-use crate::plumbing::{force_from_dep_node_inner, try_load_from_on_disk_cache_inner};
+use crate::plumbing::{force_from_dep_node_inner, promote_from_disk_inner};
 
 /// [`DepKindVTable`] constructors for special dep kinds that aren't queries.
 #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")]
@@ -16,10 +16,10 @@ mod non_query {
             is_anon: false,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Unit,
-            force_from_dep_node: Some(|_, dep_node, _| {
+            force_from_dep_node_fn: Some(|_, dep_node, _| {
                 bug!("force_from_dep_node: encountered {dep_node:?}")
             }),
-            try_load_from_on_disk_cache: None,
+            promote_from_disk_fn: None,
         }
     }
 
@@ -29,10 +29,10 @@ mod non_query {
             is_anon: false,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Unit,
-            force_from_dep_node: Some(|_, dep_node, _| {
+            force_from_dep_node_fn: Some(|_, dep_node, _| {
                 bug!("force_from_dep_node: encountered {dep_node:?}")
             }),
-            try_load_from_on_disk_cache: None,
+            promote_from_disk_fn: None,
         }
     }
 
@@ -41,11 +41,11 @@ mod non_query {
             is_anon: false,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Unit,
-            force_from_dep_node: Some(|tcx, _, prev_index| {
+            force_from_dep_node_fn: Some(|tcx, _, prev_index| {
                 tcx.dep_graph.force_diagnostic_node(tcx, prev_index);
                 true
             }),
-            try_load_from_on_disk_cache: None,
+            promote_from_disk_fn: None,
         }
     }
 
@@ -54,8 +54,8 @@ mod non_query {
             is_anon: true,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Opaque,
-            force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
-            try_load_from_on_disk_cache: None,
+            force_from_dep_node_fn: Some(|_, _, _| bug!("cannot force an anon node")),
+            promote_from_disk_fn: None,
         }
     }
 
@@ -64,8 +64,8 @@ mod non_query {
             is_anon: true,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Unit,
-            force_from_dep_node: None,
-            try_load_from_on_disk_cache: None,
+            force_from_dep_node_fn: None,
+            promote_from_disk_fn: None,
         }
     }
 
@@ -74,8 +74,8 @@ mod non_query {
             is_anon: false,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Opaque,
-            force_from_dep_node: None,
-            try_load_from_on_disk_cache: None,
+            force_from_dep_node_fn: None,
+            promote_from_disk_fn: None,
         }
     }
 
@@ -84,8 +84,8 @@ mod non_query {
             is_anon: false,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Opaque,
-            force_from_dep_node: None,
-            try_load_from_on_disk_cache: None,
+            force_from_dep_node_fn: None,
+            promote_from_disk_fn: None,
         }
     }
 
@@ -94,8 +94,8 @@ mod non_query {
             is_anon: false,
             is_eval_always: false,
             key_fingerprint_style: KeyFingerprintStyle::Unit,
-            force_from_dep_node: None,
-            try_load_from_on_disk_cache: None,
+            force_from_dep_node_fn: None,
+            promote_from_disk_fn: None,
         }
     }
 }
@@ -104,6 +104,7 @@ mod non_query {
 /// Called from macro-generated code for each query.
 pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q>(
     is_anon: bool,
+    is_cache_on_disk: bool,
     is_eval_always: bool,
 ) -> DepKindVTable<'tcx>
 where
@@ -115,26 +116,20 @@ where
         <Q::Cache as QueryCache>::Key::key_fingerprint_style()
     };
 
-    if is_anon || !key_fingerprint_style.reconstructible() {
-        return DepKindVTable {
-            is_anon,
-            is_eval_always,
-            key_fingerprint_style,
-            force_from_dep_node: None,
-            try_load_from_on_disk_cache: None,
-        };
+    // A query dep-node can only be forced or promoted if it can recover a key
+    // from its key fingerprint.
+    let can_recover = key_fingerprint_style.is_maybe_recoverable();
+    if is_anon {
+        assert!(!can_recover);
     }
 
     DepKindVTable {
         is_anon,
         is_eval_always,
         key_fingerprint_style,
-        force_from_dep_node: Some(|tcx, dep_node, _| {
-            force_from_dep_node_inner(Q::query_vtable(tcx), tcx, dep_node)
-        }),
-        try_load_from_on_disk_cache: Some(|tcx, dep_node| {
-            try_load_from_on_disk_cache_inner(Q::query_vtable(tcx), tcx, dep_node)
-        }),
+        force_from_dep_node_fn: can_recover.then_some(force_from_dep_node_inner::<Q>),
+        promote_from_disk_fn: (can_recover && is_cache_on_disk)
+            .then_some(promote_from_disk_inner::<Q>),
     }
 }
 
diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
@@ -572,7 +572,7 @@ fn load_from_disk_or_invoke_provider_green<'tcx, C: QueryCache>(
     // can be forced from `DepNode`.
     debug_assert!(
         !query.will_cache_on_disk_for_key(tcx, key)
-            || !tcx.key_fingerprint_style(dep_node.kind).reconstructible(),
+            || !tcx.key_fingerprint_style(dep_node.kind).is_maybe_recoverable(),
         "missing on-disk cache entry for {dep_node:?}"
     );
 
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
@@ -29,10 +29,10 @@ use rustc_middle::ty::{self, TyCtxt};
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::def_id::LOCAL_CRATE;
 
-use crate::collect_active_jobs_from_all_queries;
 use crate::error::{QueryOverflow, QueryOverflowNote};
 use crate::execution::{all_inactive, force_query};
 use crate::job::find_dep_kind_root;
+use crate::{GetQueryVTable, collect_active_jobs_from_all_queries};
 
 fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) {
     let job_map =
@@ -326,15 +326,15 @@ pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache>(
     });
 }
 
-/// Implementation of [`DepKindVTable::try_load_from_on_disk_cache`] for queries.
-pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache>(
-    query: &'tcx QueryVTable<'tcx, C>,
+/// Implementation of [`DepKindVTable::promote_from_disk_fn`] for queries.
+pub(crate) fn promote_from_disk_inner<'tcx, Q: GetQueryVTable<'tcx>>(
     tcx: TyCtxt<'tcx>,
     dep_node: DepNode,
 ) {
+    let query = Q::query_vtable(tcx);
     debug_assert!(tcx.dep_graph.is_green(&dep_node));
 
-    let key = C::Key::try_recover_key(tcx, &dep_node).unwrap_or_else(|| {
+    let key = <Q::Cache as QueryCache>::Key::try_recover_key(tcx, &dep_node).unwrap_or_else(|| {
         panic!(
             "Failed to recover key for {dep_node:?} with key fingerprint {}",
             dep_node.key_fingerprint
@@ -379,12 +379,15 @@ where
     value
 }
 
-/// Implementation of [`DepKindVTable::force_from_dep_node`] for queries.
-pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache>(
-    query: &'tcx QueryVTable<'tcx, C>,
+/// Implementation of [`DepKindVTable::force_from_dep_node_fn`] for queries.
+pub(crate) fn force_from_dep_node_inner<'tcx, Q: GetQueryVTable<'tcx>>(
     tcx: TyCtxt<'tcx>,
     dep_node: DepNode,
+    // Needed by the vtable function signature, but not used when forcing queries.
+    _prev_index: SerializedDepNodeIndex,
 ) -> bool {
+    let query = Q::query_vtable(tcx);
+
     // We must avoid ever having to call `force_from_dep_node()` for a
     // `DepNode::codegen_unit`:
     // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
@@ -403,7 +406,7 @@ pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache>(
         "calling force_from_dep_node() on dep_kinds::codegen_unit"
     );
 
-    if let Some(key) = C::Key::try_recover_key(tcx, &dep_node) {
+    if let Some(key) = <Q::Cache as QueryCache>::Key::try_recover_key(tcx, &dep_node) {
         force_query(query, tcx, key, dep_node);
         true
     } else {
@@ -697,6 +700,7 @@ macro_rules! define_queries {
                     use $crate::query_impl::$name::VTableGetter;
                     make_dep_kind_vtable_for_query::<VTableGetter>(
                         is_anon!([$($modifiers)*]),
+                        if_cache_on_disk!([$($modifiers)*] true false),
                         is_eval_always!([$($modifiers)*]),
                     )
                 }
