Make path inputs lazy

src/libfetchers/fetch-to-store.cc

@@ -58,7 +58,7 @@ std::pair<StorePath, Hash> fetchToStore2(
         }
     } else {
         static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
-        if (barf && !filter)
+        if (barf && !filter && !(path.to_string().starts_with("/") || path.to_string().starts_with("«path:/")))
             throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
         // FIXME: could still provide in-memory caching keyed on `SourcePath`.
         debug("source path '%s' is uncacheable", path);

src/libfetchers/filtering-source-accessor.cc

@@ -68,6 +68,11 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
     return next->getFingerprint(prefix / path);
 }
 
+void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
+{
+    next->invalidateCache(prefix / path);
+}
+
 void FilteringSourceAccessor::checkAccess(const CanonPath & path)
 {
     if (!isAllowed(path))

src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh

@@ -52,6 +52,8 @@ struct FilteringSourceAccessor : SourceAccessor
 
     std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
 
+    void invalidateCache(const CanonPath & path) override;
+
     /**
      * Call `makeNotAllowedError` to throw a `RestrictedPathError`
      * exception if `isAllowed()` returns `false` for `path`.

src/libfetchers/path.cc

@@ -142,41 +142,27 @@ struct PathInputScheme : InputScheme
     getAccessor(const Settings & settings, Store & store, const Input & _input) const override
     {
         Input input(_input);
-        auto path = getStrAttr(input.attrs, "path");
 
         auto absPath = getAbsPath(input);
 
         // FIXME: check whether access to 'path' is allowed.
+
+        auto accessor = makeFSSourceAccessor(absPath);
+
         auto storePath = store.maybeParseStorePath(absPath.string());
 
-        if (storePath)
+        if (storePath) {
             store.addTempRoot(*storePath);
 
-        time_t mtime = 0;
-        if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
-            // FIXME: try to substitute storePath.
-            auto src = sinkToSource(
-                [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
-            storePath = store.addToStoreFromDump(*src, "source");
+            // To prevent `fetchToStore()` copying the path again to Nix
+            // store, pre-create an entry in the fetcher cache.
+            auto info = store.queryPathInfo(*storePath);
+            accessor->fingerprint = fmt("path:%s", info->narHash.to_string(HashFormat::SRI, true));
+            settings.getCache()->upsert(
+                makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
+                {{"hash", info->narHash.to_string(HashFormat::SRI, true)}});
         }
 
-        auto accessor = store.requireStoreObjectAccessor(*storePath);
-
-        // To prevent `fetchToStore()` copying the path again to Nix
-        // store, pre-create an entry in the fetcher cache.
-        auto info = store.queryPathInfo(*storePath);
-        accessor->fingerprint =
-            fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
-        settings.getCache()->upsert(
-            makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
-            {{"hash", info->narHash.to_string(HashFormat::SRI, true)}});
-
-        /* Trust the lastModified value supplied by the user, if
-           any. It's not a "secure" attribute so we don't care. */
-        if (!input.getLastModified())
-            input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
-
         return {accessor, std::move(input)};
     }
 };

src/libflake/flake.cc

@@ -436,7 +436,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
     auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No;
     auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No;
 
-    auto flake = getFlake(state, topRef, useRegistriesTop, {}, lockFlags.requireLockable);
+    auto flake = getFlake(state, topRef, useRegistriesTop, {}, false);
 
     if (lockFlags.applyNixConfig) {
         flake.config.apply(settings);
@@ -919,6 +919,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
                                 CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
                                 newLockFileS,
                                 commitMessage);
+
+                            flake.lockFilePath().invalidateCache();
                         }
 
                         /* Rewriting the lockfile changed the top-level

src/libutil/include/nix/util/posix-source-accessor.hh

@@ -78,6 +78,8 @@ public:
         return trackLastModified ? std::optional{mtime} : std::nullopt;
     }
 
+    void invalidateCache(const CanonPath & path) override;
+
 private:
 
     /**

src/libutil/include/nix/util/source-accessor.hh

@@ -209,6 +209,11 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
     {
         return std::nullopt;
     }
+
+    /**
+     * Invalidate any cached value the accessor may have for the specified path.
+     */
+    virtual void invalidateCache(const CanonPath & path) {}
 };
 
 /**

src/libutil/include/nix/util/source-path.hh

@@ -114,6 +114,11 @@ struct SourcePath
         return {accessor, accessor->resolveSymlinks(path, mode)};
     }
 
+    void invalidateCache() const
+    {
+        accessor->invalidateCache(path);
+    }
+
     friend class std::hash<nix::SourcePath>;
 };
 

src/libutil/mounted-source-accessor.cc

@@ -99,6 +99,12 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
         auto [accessor, subpath] = resolve(path);
         return accessor->getFingerprint(subpath);
     }
+
+    void invalidateCache(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        accessor->invalidateCache(subpath);
+    }
 };
 
 ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)

src/libutil/posix-source-accessor.cc

@@ -87,11 +87,11 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path)
     return nix::pathExists(makeAbsPath(path).string());
 }
 
+using Cache = boost::concurrent_flat_map<Path, std::optional<struct stat>>;
+static Cache cache;
+
 std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & path)
 {
-    using Cache = boost::concurrent_flat_map<Path, std::optional<struct stat>>;
-    static Cache cache;
-
     // Note: we convert std::filesystem::path to Path because the
     // former is not hashable on libc++.
     Path absPath = makeAbsPath(path).string();
@@ -108,6 +108,11 @@ std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & pa
     return st;
 }
 
+void PosixSourceAccessor::invalidateCache(const CanonPath & path)
+{
+    cache.erase(makeAbsPath(path).string());
+}
+
 std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
 {
     if (auto parent = path.parent())

src/libutil/union-source-accessor.cc

@@ -84,6 +84,12 @@ struct UnionSourceAccessor : SourceAccessor
         }
         return {path, std::nullopt};
     }
+
+    void invalidateCache(const CanonPath & path) override
+    {
+        for (auto & accessor : accessors)
+            accessor->invalidateCache(path);
+    }
 };
 
 ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors)

tests/functional/common/functions.sh

@@ -361,4 +361,25 @@ execUnshare () {
   exec unshare --mount --map-root-user "$SHELL" "$@"
 }
 
+initGitRepo() {
+    local repo="$1"
+    local extraArgs="${2-}"
+
+    # shellcheck disable=SC2086 # word splitting of extraArgs is intended
+    git -C "$repo" init $extraArgs
+    git -C "$repo" config user.email "foobar@example.com"
+    git -C "$repo" config user.name "Foobar"
+}
+
+createGitRepo() {
+    local repo="$1"
+    local extraArgs="${2-}"
+
+    rm -rf "$repo" "$repo".tmp
+    mkdir -p "$repo"
+
+    # shellcheck disable=SC2086 # word splitting of extraArgs is intended
+    initGitRepo "$repo" $extraArgs
+}
+
 fi # COMMON_FUNCTIONS_SH_SOURCED

tests/functional/fetchGit.sh

@@ -12,11 +12,9 @@ repo=$TEST_ROOT/./git
 
 export _NIX_FORCE_HTTP=1
 
-rm -rf "$repo" "${repo}"-tmp "$TEST_HOME"/.cache/nix "$TEST_ROOT"/worktree "$TEST_ROOT"/minimal
+rm -rf "${repo}"-tmp "$TEST_HOME"/.cache/nix "$TEST_ROOT"/worktree "$TEST_ROOT"/minimal
 
-git init "$repo"
-git -C "$repo" config user.email "foobar@example.com"
-git -C "$repo" config user.name "Foobar"
+createGitRepo "$repo"
 
 echo utrecht > "$repo"/hello
 touch "$repo"/.gitignore
@@ -213,8 +211,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref =
 
 # Fetching from a repo with only a specific revision and no branches should
 # not fall back to copying files and record correct revision information. See: #5302
-mkdir "$TEST_ROOT"/minimal
-git -C "$TEST_ROOT"/minimal init
+createGitRepo "$TEST_ROOT"/minimal
 git -C "$TEST_ROOT"/minimal fetch "$repo" "$rev2"
 git -C "$TEST_ROOT"/minimal checkout "$rev2"
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; }).rev") = "$rev2" ]]
@@ -267,17 +264,15 @@ rm -rf "$TEST_HOME"/.cache/nix
 (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")
 
 # should succeed for a repo without commits
-git init "$repo"
+initGitRepo "$repo"
 git -C "$repo" add hello # need to add at least one file to cause the root of the repo to be visible
 # shellcheck disable=SC2034
 path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")
 
 # should succeed for a path with a space
 # regression test for #7707
 repo="$TEST_ROOT/a b"
-git init "$repo"
-git -C "$repo" config user.email "foobar@example.com"
-git -C "$repo" config user.name "Foobar"
+createGitRepo "$repo"
 
 echo utrecht > "$repo/hello"
 touch "$repo/.gitignore"
@@ -289,7 +284,7 @@ path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath")
 
 # Test a workdir with no commits.
 empty="$TEST_ROOT/empty"
-git init "$empty"
+createGitRepo "$empty"
 
 emptyAttrs="{ lastModified = 0; lastModifiedDate = \"19700101000000\"; narHash = \"sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo=\"; rev = \"0000000000000000000000000000000000000000\"; revCount = 0; shortRev = \"0000000\"; submodules = false; }"
 result=$(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]")
@@ -317,10 +312,7 @@ nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs.
 
 # Test backward compatibility hack for Nix < 2.20 locks / fetchTree calls that expect Git filters to be applied.
 eol="$TEST_ROOT/git-eol"
-mkdir -p "$eol"
-git init "$eol"
-git -C "$eol" config user.email "foobar@example.com"
-git -C "$eol" config user.name "Foobar"
+createGitRepo "$eol"
 printf "Hello\nWorld\n" > "$eol/crlf"
 printf "ignore me" > "$eol/ignored"
 git -C "$eol" add crlf ignored

tests/functional/fetchGitRefs.sh

@@ -8,11 +8,9 @@ clearStoreIfPossible
 
 repo="$TEST_ROOT/git"
 
-rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix"
+rm -rf "${repo}-tmp" "$TEST_HOME/.cache/nix"
 
-git init "$repo"
-git -C "$repo" config user.email "foobar@example.com"
-git -C "$repo" config user.name "Foobar"
+createGitRepo "$repo"
 
 echo utrecht > "$repo/hello"
 git -C "$repo" add hello

tests/functional/fetchGitShallow.sh

@@ -6,9 +6,7 @@ source common.sh
 requireGit
 
 # Create a test repo with multiple commits for all our tests
-git init "$TEST_ROOT/shallow-parent"
-git -C "$TEST_ROOT/shallow-parent" config user.email "foobar@example.com"
-git -C "$TEST_ROOT/shallow-parent" config user.name "Foobar"
+createGitRepo "$TEST_ROOT/shallow-parent"
 
 # Add several commits to have history
 echo "{ outputs = _: {}; }" > "$TEST_ROOT/shallow-parent/flake.nix"

tests/functional/fetchGitSubmodules.sh

@@ -22,22 +22,16 @@ rm -rf "${rootRepo}" "${subRepo}" "$TEST_HOME"/.cache/nix
 export XDG_CONFIG_HOME=$TEST_HOME/.config
 git config --global protocol.file.allow always
 
-initGitRepo() {
-    git init "$1"
-    git -C "$1" config user.email "foobar@example.com"
-    git -C "$1" config user.name "Foobar"
-}
-
 addGitContent() {
     echo "lorem ipsum" > "$1"/content
     git -C "$1" add content
     git -C "$1" commit -m "Initial commit"
 }
 
-initGitRepo "$subRepo"
+createGitRepo "$subRepo"
 addGitContent "$subRepo"
 
-initGitRepo "$rootRepo"
+createGitRepo "$rootRepo"
 
 git -C "$rootRepo" submodule init
 git -C "$rootRepo" submodule add "$subRepo" sub
@@ -199,19 +193,19 @@ test_submodule_nested() {
   local repoB=$TEST_ROOT/submodule_nested/b
   local repoC=$TEST_ROOT/submodule_nested/c
 
-  rm -rf "$repoA" "$repoB" "$repoC" "$TEST_HOME"/.cache/nix
+  rm -rf "$TEST_HOME"/.cache/nix
 
-  initGitRepo "$repoC"
+  createGitRepo "$repoC"
   touch "$repoC"/inside-c
   git -C "$repoC" add inside-c
   addGitContent "$repoC"
 
-  initGitRepo "$repoB"
+  createGitRepo "$repoB"
   git -C "$repoB" submodule add "$repoC" c
   git -C "$repoB" add c
   addGitContent "$repoB"
 
-  initGitRepo "$repoA"
+  createGitRepo "$repoA"
   git -C "$repoA" submodule add "$repoB" b
   git -C "$repoA" add b
   addGitContent "$repoA"

tests/functional/fetchGitVerification.sh

@@ -21,9 +21,7 @@ ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2"
 key2File="$keysDir/testkey2.pub"
 publicKey2=$(awk '{print $2}' "$key2File")
 
-git init "$repo"
-git -C "$repo" config user.email "foobar@example.com"
-git -C "$repo" config user.name "Foobar"
+createGitRepo "$repo"
 git -C "$repo" config gpg.format ssh
 
 echo 'hello' > "$repo"/text

tests/functional/fetchPath.sh

@@ -3,9 +3,9 @@
 source common.sh
 
 touch "$TEST_ROOT/foo" -t 202211111111
-# We only check whether 2022-11-1* **:**:** is the last modified date since
-# `lastModified` is transformed into UTC in `builtins.fetchTarball`.
-[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]]
+
+# The path fetcher does not return lastModified.
+[[ "$(nix eval --impure --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\") ? lastModifiedDate")" = false ]]
 
 # Check that we can override lastModified for "path:" inputs.
 [[ "$(nix eval --impure --expr "(builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; lastModified = 123; }).lastModified")" = 123 ]]