diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index c38ea2d2b80f..4dd3f67bb313 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -57,7 +57,8 @@ void setVerbosity(int level) int isValidPath(char * path) CODE: try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + RETVAL = store()->isValidPath(storePath); } catch (Error & e) { croak("%s", e.what()); } @@ -68,7 +69,8 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + auto storePath = store()->parseStorePath(path); + for (auto & i : store()->queryPathInfo(storePath)->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -78,7 +80,8 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto storePath = store()->parseStorePath(path); + auto s = store()->queryPathInfo(storePath)->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -88,7 +91,8 @@ SV * queryPathHash(char * path) SV * queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XSRETURN_UNDEF; XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { @@ -99,7 +103,8 @@ SV * queryDeriver(char * path) SV * queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XPUSHs(&PL_sv_undef); else @@ -109,7 +114,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 4fc1979563d5..813d3d462529 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -52,7 +52,8 @@ ref StoreCommand::createStore() void StoreCommand::run() { - run(getStore()); + auto store = getStore(); + run(store); } CopyCommand::CopyCommand() diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e36bda52fd92..de9dcb815ac4 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -167,15 +167,17 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { - auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + auto storePath = state.store->makeFixedOutputPathFromCA( + fetchers::downloadTarball( + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = state.store->makeFixedOutputPathFromCA( + flakeRef.resolve(state.store).fetchTree(state.store).first.storePath); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eb19030847d5..671dceecfe26 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -403,7 +403,7 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); + auto fingerprint = lockedFlake->getFingerprint(*state.store); return make_ref( evalSettings.useEvalCache && evalSettings.pureEval ? std::optional { std::cref(fingerprint) } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 6a27ea2e8dc9..54c17ae2cfc1 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -63,10 +63,12 @@ static std::tuple fetchOrSubstituteTree( auto [tree, lockedRef] = *fetched; + auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath); + debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); @@ -210,7 +212,8 @@ static Flake getFlake( auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, sourceInfo.actualPath)) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, + state.store->printStorePath(state.store->makeFixedOutputPathFromCA(sourceInfo.storePath))); Flake flake { .originalRef = originalRef, @@ -891,14 +894,14 @@ static RegisterPrimOp r4({ } -Fingerprint LockedFlake::getFingerprint() const +Fingerprint LockedFlake::getFingerprint(const Store & store) const { // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + store.makeFixedOutputPathFromCA(flake.sourceInfo->storePath).to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e5096..9dd7b05741ae 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -105,7 +105,7 @@ struct LockedFlake Flake flake; LockFile lockFile; - Fingerprint getFingerprint() const; + Fingerprint getFingerprint(const Store & store) const; }; struct LockFlags diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3c202967a025..96e76647a5ce 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -40,7 +40,7 @@ LockedNode::LockedNode(const nlohmann::json & json) fetchers::attrsToJSON(lockedRef.input.toAttrs())); } -StorePath LockedNode::computeStorePath(Store & store) const +StorePathDescriptor LockedNode::computeStorePath(Store & store) const { return lockedRef.input.computeStorePath(store); } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c8485c0..716ddced7525 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -2,6 +2,7 @@ ///@file #include "flakeref.hh" +#include "content-address.hh" #include @@ -47,7 +48,7 @@ struct LockedNode : Node LockedNode(const nlohmann::json & json); - StorePath computeStorePath(Store & store) const; + StorePathDescriptor computeStorePath(Store & store) const; }; struct LockFile diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde323..33b48598db66 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -781,8 +781,12 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + auto storePath = store->makeFixedOutputPathFromCA( + fetchers::downloadTarball( + store, + EvalSettings::resolvePseudoUrl(value), + "source", + false).tree.storePath); res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ @@ -796,7 +800,8 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + auto storePath = store->makeFixedOutputPathFromCA( + flakeRef.resolve(store).fetchTree(store).first.storePath); res = { store->toRealPath(storePath) }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e2b1ac4f65eb..68c19df3350e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1618,8 +1618,9 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); StorePathSet refs; if (state.store->isInStore(path.path.abs())) { + auto p = state.store->toStorePath(path.path.abs()).first; try { - refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; + refs = state.store->queryPathInfo(p)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -2188,7 +2189,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index e8542503a42b..2c99eb08412f 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -187,8 +187,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .errPos = state.positions[i.pos] }); auto namePath = state.store->parseStorePath(name); - if (!settings.readOnlyMode) + if (!settings.readOnlyMode) { state.store->ensurePath(namePath); + } state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index b9ff01c16e1b..f8817b4b94d9 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -74,7 +74,8 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -86,7 +87,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(state.store->makeFixedOutputPathFromCA(tree.storePath)); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f040a35109a8..1bf22ad3ae35 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -25,8 +25,9 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); + auto storePath = state.store->makeFixedOutputPathFromCA(tree.storePath); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -188,7 +189,7 @@ static void fetchTree( auto [tree, input2] = input.fetch(state.store); - state.allowPath(tree.storePath); + state.allowPath(state.store->makeFixedOutputPathFromCA(tree.storePath)); emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } @@ -250,12 +251,14 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (evalSettings.pureEval && !expectedHash) state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); - // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { + // early exit if pinned and already in the store, or substituted successfully + if (expectedHash) { + auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .method = method, .hash = *expectedHash, .references = {} }); @@ -264,6 +267,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.allowAndSetStorePathString(expectedPath, v); return; } + + // try to substitute if we can + + auto substitutableStorePath = fetchers::trySubstitute(state.store, method, *expectedHash, name); + + if (substitutableStorePath) { + state.allowAndSetStorePathString(*substitutableStorePath, v); + return; + } } // TODO: fetching may fail, yet the path may be substitutable. @@ -273,16 +285,18 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + auto actualStorePath = state.store->makeFixedOutputPathFromCA(storePath); + if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); + : hashFile(htSHA256, state.store->toRealPath(actualStorePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); } - state.allowAndSetStorePathString(storePath, v); + state.allowAndSetStorePathString(actualStorePath, v); } static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, Value & v) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d48a..2a55f085fde0 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -51,18 +51,19 @@ struct CacheImpl : Cache ref store, const Attrs & inAttrs, const Attrs & infoAttrs, - const StorePath & storePath, + const StorePathDescriptor & storePathDesc, bool locked) override { _state.lock()->add.use() (attrsToJSON(inAttrs).dump()) (attrsToJSON(infoAttrs).dump()) - (store->printStorePath(storePath)) + // FIXME should use JSON for store path descriptor + (renderStorePathDescriptor(storePathDesc)) (locked) (time(0)).exec(); } - std::optional> lookup( + std::optional> lookup( ref store, const Attrs & inAttrs) override { @@ -90,9 +91,10 @@ struct CacheImpl : Cache } auto infoJSON = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); + auto storePathDesc = parseStorePathDescriptor(stmt.getStr(1)); auto locked = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); store->addTempRoot(storePath); if (!store->isValidPath(storePath)) { @@ -107,7 +109,7 @@ struct CacheImpl : Cache return Result { .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), - .storePath = std::move(storePath) + .storePath = std::move(storePathDesc) }; } }; diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d0404ba..c92792843bef 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -13,10 +13,10 @@ struct Cache ref store, const Attrs & inAttrs, const Attrs & infoAttrs, - const StorePath & storePath, + const StorePathDescriptor & storePath, bool locked) = 0; - virtual std::optional> lookup( + virtual std::optional> lookup( ref store, const Attrs & inAttrs) = 0; @@ -24,7 +24,7 @@ struct Cache { bool expired = false; Attrs infoAttrs; - StorePath storePath; + StorePathDescriptor storePath; }; virtual std::optional lookupExpired( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e683b9f804a6..52b774ee95b5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "archive.hh" #include @@ -117,20 +118,28 @@ std::pair Input::fetch(ref store) const original source). So check that. */ if (hasAllInfo()) { try { - auto storePath = computeStorePath(*store); + auto storePathDesc = computeStorePath(*store); - store->ensurePath(storePath); + store->ensurePath(std::cref(storePathDesc)); + + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePathDesc), + }, + *this, + }; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } } - auto [storePath, input] = [&]() -> std::pair { + auto [storePath, input] = [&]() -> std::pair { try { return scheme->fetch(store, *this); } catch (Error & e) { @@ -140,7 +149,7 @@ std::pair Input::fetch(ref store) const }(); Tree tree { - .actualPath = store->toRealPath(storePath), + .actualPath = store->toRealPath(store->makeFixedOutputPathFromCA(storePath)), .storePath = storePath, }; @@ -211,16 +220,19 @@ std::string Input::getName() const return maybeGetStrAttr(attrs, "name").value_or("source"); } -StorePath Input::computeStorePath(Store & store) const +StorePathDescriptor Input::computeStorePath(Store & store) const { auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - .references = {}, - }); + return StorePathDescriptor { + getName(), + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + .references = {}, + }, + }; } std::string Input::getType() const @@ -308,4 +320,30 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name) +{ + auto ca = StorePathDescriptor { + .name = std::string { name }, + .info = FixedOutputInfo { + ingestionMethod, + hash, + {} + }, + }; + auto substitutablePath = store->makeFixedOutputPathFromCA(ca); + + try { + store->ensurePath(ca); + + debug("using substituted path '%s'", store->printStorePath(substitutablePath)); + + return substitutablePath; + } catch (Error & e) { + debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what()); + } + + return std::nullopt; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6e10e95134fd..482671296274 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -6,6 +6,7 @@ #include "path.hh" #include "attrs.hh" #include "url.hh" +#include "content-address.hh" #include @@ -16,7 +17,7 @@ namespace nix::fetchers { struct Tree { Path actualPath; - StorePath storePath; + StorePathDescriptor storePath; }; struct InputScheme; @@ -103,7 +104,7 @@ public: std::string getName() const; - StorePath computeStorePath(Store & store) const; + StorePathDescriptor computeStorePath(Store & store) const; // Convenience functions for common attributes. std::string getType() const; @@ -148,14 +149,14 @@ struct InputScheme virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); - virtual std::pair fetch(ref store, const Input & input) = 0; + virtual std::pair fetch(ref store, const Input & input) = 0; }; void registerInputScheme(std::shared_ptr && fetcher); struct DownloadFileResult { - StorePath storePath; + StorePathDescriptor storePath; std::string etag; std::string effectiveUrl; std::optional immutableUrl; @@ -182,4 +183,7 @@ DownloadTarballResult downloadTarball( bool locked, const Headers & headers = {}); +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name); + } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f8d89ab2fcdd..8ea31287d8f6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -200,7 +200,7 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) return WorkdirInfo { .clean = clean, .hasHead = hasHead }; } -std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) +std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); auto gitDir = ".git"; @@ -236,6 +236,9 @@ std::pair fetchFromWorkdir(ref store, Input & input, co }; auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); // FIXME: maybe we should use the timestamp of the last // modified dirty file? @@ -250,7 +253,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); } - return {std::move(storePath), input}; + return {std::move(storePathDesc), input}; } } // end namespace @@ -396,7 +399,7 @@ struct GitInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); auto gitDir = ".git"; @@ -429,15 +432,15 @@ struct GitInputScheme : InputScheme }); }; - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult = [&](const Attrs & infoAttrs, StorePathDescriptor && storePathDesc) + -> std::pair { assert(input.getRev()); assert(!_input.getRev() || _input.getRev() == input.getRev()); if (!shallow) input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - return {std::move(storePath), input}; + return {std::move(storePathDesc), input}; }; if (input.getRev()) { @@ -676,6 +679,9 @@ struct GitInputScheme : InputScheme } auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); @@ -693,17 +699,17 @@ struct GitInputScheme : InputScheme store, unlockedAttrs, infoAttrs, - storePath, + storePathDesc, false); getCache()->add( store, getLockedAttrs(), infoAttrs, - storePath, + storePathDesc, true); - return makeResult(infoAttrs, std::move(storePath)); + return makeResult(infoAttrs, std::move(storePathDesc)); } }; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 291f457f0d35..5763c37d2a70 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -183,7 +183,7 @@ struct GitArchiveInputScheme : InputScheme virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); @@ -267,9 +267,8 @@ struct GitHubInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + readFile(store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, url, "source", false, headers).storePath)))); auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; @@ -338,10 +337,9 @@ struct GitLabInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + auto json = nlohmann::json::parse(readFile( + store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, url, "source", false, headers).storePath)))); auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; @@ -403,8 +401,8 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { - auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + auto file = store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath)); std::ifstream is(file); std::string line; getline(is, line); @@ -419,8 +417,8 @@ struct SourceHutInputScheme : GitArchiveInputScheme } std::regex refRegex(refUri); - auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + auto file = store->toRealPath(store->makeFixedOutputPathFromCA( + downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath)); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 4874a43ff97b..2e56520296c8 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -94,7 +94,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & input) override { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51fd1ed428b0..83e6c598ba2f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -145,7 +145,7 @@ struct MercurialInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); @@ -196,8 +196,11 @@ struct MercurialInputScheme : InputScheme }; auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); - return {std::move(storePath), input}; + return {std::move(storePathDesc), input}; } } @@ -221,8 +224,8 @@ struct MercurialInputScheme : InputScheme }); }; - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult = [&](const Attrs & infoAttrs, StorePathDescriptor && storePath) + -> std::pair { assert(input.getRev()); assert(!_input.getRev() || _input.getRev() == input.getRev()); @@ -301,6 +304,9 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(storePath)->fullStorePathDescriptorOpt().value(); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -312,17 +318,17 @@ struct MercurialInputScheme : InputScheme store, unlockedAttrs, infoAttrs, - storePath, + storePathDesc, false); getCache()->add( store, getLockedAttrs(), infoAttrs, - storePath, + storePathDesc, true); - return makeResult(infoAttrs, std::move(storePath)); + return makeResult(infoAttrs, std::move(storePathDesc)); } }; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 01f1be97822c..3101ed3b575c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -81,7 +81,7 @@ struct PathInputScheme : InputScheme // nothing to do } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); std::string absPath; @@ -123,7 +123,11 @@ struct PathInputScheme : InputScheme } input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {std::move(*storePath), input}; + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + auto storePathDesc = store->queryPathInfo(*storePath)->fullStorePathDescriptorOpt().value(); + + return {std::move(storePathDesc), input}; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec17f..74df2cd3a6e6 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -158,7 +158,8 @@ static std::shared_ptr getGlobalRegistry(ref store) } if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePathDesc = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e3a41e75e445..8d33290a8915 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -64,7 +64,7 @@ DownloadFileResult downloadFile( if (res.immutableUrl) infoAttrs.emplace("immutableUrl", *res.immutableUrl); - std::optional storePath; + std::optional storePath; if (res.cached) { assert(cached); @@ -73,20 +73,22 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(res.data, sink); auto hash = hashString(htSHA256, res.data); - ValidPathInfo info { - *store, - name, - FixedOutputInfo { + storePath = { + .name = name, + .info = FixedOutputInfo { .method = FileIngestionMethod::Flat, .hash = hash, .references = {}, }, + }; + ValidPathInfo info { + *store, + StorePathDescriptor { *storePath }, hashString(htSHA256, sink.s), }; info.narSize = sink.s.size(); auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); - storePath = std::move(info.path); } getCache()->add( @@ -133,14 +135,17 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .tree = Tree { + .actualPath = store->toRealPath(store->makeFixedOutputPathFromCA(cached->storePath)), + .storePath = std::move(cached->storePath), + }, .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; auto res = downloadFile(store, url, name, locked, headers); - std::optional unpackedStorePath; + std::optional unpackedStorePath; time_t lastModified; if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { @@ -149,13 +154,18 @@ DownloadTarballResult downloadTarball( } else { Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(res.storePath), tmpDir); + unpackTarfile( + store->toRealPath(store->makeFixedOutputPathFromCA(res.storePath)), + tmpDir); auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + auto temp = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + // FIXME: just have Store::addToStore return a StorePathDescriptor, as + // it has the underlying information. + unpackedStorePath = store->queryPathInfo(temp)->fullStorePathDescriptorOpt().value(); } Attrs infoAttrs({ @@ -174,7 +184,10 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .tree = Tree { + .actualPath = store->toRealPath(store->makeFixedOutputPathFromCA(*unpackedStorePath)), + .storePath = std::move(*unpackedStorePath) + }, .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -274,7 +287,7 @@ struct FileInputScheme : CurlInputScheme : (!requireTree && !hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & input) override { auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); return {std::move(file.storePath), input}; @@ -295,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme : (requireTree || hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & _input) override { Input input(_input); auto url = getStrAttr(input.attrs, "url"); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b4fea693f5b8..7dbabda4e26d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); @@ -307,14 +307,16 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - .method = method, - .hash = nar.first, - .references = { - .others = references, + { + .name = std::string { name }, + .info = FixedOutputInfo { + .method = method, + .hash = nar.first, + .references = { + .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, + .self = false, + }, }, }, nar.first, @@ -324,12 +326,12 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n })->path; } -bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) +bool BinaryCacheStore::isValidPathUncached(StorePathOrDesc storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. - return fileExists(narInfoFileFor(storePath)); + return fileExists(narInfoFileFor(bakeCaIfNeeded(storePath))); } std::optional BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart) @@ -343,7 +345,7 @@ std::optional BinaryCacheStore::queryPathFromHashPart(const std::stri } } -void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(StorePathOrDesc storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); @@ -365,16 +367,17 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize.length; } -void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, +void BinaryCacheStore::queryPathInfoUncached(StorePathOrDesc storePath, Callback> callback) noexcept { auto uri = getUri(); - auto storePathS = printStorePath(storePath); + auto actualStorePath = bakeCaIfNeeded(storePath); + auto storePathS = printStorePath(actualStorePath); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); - auto narInfoFile = narInfoFileFor(storePath); + auto narInfoFile = narInfoFileFor(actualStorePath); auto callbackPtr = std::make_shared(std::move(callback)); @@ -424,14 +427,16 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - .method = method, - .hash = h, - .references = { - .others = references, - // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, + { + .name = std::string { name }, + .info = FixedOutputInfo { + .method = method, + .hash = h, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, }, }, nar.first, @@ -459,10 +464,12 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - std::string { name }, - TextInfo { - .hash = textHash, - .references = references, + { + .name = std::string { name }, + .info = TextInfo { + .hash = textHash, + .references = references, + }, }, nar.first, }; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d248c2..2bc2a4c6f208 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -113,9 +113,9 @@ private: public: - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override; std::optional queryPathFromHashPart(const std::string & hashPart) override; @@ -146,7 +146,7 @@ public: void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override; - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; ref getFSAccessor() override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bec0bc538ab2..b4f02aa42d6d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -258,11 +258,12 @@ void DerivationGoal::haveDerivation() ) ); else { - auto * cap = getDerivationCA(*drv); + auto optCA = getDerivationCA(*drv); + auto storePathOrCA = + optCA ? StorePathOrDesc { *optCA } : status.known->path; addWaitee(upcast_goal(worker.makePathSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - cap ? std::optional { *cap } : std::nullopt))); + storePathOrCA, + buildMode == bmRepair ? Repair : NoRepair))); } } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f0f0e5519358..8b35474f2093 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -92,7 +92,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat } -void Store::ensurePath(const StorePath & path) +void Store::ensurePath(StorePathOrDesc path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; @@ -107,8 +107,10 @@ void Store::ensurePath(const StorePath & path) if (goal->ex) { goal->ex->status = worker.failingExitStatus(); throw std::move(*goal->ex); - } else - throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } else { + auto p = this->bakeCaIfNeeded(path); + throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(p)); + } } } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6ac2d..df3a24e516a9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1244,10 +1244,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In return paths; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override { - if (goal.isAllowed(path)) { + if (goal.isAllowed(bakeCaIfNeeded(path))) { try { /* Censor impure information. */ auto info = std::make_shared(*next->queryPathInfo(path)); @@ -1319,15 +1319,17 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In return path; } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + LocalFSStore::narFromPath(pathOrDesc, sink); } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc pathOrDesc) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ @@ -2510,8 +2512,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() } ValidPathInfo newInfo0 { worker.store, - outputPathName(drv->name, outputName), - *std::move(optCA), + { + .name = outputPathName(drv->name, outputName), + .info = *std::move(optCA), + }, Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2548,10 +2552,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = std::move(refs.others); - if (refs.self) - newInfo0.references.insert(newInfo0.path); + newInfo0.references = rewriteRefs(); return newInfo0; }, @@ -2807,12 +2808,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -2852,7 +2853,7 @@ void LocalDerivationGoal::checkOutputs(const std::map ca) +PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker, DerivedPath::Opaque { storePath }) , storePath(storePath) , repair(repair) @@ -95,9 +95,7 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - ContentAddressWithReferences::withoutRefs(*ca)); + auto subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -107,7 +105,8 @@ void PathSubstitutionGoal::tryNext() try { // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); + info = sub->queryPathInfo(p); } catch (InvalidPath &) { tryNext(); return; @@ -164,9 +163,8 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references.others) + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -187,9 +185,8 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + for (auto & i : info->references.others) + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); @@ -223,8 +220,10 @@ void PathSubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); + copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + p, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1d389d328ffc..efb8c6cf8d79 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -13,15 +13,11 @@ struct PathSubstitutionGoal : public Goal { /** * The store path that should be realised through a substitute. + * + * @todo should be `OwnedStorePathOrDesc`. */ StorePath storePath; - /** - * The path the substituter refers to the path as. This will be - * different when the stores have different names. - */ - std::optional subPath; - /** * The remaining substituters. */ @@ -73,8 +69,10 @@ struct PathSubstitutionGoal : public Goal /** * Content address for recomputing store path + * + * @todo delete once `storePath` is a `std::variant`. */ - std::optional ca; + std::optional ca; void done( ExitCode result, @@ -82,7 +80,7 @@ struct PathSubstitutionGoal : public Goal std::optional errorMsg = {}); public: - PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f65f63b99473..07165c9dfee5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -100,25 +100,30 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makePathSubstitutionGoal(StorePathOrDesc path, RepairFlag repair) { - std::weak_ptr & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.lock(); // FIXME + auto p = store.bakeCaIfNeeded(path); + std::weak_ptr & goal_weak = substitutionGoals[p]; + std::shared_ptr goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); + auto optCA = std::get_if<1>(&path); + goal = std::make_shared( + p, + *this, + repair, + optCA ? std::optional { *optCA } : std::nullopt); goal_weak = goal; wakeUp(goal); } return goal; } - -std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(id, *this, repair, ca); + goal = std::make_shared(id, *this, repair); goal_weak = goal; wakeUp(goal); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a778e311c188..3768b4ebcb68 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -225,8 +225,8 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makePathSubstitutionGoal(StorePathOrDesc storePath, RepairFlag repair = NoRepair); + std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair); /** * Make a goal corresponding to the `DerivedPath`. diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e290a8d387eb..1dab6bca9b48 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -63,6 +63,19 @@ std::string ContentAddress::render() const + this->hash.to_string(Base32, true); } +static HashType parseHashType_(std::string_view & rest) { + auto hashTypeRaw = splitPrefixTo(rest, ':'); + if (!hashTypeRaw) + throw UsageError("hash must be in form \":\", but found: %s", rest); + return parseHashType(*hashTypeRaw); +}; + +static FileIngestionMethod parseFileIngestionMethod_(std::string_view & rest) { + if (splitPrefix(rest, "r:")) + return FileIngestionMethod::Recursive; + return FileIngestionMethod::Flat; +} + /** * Parses content address strings up to the hash. */ @@ -74,22 +87,14 @@ static std::pair parseContentAddressMethodPrefix { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form ':': %s", wholeInput); + throw UsageError("not a path-info content address because it is not in the form \":\": %s", wholeInput); prefix = *optPrefix; } - auto parseHashType_ = [&](){ - auto hashTypeRaw = splitPrefixTo(rest, ':'); - if (!hashTypeRaw) - throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); - }; - // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashType hashType = parseHashType_(); + HashType hashType = parseHashType_(rest); return { TextIngestionMethod {}, std::move(hashType), @@ -99,13 +104,13 @@ static std::pair parseContentAddressMethodPrefix auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + HashType hashType = parseHashType_(rest); return { std::move(method), std::move(hashType), }; } else - throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); + throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); } ContentAddress ContentAddress::parse(std::string_view rawCa) @@ -140,20 +145,119 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -std::string ContentAddress::printMethodAlgo() const + +// FIXME Deduplicate with store-api.cc path computation +std::string renderStorePathDescriptor(StorePathDescriptor ca) { - return method.renderPrefix() - + printHashType(hash.type); + std::string result { ca.name }; + result += ":"; + + auto dumpRefs = [&](auto references, bool hasSelfReference) { + result += "refs:"; + result += std::to_string(references.size()); + for (auto & i : references) { + result += ":"; + result += i.to_string(); + } + if (hasSelfReference) result += ":self"; + result += ":"; + }; + + std::visit(overloaded { + [&](const TextInfo & th) { + result += "text:"; + dumpRefs(th.references, false); + result += th.hash.to_string(Base32, true); + }, + [&](const FixedOutputInfo & fsh) { + result += "fixed:"; + dumpRefs(fsh.references.others, fsh.references.self); + result += makeFileIngestionPrefix(fsh.method); + result += fsh.hash.to_string(Base32, true); + }, + }, ca.info.raw); + + return result; } -bool StoreReferences::empty() const + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) { - return !self && others.empty(); + warn("%s", rawCa); + auto rest = rawCa; + + std::string_view name; + std::string_view tag; + { + auto optName = splitPrefixTo(rest, ':'); + auto optTag = splitPrefixTo(rest, ':'); + if (!(optTag && optName)) + throw UsageError("not a content address because it is not in the form \"::\": %s", rawCa); + tag = *optTag; + name = *optName; + } + + auto parseRefs = [&]() -> StoreReferences { + if (!splitPrefix(rest, "refs:")) + throw Error("Invalid CA \"%s\", \"%s\" should begin with \"refs:\"", rawCa, rest); + StoreReferences ret; + size_t numReferences = 0; + { + auto countRaw = splitPrefixTo(rest, ':'); + if (!countRaw) + throw UsageError("Invalid count"); + numReferences = std::stoi(std::string { *countRaw }); + } + for (size_t i = 0; i < numReferences; i++) { + auto s = splitPrefixTo(rest, ':'); + if (!s) + throw UsageError("Missing reference no. %d", i); + ret.others.insert(StorePath(*s)); + } + if (splitPrefix(rest, "self:")) + ret.self = true; + return ret; + }; + + // Dummy value + ContentAddressWithReferences info = TextInfo { Hash(htSHA256), {} }; + + // Switch on tag + if (tag == "text") { + auto refs = parseRefs(); + if (refs.self) + throw UsageError("Text content addresses cannot have self references"); + auto hashType = parseHashType_(rest); + if (hashType != htSHA256) + throw Error("Text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + info = TextInfo { + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + .references = refs.others, + }; + } else if (tag == "fixed") { + auto refs = parseRefs(); + auto method = parseFileIngestionMethod_(rest); + auto hashType = parseHashType_(rest); + info = FixedOutputInfo { + .method = method, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + .references = refs, + }; + } else + throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\" or \"fixed\"", tag); + + return StorePathDescriptor { + .name = std::string { name }, + .info = info, + }; } -size_t StoreReferences::size() const + +std::string ContentAddress::printMethodAlgo() const { - return (self ? 1 : 0) + others.size(); + return method.renderPrefix() + + printHashType(hash.type); } ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c4d619bdc63f..5c36fcfd0d70 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" #include "variant-wrapper.hh" namespace nix { @@ -163,38 +164,7 @@ std::string renderContentAddress(std::optional ca); * See the schema for store paths in store-api.cc */ -/** - * A set of references to other store objects. - * - * References to other store objects are tracked with store paths, self - * references however are tracked with a boolean. - */ -struct StoreReferences -{ - /** - * References to other store objects - */ - StorePathSet others; - - /** - * Reference to this store object - */ - bool self = false; - - /** - * @return true iff no references, i.e. others is empty and self is - * false. - */ - bool empty() const; - - /** - * Returns the numbers of references, i.e. the size of others + 1 - * iff self is true. - */ - size_t size() const; - - GENERATE_CMP(StoreReferences, me->self, me->others); -}; +using StoreReferences = References; // This matches the additional info that we need for makeTextPath struct TextInfo @@ -277,4 +247,15 @@ struct ContentAddressWithReferences Hash getHash() const; }; +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + GENERATE_CMP(StorePathDescriptor, me->name, me->info); +}; + +std::string renderStorePathDescriptor(StorePathDescriptor ca); + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044faf..6adff1918fc9 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -346,7 +346,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == WorkerProto::Op::QueryReferrers) store->queryReferrers(path, paths); @@ -776,7 +776,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); + store->querySubstitutablePathInfos({path}, {}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -784,7 +784,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - WorkerProto::write(*store, wconn, i->second.references); + WorkerProto::write(*store, wconn, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -793,21 +793,34 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; - StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = WorkerProto::Serialise::read(*store, rconn); - for (auto & path : paths) - pathsMap.emplace(path, std::nullopt); - } else - pathsMap = WorkerProto::Serialise::read(*store, rconn); + StorePathSet paths; + std::set caPaths; + if (GET_PROTOCOL_MINOR(clientVersion) >= 36) { + paths = WorkerProto::Serialise::read(*store, rconn); + caPaths = WorkerProto::Serialise>::read(*store, rconn); + } else if (GET_PROTOCOL_MINOR(clientVersion) >= 22) { + auto pathsMap = WorkerProto::Serialise::read(*store, rconn); + for (auto & [storePath, optCA] : pathsMap) { + if (!optCA) { + paths.insert(storePath); + } else { + caPaths.insert(StorePathDescriptor { + .name = std::string { storePath.name() }, + .info = ContentAddressWithReferences::withoutRefs(*optCA), + }); + } + } + } else { + paths = WorkerProto::Serialise::read(*store, rconn); + } logger->startWork(); - store->querySubstitutablePathInfos(pathsMap, infos); + store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - WorkerProto::write(*store, wconn, i.second.references); + WorkerProto::write(*store, wconn, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -887,7 +900,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.setReferencesPossiblyToSelf(WorkerProto::Serialise::read(*store, rconn)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = ContentAddress::parseOpt(readString(from)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index dc32c3847de9..8a8c1346f60c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -38,9 +38,10 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { - return store.makeFixedOutputPathFromCA( - outputPathName(drvName, outputName), - ContentAddressWithReferences::withoutRefs(ca)); + return store.makeFixedOutputPathFromCA({ + .name = outputPathName(drvName, outputName), + .info = ContentAddressWithReferences::withoutRefs(ca), + }); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b518a..42a183b6ea03 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -33,7 +33,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store return *uriSchemes().begin(); } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override { callback(nullptr); @@ -65,7 +65,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair) override { unsupported("addTextToStore"); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc path, Sink & sink) override { unsupported("narFromPath"); } void queryRealisationUncached(const DrvOutput &, diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42d27..d45a91146d35 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -48,7 +48,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) << printStorePath(path); WorkerProto::write(*this, WorkerProto::WriteConn { .to = teeSink }, - info->references); + info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -84,7 +84,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031ae69..0e7eee502e89 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -159,9 +159,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor return *uriSchemes().begin() + "://" + host; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept override { + auto path = this->bakeCaIfNeeded(pathOrDesc); try { auto conn(connections->get()); @@ -183,7 +184,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->setReferencesPossiblyToSelf(WorkerProto::Serialise::read(*this, *conn)); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +218,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -246,7 +247,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -259,8 +260,9 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = this->bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); @@ -378,7 +380,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor } } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc desc) override { unsupported("ensurePath"); } virtual ref getFSAccessor() override @@ -416,19 +418,27 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor out.insert(i); } - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + conn->to << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths2); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + auto res = WorkerProto::Serialise::read(*this, *conn); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } void connect() override diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e989b..fed638a7a5e8 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -78,11 +78,12 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePathOrDesc pathOrDesc, Sink & sink) { - if (!isValidPath(path)) - throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); + auto p = this->bakeCaIfNeeded(pathOrDesc); + if (!isValidPath(pathOrDesc)) + throw Error("path '%s' is not valid", printStorePath(p)); + dumpPath(getRealStoreDir() + std::string(printStorePath(p), storeDir.size()), sink); } const std::string LocalFSStore::drvsLogDir = "drvs"; diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 48810950113b..e3dafe9783ff 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -46,7 +46,7 @@ public: LocalFSStore(const Params & params); - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; ref getFSAccessor() override; /** diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17b4ecc73125..60ee7fc8f827 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -868,9 +868,10 @@ uint64_t LocalStore::addValidPath(State & state, } -void LocalStore::queryPathInfoUncached(const StorePath & path, +void LocalStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { callback(retrySQLite>([&]() { auto state(_state.lock()); @@ -922,7 +923,8 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -957,8 +959,9 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) } -bool LocalStore::isValidPathUncached(const StorePath & path) +bool LocalStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); return retrySQLite([&]() { auto state(_state.lock()); return isValidPath_(*state, path); @@ -966,11 +969,11 @@ bool LocalStore::isValidPathUncached(const StorePath & path) } -StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set LocalStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) res.insert(i); return res; } @@ -1121,7 +1124,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1141,7 +1144,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1332,21 +1335,24 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - .method = method, - .hash = hash, - .references = { - .others = references, - // caller is not capable of creating a self-reference, because this is content-addressed without modulus - .self = false, + auto desc = StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + .method = method, + .hash = hash, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, }, }; - auto dstPath = makeFixedOutputPathFromCA(name, desc); + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); - if (repair || !isValidPath(dstPath)) { + if (repair || !isValidPath(desc)) { /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ @@ -1355,7 +1361,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name PathLocks outputLock({realPath}); - if (repair || !isValidPath(dstPath)) { + if (repair || !isValidPath(desc)) { deletePath(realPath); @@ -1386,12 +1392,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { - *this, - name, - std::move(desc), - narHash.first - }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; registerValidPath(info); } @@ -1440,7 +1441,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = { .method = TextIngestionMethod {}, .hash = hash, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e97195f5b88c..41e48f6600f0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -152,14 +152,14 @@ public: std::string getUri() override; - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 253609ed2a9d..1fe8be40b369 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,15 +28,13 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - for (auto & ref : oldInfo->references) { - if (ref == path) - refs.self = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); refs.others.insert(std::move(replacement)); } } @@ -50,11 +48,13 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - path.name(), - FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - .references = std::move(refs), + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + .references = std::move(refs), + }, }, Hash::dummy, }; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c043b9b93056..249a9ca5b65e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,9 +40,8 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto& ref : info->references) - if (ref != path) - res.insert(ref); + for (auto & ref : info->references.others) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -83,15 +82,18 @@ void Store::computeFSClosure(const StorePath & startPath, } -const ContentAddress * getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out == drv.outputs.end()) - return nullptr; + return std::nullopt; if (auto dof = std::get_if(&out->second.raw)) { - return &dof->ca; + return StorePathDescriptor { + .name = drv.name, + .info = ContentAddressWithReferences::withoutRefs(dof->ca), + }; } - return nullptr; + return std::nullopt; } void Store::queryMissing(const std::vector & targets, @@ -141,13 +143,11 @@ void Store::queryMissing(const std::vector & targets, if (drvState_->lock()->done) return; SubstitutablePathInfos infos; - auto * cap = getDerivationCA(*drv); - querySubstitutablePathInfos({ - { - outPath, - cap ? std::optional { *cap } : std::nullopt, - }, - }, infos); + auto caOpt = getDerivationCA(*drv); + if (caOpt) + querySubstitutablePathInfos({}, { *std::move(caOpt) }, infos); + else + querySubstitutablePathInfos({outPath}, {}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -251,7 +251,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(bo.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); + querySubstitutablePathInfos({bo.path}, {}, infos); if (infos.empty()) { auto state(state_.lock()); @@ -269,7 +269,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -287,7 +287,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -300,6 +300,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } + std::map drvOutputReferences( const std::set & inputRealisations, const StorePathSet & pathReferences) @@ -343,7 +344,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f7af..3114ac972fbe 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -268,7 +268,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741a6a..c035de748fb7 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -66,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3ebd..b99523032720 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,8 +17,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) - : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f91a..97f0e4bbfb18 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -14,7 +14,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -23,46 +23,40 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferences() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - .hash = ca->hash, - .references = references, - }; - }, - [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - .method = m2, - .hash = ca->hash, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, ca->method.raw); + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { + assert(!references.self); + return TextInfo { + .hash = ca->hash, + .references = references.others, + }; + }, + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { + return FixedOutputInfo { + .method = m2, + .hash = ca->hash, + .references = references, + }; + }, + }, ca->method.raw), + }; } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferences(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -94,7 +88,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } @@ -102,30 +96,46 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - std::string_view name, - ContentAddressWithReferences && ca, + StorePathDescriptor && info, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) + : path(store.makeFixedOutputPathFromCA(info)) , narHash(narHash) { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = ContentAddress { .method = TextIngestionMethod {}, .hash = std::move(ti.hash), }; }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references.others); - if (foi.references.self) - this->references.insert(path); + this->references = std::move(foi.references); this->ca = ContentAddress { .method = std::move(foi.method), .hash = std::move(foi.hash), }; }, - }, std::move(ca).raw); + }, std::move(info.info).raw); +} + + +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); } @@ -140,8 +150,8 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); + info.setReferencesPossiblyToSelf(WorkerProto::Serialise::read(store, + WorkerProto::ReadConn { .from = source })); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -164,7 +174,7 @@ void ValidPathInfo::write( << narHash.to_string(Base16, false); WorkerProto::write(store, WorkerProto::WriteConn { .to = sink }, - references); + referencesPossiblyToSelf()); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 22152362206e..18baae9eff18 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -18,7 +18,7 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StorePathSet references; + StoreReferences references; /** * 0 = unknown or inapplicable */ @@ -31,7 +31,6 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; @@ -40,7 +39,7 @@ struct ValidPathInfo * \todo document this */ Hash narHash; - StorePathSet references; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -93,17 +92,23 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); /** - * @return The `ContentAddressWithReferences` that determines the + * @return The `StorePathDescriptor` that determines the * store path for a content-addressed store object, `std::nullopt` * for an input-addressed store object. */ - std::optional contentAddressWithReferences() const; + std::optional fullStorePathDescriptorOpt() const; /** * @return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /** @@ -126,7 +131,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + StorePathDescriptor && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 58f72beb90c1..b32e708217fc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -187,8 +187,9 @@ void RemoteStore::setOptions() setOptions(*(getConnection().handle)); } -bool RemoteStore::isValidPathUncached(const StorePath & path) +bool RemoteStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << WorkerProto::Op::IsValidPath << printStorePath(path); conn.processStderr(); @@ -196,22 +197,31 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) } -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set RemoteStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) + res.insert(i); return res; } else { conn->to << WorkerProto::Op::QueryValidPaths; - WorkerProto::write(*this, *conn, paths); + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + // FIXME make new version to take advantage of desc case + WorkerProto::write(*this, *conn, paths2); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return WorkerProto::Serialise::read(*this, *conn); + auto res = WorkerProto::Serialise::read(*this, *conn); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } } @@ -245,58 +255,76 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { - if (pathsMap.empty()) return; + if (paths.empty() && caPaths.empty()) return; auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + auto combineSet = [&]() { + std::set combined { paths }; + for (auto & ca : caPaths) + combined.insert(makeFixedOutputPathFromCA(ca)); + return combined; + }; - for (auto & i : pathsMap) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + for (auto & path : combineSet()) { SubstitutablePathInfo info; - conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(path); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, *conn); + info.references.setPossiblyToSelf(path, WorkerProto::Serialise::read(*this, *conn)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); + infos.insert_or_assign(path, std::move(info)); } } else { conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); + WorkerProto::write(*this, *conn, combineSet()); + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 36) { + WorkerProto::StorePathCAMap pathMap; + for (auto & path : paths) + pathMap[path]; + for (auto & desc : caPaths) { + auto path = makeFixedOutputPathFromCA(desc); + pathMap[path] = { + .method = desc.info.getMethod(), + .hash = desc.info.getHash(), + }; + } + WorkerProto::write(*this, *conn, pathMap); + } else { WorkerProto::write(*this, *conn, paths); - } else - WorkerProto::write(*this, *conn, pathsMap); + WorkerProto::write(*this, *conn, caPaths); + } conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, *conn); + info.references.setPossiblyToSelf(path, WorkerProto::Serialise::read(*this, *conn)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } - } } -void RemoteStore::queryPathInfoUncached(const StorePath & path, +void RemoteStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { std::shared_ptr info; { @@ -506,7 +534,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -525,7 +553,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + WorkerProto::write(*this, *conn, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -815,8 +843,9 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } -void RemoteStore::ensurePath(const StorePath & path) +void RemoteStore::ensurePath(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); conn.processStderr(); @@ -979,8 +1008,9 @@ RemoteStore::Connection::~Connection() } } -void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +void RemoteStore::narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); conn->processStderr(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a1ae82a0f3f0..8b2483d88616 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -46,14 +46,14 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -67,7 +67,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; /** @@ -122,7 +123,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(StorePathOrDesc path) override; void addTempRoot(const StorePath & path) override; @@ -187,7 +188,7 @@ protected: virtual ref getFSAccessor() override; - virtual void narFromPath(const StorePath & path, Sink & sink) override; + virtual void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override; private: diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abafe46..a2d743b3946a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -309,8 +309,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(StorePathOrDesc storePathOrDesc) override { + auto storePath = bakeCaIfNeeded(storePathOrDesc); try { queryPathInfo(storePath); return true; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100e24..948a3c8e6bab 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -181,6 +181,31 @@ static std::string makeType( return std::move(type); } +StorePath Store::bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const +{ + return bakeCaIfNeeded(borrowStorePathOrDesc(path)); +} + +StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const +{ + return std::visit(overloaded { + [this](std::reference_wrapper storePath) { + return StorePath {storePath}; + }, + [this](std::reference_wrapper ca) { + return makeFixedOutputPathFromCA(ca); + }, + }, path); +} + +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc & storePathOrDesc) { + // Can't use std::visit as it would copy :( + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + abort(); +} StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { @@ -202,26 +227,23 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { - .others = info.references, - .self = false, - }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(name, ti); + return makeTextPath(desc.name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(name, foi); + return makeFixedOutputPath(desc.name, foi); } - }, ca.raw); + }, desc.info.raw); } @@ -313,7 +335,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -437,11 +459,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - name, - FixedOutputInfo { - .method = method, - .hash = hash, - .references = {}, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + .method = method, + .hash = hash, + .references = {}, + }, }, narHash, }; @@ -552,58 +576,66 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) } -void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +void Store::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA( - path.first.name(), - ContentAddressWithReferences::withoutRefs(*path.second)); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; + auto query = [&](auto & sub, const StorePath & localPath, const StorePath & subPath) { + if (infos.count(subPath)) + // Choose first succeeding substituter. + return; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - .deriver = info->deriver, - .references = info->references, - .downloadSize = narInfo ? narInfo->fileSize : 0, - .narSize = info->narSize, - }); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + return; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(localPath, SubstitutablePathInfo { + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + }; + + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (sub->storeDir != storeDir) continue; + query(sub, path, path); + } + for (auto & ca : caPaths) { + // TODO Deal with references: either disallow, or require the + // store path lengths be the same and rewrite strings. + auto localPath = makeFixedOutputPathFromCA(ca); + auto subPath = sub->makeFixedOutputPathFromCA(ca); + if (sub->storeDir == storeDir) + assert(localPath == subPath); + if (localPath != subPath) + // TODO print CA too + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(localPath), sub->printStorePath(subPath), sub->getUri()); + query(sub, localPath, subPath); } } } -bool Store::isValidPath(const StorePath & storePath) +bool Store::isValidPath(StorePathOrDesc storePathOrDesc) { + auto storePath = bakeCaIfNeeded(storePathOrDesc); + { auto state_(state.lock()); auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); @@ -636,7 +668,7 @@ bool Store::isValidPath(const StorePath & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const StorePath & path) +bool Store::isValidPathUncached(StorePathOrDesc path) { try { queryPathInfo(path); @@ -647,7 +679,7 @@ bool Store::isValidPathUncached(const StorePath & path) } -ref Store::queryPathInfo(const StorePath & storePath) +ref Store::queryPathInfo(StorePathOrDesc storePath) { std::promise> promise; @@ -672,9 +704,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) } -void Store::queryPathInfo(const StorePath & storePath, +void Store::queryPathInfo(StorePathOrDesc pathOrCa, Callback> callback) noexcept { + auto storePath = bakeCaIfNeeded(pathOrCa); + auto hashPart = std::string(storePath.hashPart()); try { @@ -708,7 +742,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); - queryPathInfoUncached(storePath, + queryPathInfoUncached(pathOrCa, {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { @@ -824,22 +858,38 @@ void Store::substitutePaths(const StorePathSet & paths) StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + std::set paths2; + for (auto & p : paths) + paths2.insert(p); + auto res = queryValidPaths(paths2, maybeSubstitute); + StorePathSet res2; + for (auto & r : res) { + auto p = std::get_if(&r); + assert(p); + res2.insert(*p); + } + return res2; +} + +std::set Store::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - StorePathSet valid; + std::set valid; std::exception_ptr exc; }; - Sync state_(State{paths.size(), StorePathSet()}); + Sync state_(State{paths.size(), {}}); std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const StorePath & path) { + auto doQuery = [&](const OwnedStorePathOrDesc & path) { checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { + auto path2 = borrowStorePathOrDesc(path); + queryPathInfo(path2, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); @@ -893,7 +943,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += fmt("%1%\n", info->references.size()); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -956,7 +1006,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1042,6 +1092,7 @@ static std::string makeCopyPathMessage( std::string_view dstUri, std::string_view storePath) { + // FIXME Use CA when we have it in messages below return srcUri == "local" || srcUri == "daemon" ? fmt("copying path '%s' to '%s'", storePath, dstUri) : dstUri == "local" || dstUri == "daemon" @@ -1049,17 +1100,19 @@ static std::string makeCopyPathMessage( : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri); } - void copyStorePath( Store & srcStore, Store & dstStore, - const StorePath & storePath, + StorePathOrDesc storePath, RepairFlag repair, CheckSigsFlag checkSigs) { + + auto actualStorePath = srcStore.bakeCaIfNeeded(storePath); + auto srcUri = srcStore.getUri(); auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); + auto storePathS = srcStore.printStorePath(actualStorePath); Activity act(*logger, lvlInfo, actCopyPath, makeCopyPathMessage(srcUri, dstUri, storePathS), {storePathS, srcUri, dstUri}); @@ -1070,14 +1123,21 @@ void copyStorePath( uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->ca && info->references.empty()) { - auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA( - info->path.name(), - info->contentAddressWithReferences().value()); - if (dstStore.storeDir == srcStore.storeDir) - assert(info->path == info2->path); - info = info2; + if (auto p = std::get_if>(&storePath)) { + auto ca = static_cast(*p); + // { + // ValidPathInfo srcInfoCA { *srcStore, StorePathDescriptor { ca } }; + // assert((StoreReferences &)(*info) == (StoreReferences &)srcInfoCA); + // } + if (info->references.empty()) { + auto info2 = make_ref(*info); + ValidPathInfo dstInfoCA { dstStore, StorePathDescriptor { ca }, info->narHash }; + if (dstStore.storeDir == srcStore.storeDir) + assert(info2->path == info2->path); + info2->path = std::move(dstInfoCA.path); + info2->ca = std::move(dstInfoCA.ca); + info = info2; + } } if (info->ultimate) { @@ -1094,14 +1154,14 @@ void copyStorePath( TeeSink tee { sink, progressSink }; srcStore.narFromPath(storePath, tee); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(actualStorePath), srcStore.getUri()); }); dstStore.addToStore(*info, *source, repair, checkSigs); } -std::map copyPaths( +void copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, @@ -1118,7 +1178,7 @@ std::map copyPaths( toplevelRealisations.insert(*realisation); } } - auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); ThreadPool pool; @@ -1151,11 +1211,10 @@ std::map copyPaths( else throw; } - - return pathsMap; } -std::map copyPaths( + +void copyPaths( Store & srcStore, Store & dstStore, const StorePathSet & storePaths, @@ -1171,74 +1230,149 @@ std::map copyPaths( Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + uint64_t total = 0; + + auto sorted = srcStore.topoSortPaths(missing); + std::reverse(sorted.begin(), sorted.end()); + + auto source = sinkToSource([&](Sink & sink) { + sink << sorted.size(); + for (auto & storePath : sorted) { + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore.queryPathInfo(storePath); + info->write(sink, srcStore, 16); + + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); + act.progress(total, info->narSize); + }); + TeeSink tee { sink, progressSink }; + + srcStore.narFromPath(storePath, tee); + } + }); + + dstStore.addMultipleToStore(*source, repair, checkSigs); +} + + +void copyPaths( + Store & srcStore, + Store & dstStore, + const std::set & storePaths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + auto valid = dstStore.queryValidPaths(storePaths, substitute); + + std::set missing; + + // Filter to just the ones which are missing. + + // While doing so, if we can "upgrade" the store path into a store + // path descriptor because it is content-addressed, do that. + for (auto & path : storePaths) { + if (valid.count(path)) continue; + + auto pathDescOpt = [&]() -> std::optional> { + auto storePathP = std::get_if(&path); + // Already store path descriptor + if (!storePathP) return std::nullopt; + auto & storePath = *storePathP; + + auto info = srcStore.queryPathInfo(borrowStorePathOrDesc(path)); + + auto descOpt = info->fullStorePathDescriptorOpt(); + if (!descOpt) return std::nullopt; + + return { { storePath, *std::move(descOpt) } }; + }(); + + if (pathDescOpt) { + auto [storePath, desc] = *std::move(pathDescOpt); + + debug("found CA description '%s' for path '%s'", + renderStorePathDescriptor(desc), + srcStore.printStorePath(storePath)); + + missing.insert(std::move(desc)); + } else { + missing.insert(path); + } + } + + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + uint64_t total = 0; + // In the general case, `addMultipleToStore` requires a sorted list of // store paths to add, so sort them right now - auto sortedMissing = srcStore.topoSortPaths(missing); - std::reverse(sortedMissing.begin(), sortedMissing.end()); - std::map pathsMap; - for (auto & path : storePaths) - pathsMap.insert_or_assign(path, path); + // Only paths, no descriptors + StorePathSet missingPaths; + // Map from paths back to original + std::map rich; - Store::PathsSource pathsToCopy; + // Build + for (auto & pathOrDesc : missing) { + auto path = srcStore.bakeCaIfNeeded(pathOrDesc); + rich.insert_or_assign(path, &pathOrDesc); + } - auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { - auto storePathForSrc = currentPathInfo.path; - auto storePathForDst = storePathForSrc; - if (currentPathInfo.ca && currentPathInfo.references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferences().value()); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePathForSrc); - if (storePathForDst != storePathForSrc) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePathForSrc), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - return storePathForDst; - }; + // Sort paths + auto sortedMissingPaths = srcStore.topoSortPaths(missingPaths); + std::reverse(sortedMissingPaths.begin(), sortedMissingPaths.end()); - // total is accessed by each copy, which are each handled in separate threads - std::atomic total = 0; + // Sorted originals + std::vector sortedMissing; + for (auto & path : sortedMissingPaths) + sortedMissing.push_back(rich.at(path)); - for (auto & missingPath : sortedMissing) { - auto info = srcStore.queryPathInfo(missingPath); + auto source = sinkToSource([&](Sink & sink) { + sink << sortedMissing.size(); + for (auto * missingPathP : sortedMissing) { + auto & missingPath = *missingPathP; - auto storePathForDst = computeStorePathForDst(*info); - pathsMap.insert_or_assign(missingPath, storePathForDst); + // For destination store + auto info = *srcStore.queryPathInfo(borrowStorePathOrDesc(missingPath)); - ValidPathInfo infoForDst = *info; - infoForDst.path = storePathForDst; + if (auto * pDesc = std::get_if(&missingPath)) + info.path = dstStore.makeFixedOutputPathFromCA(*pDesc); - auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we // read the path, so log something about that at that point auto srcUri = srcStore.getUri(); auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(missingPath); + auto storePathS = srcStore.printStorePath(info.path); Activity act(*logger, lvlInfo, actCopyPath, makeCopyPathMessage(srcUri, dstUri, storePathS), {storePathS, srcUri, dstUri}); PushActivity pact(act.id); + info.write(sink, srcStore, 16); + LambdaSink progressSink([&](std::string_view data) { total += data.size(); - act.progress(total, info->narSize); + act.progress(total, info.narSize); }); TeeSink tee { sink, progressSink }; - srcStore.narFromPath(missingPath, tee); - }); - pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); - } - - dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); + srcStore.narFromPath(borrowStorePathOrDesc(missingPath), tee); + } + }); - return pathsMap; + dstStore.addMultipleToStore(*source, repair, checkSigs); } + void copyClosure( Store & srcStore, Store & dstStore, @@ -1295,7 +1429,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1318,7 +1452,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index da1a3eefbbc4..95e402d6131c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -95,7 +95,15 @@ struct BuildResult; struct KeyedBuildResult; -typedef std::map> StorePathCAMap; +/* Useful for many store functions which can take advantage of content + addresses or work with regular store paths */ +typedef std::variant OwnedStorePathOrDesc; +typedef std::variant< + std::reference_wrapper, + std::reference_wrapper +> StorePathOrDesc; + +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc &); struct StoreConfig : public Config { @@ -286,7 +294,10 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + + StorePath bakeCaIfNeeded(StorePathOrDesc path) const; + StorePath bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const; /** * Preparatory part of addToStore(). @@ -322,11 +333,11 @@ public: /** * Check whether a path is valid. */ - bool isValidPath(const StorePath & path); + bool isValidPath(StorePathOrDesc desc); protected: - virtual bool isValidPathUncached(const StorePath & path); + virtual bool isValidPathUncached(StorePathOrDesc desc); public: @@ -341,6 +352,8 @@ public: * Query which of the given paths is valid. Optionally, try to * substitute missing paths. */ + virtual std::set queryValidPaths(const std::set & paths, + SubstituteFlag maybeSubstitute = NoSubstitute); virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); @@ -361,12 +374,12 @@ public: * Query information about a valid path. It is permitted to omit * the name part of the store path. */ - ref queryPathInfo(const StorePath & path); + ref queryPathInfo(StorePathOrDesc path); /** * Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const StorePath & path, + void queryPathInfo(StorePathOrDesc path, Callback> callback) noexcept; /** @@ -404,7 +417,7 @@ public: protected: - virtual void queryPathInfoUncached(const StorePath & path, + virtual void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept = 0; virtual void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept = 0; @@ -477,7 +490,8 @@ public: * If a path does not have substitute info, it's omitted from the * resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + virtual void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos); /** @@ -573,7 +587,7 @@ public: /** * Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(StorePathOrDesc desc, Sink & sink) = 0; /** * For each path, if it's a derivation, build it. Building a @@ -644,7 +658,7 @@ public: * may be made valid by running a substitute (if defined for the * path). */ - virtual void ensurePath(const StorePath & path); + virtual void ensurePath(StorePathOrDesc desc); /** * Add a store path as a temporary root of the garbage collector. @@ -889,7 +903,7 @@ protected: void copyStorePath( Store & srcStore, Store & dstStore, - const StorePath & storePath, + StorePathOrDesc storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); @@ -899,19 +913,24 @@ void copyStorePath( * in parallel. They are copied in a topologically sorted order (i.e. if * A is a reference of B, then A is copied before B), but the set of * store paths is not automatically closed; use copyClosure() for that. - * - * @return a map of what each path was copied to the dstStore as. */ -std::map copyPaths( +void copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -std::map copyPaths( +void copyPaths( Store & srcStore, Store & dstStore, - const StorePathSet & paths, + const std::set & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + +void copyPaths( + Store & srcStore, Store & dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -1048,7 +1067,7 @@ std::optional decodeValidPathInfo( */ std::pair splitUriAndParams(const std::string & uri); -const ContentAddress * getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001cf4..057127e5fdad 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -38,7 +38,7 @@ public: ref getFSAccessor() override { return LocalFSStore::getFSAccessor(); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } /** diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743ee0..6d11b0bc0d20 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -79,6 +79,17 @@ void WorkerProto::Serialise::write(const Store & store, WorkerPr } +StorePathDescriptor WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +{ + return parseStorePathDescriptor(readString(conn.from)); +} + +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePathDescriptor & ca) +{ + conn.to << renderStorePathDescriptor(ca); +} + + DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ff762c924ab1..63754364ef2a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 35) +#define PROTOCOL_VERSION (1 << 8 | 36) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -111,6 +111,11 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + /** + * For an older version of the protocol + */ + typedef std::map> StorePathCAMap; }; enum struct WorkerProto::Op : uint64_t @@ -204,6 +209,8 @@ MAKE_WORKER_PROTO(StorePath); template<> MAKE_WORKER_PROTO(ContentAddress); template<> +MAKE_WORKER_PROTO(StorePathDescriptor); +template<> MAKE_WORKER_PROTO(DerivedPath); template<> MAKE_WORKER_PROTO(Realisation); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 000000000000..ac4a9994e5ac --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19fa6..e5205ce79f00 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea8e3..1a714e87cddf 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -72,6 +72,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and @@ -232,6 +233,7 @@ struct TeeSource : Source } }; + /** * A reader that consumes the original Source until 'size'. */ diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 95f4014411f1..8eab069c9d58 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,7 +113,8 @@ static void update(const StringSet & channelNames) // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); - auto filename = store->toRealPath(result.storePath); + auto filename = store->toRealPath( + store->makeFixedOutputPathFromCA(result.storePath)); url = result.effectiveUrl; bool unpacked = false; @@ -124,12 +125,21 @@ static void update(const StringSet & channelNames) } if (!unpacked) { + StorePathDescriptor storePathDesc { + .name = "t e m p", + .info = FixedOutputInfo { + .method = FileIngestionMethod::Flat, + .hash = Hash(htSHA256), + .references = {}, + }, + }; // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + storePathDesc = fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath; } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + storePathDesc = fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath; } + filename = store->toRealPath(store->makeFixedOutputPathFromCA(storePathDesc)); } // Regardless of where it came from, add the expression representing this channel to accumulated expression exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb313..36d774dca766 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 4395576589c3..01a3a605fb34 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e038..e41652f7639e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -265,7 +265,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -347,7 +347,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -368,7 +368,8 @@ static void opQuery(Strings opFlags, Strings opArgs) case qDeriver: for (auto & i : opArgs) { - auto info = store->queryPathInfo(store->followLinksToStorePath(i)); + auto path = store->followLinksToStorePath(i); + auto info = store->queryPathInfo(path); cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; @@ -888,7 +889,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + WorkerProto::write(*store, wconn, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -903,9 +904,11 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case ServeProto::Command::DumpStorePath: - store->narFromPath(store->parseStorePath(readString(in)), out); + case ServeProto::Command::DumpStorePath: { + auto path = store->parseStorePath(readString(in)); + store->narFromPath(path, out); break; + } case ServeProto::Command::ImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); @@ -988,7 +991,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.setReferencesPossiblyToSelf(WorkerProto::Serialise::read(*store, rconn)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99dd2f..a42864d47a8c 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,11 +43,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - std::move(*namePart), - FixedOutputInfo { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - .references = {}, + StorePathDescriptor { + .name = std::move(*namePart), + .info = FixedOutputInfo { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + .references = {}, + }, }, narHash, }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1bec8..3b0e8c934d6d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -186,7 +186,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath( + store->makeFixedOutputPathFromCA(flake.sourceInfo->storePath)); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -202,7 +203,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath( + store->makeFixedOutputPathFromCA(flake.sourceInfo->storePath))); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -972,7 +974,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert( + store->makeFixedOutputPathFromCA(flake.flake.sourceInfo->storePath)); // FIXME: use graph output, handle cycles. std::function traverse; @@ -981,10 +984,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { - auto storePath = + auto storePathDesc = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + auto storePath = store->makeFixedOutputPathFromCA(storePathDesc); if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1001,8 +1005,16 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, - {"inputs", traverse(*flake.lockFile.root)}, + { + "path", + store->printStorePath( + store->makeFixedOutputPathFromCA( + flake.flake.sourceInfo->storePath)), + }, + { + "inputs", + traverse(*flake.lockFile.root), + }, }; logger->cout("%s", jsonRoot); } else { @@ -1338,15 +1350,16 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto [tree, lockedRef] = resolvedRef.fetchTree(store); auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto actualStorePath = store->bakeCaIfNeeded((const OwnedStorePathOrDesc &)tree.storePath); if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); + res["storePath"] = store->printStorePath(actualStorePath); res["hash"] = hash.to_string(SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), + store->printStorePath(actualStorePath), hash.to_string(SRI, true)); } } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd609b5..65df6fdc863b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -220,14 +220,16 @@ struct ProfileManifest ValidPathInfo info { *store, - "profile", - FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - .references = { - .others = std::move(references), - // profiles never refer to themselves - .self = false, + StorePathDescriptor { + "profile", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + .references = { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, }, }, narHash, diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d238456db5bc..c3010711856b 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -132,7 +132,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); - if (!store->isValidPath(store->parseStorePath(userEnv))) + auto path = store->parseStorePath(userEnv); + if (!store->isValidPath(path)) throw Error("directory '%s' is not in the Nix store", userEnv); return profileDir; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 592de773ce90..7818fda9ff62 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf });