From 19e9ada5643360d6acb139b14f7ffcade656b3ce Mon Sep 17 00:00:00 2001 From: Qyriad Date: Sat, 27 Apr 2024 13:24:39 -0600 Subject: [PATCH] refactor some nix-env and 3-profile code libcmd Notably, ProfileManifest and ProfileElement are useful generic profile management code, and nix profile is not the only place in the codebase where profiles are relevant. This commit is in preparation for fixing upgrade-nix's interaction with new-style profiles. Change-Id: Iefc8bbd34b4bc6012175cb3d6e6a8207973bc792 --- src/libcmd/cmd-profiles.cc | 277 +++++++++++++++++++++++++++++++++++++ src/libcmd/cmd-profiles.hh | 73 ++++++++++ src/libcmd/command.hh | 2 - src/libcmd/meson.build | 2 + src/nix-env/user-env.cc | 16 --- src/nix/diff-closures.cc | 10 +- src/nix/profile.cc | 268 +---------------------------------- 7 files changed, 356 insertions(+), 292 deletions(-) create mode 100644 src/libcmd/cmd-profiles.cc create mode 100644 src/libcmd/cmd-profiles.hh diff --git a/src/libcmd/cmd-profiles.cc b/src/libcmd/cmd-profiles.cc new file mode 100644 index 000000000..b487d2a77 --- /dev/null +++ b/src/libcmd/cmd-profiles.cc @@ -0,0 +1,277 @@ +#include "cmd-profiles.hh" +#include "built-path.hh" +#include "builtins/buildenv.hh" +#include "names.hh" +#include "store-api.hh" + +namespace nix +{ + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv) +{ + DrvInfos elems; + if (pathExists(userEnv + "/manifest.json")) + throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); + auto manifestFile = userEnv + "/manifest.nix"; + if (pathExists(manifestFile)) { + Value v; + state.evalFile(state.rootPath(CanonPath(manifestFile)), v); + Bindings & bindings(*state.allocBindings(0)); + getDerivations(state, v, "", bindings, elems, false); + } + return elems; +} + +std::string showVersions(const std::set & versions) +{ + if (versions.empty()) return "∅"; + std::set versions2; + for (auto & version : versions) + versions2.insert(version.empty() ? "ε" : version); + return concatStringsSep(", ", versions2); +} + +bool ProfileElementSource::operator<(const ProfileElementSource & other) const +{ + return std::tuple(originalRef.to_string(), attrPath, outputs) + < std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); +} + +std::string ProfileElementSource::to_string() const +{ + return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); +} + +std::string ProfileElement::identifier() const +{ + if (source) { + return source->to_string(); + } + StringSet names; + for (auto & path : storePaths) { + names.insert(DrvName(path.name()).name); + } + return concatStringsSep(", ", names); +} + +std::set ProfileElement::toInstallables(Store & store) +{ + if (source) { + return {source->to_string()}; + } + StringSet rawPaths; + for (auto & path : storePaths) { + rawPaths.insert(store.printStorePath(path)); + } + return rawPaths; +} + +std::string ProfileElement::versions() const +{ + StringSet versions; + for (auto & path : storePaths) { + versions.insert(DrvName(path.name()).version); + } + return showVersions(versions); +} + +bool ProfileElement::operator<(const ProfileElement & other) const +{ + return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); +} + +void ProfileElement::updateStorePaths( + ref evalStore, ref store, const BuiltPaths & builtPaths +) +{ + storePaths.clear(); + for (auto & buildable : builtPaths) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { storePaths.insert(bo.path); }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + storePaths.insert(output.second); + } + }, + }, + buildable.raw() + ); + } +} + +ProfileManifest::ProfileManifest(EvalState & state, const Path & profile) +{ + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + std::string sUrl; + std::string sOriginalUrl; + switch (version) { + case 1: + sUrl = "uri"; + sOriginalUrl = "originalUri"; + break; + case 2: + sUrl = "url"; + sOriginalUrl = "originalUrl"; + break; + default: + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + } + + for (auto & e : json["elements"]) { + ProfileElement element; + for (auto & p : e["storePaths"]) { + element.storePaths.insert(state.store->parseStorePath((std::string) p)); + } + element.active = e["active"]; + if (e.contains("priority")) { + element.priority = e["priority"]; + } + if (e.value(sUrl, "") != "") { + element.source = ProfileElementSource{ + parseFlakeRef(e[sOriginalUrl]), + parseFlakeRef(e[sUrl]), + e["attrPath"], + e["outputs"].get()}; + } + elements.emplace_back(std::move(element)); + } + } + + else if (pathExists(profile + "/manifest.nix")) + { + // FIXME: needed because of pure mode; ugly. + state.allowPath(state.store->followLinksToStore(profile)); + state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = {drvInfo.queryOutPath()}; + elements.emplace_back(std::move(element)); + } + } +} + +nlohmann::json ProfileManifest::toJSON(Store & store) const +{ + auto array = nlohmann::json::array(); + for (auto & element : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) { + paths.push_back(store.printStorePath(path)); + } + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + obj["priority"] = element.priority; + if (element.source) { + obj["originalUrl"] = element.source->originalRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + obj["outputs"] = element.source->outputs; + } + array.push_back(obj); + } + nlohmann::json json; + json["version"] = 2; + json["elements"] = array; + return json; +} + +StorePath ProfileManifest::build(ref store) +{ + auto tempDir = createTempDir(); + + StorePathSet references; + + Packages pkgs; + for (auto & element : elements) { + for (auto & path : element.storePaths) { + if (element.active) { + pkgs.emplace_back(store->printStorePath(path), true, element.priority); + } + references.insert(path); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + auto narHash = hashString(htSHA256, sink.s); + + ValidPathInfo info{ + *store, + "profile", + FixedOutputInfo{ + .method = FileIngestionMethod::Recursive, + .hash = narHash, + .references = + { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, + }, + narHash, + }; + info.narSize = sink.s.size(); + + StringSource source(sink.s); + store->addToStore(info, source); + + return std::move(info.path); +} + +void ProfileManifest::printDiff( + const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent +) +{ + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) { + logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions()); + changes = true; + ++j; + } else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) { + logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions()); + changes = true; + ++i; + } else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) { + logger->cout("%sNo changes.", indent); + } +} +} diff --git a/src/libcmd/cmd-profiles.hh b/src/libcmd/cmd-profiles.hh new file mode 100644 index 000000000..d03f9b1c2 --- /dev/null +++ b/src/libcmd/cmd-profiles.hh @@ -0,0 +1,73 @@ +#pragma once +///@file + +#include "built-path.hh" +#include "eval.hh" +#include "flake/flakeref.hh" +#include "get-drvs.hh" +#include "types.hh" + +#include +#include + +#include + +namespace nix +{ + +struct ProfileElementSource +{ + FlakeRef originalRef; + // FIXME: record original attrpath. + FlakeRef lockedRef; + std::string attrPath; + ExtendedOutputsSpec outputs; + + bool operator<(const ProfileElementSource & other) const; + + std::string to_string() const; +}; + +constexpr int DEFAULT_PRIORITY = 5; + +struct ProfileElement +{ + StorePathSet storePaths; + std::optional source; + bool active = true; + int priority = DEFAULT_PRIORITY; + + std::string identifier() const; + + /** + * Return a string representing an installable corresponding to the current + * element, either a flakeref or a plain store path + */ + std::set toInstallables(Store & store); + + std::string versions() const; + + bool operator<(const ProfileElement & other) const; + + void updateStorePaths(ref evalStore, ref store, const BuiltPaths & builtPaths); +}; + +struct ProfileManifest +{ + std::vector elements; + + ProfileManifest() { } + + ProfileManifest(EvalState & state, const Path & profile); + + nlohmann::json toJSON(Store & store) const; + + StorePath build(ref store); + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent); +}; + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv); +std::string showVersions(const std::set & versions); + +} diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 120c832ac..6c6c81718 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -342,8 +342,6 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); -std::string showVersions(const std::set & versions); - void printClosureDiff( ref store, const StorePath & beforePath, diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 6b1b44d84..5a0e61503 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -1,6 +1,7 @@ libcmd_sources = files( 'built-path.cc', 'command-installable-value.cc', + 'cmd-profiles.cc', 'command.cc', 'common-eval-args.cc', 'editor-for.cc', @@ -18,6 +19,7 @@ libcmd_sources = files( libcmd_headers = files( 'built-path.hh', 'command-installable-value.hh', + 'cmd-profiles.hh', 'command.hh', 'common-eval-args.hh', 'editor-for.hh', diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 757938914..f0131a458 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -16,22 +16,6 @@ namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv) -{ - DrvInfos elems; - if (pathExists(userEnv + "/manifest.json")) - throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); - auto manifestFile = userEnv + "/manifest.nix"; - if (pathExists(manifestFile)) { - Value v; - state.evalFile(state.rootPath(CanonPath(manifestFile)), v); - Bindings & bindings(*state.allocBindings(0)); - getDerivations(state, v, "", bindings, elems, false); - } - return elems; -} - - bool createUserEnv(EvalState & state, DrvInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken) diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c7c37b66f..736cbf55d 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "cmd-profiles.hh" #include "shared.hh" #include "store-api.hh" #include "common-args.hh" @@ -43,15 +44,6 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) return groupedPaths; } -std::string showVersions(const std::set & versions) -{ - if (versions.empty()) return "∅"; - std::set versions2; - for (auto & version : versions) - versions2.insert(version.empty() ? "ε" : version); - return concatStringsSep(", ", versions2); -} - void printClosureDiff( ref store, const StorePath & beforePath, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd60..67f97ca9b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "cmd-profiles.hh" #include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" @@ -17,269 +18,6 @@ using namespace nix; -struct ProfileElementSource -{ - FlakeRef originalRef; - // FIXME: record original attrpath. - FlakeRef lockedRef; - std::string attrPath; - ExtendedOutputsSpec outputs; - - bool operator < (const ProfileElementSource & other) const - { - return - std::tuple(originalRef.to_string(), attrPath, outputs) < - std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); - } - - std::string to_string() const - { - return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); - } -}; - -const int defaultPriority = 5; - -struct ProfileElement -{ - StorePathSet storePaths; - std::optional source; - bool active = true; - int priority = defaultPriority; - - std::string identifier() const - { - if (source) - return source->to_string(); - StringSet names; - for (auto & path : storePaths) - names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); - } - - /** - * Return a string representing an installable corresponding to the current - * element, either a flakeref or a plain store path - */ - std::set toInstallables(Store & store) - { - if (source) - return {source->to_string()}; - StringSet rawPaths; - for (auto & path : storePaths) - rawPaths.insert(store.printStorePath(path)); - return rawPaths; - } - - std::string versions() const - { - StringSet versions; - for (auto & path : storePaths) - versions.insert(DrvName(path.name()).version); - return showVersions(versions); - } - - bool operator < (const ProfileElement & other) const - { - return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); - } - - void updateStorePaths( - ref evalStore, - ref store, - const BuiltPaths & builtPaths) - { - storePaths.clear(); - for (auto & buildable : builtPaths) { - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - storePaths.insert(bo.path); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) - storePaths.insert(output.second); - }, - }, buildable.raw()); - } - } -}; - -struct ProfileManifest -{ - std::vector elements; - - ProfileManifest() { } - - ProfileManifest(EvalState & state, const Path & profile) - { - auto manifestPath = profile + "/manifest.json"; - - if (pathExists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath)); - - auto version = json.value("version", 0); - std::string sUrl; - std::string sOriginalUrl; - switch (version) { - case 1: - sUrl = "uri"; - sOriginalUrl = "originalUri"; - break; - case 2: - sUrl = "url"; - sOriginalUrl = "originalUrl"; - break; - default: - throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); - } - - for (auto & e : json["elements"]) { - ProfileElement element; - for (auto & p : e["storePaths"]) - element.storePaths.insert(state.store->parseStorePath((std::string) p)); - element.active = e["active"]; - if(e.contains("priority")) { - element.priority = e["priority"]; - } - if (e.value(sUrl, "") != "") { - element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), - e["attrPath"], - e["outputs"].get() - }; - } - elements.emplace_back(std::move(element)); - } - } - - else if (pathExists(profile + "/manifest.nix")) { - // FIXME: needed because of pure mode; ugly. - state.allowPath(state.store->followLinksToStore(profile)); - state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); - - auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); - - for (auto & drvInfo : drvInfos) { - ProfileElement element; - element.storePaths = {drvInfo.queryOutPath()}; - elements.emplace_back(std::move(element)); - } - } - } - - nlohmann::json toJSON(Store & store) const - { - auto array = nlohmann::json::array(); - for (auto & element : elements) { - auto paths = nlohmann::json::array(); - for (auto & path : element.storePaths) - paths.push_back(store.printStorePath(path)); - nlohmann::json obj; - obj["storePaths"] = paths; - obj["active"] = element.active; - obj["priority"] = element.priority; - if (element.source) { - obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->lockedRef.to_string(); - obj["attrPath"] = element.source->attrPath; - obj["outputs"] = element.source->outputs; - } - array.push_back(obj); - } - nlohmann::json json; - json["version"] = 2; - json["elements"] = array; - return json; - } - - StorePath build(ref store) - { - auto tempDir = createTempDir(); - - StorePathSet references; - - Packages pkgs; - for (auto & element : elements) { - for (auto & path : element.storePaths) { - if (element.active) - pkgs.emplace_back(store->printStorePath(path), true, element.priority); - references.insert(path); - } - } - - buildProfile(tempDir, std::move(pkgs)); - - writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); - - /* Add the symlink tree to the store. */ - StringSink sink; - dumpPath(tempDir, sink); - - auto narHash = hashString(htSHA256, sink.s); - - ValidPathInfo info { - *store, - "profile", - FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - .references = { - .others = std::move(references), - // profiles never refer to themselves - .self = false, - }, - }, - narHash, - }; - info.narSize = sink.s.size(); - - StringSource source(sink.s); - store->addToStore(info, source); - - return std::move(info.path); - } - - static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) - { - auto prevElems = prev.elements; - std::sort(prevElems.begin(), prevElems.end()); - - auto curElems = cur.elements; - std::sort(curElems.begin(), curElems.end()); - - auto i = prevElems.begin(); - auto j = curElems.begin(); - - bool changes = false; - - while (i != prevElems.end() || j != curElems.end()) { - if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) { - logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions()); - changes = true; - ++j; - } - else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) { - logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions()); - changes = true; - ++i; - } - else { - auto v1 = i->versions(); - auto v2 = j->versions(); - if (v1 != v2) { - logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2); - changes = true; - } - ++i; - ++j; - } - } - - if (!changes) - logger->cout("%sNo changes.", indent); - } -}; static std::map>> builtPathsPerInstallable( @@ -361,8 +99,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile : ({ auto * info2 = dynamic_cast(&*info); info2 - ? info2->value.priority.value_or(defaultPriority) - : defaultPriority; + ? info2->value.priority.value_or(DEFAULT_PRIORITY) + : DEFAULT_PRIORITY; }); element.updateStorePaths(getEvalStore(), store, res);