From 8414685c0f0c0d744c70d438894e4ecd7a078808 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jan 2020 19:16:40 +0100 Subject: [PATCH] Change lock file format to use an attribute representation of flake refs rather than URLs --- src/libexpr/flake/flakeref.cc | 20 +++++++++-- src/libexpr/flake/flakeref.hh | 7 ++-- src/libexpr/flake/lockfile.cc | 54 +++++++++++++++++++++++++----- src/libstore/fetchers/fetchers.cc | 50 +++++++++++++++++++++++++++ src/libstore/fetchers/fetchers.hh | 23 ++++++++++++- src/libstore/fetchers/git.cc | 27 +++++++++++++-- src/libstore/fetchers/github.cc | 27 ++++++++++++++- src/libstore/fetchers/indirect.cc | 25 +++++++++++++- src/libstore/fetchers/mercurial.cc | 27 +++++++++++++-- src/libstore/fetchers/tarball.cc | 47 ++++++++++++++++++++++---- src/libutil/util.hh | 5 +-- tests/flakes.sh | 20 +++++------ 12 files changed, 293 insertions(+), 39 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 7b840bce5..194332674 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -14,12 +14,19 @@ const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; #endif - std::string FlakeRef::to_string() const { return input->to_string(); } +fetchers::Input::Attrs FlakeRef::toAttrs() const +{ + auto attrs = input->toAttrs(); + if (subdir != "") + attrs.emplace("subdir", subdir); + return attrs; +} + bool FlakeRef::isDirect() const { return input->isDirect(); @@ -36,7 +43,7 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -bool FlakeRef::operator==(const FlakeRef & other) const +bool FlakeRef::operator ==(const FlakeRef & other) const { return *input == *other.input && subdir == other.subdir; } @@ -166,4 +173,13 @@ std::optional> maybeParseFlakeRefWithFragment( } } +FlakeRef FlakeRef::fromAttrs(const fetchers::Input::Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("subdir"); + return FlakeRef( + fetchers::inputFromAttrs(attrs2), + fetchers::maybeGetStrAttr(attrs, "subdir").value_or("")); +} + } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index f552c99d8..9febc639d 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "hash.hh" +#include "fetchers/fetchers.hh" #include @@ -9,8 +10,6 @@ namespace nix { class Store; -namespace fetchers { struct Input; } - typedef std::string FlakeId; struct FlakeRef @@ -30,6 +29,8 @@ struct FlakeRef // FIXME: change to operator <<. std::string to_string() const; + fetchers::Input::Attrs toAttrs() const; + /* Check whether this is a "direct" flake reference, that is, not a flake ID, which requires a lookup in the flake registry. */ bool isDirect() const; @@ -39,6 +40,8 @@ struct FlakeRef bool isImmutable() const; FlakeRef resolve(ref store) const; + + static FlakeRef fromAttrs(const fetchers::Input::Attrs & attrs); }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 2696d4710..60fb914f9 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -6,10 +6,48 @@ namespace nix::flake { +FlakeRef flakeRefFromJson(const nlohmann::json & json) +{ + fetchers::Input::Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return FlakeRef::fromAttrs(attrs); +} + +FlakeRef getFlakeRef( + const nlohmann::json & json, + const char * version3Attr1, + const char * version3Attr2, + const char * version4Attr) +{ + auto i = json.find(version4Attr); + if (i != json.end()) + return flakeRefFromJson(*i); + + // FIXME: remove these. + i = json.find(version3Attr1); + if (i != json.end()) + return parseFlakeRef(*i); + + i = json.find(version3Attr2); + if (i != json.end()) + return parseFlakeRef(*i); + + throw Error("attribute '%s' missing in lock file", version4Attr); +} + LockedInput::LockedInput(const nlohmann::json & json) : LockedInputs(json) - , ref(parseFlakeRef(json.value("url", json.value("uri", "")))) - , originalRef(parseFlakeRef(json.value("originalUrl", json.value("originalUri", "")))) + , ref(getFlakeRef(json, "url", "uri", "resolvedRef")) + , originalRef(getFlakeRef(json, "originalUrl", "originalUri", "originalRef")) , narHash(Hash((std::string) json["narHash"])) { if (!ref.isImmutable()) @@ -19,9 +57,9 @@ LockedInput::LockedInput(const nlohmann::json & json) nlohmann::json LockedInput::toJson() const { auto json = LockedInputs::toJson(); - json["url"] = ref.to_string(); - json["originalUrl"] = originalRef.to_string(); - json["narHash"] = narHash.to_string(SRI); + json["originalRef"] = fetchers::attrsToJson(originalRef.toAttrs()); + json["resolvedRef"] = fetchers::attrsToJson(ref.toAttrs()); + json["narHash"] = narHash.to_string(SRI); // FIXME return json; } @@ -91,7 +129,7 @@ void LockedInputs::removeInput(const InputPath & path) nlohmann::json LockFile::toJson() const { auto json = LockedInputs::toJson(); - json["version"] = 3; + json["version"] = 4; return json; } @@ -101,7 +139,7 @@ LockFile LockFile::read(const Path & path) auto json = nlohmann::json::parse(readFile(path)); auto version = json.value("version", 0); - if (version != 3) + if (version != 3 && version != 4) throw Error("lock file '%s' has unsupported version %d", path, version); return LockFile(json); @@ -111,7 +149,7 @@ LockFile LockFile::read(const Path & path) std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - stream << lockFile.toJson().dump(4); // '4' = indentation in json file + stream << lockFile.toJson().dump(2); return stream; } diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index efc18dbb8..16f674401 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -2,6 +2,8 @@ #include "parse.hh" #include "store-api.hh" +#include + namespace nix::fetchers { std::unique_ptr>> inputSchemes = nullptr; @@ -26,6 +28,54 @@ std::unique_ptr inputFromURL(const std::string & url) return inputFromURL(parseURL(url)); } +std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs); + if (res) return res; + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else abort(); + } + return json; +} + +Input::Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("type", type()); + return attrs; +} + +std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string", name); +} + +std::string getStrAttr(const Input::Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + std::pair> Input::fetchTree(ref store) const { auto [tree, input] = fetchTreeInternal(store); diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index ccc1683ba..39e004240 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -5,6 +5,9 @@ #include "path.hh" #include +#include + +#include namespace nix { class Store; } @@ -24,9 +27,10 @@ struct Tree struct Input : std::enable_shared_from_this { - std::string type; std::optional narHash; // FIXME: implement + virtual std::string type() const = 0; + virtual ~Input() { } virtual bool operator ==(const Input & other) const { return false; } @@ -43,6 +47,11 @@ struct Input : std::enable_shared_from_this virtual std::string to_string() const = 0; + typedef std::variant Attr; + typedef std::map Attrs; + + Attrs toAttrs() const; + std::pair> fetchTree(ref store) const; virtual std::shared_ptr applyOverrides( @@ -59,6 +68,8 @@ struct Input : std::enable_shared_from_this private: virtual std::pair> fetchTreeInternal(ref store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; }; struct ParsedURL; @@ -68,12 +79,22 @@ struct InputScheme virtual ~InputScheme() { } virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) = 0; }; std::unique_ptr inputFromURL(const ParsedURL & url); std::unique_ptr inputFromURL(const std::string & url); +std::unique_ptr inputFromAttrs(const Input::Attrs & attrs); + void registerInputScheme(std::unique_ptr && fetcher); +nlohmann::json attrsToJson(const Input::Attrs & attrs); + +std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Input::Attrs & attrs, const std::string & name); + } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 1350c5754..2a7ce5432 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -74,9 +74,9 @@ struct GitInput : Input std::optional rev; GitInput(const ParsedURL & url) : url(url) - { - type = "git"; - } + { } + + std::string type() const override { return "git"; } bool operator ==(const Input & other) const override { @@ -105,6 +105,17 @@ struct GitInput : Input return url2.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + void clone(const Path & destDir) const override { auto [isLocal, actualUrl] = getActualUrl(); @@ -379,6 +390,16 @@ struct GitInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index c75680649..13ac4e2f1 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -19,6 +19,8 @@ struct GitHubInput : Input std::optional ref; std::optional rev; + std::string type() const override { return "github"; } + bool operator ==(const Input & other) const override { auto other2 = dynamic_cast(&other); @@ -48,6 +50,18 @@ struct GitHubInput : Input return s; } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + void clone(const Path & destDir) const override { std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); @@ -138,7 +152,6 @@ struct GitHubInputScheme : InputScheme auto path = tokenizeString>(url.path, "/"); auto input = std::make_unique(); - input->type = "github"; if (path.size() == 2) { } else if (path.size() == 3) { @@ -176,6 +189,18 @@ struct GitHubInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc index 1f9d1e24f..d079b3ad3 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libstore/fetchers/indirect.cc @@ -12,6 +12,8 @@ struct IndirectInput : Input std::optional rev; std::optional ref; + std::string type() const override { return "indirect"; } + bool operator ==(const Input & other) const override { auto other2 = dynamic_cast(&other); @@ -51,6 +53,17 @@ struct IndirectInput : Input return url.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("id", id); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + std::shared_ptr applyOverrides( std::optional ref, std::optional rev) const override @@ -79,7 +92,6 @@ struct IndirectInputScheme : InputScheme auto path = tokenizeString>(url.path, "/"); auto input = std::make_unique(); - input->type = "indirect"; if (path.size() == 1) { } else if (path.size() == 2) { @@ -107,6 +119,17 @@ struct IndirectInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + auto input = std::make_unique(); + input->id = getStrAttr(attrs, "id"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index e012f98fc..f0135d512 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -20,9 +20,9 @@ struct MercurialInput : Input std::optional rev; MercurialInput(const ParsedURL & url) : url(url) - { - type = "hg"; - } + { } + + std::string type() const override { return "hg"; } bool operator ==(const Input & other) const override { @@ -51,6 +51,17 @@ struct MercurialInput : Input return url2.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + std::shared_ptr applyOverrides( std::optional ref, std::optional rev) const override @@ -273,6 +284,16 @@ struct MercurialInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index e82066089..21c785ada 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -11,6 +11,11 @@ struct TarballInput : Input ParsedURL url; std::optional hash; + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + bool operator ==(const Input & other) const override { auto other2 = dynamic_cast(&other); @@ -22,17 +27,32 @@ struct TarballInput : Input bool isImmutable() const override { - return (bool) hash; + return hash || narHash; } std::string to_string() const override { auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. if (narHash) url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI)); return url2.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (narHash) + attrs.emplace("narHash", hash->to_string(SRI)); + else if (hash) + attrs.emplace("hash", hash->to_string(SRI)); + return attrs; + } + std::pair> fetchTreeInternal(nix::ref store) const override { CachedDownloadRequest request(url.to_string()); @@ -72,18 +92,33 @@ struct TarballInputScheme : InputScheme && !hasSuffix(url.path, ".tar.bz2")) return nullptr; - auto input = std::make_unique(); - input->type = "tarball"; - input->url = url; + auto input = std::make_unique(url); + + auto hash = url.query.find("hash"); + if (hash != url.query.end()) + // FIXME: require SRI hash. + input->hash = Hash(hash->second); auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) { + if (narHash != url.query.end()) // FIXME: require SRI hash. input->narHash = Hash(narHash->second); - } return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + // FIXME: require SRI hash. + input->hash = Hash(*hash); + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + input->narHash = Hash(*narHash); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 320590836..0aee67008 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -483,10 +483,11 @@ string base64Decode(const string & s); /* Get a value for the specified key from an associate container, or a default value if the key doesn't exist. */ template -std::optional get(const T & map, const std::string & key) +std::optional get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - return i == map.end() ? std::optional() : i->second; + if (i == map.end()) return {}; + return std::optional(i->second); } diff --git a/tests/flakes.sh b/tests/flakes.sh index 4642a7c2e..570f4f468 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -477,7 +477,7 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix <