lix-project/lix
20
57
Fork 31
Code Issues 349 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Change lock file format to use an attribute representation of flake refs rather than URLs

Browse source
This commit is contained in:
Eelco Dolstra 2020-01-31 19:16:40 +01:00
parent dbefe9e6b8
commit 8414685c0f
12 changed files with 293 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/libexpr/flake/flakeref.cc
View file

@ -14,12 +14,19 @@ const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
#endif
std::string FlakeRef::to_string() const
{
return input->to_string();
}
fetchers::Input::Attrs FlakeRef::toAttrs() const
{
auto attrs = input->toAttrs();
if (subdir != "")
attrs.emplace("subdir", subdir);
return attrs;
}
bool FlakeRef::isDirect() const
{
return input->isDirect();
@ -166,4 +173,13 @@ std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
}
}
FlakeRef FlakeRef::fromAttrs(const fetchers::Input::Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("subdir");
return FlakeRef(
fetchers::inputFromAttrs(attrs2),
fetchers::maybeGetStrAttr(attrs, "subdir").value_or(""));
}
}

7
src/libexpr/flake/flakeref.hh
View file

@ -2,6 +2,7 @@
#include "types.hh"
#include "hash.hh"
#include "fetchers/fetchers.hh"
#include <variant>
@ -9,8 +10,6 @@ namespace nix {
class Store;
namespace fetchers { struct Input; }
typedef std::string FlakeId;
struct FlakeRef
@ -30,6 +29,8 @@ struct FlakeRef
// FIXME: change to operator <<.
std::string to_string() const;
fetchers::Input::Attrs toAttrs() const;
/* Check whether this is a "direct" flake reference, that is, not
a flake ID, which requires a lookup in the flake registry. */
bool isDirect() const;
@ -39,6 +40,8 @@ struct FlakeRef
bool isImmutable() const;
FlakeRef resolve(ref<Store> store) const;
static FlakeRef fromAttrs(const fetchers::Input::Attrs & attrs);
};
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);

54
src/libexpr/flake/lockfile.cc
View file

@ -6,10 +6,48 @@
namespace nix::flake {
FlakeRef flakeRefFromJson(const nlohmann::json & json)
{
fetchers::Input::Attrs attrs;
for (auto & i : json.items()) {
if (i.value().is_number())
attrs.emplace(i.key(), i.value().get<int64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else
throw Error("unsupported input attribute type in lock file");
}
return FlakeRef::fromAttrs(attrs);
}
FlakeRef getFlakeRef(
const nlohmann::json & json,
const char * version3Attr1,
const char * version3Attr2,
const char * version4Attr)
{
auto i = json.find(version4Attr);
if (i != json.end())
return flakeRefFromJson(*i);
// FIXME: remove these.
i = json.find(version3Attr1);
if (i != json.end())
return parseFlakeRef(*i);
i = json.find(version3Attr2);
if (i != json.end())
return parseFlakeRef(*i);
throw Error("attribute '%s' missing in lock file", version4Attr);
}
LockedInput::LockedInput(const nlohmann::json & json)
: LockedInputs(json)
, ref(parseFlakeRef(json.value("url", json.value("uri", ""))))
, originalRef(parseFlakeRef(json.value("originalUrl", json.value("originalUri", ""))))
, ref(getFlakeRef(json, "url", "uri", "resolvedRef"))
, originalRef(getFlakeRef(json, "originalUrl", "originalUri", "originalRef"))
, narHash(Hash((std::string) json["narHash"]))
{
if (!ref.isImmutable())
@ -19,9 +57,9 @@ LockedInput::LockedInput(const nlohmann::json & json)
nlohmann::json LockedInput::toJson() const
{
auto json = LockedInputs::toJson();
json["url"] = ref.to_string();
json["originalUrl"] = originalRef.to_string();
json["narHash"] = narHash.to_string(SRI);
json["originalRef"] = fetchers::attrsToJson(originalRef.toAttrs());
json["resolvedRef"] = fetchers::attrsToJson(ref.toAttrs());
json["narHash"] = narHash.to_string(SRI); // FIXME
return json;
}
@ -91,7 +129,7 @@ void LockedInputs::removeInput(const InputPath & path)
nlohmann::json LockFile::toJson() const
{
auto json = LockedInputs::toJson();
json["version"] = 3;
json["version"] = 4;
return json;
}
@ -101,7 +139,7 @@ LockFile LockFile::read(const Path & path)
auto json = nlohmann::json::parse(readFile(path));
auto version = json.value("version", 0);
if (version != 3)
if (version != 3 && version != 4)
throw Error("lock file '%s' has unsupported version %d", path, version);
return LockFile(json);
@ -111,7 +149,7 @@ LockFile LockFile::read(const Path & path)
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
stream << lockFile.toJson().dump(4); // '4' = indentation in json file
stream << lockFile.toJson().dump(2);
return stream;
}

50
src/libstore/fetchers/fetchers.cc
View file

@ -2,6 +2,8 @@
#include "parse.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
@ -26,6 +28,54 @@ std::unique_ptr<Input> inputFromURL(const std::string & url)
return inputFromURL(parseURL(url));
}
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs);
if (res) return res;
}
throw Error("input '%s' is unsupported", attrsToJson(attrs));
}
nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
} else abort();
}
return json;
}
Input::Attrs Input::toAttrs() const
{
auto attrs = toAttrsInternal();
if (narHash)
attrs.emplace("narHash", narHash->to_string(SRI));
attrs.emplace("type", type());
return attrs;
}
std::optional<std::string> maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string", name);
}
std::string getStrAttr(const Input::Attrs & attrs, const std::string & name)
{
auto s = maybeGetStrAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
{
auto [tree, input] = fetchTreeInternal(store);

23
src/libstore/fetchers/fetchers.hh
View file

@ -5,6 +5,9 @@
#include "path.hh"
#include <memory>
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; }
@ -24,9 +27,10 @@ struct Tree
struct Input : std::enable_shared_from_this<Input>
{
std::string type;
std::optional<Hash> narHash; // FIXME: implement
virtual std::string type() const = 0;
virtual ~Input() { }
virtual bool operator ==(const Input & other) const { return false; }
@ -43,6 +47,11 @@ struct Input : std::enable_shared_from_this<Input>
virtual std::string to_string() const = 0;
typedef std::variant<std::string, int64_t> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs toAttrs() const;
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
virtual std::shared_ptr<const Input> applyOverrides(
@ -59,6 +68,8 @@ struct Input : std::enable_shared_from_this<Input>
private:
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
virtual Attrs toAttrsInternal() const = 0;
};
struct ParsedURL;
@ -68,12 +79,22 @@ struct InputScheme
virtual ~InputScheme() { }
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) = 0;
};
std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
std::unique_ptr<Input> inputFromURL(const std::string & url);
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs);
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
nlohmann::json attrsToJson(const Input::Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name);
std::string getStrAttr(const Input::Attrs & attrs, const std::string & name);
}

27
src/libstore/fetchers/git.cc
View file

@ -74,9 +74,9 @@ struct GitInput : Input
std::optional<Hash> rev;
GitInput(const ParsedURL & url) : url(url)
{
type = "git";
}
{ }
std::string type() const override { return "git"; }
bool operator ==(const Input & other) const override
{
@ -105,6 +105,17 @@ struct GitInput : Input
return url2.to_string();
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
void clone(const Path & destDir) const override
{
auto [isLocal, actualUrl] = getActualUrl();
@ -379,6 +390,16 @@ struct GitInputScheme : InputScheme
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });

27
src/libstore/fetchers/github.cc
View file

@ -19,6 +19,8 @@ struct GitHubInput : Input
std::optional<std::string> ref;
std::optional<Hash> rev;
std::string type() const override { return "github"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const GitHubInput *>(&other);
@ -48,6 +50,18 @@ struct GitHubInput : Input
return s;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("owner", owner);
attrs.emplace("repo", repo);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
void clone(const Path & destDir) const override
{
std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo));
@ -138,7 +152,6 @@ struct GitHubInputScheme : InputScheme
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
auto input = std::make_unique<GitHubInput>();
input->type = "github";
if (path.size() == 2) {
} else if (path.size() == 3) {
@ -176,6 +189,18 @@ struct GitHubInputScheme : InputScheme
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "github") return {};
auto input = std::make_unique<GitHubInput>();
input->owner = getStrAttr(attrs, "owner");
input->repo = getStrAttr(attrs, "repo");
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });

25
src/libstore/fetchers/indirect.cc
View file

@ -12,6 +12,8 @@ struct IndirectInput : Input
std::optional<Hash> rev;
std::optional<std::string> ref;
std::string type() const override { return "indirect"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const IndirectInput *>(&other);
@ -51,6 +53,17 @@ struct IndirectInput : Input
return url.to_string();
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("id", id);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::shared_ptr<const Input> applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const override
@ -79,7 +92,6 @@ struct IndirectInputScheme : InputScheme
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
auto input = std::make_unique<IndirectInput>();
input->type = "indirect";
if (path.size() == 1) {
} else if (path.size() == 2) {
@ -107,6 +119,17 @@ struct IndirectInputScheme : InputScheme
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
auto input = std::make_unique<IndirectInput>();
input->id = getStrAttr(attrs, "id");
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });

27
src/libstore/fetchers/mercurial.cc
View file

@ -20,9 +20,9 @@ struct MercurialInput : Input
std::optional<Hash> rev;
MercurialInput(const ParsedURL & url) : url(url)
{
type = "hg";
}
{ }
std::string type() const override { return "hg"; }
bool operator ==(const Input & other) const override
{
@ -51,6 +51,17 @@ struct MercurialInput : Input
return url2.to_string();
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::shared_ptr<const Input> applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const override
@ -273,6 +284,16 @@ struct MercurialInputScheme : InputScheme
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });

45
src/libstore/fetchers/tarball.cc
View file

@ -11,6 +11,11 @@ struct TarballInput : Input
ParsedURL url;
std::optional<Hash> hash;
TarballInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "tarball"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const TarballInput *>(&other);
@ -22,17 +27,32 @@ struct TarballInput : Input
bool isImmutable() const override
{
return (bool) hash;
return hash || narHash;
}
std::string to_string() const override
{
auto url2(url);
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (narHash)
url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
else if (hash)
url2.query.insert_or_assign("hash", hash->to_string(SRI));
return url2.to_string();
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (narHash)
attrs.emplace("narHash", hash->to_string(SRI));
else if (hash)
attrs.emplace("hash", hash->to_string(SRI));
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
CachedDownloadRequest request(url.to_string());
@ -72,16 +92,31 @@ struct TarballInputScheme : InputScheme
&& !hasSuffix(url.path, ".tar.bz2"))
return nullptr;
auto input = std::make_unique<TarballInput>();
input->type = "tarball";
input->url = url;
auto input = std::make_unique<TarballInput>(url);
auto hash = url.query.find("hash");
if (hash != url.query.end())
// FIXME: require SRI hash.
input->hash = Hash(hash->second);
auto narHash = url.query.find("narHash");
if (narHash != url.query.end()) {
if (narHash != url.query.end())
// FIXME: require SRI hash.
input->narHash = Hash(narHash->second);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
if (auto hash = maybeGetStrAttr(attrs, "hash"))
// FIXME: require SRI hash.
input->hash = Hash(*hash);
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
// FIXME: require SRI hash.
input->narHash = Hash(*narHash);
return input;
}
};

5
src/libutil/util.hh
View file

@ -483,10 +483,11 @@ string base64Decode(const string & s);
/* Get a value for the specified key from an associate container, or a
default value if the key doesn't exist. */
template <class T>
std::optional<std::string> get(const T & map, const std::string & key)
std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
{
auto i = map.find(key);
return i == map.end() ? std::optional<std::string>() : i->second;
if (i == map.end()) return {};
return std::optional<typename T::mapped_type>(i->second);
}

20
tests/flakes.sh
View file

@ -477,7 +477,7 @@ cat > $flake3Dir/flake.nix <<EOF
EOF
nix flake update $flake3Dir
[[ $(jq .inputs.foo.url $flake3Dir/flake.lock) = $(jq .inputs.bar.url $flake3Dir/flake.lock) ]]
[[ $(jq .inputs.foo.resolvedRef $flake3Dir/flake.lock) = $(jq .inputs.bar.resolvedRef $flake3Dir/flake.lock) ]]
cat > $flake3Dir/flake.nix <<EOF
{
@ -491,7 +491,7 @@ cat > $flake3Dir/flake.nix <<EOF
EOF
nix flake update $flake3Dir
[[ $(jq .inputs.bar.url $flake3Dir/flake.lock) =~ flake1 ]]
[[ $(jq .inputs.bar.resolvedRef.url $flake3Dir/flake.lock) =~ flake1 ]]
cat > $flake3Dir/flake.nix <<EOF
{
@ -505,7 +505,7 @@ cat > $flake3Dir/flake.nix <<EOF
EOF
nix flake update $flake3Dir
[[ $(jq .inputs.bar.url $flake3Dir/flake.lock) =~ flake2 ]]
[[ $(jq .inputs.bar.resolvedRef.url $flake3Dir/flake.lock) =~ flake2 ]]
# Test overriding inputs of inputs.
cat > $flake3Dir/flake.nix <<EOF
@ -520,7 +520,7 @@ cat > $flake3Dir/flake.nix <<EOF
EOF
nix flake update $flake3Dir
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake7 ]]
[[ $(jq .inputs.flake2.inputs.flake1.resolvedRef.url $flake3Dir/flake.lock) =~ flake7 ]]
cat > $flake3Dir/flake.nix <<EOF
{
@ -535,7 +535,7 @@ cat > $flake3Dir/flake.nix <<EOF
EOF
nix flake update $flake3Dir --recreate-lock-file
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake7 ]]
[[ $(jq .inputs.flake2.inputs.flake1.resolvedRef.url $flake3Dir/flake.lock) =~ flake7 ]]
# Test Mercurial flakes.
if [[ -z $(type -p hg) ]]; then
@ -603,20 +603,20 @@ nix build -o $TEST_ROOT/result "file://$TEST_ROOT/flake.tar.gz?narHash=sha256-qQ
# Test --override-input.
git -C $flake3Dir reset --hard
nix flake update $flake3Dir --override-input flake2/flake1 flake5
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake5 ]]
[[ $(jq .inputs.flake2.inputs.flake1.resolvedRef.url $flake3Dir/flake.lock) =~ flake5 ]]
nix flake update $flake3Dir --override-input flake2/flake1 flake1
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake1.*rev=$hash2 ]]
[[ $(jq -r .inputs.flake2.inputs.flake1.resolvedRef.rev $flake3Dir/flake.lock) =~ $hash2 ]]
nix flake update $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake1.*rev=$hash1 ]]
[[ $(jq -r .inputs.flake2.inputs.flake1.resolvedRef.rev $flake3Dir/flake.lock) =~ $hash1 ]]
# Test --update-input.
nix flake update $flake3Dir
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake1.*rev=$hash1 ]]
[[ $(jq -r .inputs.flake2.inputs.flake1.resolvedRef.rev $flake3Dir/flake.lock) = $hash1 ]]
nix flake update $flake3Dir --update-input flake2/flake1
[[ $(jq .inputs.flake2.inputs.flake1.url $flake3Dir/flake.lock) =~ flake1.*rev=$hash2 ]]
[[ $(jq -r .inputs.flake2.inputs.flake1.resolvedRef.rev $flake3Dir/flake.lock) =~ $hash2 ]]
# Test 'nix flake list-inputs'.
[[ $(nix flake list-inputs $flake3Dir | wc -l) == 5 ]]