Merge pull request #4372 from tweag/ca/drvoutputs-commands

Add a new Cmd type working on RealisedPaths
This commit is contained in:
Eelco Dolstra 2021-02-05 13:03:50 +01:00 committed by GitHub
commit d7c27f21ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 189 additions and 36 deletions

View file

@ -54,7 +54,7 @@ void StoreCommand::run()
run(getStore()); run(getStore());
} }
StorePathsCommand::StorePathsCommand(bool recursive) RealisedPathsCommand::RealisedPathsCommand(bool recursive)
: recursive(recursive) : recursive(recursive)
{ {
if (recursive) if (recursive)
@ -81,30 +81,40 @@ StorePathsCommand::StorePathsCommand(bool recursive)
}); });
} }
void StorePathsCommand::run(ref<Store> store) void RealisedPathsCommand::run(ref<Store> store)
{ {
StorePaths storePaths; std::vector<RealisedPath> paths;
if (all) { if (all) {
if (installables.size()) if (installables.size())
throw UsageError("'--all' does not expect arguments"); throw UsageError("'--all' does not expect arguments");
// XXX: Only uses opaque paths, ignores all the realisations
for (auto & p : store->queryAllValidPaths()) for (auto & p : store->queryAllValidPaths())
storePaths.push_back(p); paths.push_back(p);
} } else {
auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
else {
for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
storePaths.push_back(p);
if (recursive) { if (recursive) {
StorePathSet closure; auto roots = std::move(pathSet);
store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false); pathSet = {};
storePaths.clear(); RealisedPath::closure(*store, roots, pathSet);
for (auto & p : closure)
storePaths.push_back(p);
} }
for (auto & path : pathSet)
paths.push_back(path);
} }
run(store, std::move(paths));
}
StorePathsCommand::StorePathsCommand(bool recursive)
: RealisedPathsCommand(recursive)
{
}
void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
{
StorePaths storePaths;
for (auto & p : paths)
storePaths.push_back(p.path());
run(store, std::move(storePaths)); run(store, std::move(storePaths));
} }

View file

@ -141,7 +141,7 @@ private:
}; };
/* A command that operates on zero or more store paths. */ /* A command that operates on zero or more store paths. */
struct StorePathsCommand : public InstallablesCommand struct RealisedPathsCommand : public InstallablesCommand
{ {
private: private:
@ -154,17 +154,28 @@ protected:
public: public:
StorePathsCommand(bool recursive = false); RealisedPathsCommand(bool recursive = false);
using StoreCommand::run; using StoreCommand::run;
virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0; virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
void run(ref<Store> store) override; void run(ref<Store> store) override;
bool useDefaultInstallables() override { return !all; } bool useDefaultInstallables() override { return !all; }
}; };
struct StorePathsCommand : public RealisedPathsCommand
{
StorePathsCommand(bool recursive = false);
using RealisedPathsCommand::run;
virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
void run(ref<Store> store, std::vector<RealisedPath> paths) override;
};
/* A command that operates on exactly one store path. */ /* A command that operates on exactly one store path. */
struct StorePathCommand : public InstallablesCommand struct StorePathCommand : public InstallablesCommand
{ {
@ -218,6 +229,12 @@ std::set<StorePath> toDerivations(ref<Store> store,
std::vector<std::shared_ptr<Installable>> installables, std::vector<std::shared_ptr<Installable>> installables,
bool useDeriver = false); bool useDeriver = false);
std::set<RealisedPath> toRealisedPaths(
ref<Store> store,
Realise mode,
OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables);
/* Helper function to generate args that invoke $EDITOR on /* Helper function to generate args that invoke $EDITOR on
filename:lineno. */ filename:lineno. */
Strings editorFor(const Pos & pos); Strings editorFor(const Pos & pos);

View file

@ -704,23 +704,42 @@ Buildables build(ref<Store> store, Realise mode,
return buildables; return buildables;
} }
StorePathSet toStorePaths(ref<Store> store, std::set<RealisedPath> toRealisedPaths(
Realise mode, OperateOn operateOn, ref<Store> store,
Realise mode,
OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables) std::vector<std::shared_ptr<Installable>> installables)
{ {
StorePathSet outPaths; std::set<RealisedPath> res;
if (operateOn == OperateOn::Output) { if (operateOn == OperateOn::Output) {
for (auto & b : build(store, mode, installables)) for (auto & b : build(store, mode, installables))
std::visit(overloaded { std::visit(overloaded {
[&](BuildableOpaque bo) { [&](BuildableOpaque bo) {
outPaths.insert(bo.path); res.insert(bo.path);
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
auto drv = store->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*store, drv);
for (auto & output : bfd.outputs) { for (auto & output : bfd.outputs) {
if (!output.second) if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
throw Error("Cannot operate on output of unbuilt CA drv"); if (!outputHashes.count(output.first))
outPaths.insert(*output.second); throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath),
output.first);
auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
auto realisation = store->queryRealisation(outputId);
if (!realisation)
throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
res.insert(RealisedPath{*realisation});
}
else {
// If ca-derivations isn't enabled, behave as if
// all the paths are opaque to keep the default
// behavior
assert(output.second);
res.insert(*output.second);
}
} }
}, },
}, b); }, b);
@ -731,9 +750,19 @@ StorePathSet toStorePaths(ref<Store> store,
for (auto & i : installables) for (auto & i : installables)
for (auto & b : i->toBuildables()) for (auto & b : i->toBuildables())
if (auto bfd = std::get_if<BuildableFromDrv>(&b)) if (auto bfd = std::get_if<BuildableFromDrv>(&b))
outPaths.insert(bfd->drvPath); res.insert(bfd->drvPath);
} }
return res;
}
StorePathSet toStorePaths(ref<Store> store,
Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
{
StorePathSet outPaths;
for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
outPaths.insert(path.path());
return outPaths; return outPaths;
} }

View file

@ -46,4 +46,35 @@ Realisation Realisation::fromJSON(
}; };
} }
StorePath RealisedPath::path() const {
return std::visit([](auto && arg) { return arg.getPath(); }, raw);
}
void RealisedPath::closure(
Store& store,
const RealisedPath::Set& startPaths,
RealisedPath::Set& ret)
{
// FIXME: This only builds the store-path closure, not the real realisation
// closure
StorePathSet initialStorePaths, pathsClosure;
for (auto& path : startPaths)
initialStorePaths.insert(path.path());
store.computeFSClosure(initialStorePaths, pathsClosure);
ret.insert(startPaths.begin(), startPaths.end());
ret.insert(pathsClosure.begin(), pathsClosure.end());
}
void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const
{
RealisedPath::closure(store, {*this}, ret);
}
RealisedPath::Set RealisedPath::closure(Store& store) const
{
RealisedPath::Set ret;
closure(store, ret);
return ret;
}
} // namespace nix } // namespace nix

View file

@ -2,6 +2,7 @@
#include "path.hh" #include "path.hh"
#include <nlohmann/json_fwd.hpp> #include <nlohmann/json_fwd.hpp>
#include "comparator.hh"
namespace nix { namespace nix {
@ -17,13 +18,7 @@ struct DrvOutput {
static DrvOutput parse(const std::string &); static DrvOutput parse(const std::string &);
bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } GENERATE_CMP(DrvOutput, me->drvHash, me->outputName);
bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
private:
// Just to make comparison operators easier to write
std::pair<Hash, std::string> to_pair() const
{ return std::make_pair(drvHash, outputName); }
}; };
struct Realisation { struct Realisation {
@ -32,8 +27,47 @@ struct Realisation {
nlohmann::json toJSON() const; nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
StorePath getPath() const { return outPath; }
GENERATE_CMP(Realisation, me->id, me->outPath);
}; };
typedef std::map<DrvOutput, Realisation> DrvOutputs; struct OpaquePath {
StorePath path;
StorePath getPath() const { return path; }
GENERATE_CMP(OpaquePath, me->path);
};
/**
* A store path with all the history of how it went into the store
*/
struct RealisedPath {
/*
* A path is either the result of the realisation of a derivation or
* an opaque blob that has been directly added to the store
*/
using Raw = std::variant<Realisation, OpaquePath>;
Raw raw;
using Set = std::set<RealisedPath>;
RealisedPath(StorePath path) : raw(OpaquePath{path}) {}
RealisedPath(Realisation r) : raw(r) {}
/**
* Get the raw store path associated to this
*/
StorePath path() const;
void closure(Store& store, Set& ret) const;
static void closure(Store& store, const Set& startPaths, Set& ret);
Set closure(Store& store) const;
GENERATE_CMP(RealisedPath, me->raw);
};
} }

30
src/libutil/comparator.hh Normal file
View file

@ -0,0 +1,30 @@
#pragma once
/* Awfull hacky generation of the comparison operators by doing a lexicographic
* comparison between the choosen fields.
*
* ```
* GENERATE_CMP(ClassName, me->field1, me->field2, ...)
* ```
*
* will generate comparison operators semantically equivalent to:
*
* ```
* bool operator<(const ClassName& other) {
* return field1 < other.field1 && field2 < other.field2 && ...;
* }
* ```
*/
#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
bool operator COMPARATOR(const MY_TYPE& other) const { \
const MY_TYPE* me = this; \
auto fields1 = std::make_tuple( FIELDS ); \
me = &other; \
auto fields2 = std::make_tuple( FIELDS ); \
return fields1 COMPARATOR fields2; \
}
#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
#define GENERATE_CMP(args...) \
GENERATE_EQUAL(args) \
GENERATE_LEQ(args)

View file

@ -16,6 +16,8 @@ struct CmdCopy : StorePathsCommand
SubstituteFlag substitute = NoSubstitute; SubstituteFlag substitute = NoSubstitute;
using StorePathsCommand::run;
CmdCopy() CmdCopy()
: StorePathsCommand(true) : StorePathsCommand(true)
{ {