Merge remote-tracking branch 'upstream/master' into single-ca-drv-build

This commit is contained in:
John Ericson 2020-09-03 15:43:17 +00:00
commit e7d93e7ece
20 changed files with 55 additions and 104 deletions

View file

@ -12,7 +12,6 @@ Title: nix-instantiate
[`--arg` *name* *value*] [`--arg` *name* *value*]
[{`--attr`| `-A`} *attrPath*] [{`--attr`| `-A`} *attrPath*]
[`--add-root` *path*] [`--add-root` *path*]
[`--indirect`]
[`--expr` | `-E`] [`--expr` | `-E`]
*files…* *files…*
@ -32,8 +31,8 @@ standard input.
# Options # Options
- `--add-root` *path*; `--indirect` - `--add-root` *path*
See the [corresponding options](nix-store.md) in `nix-store`. See the [corresponding option](nix-store.md) in `nix-store`.
- `--parse` - `--parse`
Just parse the input files, and print their abstract syntax trees on Just parse the input files, and print their abstract syntax trees on

View file

@ -9,7 +9,6 @@ Title: nix-store
`nix-store` *operation* [*options…*] [*arguments…*] `nix-store` *operation* [*options…*] [*arguments…*]
[`--option` *name* *value*] [`--option` *name* *value*]
[`--add-root` *path*] [`--add-root` *path*]
[`--indirect`]
# Description # Description
@ -28,27 +27,12 @@ have an effect.
- `--add-root` *path* - `--add-root` *path*
Causes the result of a realisation (`--realise` and Causes the result of a realisation (`--realise` and
`--force-realise`) to be registered as a root of the garbage `--force-realise`) to be registered as a root of the garbage
collector. The root is stored in *path*, which must be inside a collector. *path* will be created as a symlink to the resulting
directory that is scanned for roots by the garbage collector store path. In addition, a uniquely named symlink to *path* will
(i.e., typically in a subdirectory of `/nix/var/nix/gcroots/`) be created in `/nix/var/nix/gcroots/auto/`. For instance,
*unless* the `--indirect` flag is used.
If there are multiple results, then multiple symlinks will be
created by sequentially numbering symlinks beyond the first one
(e.g., `foo`, `foo-2`, `foo-3`, and so on).
- `--indirect`
In conjunction with `--add-root`, this option allows roots to be
stored *outside* of the GC roots directory. This is useful for
commands such as `nix-build` that place a symlink to the build
result in the current directory; such a build result should not be
garbage-collected unless the symlink is removed.
The `--indirect` flag causes a uniquely named symlink to *path* to
be stored in `/nix/var/nix/gcroots/auto/`. For instance,
```console ```console
$ nix-store --add-root /home/eelco/bla/result --indirect -r ... $ nix-store --add-root /home/eelco/bla/result -r ...
$ ls -l /nix/var/nix/gcroots/auto $ ls -l /nix/var/nix/gcroots/auto
lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
@ -63,11 +47,13 @@ have an effect.
> **Warning** > **Warning**
> >
> Note that it is not possible to move or rename indirect GC roots, > Note that it is not possible to move or rename GC roots, since
> since the symlink in the `auto` directory will still point to the > the symlink in the `auto` directory will still point to the old
> old location. > location.
<!-- end list --> If there are multiple results, then multiple symlinks will be
created by sequentially numbering symlinks beyond the first one
(e.g., `foo`, `foo-2`, `foo-3`, and so on).
# Operation `--realise` # Operation `--realise`

View file

@ -391,7 +391,8 @@ Value & AttrCursor::forceValue()
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) { if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type == tString) if (v.type == tString)
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s}; cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
string_t{v.string.s, {}}};
else if (v.type == tPath) else if (v.type == tPath)
cachedValue = {root->db->setString(getKey(), v.path), v.path}; cachedValue = {root->db->setString(getKey(), v.path), v.path};
else if (v.type == tBool) else if (v.type == tBool)

View file

@ -147,7 +147,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
if (!hasPrefix(path, "/")) { if (!hasPrefix(path, "/")) {
auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
path = store->toRealPath(storePath); path = store->toRealPath(storePath);
} }

View file

@ -85,8 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
} }
Path LocalFSStore::addPermRoot(const StorePath & storePath, Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
{ {
Path gcRoot(canonPath(_gcRoot)); Path gcRoot(canonPath(_gcRoot));
@ -95,47 +94,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
"creating a garbage collector root (%1%) in the Nix store is forbidden " "creating a garbage collector root (%1%) in the Nix store is forbidden "
"(are you running nix-build inside the store?)", gcRoot); "(are you running nix-build inside the store?)", gcRoot);
if (indirect) {
/* Don't clobber the link if it already exists and doesn't /* Don't clobber the link if it already exists and doesn't
point to the Nix store. */ point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
throw Error("cannot create symlink '%1%'; already exists", gcRoot); throw Error("cannot create symlink '%1%'; already exists", gcRoot);
makeSymlink(gcRoot, printStorePath(storePath)); makeSymlink(gcRoot, printStorePath(storePath));
addIndirectRoot(gcRoot); addIndirectRoot(gcRoot);
}
else {
if (!allowOutsideRootsDir) {
Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
throw Error(
"path '%1%' is not a valid garbage collector root; "
"it's not in the directory '%2%'",
gcRoot, rootsDir);
}
if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
writeFile(gcRoot, "");
else
makeSymlink(gcRoot, printStorePath(storePath));
}
/* Check that the root can be found by the garbage collector.
!!! This can be very slow on machines that have many roots.
Instead of reading all the roots, it would be more efficient to
check if the root is in a directory in or linked from the
gcroots directory. */
if (settings.checkRootReachability) {
auto roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0)
logWarning({
.name = "GC root",
.hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector",
gcRoot, printStorePath(storePath))
});
}
/* Grab the global GC root, causing us to block while a GC is in /* Grab the global GC root, causing us to block while a GC is in
progress. This prevents the set of permanent roots from progress. This prevents the set of permanent roots from

View file

@ -293,10 +293,6 @@ public:
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
"How often (in seconds) to poll for locks."}; "How often (in seconds) to poll for locks."};
Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
"Whether to check if new GC roots can in fact be found by the "
"garbage collector."};
Setting<bool> gcKeepOutputs{ Setting<bool> gcKeepOutputs{
this, false, "keep-outputs", this, false, "keep-outputs",
R"( R"(

View file

@ -85,7 +85,7 @@ protected:
checkEnabled(); checkEnabled();
try { try {
FileTransferRequest request(cacheUri + "/" + path); FileTransferRequest request(makeRequest(path));
request.head = true; request.head = true;
getFileTransfer()->download(request); getFileTransfer()->download(request);
return true; return true;
@ -103,7 +103,7 @@ protected:
std::shared_ptr<std::basic_iostream<char>> istream, std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override const std::string & mimeType) override
{ {
auto req = FileTransferRequest(cacheUri + "/" + path); auto req = makeRequest(path);
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain()); req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.mimeType = mimeType; req.mimeType = mimeType;
try { try {
@ -115,8 +115,11 @@ protected:
FileTransferRequest makeRequest(const std::string & path) FileTransferRequest makeRequest(const std::string & path)
{ {
FileTransferRequest request(cacheUri + "/" + path); return FileTransferRequest(
return request; hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
? path
: cacheUri + "/" + path);
} }
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string & path, Sink & sink) override

View file

@ -72,7 +72,7 @@ static void makeName(const Path & profile, GenerationNumber num,
} }
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
{ {
/* The new generation number should be higher than old the /* The new generation number should be higher than old the
previous ones. */ previous ones. */
@ -82,7 +82,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
if (gens.size() > 0) { if (gens.size() > 0) {
Generation last = gens.back(); Generation last = gens.back();
if (readLink(last.path) == outPath) { if (readLink(last.path) == store->printStorePath(outPath)) {
/* We only create a new generation symlink if it differs /* We only create a new generation symlink if it differs
from the last one. from the last one.
@ -105,7 +105,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
user environment etc. we've just built. */ user environment etc. we've just built. */
Path generation; Path generation;
makeName(profile, num + 1, generation); makeName(profile, num + 1, generation);
store->addPermRoot(store->parseStorePath(outPath), generation, false, true); store->addPermRoot(outPath, generation);
return generation; return generation;
} }

View file

@ -8,6 +8,8 @@
namespace nix { namespace nix {
class StorePath;
typedef unsigned int GenerationNumber; typedef unsigned int GenerationNumber;
@ -28,7 +30,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
class LocalFSStore; class LocalFSStore;
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath); Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
void deleteGeneration(const Path & profile, GenerationNumber gen); void deleteGeneration(const Path & profile, GenerationNumber gen);

View file

@ -651,8 +651,7 @@ public:
ref<FSAccessor> getFSAccessor() override; ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */ /* Register a permanent GC root. */
Path addPermRoot(const StorePath & storePath, Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
virtual Path getRealStoreDir() { return storeDir; } virtual Path getRealStoreDir() { return storeDir; }

View file

@ -526,7 +526,7 @@ static void _main(int argc, char * * argv)
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
std::string symlink = drvPrefix; std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName; if (outputName != "out") symlink += "-" + outputName;
store2->addPermRoot(outputPath, absPath(symlink), true); store2->addPermRoot(outputPath, absPath(symlink));
} }
outPaths.push_back(outputPath); outPaths.push_back(outputPath);

View file

@ -708,7 +708,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
} }
debug(format("switching to new user environment")); debug(format("switching to new user environment"));
Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, drv.queryOutPath()); Path generation = createGeneration(
ref<LocalFSStore>(store2), globals.profile,
store2->parseStorePath(drv.queryOutPath()));
switchLink(globals.profile, generation); switchLink(globals.profile, generation);
} }

View file

@ -151,7 +151,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
} }
debug(format("switching to new user environment")); debug(format("switching to new user environment"));
Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); Path generation = createGeneration(ref<LocalFSStore>(store2), profile,
store2->parseStorePath(topLevelOut));
switchLink(profile, generation); switchLink(profile, generation);
} }

View file

@ -20,7 +20,6 @@ using namespace nix;
static Path gcRoot; static Path gcRoot;
static int rootNr = 0; static int rootNr = 0;
static bool indirectRoot = false;
enum OutputKind { okPlain, okXML, okJSON }; enum OutputKind { okPlain, okXML, okJSON };
@ -71,11 +70,11 @@ void processExpr(EvalState & state, const Strings & attrPaths,
if (gcRoot == "") if (gcRoot == "")
printGCWarning(); printGCWarning();
else { else {
Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; Path rootName = absPath(gcRoot);
if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); if (++rootNr > 1) rootName += "-" + std::to_string(rootNr);
auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
if (store2) if (store2)
drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot); drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName);
} }
std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : "")); std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : ""));
} }
@ -127,7 +126,7 @@ static int _main(int argc, char * * argv)
else if (*arg == "--add-root") else if (*arg == "--add-root")
gcRoot = getArg(*arg, arg, end); gcRoot = getArg(*arg, arg, end);
else if (*arg == "--indirect") else if (*arg == "--indirect")
indirectRoot = true; ;
else if (*arg == "--xml") else if (*arg == "--xml")
outputKind = okXML; outputKind = okXML;
else if (*arg == "--json") else if (*arg == "--json")

View file

@ -34,7 +34,6 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs);
static Path gcRoot; static Path gcRoot;
static int rootNr = 0; static int rootNr = 0;
static bool indirectRoot = false;
static bool noOutput = false; static bool noOutput = false;
static std::shared_ptr<Store> store; static std::shared_ptr<Store> store;
@ -87,7 +86,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
Path rootName = gcRoot; Path rootName = gcRoot;
if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
if (i->first != "out") rootName += "-" + i->first; if (i->first != "out") rootName += "-" + i->first;
retPath = store2->addPermRoot(outPath, rootName, indirectRoot); retPath = store2->addPermRoot(outPath, rootName);
} }
} }
outputs.insert(retPath); outputs.insert(retPath);
@ -106,7 +105,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
Path rootName = gcRoot; Path rootName = gcRoot;
rootNr++; rootNr++;
if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
return {store2->addPermRoot(path.path, rootName, indirectRoot)}; return {store2->addPermRoot(path.path, rootName)};
} }
} }
return {store->printStorePath(path.path)}; return {store->printStorePath(path.path)};
@ -1090,7 +1089,7 @@ static int _main(int argc, char * * argv)
else if (*arg == "--add-root") else if (*arg == "--add-root")
gcRoot = absPath(getArg(*arg, arg, end)); gcRoot = absPath(getArg(*arg, arg, end));
else if (*arg == "--indirect") else if (*arg == "--indirect")
indirectRoot = true; ;
else if (*arg == "--no-output") else if (*arg == "--no-output")
noOutput = true; noOutput = true;
else if (*arg != "" && arg->at(0) == '-') { else if (*arg != "" && arg->at(0) == '-') {

View file

@ -71,7 +71,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
[&](BuildableOpaque bo) { [&](BuildableOpaque bo) {
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
store2->addPermRoot(bo.path, absPath(symlink), true); store2->addPermRoot(bo.path, absPath(symlink));
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
@ -79,7 +79,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first); if (output.first != "out") symlink += fmt("-%s", output.first);
store2->addPermRoot(output.second, absPath(symlink), true); store2->addPermRoot(output.second, absPath(symlink));
} }
}, },
}, buildables[i]); }, buildables[i]);

View file

@ -122,7 +122,7 @@ struct CmdBundle : InstallableCommand
if (!outLink) if (!outLink)
outLink = baseNameOf(app.program); outLink = baseNameOf(app.program);
store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink), true); store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink));
} }
}; };

View file

@ -134,7 +134,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
switchLink(profile2, switchLink(profile2,
createGeneration( createGeneration(
ref<LocalFSStore>(store), ref<LocalFSStore>(store),
profile2, store->printStorePath(storePath))); profile2, storePath));
} }
void MixProfile::updateProfile(const Buildables & buildables) void MixProfile::updateProfile(const Buildables & buildables)

View file

@ -24,5 +24,5 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link)
outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link) outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
[[ $outPath = $outPath2 ]] [[ $outPath = $outPath2 ]]
outPath2=$(nix-store -r $(nix-instantiate --indirect --add-root $TEST_ROOT/indirect dependencies.nix)!out) outPath2=$(nix-store -r $(nix-instantiate --add-root $TEST_ROOT/indirect dependencies.nix)!out)
[[ $outPath = $outPath2 ]] [[ $outPath = $outPath2 ]]

View file

@ -27,12 +27,12 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run
# Test nix-shell on a .drv symlink # Test nix-shell on a .drv symlink
# Legacy: absolute path and .drv extension required # Legacy: absolute path and .drv extension required
nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell.drv
[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ [[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \
'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
# New behaviour: just needs to resolve to a derivation in the store # New behaviour: just needs to resolve to a derivation in the store
nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell
[[ $(nix-shell --pure $TEST_ROOT/shell --run \ [[ $(nix-shell --pure $TEST_ROOT/shell --run \
'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]