Merge remote-tracking branch 'obsidian/master' into binary-cache-addToStoreFromDump

This commit is contained in:
John Ericson 2020-09-26 00:21:45 +00:00
commit c40c832f19
13 changed files with 84 additions and 129 deletions

View file

@ -58,6 +58,7 @@
configureFlags = configureFlags =
lib.optionals stdenv.isLinux [ lib.optionals stdenv.isLinux [
"--with-sandbox-shell=${sh}/bin/busybox" "--with-sandbox-shell=${sh}/bin/busybox"
"LDFLAGS=-fuse-ld=gold"
]; ];
buildDeps = buildDeps =

View file

@ -215,10 +215,9 @@ static Flake getFlake(
if (auto outputs = vInfo.attrs->get(sOutputs)) { if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, tLambda, *outputs->value, *outputs->pos); expectType(state, tLambda, *outputs->value, *outputs->pos);
flake.vOutputs = allocRootValue(outputs->value);
if ((*flake.vOutputs)->lambda.fun->matchAttrs) { if (outputs->value->lambda.fun->matchAttrs) {
for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) { for (auto & formal : outputs->value->lambda.fun->formals->formals) {
if (formal.name != state.sSelf) if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput { flake.inputs.emplace(formal.name, FlakeInput {
.ref = parseFlakeRef(formal.name) .ref = parseFlakeRef(formal.name)
@ -367,7 +366,7 @@ LockedFlake lockFlake(
/* If we have an --update-input flag for an input /* If we have an --update-input flag for an input
of this input, then we must fetch the flake to of this input, then we must fetch the flake to
to update it. */ update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto hasChildUpdate = auto hasChildUpdate =

View file

@ -34,7 +34,6 @@ struct Flake
std::optional<std::string> description; std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo; std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs; FlakeInputs inputs;
RootValue vOutputs;
~Flake(); ~Flake();
}; };

View file

@ -614,8 +614,7 @@ Path resolveExprPath(Path path)
// Basic cycle/depth limit to avoid infinite loops. // Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow) if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path); throw Error("too many symbolic links encountered while traversing the path '%s'", path);
if (lstat(path.c_str(), &st)) st = lstat(path);
throw SysError("getting status of '%s'", path);
if (!S_ISLNK(st.st_mode)) break; if (!S_ISLNK(st.st_mode)) break;
path = absPath(readLink(path), dirOf(path)); path = absPath(readLink(path), dirOf(path));
} }

View file

@ -1675,7 +1675,34 @@ void DerivationGoal::tryLocalBuild() {
} }
void replaceValidPath(const Path & storePath, const Path tmpPath) static void chmod_(const Path & path, mode_t mode)
{
if (chmod(path.c_str(), mode) == -1)
throw SysError("setting permissions on '%s'", path);
}
/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
it's a directory and we're not root (to be able to update the
directory's parent link ".."). */
static void movePath(const Path & src, const Path & dst)
{
auto st = lstat(src);
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
if (rename(src.c_str(), dst.c_str()))
throw SysError("renaming '%1%' to '%2%'", src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
}
void replaceValidPath(const Path & storePath, const Path & tmpPath)
{ {
/* We can't atomically replace storePath (the original) with /* We can't atomically replace storePath (the original) with
tmpPath (the replacement), so we have to move it out of the tmpPath (the replacement), so we have to move it out of the
@ -1683,11 +1710,20 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
we're repairing (say) Glibc, we end up with a broken system. */ we're repairing (say) Glibc, we end up with a broken system. */
Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
if (pathExists(storePath)) if (pathExists(storePath))
rename(storePath.c_str(), oldPath.c_str()); movePath(storePath, oldPath);
if (rename(tmpPath.c_str(), storePath.c_str()) == -1) {
rename(oldPath.c_str(), storePath.c_str()); // attempt to recover try {
throw SysError("moving '%s' to '%s'", tmpPath, storePath); movePath(tmpPath, storePath);
} catch (...) {
try {
// attempt to recover
movePath(oldPath, storePath);
} catch (...) {
ignoreException();
} }
throw;
}
deletePath(oldPath); deletePath(oldPath);
} }
@ -2005,13 +2041,6 @@ HookReply DerivationGoal::tryBuildHook()
} }
static void chmod_(const Path & path, mode_t mode)
{
if (chmod(path.c_str(), mode) == -1)
throw SysError("setting permissions on '%s'", path);
}
int childEntry(void * arg) int childEntry(void * arg)
{ {
((DerivationGoal *) arg)->runChild(); ((DerivationGoal *) arg)->runChild();
@ -2367,10 +2396,7 @@ void DerivationGoal::startBuilder()
for (auto & i : inputPaths) { for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i); auto p = worker.store.printStorePath(i);
Path r = worker.store.toRealPath(p); Path r = worker.store.toRealPath(p);
struct stat st; if (S_ISDIR(lstat(r).st_mode))
if (lstat(r.c_str(), &st))
throw SysError("getting attributes of path '%s'", p);
if (S_ISDIR(st.st_mode))
dirsInChroot.insert_or_assign(p, r); dirsInChroot.insert_or_assign(p, r);
else else
linkOrCopy(r, chrootRootDir + p); linkOrCopy(r, chrootRootDir + p);
@ -3144,9 +3170,7 @@ void DerivationGoal::addDependency(const StorePath & path)
if (pathExists(target)) if (pathExists(target))
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
struct stat st; auto st = lstat(source);
if (lstat(source.c_str(), &st))
throw SysError("getting attributes of path '%s'", source);
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
@ -3735,29 +3759,6 @@ void DerivationGoal::runChild()
} }
static void moveCheckToStore(const Path & src, const Path & dst)
{
/* For the rename of directory to succeed, we must be running as root or
the directory must be made temporarily writable (to update the
directory's parent link ".."). */
struct stat st;
if (lstat(src.c_str(), &st) == -1) {
throw SysError("getting attributes of path '%1%'", src);
}
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
if (rename(src.c_str(), dst.c_str()))
throw SysError("renaming '%1%' to '%2%'", src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
}
void DerivationGoal::registerOutputs() void DerivationGoal::registerOutputs()
{ {
/* When using a build hook, the build hook can register the output /* When using a build hook, the build hook can register the output
@ -3858,7 +3859,7 @@ void DerivationGoal::registerOutputs()
something like that. */ something like that. */
canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath); debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
/* Pass blank Sink as we are not ready to hash data at this stage. */ /* Pass blank Sink as we are not ready to hash data at this stage. */
NullSink blank; NullSink blank;
@ -3913,7 +3914,6 @@ void DerivationGoal::registerOutputs()
outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
}; };
bool rewritten = false;
std::optional<StorePathSet> referencesOpt = std::visit(overloaded { std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
[&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> { [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
finish(skippedFinalPath.path); finish(skippedFinalPath.path);
@ -3944,7 +3944,9 @@ void DerivationGoal::registerOutputs()
StringSource source(*sink.s); StringSource source(*sink.s);
restorePath(actualPath, source); restorePath(actualPath, source);
rewritten = true; /* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, -1, inodesSeen);
} }
}; };
@ -4027,7 +4029,7 @@ void DerivationGoal::registerOutputs()
[&](DerivationOutputInputAddressed output) { [&](DerivationOutputInputAddressed output) {
/* input-addressed case */ /* input-addressed case */
auto requiredFinalPath = output.path; auto requiredFinalPath = output.path;
/* Preemtively add rewrite rule for final hash, as that is /* Preemptively add rewrite rule for final hash, as that is
what the NAR hash will use rather than normalized-self references */ what the NAR hash will use rather than normalized-self references */
if (scratchPath != requiredFinalPath) if (scratchPath != requiredFinalPath)
outputRewrites.insert_or_assign( outputRewrites.insert_or_assign(
@ -4101,44 +4103,21 @@ void DerivationGoal::registerOutputs()
else. No moving needed. */ else. No moving needed. */
assert(newInfo.ca); assert(newInfo.ca);
} else { } else {
/* Temporarily add write perm so we can move, will be fixed auto destPath = worker.store.toRealPath(finalDestPath);
later. */ movePath(actualPath, destPath);
{ actualPath = destPath;
struct stat st;
auto & mode = st.st_mode;
if (lstat(actualPath.c_str(), &st))
throw SysError("getting attributes of path '%1%'", actualPath);
mode |= 0200;
/* Try to change the perms, but only if the file isn't a
symlink as symlinks permissions are mostly ignored and
calling `chmod` on it will just forward the call to the
target of the link. */
if (!S_ISLNK(st.st_mode))
if (chmod(actualPath.c_str(), mode) == -1)
throw SysError("changing mode of '%1%' to %2$o", actualPath, mode);
}
if (rename(
actualPath.c_str(),
worker.store.toRealPath(finalDestPath).c_str()) == -1)
throw SysError("moving build output '%1%' from it's temporary location to the Nix store", finalDestPath);
actualPath = worker.store.toRealPath(finalDestPath);
} }
} }
/* Get rid of all weird permissions. This also checks that
all files are owned by the build user, if applicable. */
canonicalisePathMetaData(actualPath,
buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen);
if (buildMode == bmCheck) { if (buildMode == bmCheck) {
if (!worker.store.isValidPath(newInfo.path)) continue; if (!worker.store.isValidPath(newInfo.path)) continue;
ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
if (newInfo.narHash != oldInfo.narHash) { if (newInfo.narHash != oldInfo.narHash) {
worker.checkMismatch = true; worker.checkMismatch = true;
if (settings.runDiffHook || settings.keepFailed) { if (settings.runDiffHook || settings.keepFailed) {
Path dst = worker.store.toRealPath(finalDestPath + checkSuffix); auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
deletePath(dst); deletePath(dst);
moveCheckToStore(actualPath, dst); movePath(actualPath, dst);
handleDiffHook( handleDiffHook(
buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getUID() : getuid(),

View file

@ -663,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
if (name == "." || name == "..") continue; if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name; Path path = linksDir + "/" + name;
struct stat st; auto st = lstat(path);
if (lstat(path.c_str(), &st) == -1)
throw SysError("statting '%1%'", path);
if (st.st_nlink != 1) { if (st.st_nlink != 1) {
actualSize += st.st_size; actualSize += st.st_size;

View file

@ -114,8 +114,7 @@ LocalStore::LocalStore(const Params & params)
Path path = realStoreDir; Path path = realStoreDir;
struct stat st; struct stat st;
while (path != "/") { while (path != "/") {
if (lstat(path.c_str(), &st)) st = lstat(path);
throw SysError("getting status of '%1%'", path);
if (S_ISLNK(st.st_mode)) if (S_ISLNK(st.st_mode))
throw Error( throw Error(
"the path '%1%' is a symlink; " "the path '%1%' is a symlink; "
@ -419,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
void canonicaliseTimestampAndPermissions(const Path & path) void canonicaliseTimestampAndPermissions(const Path & path)
{ {
struct stat st; canonicaliseTimestampAndPermissions(path, lstat(path));
if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path '%1%'", path);
canonicaliseTimestampAndPermissions(path, st);
} }
@ -440,9 +436,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
} }
#endif #endif
struct stat st; auto st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path '%1%'", path);
/* Really make sure that the path is of a supported type. */ /* Really make sure that the path is of a supported type. */
if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
@ -478,8 +472,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
ensure that we don't fail on hard links within the same build ensure that we don't fail on hard links within the same build
(i.e. "touch $out/foo; ln $out/foo $out/bar"). */ (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
assert(!S_ISDIR(st.st_mode)); if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
throw BuildError("invalid ownership on file '%1%'", path); throw BuildError("invalid ownership on file '%1%'", path);
mode_t mode = st.st_mode & ~S_IFMT; mode_t mode = st.st_mode & ~S_IFMT;
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
@ -522,9 +515,7 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
/* On platforms that don't have lchown(), the top-level path can't /* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */ be a symlink, since we can't change its ownership. */
struct stat st; auto st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path '%1%'", path);
if (st.st_uid != geteuid()) { if (st.st_uid != geteuid()) {
assert(S_ISLNK(st.st_mode)); assert(S_ISLNK(st.st_mode));
@ -1455,7 +1446,7 @@ static void makeMutable(const Path & path)
{ {
checkInterrupt(); checkInterrupt();
struct stat st = lstat(path); auto st = lstat(path);
if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;

View file

@ -17,9 +17,7 @@ namespace nix {
static void makeWritable(const Path & path) static void makeWritable(const Path & path)
{ {
struct stat st; auto st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path '%1%'", path);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError("changing writability of '%1%'", path); throw SysError("changing writability of '%1%'", path);
} }
@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
{ {
checkInterrupt(); checkInterrupt();
struct stat st; auto st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path '%1%'", path);
#if __APPLE__ #if __APPLE__
/* HFS/macOS has some undocumented security feature disabling hardlinking for /* HFS/macOS has some undocumented security feature disabling hardlinking for
@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Yes! We've seen a file with the same contents. Replace the /* Yes! We've seen a file with the same contents. Replace the
current file with a hard link to that file. */ current file with a hard link to that file. */
struct stat stLink; auto stLink = lstat(linkPath);
if (lstat(linkPath.c_str(), &stLink))
throw SysError("getting attributes of path '%1%'", linkPath);
if (st.st_ino == stLink.st_ino) { if (st.st_ino == stLink.st_ino) {
debug(format("'%1%' is already linked to '%2%'") % path % linkPath); debug(format("'%1%' is already linked to '%2%'") % path % linkPath);

View file

@ -39,13 +39,10 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
for (auto & i : readDirectory(profileDir)) { for (auto & i : readDirectory(profileDir)) {
if (auto n = parseName(profileName, i.name)) { if (auto n = parseName(profileName, i.name)) {
auto path = profileDir + "/" + i.name; auto path = profileDir + "/" + i.name;
struct stat st;
if (lstat(path.c_str(), &st) != 0)
throw SysError("statting '%1%'", path);
gens.push_back({ gens.push_back({
.number = *n, .number = *n,
.path = path, .path = path,
.creationTime = st.st_mtime .creationTime = lstat(path).st_mtime
}); });
} }
} }

View file

@ -27,6 +27,8 @@ struct ArchiveSettings : Config
#endif #endif
"use-case-hack", "use-case-hack",
"Whether to enable a Darwin-specific hack for dealing with file name collisions."}; "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
Setting<bool> preallocateContents{this, true, "preallocate-contents",
"Whether to preallocate files when writing objects with known size."};
}; };
static ArchiveSettings archiveSettings; static ArchiveSettings archiveSettings;
@ -66,9 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
{ {
checkInterrupt(); checkInterrupt();
struct stat st; auto st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path '%1%'", path);
sink << "("; sink << "(";
@ -325,6 +325,9 @@ struct RestoreSink : ParseSink
void preallocateContents(uint64_t len) void preallocateContents(uint64_t len)
{ {
if (!archiveSettings.preallocateContents)
return;
#if HAVE_POSIX_FALLOCATE #if HAVE_POSIX_FALLOCATE
if (len) { if (len) {
errno = posix_fallocate(fd.get(), 0, len); errno = posix_fallocate(fd.get(), 0, len);

View file

@ -111,11 +111,7 @@ std::set<std::string> runResolver(const Path & filename)
bool isSymlink(const Path & path) bool isSymlink(const Path & path)
{ {
struct stat st; return S_ISLNK(lstat(path).st_mode);
if (lstat(path.c_str(), &st) == -1)
throw SysError("getting attributes of path '%1%'", path);
return S_ISLNK(st.st_mode);
} }
Path resolveSymlink(const Path & path) Path resolveSymlink(const Path & path)

View file

@ -13,14 +13,14 @@ hash=$(nix-hash $path2)
chmod u+w $path2 chmod u+w $path2
touch $path2/bad touch $path2/bad
if nix-store --verify --check-contents -v; then (! nix-store --verify --check-contents -v)
echo "nix-store --verify succeeded unexpectedly" >&2
exit 1
fi
# The path can be repaired by rebuilding the derivation. # The path can be repaired by rebuilding the derivation.
nix-store --verify --check-contents --repair nix-store --verify --check-contents --repair
(! [ -e $path2/bad ])
(! [ -w $path2 ])
nix-store --verify-path $path2 nix-store --verify-path $path2
# Re-corrupt and delete the deriver. Now --verify --repair should # Re-corrupt and delete the deriver. Now --verify --repair should
@ -30,10 +30,7 @@ touch $path2/bad
nix-store --delete $(nix-store -qd $path2) nix-store --delete $(nix-store -qd $path2)
if nix-store --verify --check-contents --repair; then (! nix-store --verify --check-contents --repair)
echo "nix-store --verify --repair succeeded unexpectedly" >&2
exit 1
fi
nix-build dependencies.nix -o $TEST_ROOT/result --repair nix-build dependencies.nix -o $TEST_ROOT/result --repair

View file

@ -10,13 +10,15 @@ outPath=$(nix-store -rvv "$drvPath")
echo "output path is $outPath" echo "output path is $outPath"
(! [ -w $outPath ])
text=$(cat "$outPath"/hello) text=$(cat "$outPath"/hello)
if test "$text" != "Hello World!"; then exit 1; fi if test "$text" != "Hello World!"; then exit 1; fi
# Directed delete: $outPath is not reachable from a root, so it should # Directed delete: $outPath is not reachable from a root, so it should
# be deleteable. # be deleteable.
nix-store --delete $outPath nix-store --delete $outPath
if test -e $outPath/hello; then false; fi (! [ -e $outPath/hello ])
outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)"
if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then