Merge remote-tracking branch 'origin/master' into flakes

maintainers/upload-release.pl

@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
+#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
 
 use strict;
 use Data::Dumper;
@@ -9,12 +9,16 @@ use File::Slurp;
 use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
+use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
-my $releasesDir = "/home/eelco/mnt/releases";
+my $releasesBucketName = "nix-releases";
+my $channelsBucketName = "nix-channels";
 my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
+my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -42,13 +46,31 @@ my $version = $1;
 
 print STDERR "Nix revision is $nixRev, version is $version\n";
 
-File::Path::make_path($releasesDir);
-if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
-}
+my $releaseDir = "nix/$releaseName";
 
-my $releaseDir = "$releasesDir/nix/$releaseName";
-File::Path::make_path($releaseDir);
+my $tmpDir = "$TMPDIR/nix-release/$releaseName";
+File::Path::make_path($tmpDir);
+
+# S3 setup.
+my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
+my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
+
+my $s3 = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+      host                  => "s3-eu-west-1.amazonaws.com",
+    });
+
+my $releasesBucket = $s3->bucket($releasesBucketName) or die;
+
+my $s3_us = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+    });
+
+my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -57,40 +79,49 @@ sub downloadFile {
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
-    my $dstFile = "$releaseDir/" . $dstName;
+    my $tmpFile = "$tmpDir/$dstName";
 
-    if (! -e $dstFile) {
-        print STDERR "downloading $srcFile to $dstFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
+    if (!-e $tmpFile) {
+        print STDERR "downloading $srcFile to $tmpFile...\n";
+        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
             or die "unable to fetch $srcFile\n";
-        rename("$dstFile.tmp", $dstFile) or die;
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
+        print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$dstFile.sha256", $sha256_expected);
+    write_file("$tmpFile.sha256", $sha256_expected);
 
-    if (! -e "$dstFile.asc") {
-        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
+    if (! -e "$tmpFile.asc") {
+        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
     }
 
-    return ($dstFile, $sha256_expected);
+    return $sha256_expected;
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
-my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
+my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
+for my $fn (glob "$tmpDir/*") {
+    my $name = basename($fn);
+    my $dstKey = "$releaseDir/" . $name;
+    unless (defined $releasesBucket->head_key($dstKey)) {
+        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
+        $releasesBucket->add_key_filename($dstKey, $fn)
+            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
+    }
+}
+
 exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
@@ -125,18 +156,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
 
 system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
 
-# Extract the HTML manual.
-File::Path::make_path("$releaseDir/manual");
-
-system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
-
-if (! -e "$releaseDir/manual/index.html") {
-    symlink("manual.html", "$releaseDir/manual/index.html") or die;
-}
-
 # Update the "latest" symlink.
-symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
-rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
+$channelsBucket->add_key(
+    "nix-latest/install", "",
+    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;

src/libexpr/primops.cc

@@ -1021,7 +1021,9 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
+            throw EvalError(format(
+                "in 'toFile': the file named '%1%' must not contain a reference "
+                "to a derivation but contains (%2%), at %3%") % name % path % pos);
         refs.insert(state.store->parseStorePath(path));
     }
 

src/libmain/shared.cc

@@ -155,7 +155,7 @@ void initNix()
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 }

src/libstore/build.cc

@@ -1680,6 +1680,7 @@ void DerivationGoal::buildDone()
         }
 
         if (buildMode == bmCheck) {
+            deleteTmpDir(true);
             done(BuildResult::Built);
             return;
         }
@@ -3536,6 +3537,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
 }
 
 
+static void moveCheckToStore(const Path & src, const Path & dst)
+{
+    /* For the rename of directory to succeed, we must be running as root or
+       the directory must be made temporarily writable (to update the
+       directory's parent link ".."). */
+    struct stat st;
+    if (lstat(src.c_str(), &st) == -1) {
+        throw SysError(format("getting attributes of path '%1%'") % src);
+    }
+
+    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+    if (changePerm)
+        chmod_(src, st.st_mode | S_IWUSR);
+
+    if (rename(src.c_str(), dst.c_str()))
+        throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
+
+    if (changePerm)
+        chmod_(dst, st.st_mode);
+}
+
+
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
@@ -3714,8 +3738,7 @@ void DerivationGoal::registerOutputs()
                 if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
-                    if (rename(actualPath.c_str(), dst.c_str()))
-                        throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+                    moveCheckToStore(actualPath, dst);
 
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),
@@ -3723,10 +3746,10 @@ void DerivationGoal::registerOutputs()
                         path, dst, worker.store.printStorePath(drvPath), tmpDir);
 
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), path, dst);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
                 } else
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), path);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
             }
 
             /* Since we verified the build, it's now ultimately trusted. */

src/libstore/gc.cc

@@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
+        if (i.name[0] == '.') {
+            // Ignore hidden files. Some package managers (notably portage) create
+            // those to keep the directory alive.
+            continue;
+        }
         Path path = tempRootsDir + "/" + i.name;
 
         pid_t pid = std::stoi(i.name);

tests/check.nix

@@ -1,12 +1,45 @@
+{checkBuildId ? 0}:
+
 with import ./config.nix;
 
 {
   nondeterministic = mkDerivation {
+    inherit checkBuildId;
     name = "nondeterministic";
     buildCommand =
       ''
         mkdir $out
         date +%s.%N > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  deterministic = mkDerivation {
+    inherit checkBuildId;
+    name = "deterministic";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  failed = mkDerivation {
+    inherit checkBuildId;
+    name = "failed";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+        false
       '';
   };
 

tests/check.sh

@@ -1,14 +1,57 @@
 source common.sh
 
+checkBuildTempDirRemoved ()
+{
+    buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
+    checkBuildIdFile=${buildDir}/checkBuildId
+    [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile
+}
+
+# written to build temp directories to verify created by this instance
+checkBuildId=$(date +%s%N)
+
 clearStore
 
 nix-build dependencies.nix --no-out-link
 nix-build dependencies.nix --no-out-link --check
 
-nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
+# check for dangling temporary build directories
+# only retain if build fails and --keep-failed is specified, or...
+# ...build is non-deterministic and --check and --keep-failed are both specified
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log
+if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check 2> $TEST_ROOT/log || status=$?
 grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
+grep 'may not be deterministic' $TEST_ROOT/log
+[ "$status" = "104" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
 clearStore
 

tests/linux-sandbox.sh

@@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR
 
 # Test --check without hash rewriting.
 nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
+
+# Test that sandboxed builds with --check and -K can move .check directory to store
+nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
+
+(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
+if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi
+grep -q 'may not be deterministic' $TEST_ROOT/log