forked from lix-project/lix
Merge remote-tracking branch 'origin/master' into flakes
This commit is contained in:
Eelco Dolstra
commit
c0c2cb871d
|
|
@ -1,5 +1,5 @@
|
|||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
|
||||
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
|
||||
|
||||
use strict;
|
||||
use Data::Dumper;
|
||||
|
|
@ -9,12 +9,16 @@ use File::Slurp;
|
|||
use File::Copy;
|
||||
use JSON::PP;
|
||||
use LWP::UserAgent;
|
||||
use Net::Amazon::S3;
|
||||
|
||||
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
|
||||
|
||||
my $releasesDir = "/home/eelco/mnt/releases";
|
||||
my $releasesBucketName = "nix-releases";
|
||||
my $channelsBucketName = "nix-channels";
|
||||
my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
|
||||
|
||||
my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
|
||||
|
||||
# FIXME: cut&paste from nixos-channel-scripts.
|
||||
sub fetch {
|
||||
my ($url, $type) = @_;
|
||||
|
|
@ -42,13 +46,31 @@ my $version = $1;
|
|||
|
||||
print STDERR "Nix revision is $nixRev, version is $version\n";
|
||||
|
||||
File::Path::make_path($releasesDir);
|
||||
if (system("mountpoint -q $releasesDir") != 0) {
|
||||
system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
|
||||
}
|
||||
my $releaseDir = "nix/$releaseName";
|
||||
|
||||
my $releaseDir = "$releasesDir/nix/$releaseName";
|
||||
File::Path::make_path($releaseDir);
|
||||
my $tmpDir = "$TMPDIR/nix-release/$releaseName";
|
||||
File::Path::make_path($tmpDir);
|
||||
|
||||
# S3 setup.
|
||||
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
|
||||
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
|
||||
|
||||
my $s3 = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
retry => 1,
|
||||
host => "s3-eu-west-1.amazonaws.com",
|
||||
});
|
||||
|
||||
my $releasesBucket = $s3->bucket($releasesBucketName) or die;
|
||||
|
||||
my $s3_us = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
retry => 1,
|
||||
});
|
||||
|
||||
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
|
||||
|
||||
sub downloadFile {
|
||||
my ($jobName, $productNr, $dstName) = @_;
|
||||
|
|
@ -57,40 +79,49 @@ sub downloadFile {
|
|||
|
||||
my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
|
||||
$dstName //= basename($srcFile);
|
||||
my $dstFile = "$releaseDir/" . $dstName;
|
||||
my $tmpFile = "$tmpDir/$dstName";
|
||||
|
||||
if (! -e $dstFile) {
|
||||
print STDERR "downloading $srcFile to $dstFile...\n";
|
||||
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
|
||||
if (!-e $tmpFile) {
|
||||
print STDERR "downloading $srcFile to $tmpFile...\n";
|
||||
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
|
||||
or die "unable to fetch $srcFile\n";
|
||||
rename("$dstFile.tmp", $dstFile) or die;
|
||||
}
|
||||
|
||||
my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
|
||||
my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
|
||||
my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
|
||||
chomp $sha256_actual;
|
||||
if ($sha256_expected ne $sha256_actual) {
|
||||
print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
|
||||
print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
write_file("$dstFile.sha256", $sha256_expected);
|
||||
write_file("$tmpFile.sha256", $sha256_expected);
|
||||
|
||||
if (! -e "$dstFile.asc") {
|
||||
system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
|
||||
if (! -e "$tmpFile.asc") {
|
||||
system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
|
||||
}
|
||||
|
||||
return ($dstFile, $sha256_expected);
|
||||
return $sha256_expected;
|
||||
}
|
||||
|
||||
downloadFile("tarball", "2"); # .tar.bz2
|
||||
my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
|
||||
my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
|
||||
downloadFile("binaryTarball.i686-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-linux", "1");
|
||||
downloadFile("binaryTarball.aarch64-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-darwin", "1");
|
||||
downloadFile("installerScript", "1");
|
||||
|
||||
for my $fn (glob "$tmpDir/*") {
|
||||
my $name = basename($fn);
|
||||
my $dstKey = "$releaseDir/" . $name;
|
||||
unless (defined $releasesBucket->head_key($dstKey)) {
|
||||
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
|
||||
$releasesBucket->add_key_filename($dstKey, $fn)
|
||||
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
|
||||
}
|
||||
}
|
||||
|
||||
exit if $version =~ /pre/;
|
||||
|
||||
# Update Nixpkgs in a very hacky way.
|
||||
|
|
@ -125,18 +156,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
|
|||
|
||||
system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
|
||||
|
||||
# Extract the HTML manual.
|
||||
File::Path::make_path("$releaseDir/manual");
|
||||
|
||||
system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
|
||||
|
||||
if (! -e "$releaseDir/manual/index.html") {
|
||||
symlink("manual.html", "$releaseDir/manual/index.html") or die;
|
||||
}
|
||||
|
||||
# Update the "latest" symlink.
|
||||
symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
|
||||
rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
|
||||
$channelsBucket->add_key(
|
||||
"nix-latest/install", "",
|
||||
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
|
||||
or die $channelsBucket->err . ": " . $channelsBucket->errstr;
|
||||
|
||||
# Tag the release in Git.
|
||||
chdir("/home/eelco/Dev/nix-pristine") or die;
|
||||
|
|
|
|||
|
|
@ -1021,7 +1021,9 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
|
|||
|
||||
for (auto path : context) {
|
||||
if (path.at(0) != '/')
|
||||
throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
|
||||
throw EvalError(format(
|
||||
"in 'toFile': the file named '%1%' must not contain a reference "
|
||||
"to a derivation but contains (%2%), at %3%") % name % path % pos);
|
||||
refs.insert(state.store->parseStorePath(path));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ void initNix()
|
|||
sshd). This breaks build users because they don't have access
|
||||
to the TMPDIR, in particular in ‘nix-store --serve’. */
|
||||
#if __APPLE__
|
||||
if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
|
||||
if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
|
||||
unsetenv("TMPDIR");
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1680,6 +1680,7 @@ void DerivationGoal::buildDone()
|
|||
}
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
deleteTmpDir(true);
|
||||
done(BuildResult::Built);
|
||||
return;
|
||||
}
|
||||
|
|
@ -3536,6 +3537,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
|
|||
}
|
||||
|
||||
|
||||
static void moveCheckToStore(const Path & src, const Path & dst)
|
||||
{
|
||||
/* For the rename of directory to succeed, we must be running as root or
|
||||
the directory must be made temporarily writable (to update the
|
||||
directory's parent link ".."). */
|
||||
struct stat st;
|
||||
if (lstat(src.c_str(), &st) == -1) {
|
||||
throw SysError(format("getting attributes of path '%1%'") % src);
|
||||
}
|
||||
|
||||
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
|
||||
|
||||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
if (rename(src.c_str(), dst.c_str()))
|
||||
throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
|
|
@ -3714,8 +3738,7 @@ void DerivationGoal::registerOutputs()
|
|||
if (settings.runDiffHook || settings.keepFailed) {
|
||||
Path dst = worker.store.toRealPath(path + checkSuffix);
|
||||
deletePath(dst);
|
||||
if (rename(actualPath.c_str(), dst.c_str()))
|
||||
throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
|
||||
moveCheckToStore(actualPath, dst);
|
||||
|
||||
handleDiffHook(
|
||||
buildUser ? buildUser->getUID() : getuid(),
|
||||
|
|
@ -3723,10 +3746,10 @@ void DerivationGoal::registerOutputs()
|
|||
path, dst, worker.store.printStorePath(drvPath), tmpDir);
|
||||
|
||||
throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
|
||||
worker.store.printStorePath(drvPath), path, dst);
|
||||
worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
|
||||
} else
|
||||
throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
|
||||
worker.store.printStorePath(drvPath), path);
|
||||
worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
|
||||
}
|
||||
|
||||
/* Since we verified the build, it's now ultimately trusted. */
|
||||
|
|
|
|||
|
|
@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
|
|||
/* Read the `temproots' directory for per-process temporary root
|
||||
files. */
|
||||
for (auto & i : readDirectory(tempRootsDir)) {
|
||||
if (i.name[0] == '.') {
|
||||
// Ignore hidden files. Some package managers (notably portage) create
|
||||
// those to keep the directory alive.
|
||||
continue;
|
||||
}
|
||||
Path path = tempRootsDir + "/" + i.name;
|
||||
|
||||
pid_t pid = std::stoi(i.name);
|
||||
|
|
|
|||
|
|
@ -1,12 +1,45 @@
|
|||
{checkBuildId ? 0}:
|
||||
|
||||
with import ./config.nix;
|
||||
|
||||
{
|
||||
nondeterministic = mkDerivation {
|
||||
inherit checkBuildId;
|
||||
name = "nondeterministic";
|
||||
buildCommand =
|
||||
''
|
||||
mkdir $out
|
||||
date +%s.%N > $out/date
|
||||
echo "CHECK_TMPDIR=$TMPDIR"
|
||||
echo "checkBuildId=$checkBuildId"
|
||||
echo "$checkBuildId" > $TMPDIR/checkBuildId
|
||||
'';
|
||||
};
|
||||
|
||||
deterministic = mkDerivation {
|
||||
inherit checkBuildId;
|
||||
name = "deterministic";
|
||||
buildCommand =
|
||||
''
|
||||
mkdir $out
|
||||
echo date > $out/date
|
||||
echo "CHECK_TMPDIR=$TMPDIR"
|
||||
echo "checkBuildId=$checkBuildId"
|
||||
echo "$checkBuildId" > $TMPDIR/checkBuildId
|
||||
'';
|
||||
};
|
||||
|
||||
failed = mkDerivation {
|
||||
inherit checkBuildId;
|
||||
name = "failed";
|
||||
buildCommand =
|
||||
''
|
||||
mkdir $out
|
||||
echo date > $out/date
|
||||
echo "CHECK_TMPDIR=$TMPDIR"
|
||||
echo "checkBuildId=$checkBuildId"
|
||||
echo "$checkBuildId" > $TMPDIR/checkBuildId
|
||||
false
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,57 @@
|
|||
source common.sh
|
||||
|
||||
checkBuildTempDirRemoved ()
|
||||
{
|
||||
buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
|
||||
checkBuildIdFile=${buildDir}/checkBuildId
|
||||
[[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile
|
||||
}
|
||||
|
||||
# written to build temp directories to verify created by this instance
|
||||
checkBuildId=$(date +%s%N)
|
||||
|
||||
clearStore
|
||||
|
||||
nix-build dependencies.nix --no-out-link
|
||||
nix-build dependencies.nix --no-out-link --check
|
||||
|
||||
nix-build check.nix -A nondeterministic --no-out-link
|
||||
nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
|
||||
# check for dangling temporary build directories
|
||||
# only retain if build fails and --keep-failed is specified, or...
|
||||
# ...build is non-deterministic and --check and --keep-failed are both specified
|
||||
nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link 2> $TEST_ROOT/log || status=$?
|
||||
[ "$status" = "100" ]
|
||||
checkBuildTempDirRemoved $TEST_ROOT/log
|
||||
|
||||
nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link --keep-failed 2> $TEST_ROOT/log || status=$?
|
||||
[ "$status" = "100" ]
|
||||
if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
|
||||
|
||||
nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link 2> $TEST_ROOT/log
|
||||
checkBuildTempDirRemoved $TEST_ROOT/log
|
||||
|
||||
nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link --check --keep-failed 2> $TEST_ROOT/log
|
||||
if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
|
||||
checkBuildTempDirRemoved $TEST_ROOT/log
|
||||
|
||||
nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link 2> $TEST_ROOT/log
|
||||
checkBuildTempDirRemoved $TEST_ROOT/log
|
||||
|
||||
nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link --check 2> $TEST_ROOT/log || status=$?
|
||||
grep 'may not be deterministic' $TEST_ROOT/log
|
||||
[ "$status" = "104" ]
|
||||
checkBuildTempDirRemoved $TEST_ROOT/log
|
||||
|
||||
nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
|
||||
--no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
|
||||
grep 'may not be deterministic' $TEST_ROOT/log
|
||||
[ "$status" = "104" ]
|
||||
if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
|
||||
|
||||
clearStore
|
||||
|
||||
|
|
|
|||
|
|
@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR
|
|||
|
||||
# Test --check without hash rewriting.
|
||||
nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
|
||||
|
||||
# Test that sandboxed builds with --check and -K can move .check directory to store
|
||||
nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
|
||||
|
||||
(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
|
||||
if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi
|
||||
grep -q 'may not be deterministic' $TEST_ROOT/log
|
||||
|
|
|
|||
Loading…
Reference in a new issue