diff --git a/configure.ac b/configure.ac index acf5ac7ce..bf18abea3 100644 --- a/configure.ac +++ b/configure.ac @@ -10,8 +10,8 @@ AC_CANONICAL_HOST # Construct a Nix system name (like "i686-linux"). AC_MSG_CHECKING([for the canonical Nix system name]) -cpu_name=$(uname -p | tr 'A-Z ' 'a-z_') -machine_name=$(uname -m | tr 'A-Z ' 'a-z_') +cpu_name=$(uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_') +machine_name=$(uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_') case $machine_name in i*86) @@ -30,7 +30,7 @@ case $machine_name in ;; esac -sys_name=$(uname -s | tr 'A-Z ' 'a-z_') +sys_name=$(uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_') case $sys_name in cygwin*) @@ -128,6 +128,10 @@ AC_CHECK_HEADERS([locale], [], [], []) AC_LANG_POP(C++) +# Check for . +AC_CHECK_HEADER([err.h], [], [bsddiff_compat_include="-Icompat-include"]) +AC_SUBST([bsddiff_compat_include]) + # Check whether we have the personality() syscall, which allows us to # do i686-linux builds on x86_64-linux machines. AC_CHECK_HEADERS([sys/personality.h]) diff --git a/nix.conf.example b/nix.conf.example index e17cf3c25..7063bed54 100644 --- a/nix.conf.example +++ b/nix.conf.example @@ -59,6 +59,17 @@ #build-max-jobs = 1 +### Option `build-cores' +# +# This option defines the number of CPU cores to utilize in parallel +# within a build job, i.e. by passing an appropriate `-jN' flag to GNU +# make. The default is 1, meaning that parallel building within jobs +# is disabled. Passing the special value `0' causes Nix to try and +# auto-detect the number of available cores on the local host. This +# setting can be overridden using the `--cores' command line switch. +#build-cores = 1 + + ### Option `build-max-silent-time' # # This option defines the maximum number of seconds that a builder can diff --git a/scripts/generate-patches.pl.in b/scripts/generate-patches.pl.in index 148b7d81f..dba647350 100755 --- a/scripts/generate-patches.pl.in +++ b/scripts/generate-patches.pl.in @@ -20,11 +20,11 @@ die unless scalar @ARGV == 5; my $hashAlgo = "sha256"; -my $cacheDir = $ARGV[0]; +my $narDir = $ARGV[0]; my $patchesDir = $ARGV[1]; my $patchesURL = $ARGV[2]; -my $srcDir = $ARGV[3]; -my $dstDir = $ARGV[4]; +my $srcManifest = $ARGV[3]; +my $dstManifest = $ARGV[4]; my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; @@ -41,10 +41,10 @@ my %dstNarFiles; my %dstLocalPaths; my %dstPatches; -readManifest "$srcDir/MANIFEST", +readManifest "$srcManifest", \%srcNarFiles, \%srcLocalPaths, \%srcPatches; -readManifest "$dstDir/MANIFEST", +readManifest "$dstManifest", \%dstNarFiles, \%dstLocalPaths, \%dstPatches; @@ -55,8 +55,7 @@ sub findOutputPaths { foreach my $p (keys %{$narFiles}) { - # Ignore store expressions. - next if ($p =~ /\.store$/); + # Ignore derivations. next if ($p =~ /\.drv$/); # Ignore builders (too much ambiguity -- they're all called @@ -65,7 +64,7 @@ sub findOutputPaths { next if ($p =~ /\.patch$/); # Don't bother including tar files etc. - next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/); + next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); $outPaths{$p} = 1; } @@ -85,6 +84,7 @@ sub getNameVersion { $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; my $name = $1; my $version = $2; + return undef unless defined $name && defined $version; $name =~ s/^-//; $version =~ s/^-//; return ($name, $version); @@ -112,14 +112,14 @@ sub getNarBz2 { my $storePath = shift; my $narFileList = $$narFiles{$storePath}; - die "missing store expression $storePath" unless defined $narFileList; + die "missing path $storePath" unless defined $narFileList; my $narFile = @{$narFileList}[0]; die unless defined $narFile; $narFile->{url} =~ /\/([^\/]+)$/; die unless defined $1; - return "$cacheDir/$1"; + return "$narDir/$1"; } @@ -213,6 +213,7 @@ foreach my $p (keys %dstOutPaths) { # this path. (my $name, my $version) = getNameVersion $p; + next unless defined $name && defined $version; my @closest = (); my $closestVersion; @@ -222,6 +223,8 @@ foreach my $p (keys %dstOutPaths) { foreach my $q (keys %srcOutPaths) { (my $name2, my $version2) = getNameVersion $q; + next unless defined $name2 && defined $version2; + if ($name eq $name2) { # If the sizes differ too much, then skip. This @@ -241,11 +244,11 @@ foreach my $p (keys %dstOutPaths) { # If the numbers of weighted uses differ too much, then # skip. This disambiguates between, e.g., the bootstrap # GCC and the final GCC in Nixpkgs. - my $srcUses = computeUses \%srcNarFiles, $q; - my $dstUses = computeUses \%dstNarFiles, $p; - $ratio = $srcUses / $dstUses; - $ratio = 1 / $ratio if $ratio < 1; - print " USE $srcUses $dstUses $ratio $q\n"; +# my $srcUses = computeUses \%srcNarFiles, $q; +# my $dstUses = computeUses \%dstNarFiles, $p; +# $ratio = $srcUses / $dstUses; +# $ratio = 1 / $ratio if $ratio < 1; +# print " USE $srcUses $dstUses $ratio $q\n"; # if ($ratio >= 2) { # print " SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n"; @@ -288,6 +291,11 @@ foreach my $p (keys %dstOutPaths) { my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest; my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p; + if (! -f $srcNarBz2) { + warn "patch source archive $srcNarBz2 is missing\n"; + next; + } + system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0 or die "cannot unpack $srcNarBz2"; @@ -404,5 +412,5 @@ do { # Rewrite the manifest of the destination (with the new patches). -writeManifest "$dstDir/MANIFEST", +writeManifest "${dstManifest}", \%dstNarFiles, \%dstPatches; diff --git a/src/bsdiff-4.3/Makefile.am b/src/bsdiff-4.3/Makefile.am index 4b3af0783..57970bc55 100644 --- a/src/bsdiff-4.3/Makefile.am +++ b/src/bsdiff-4.3/Makefile.am @@ -1,3 +1,5 @@ +EXTRA_DIST = compat-include + libexec_PROGRAMS = bsdiff bspatch bsdiff_SOURCES = bsdiff.c @@ -8,4 +10,4 @@ bspatch_SOURCES = bspatch.c bspatch_LDADD = ${bzip2_lib} -AM_CFLAGS = -O3 ${bzip2_include} +AM_CFLAGS = -O3 ${bzip2_include} ${bsddiff_compat_include} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h new file mode 100644 index 000000000..a851ded6f --- /dev/null +++ b/src/bsdiff-4.3/compat-include/err.h @@ -0,0 +1,12 @@ +/* Simulate BSD's functionality. */ + +#ifndef COMPAT_ERR_H_INCLUDED +#define COMPAT_ERR_H_INCLUDED 1 + +#include +#include + +#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) +#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) + +#endif diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index eb1d8d432..96bda43a3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -750,13 +750,15 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) { - Value v2; - state.evalAttrs(env, e1, v2); - - state.cloneAttrs(v2, v); - + Value v1, v2; + state.evalAttrs(env, e1, v1); state.evalAttrs(env, e2, v2); - + + if (v1.attrs->size() == 0) { v = v2; return; } + if (v2.attrs->size() == 0) { v = v1; return; } + + state.cloneAttrs(v1, v); + foreach (Bindings::iterator, i, *v2.attrs) { Attr & a = (*v.attrs)[i->first]; mkCopy(a.value, i->second.value); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 42c858611..68f66acc7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -788,13 +788,13 @@ static void prim_intersectAttrs(EvalState & state, Value * * args, Value & v) state.forceAttrs(*args[1]); state.mkAttrs(v); - - foreach (Bindings::iterator, i, *args[1]->attrs) { - Bindings::iterator j = args[0]->attrs->find(i->first); - if (j != args[0]->attrs->end()) { - Attr & a = (*v.attrs)[i->first]; - mkCopy(a.value, i->second.value); - a.pos = i->second.pos; + + foreach (Bindings::iterator, i, *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i->first); + if (j != args[1]->attrs->end()) { + Attr & a = (*v.attrs)[j->first]; + mkCopy(a.value, j->second.value); + a.pos = j->second.pos; } } } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 3fbec4b52..eddc4e64b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -135,6 +135,7 @@ static void initAndRun(int argc, char * * argv) /* Get some settings from the configuration file. */ thisSystem = querySetting("system", SYSTEM); maxBuildJobs = queryIntSetting("build-max-jobs", 1); + buildCores = queryIntSetting("build-cores", 1); maxSilentTime = queryIntSetting("build-max-silent-time", 0); /* Catch SIGINT. */ @@ -226,6 +227,8 @@ static void initAndRun(int argc, char * * argv) tryFallback = true; else if (arg == "--max-jobs" || arg == "-j") maxBuildJobs = getIntArg(arg, i, args.end()); + else if (arg == "--cores") + buildCores = getIntArg(arg, i, args.end()); else if (arg == "--readonly-mode") readOnlyMode = true; else if (arg == "--max-silent-time") diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5818aa51d..6f0276244 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1427,6 +1428,9 @@ void DerivationGoal::startBuilder() in the store or in the build directory). */ env["NIX_STORE"] = nixStore; + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str(); + /* Add all bindings specified in the derivation. */ foreach (StringPairs::iterator, i, drv.env) env[i->first] = i->second; @@ -2654,6 +2658,7 @@ void Worker::waitForInput() timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before); } else lastWokenUp = 0; + using namespace std; /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index cc0e44e8e..75d2f69c2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -22,6 +22,7 @@ bool keepGoing = false; bool tryFallback = false; Verbosity buildVerbosity = lvlInfo; unsigned int maxBuildJobs = 1; +unsigned int buildCores = 1; bool readOnlyMode = false; string thisSystem = "unset"; time_t maxSilentTime = 0; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d3388e309..a74a741d6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -55,6 +55,11 @@ extern Verbosity buildVerbosity; /* Maximum number of parallel build jobs. 0 means unlimited. */ extern unsigned int maxBuildJobs; +/* Number of CPU cores to utilize in parallel within a build, i.e. by passing + this number to Make via '-j'. 0 means that the number of actual CPU cores on + the local host ought to be auto-detected. */ +extern unsigned int buildCores; + /* Read-only mode. Don't copy stuff to the store, don't change the database. */ extern bool readOnlyMode; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 334ad95cf..93319ebb8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,6 +13,7 @@ #include #include +#include namespace nix { @@ -158,6 +159,7 @@ void RemoteStore::connectToDaemon() addr.sun_family = AF_UNIX; if (socketPathRel.size() >= sizeof(addr.sun_path)) throw Error(format("socket path `%1%' is too long") % socketPathRel); + using namespace std; strcpy(addr.sun_path, socketPathRel.c_str()); if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e0c41a58d..9540720fe 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include "util.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ca67119e9..a67ce243a 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -247,11 +247,12 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, } /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one with the highest priority. - If there are multiple derivations with the same priority, - pick the one with the highest version. If there are - multiple derivations with the same priority and name and - version, then pick the first one. */ + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ if (newestOnly) { /* Map from package names to derivations. */ @@ -266,7 +267,12 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, Newest::iterator k = newest.find(drvName.name); if (k != newest.end()) { - d = comparePriorities(state, j->first, k->second.first); + d = j->first.system == k->second.first.system ? 0 : + j->first.system == thisSystem ? 1 : + k->second.first.system == thisSystem ? -1 : 0; + printMsg(lvlError, format("%1% %2% %3% %4%") % j->first.system % k->second.first.system % thisSystem % d); + if (d == 0) + d = comparePriorities(state, j->first, k->second.first); if (d == 0) d = compareVersions(drvName.version, DrvName(k->second.first.name).version); } @@ -1230,7 +1236,7 @@ void run(Strings args) globals.instSource.type = srcUnknown; globals.instSource.nixExprPath = getDefNixExprPath(); - globals.instSource.systemFilter = thisSystem; + globals.instSource.systemFilter = "*"; globals.dryRun = false; globals.preserveInstalled = false; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index d41877e88..b292b83ed 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include +#include #include #include #include @@ -111,6 +112,7 @@ static bool isFarSideClosed(int socket) time and wouldn't have to worry about races. */ static void sigPollHandler(int sigNo) { + using namespace std; try { /* Check that the far side actually closed. We're still getting spurious signals every once in a while. I.e., diff --git a/tests/build-hook.hook.sh b/tests/build-hook.hook.sh index 18eba283e..83fa3bf78 100755 --- a/tests/build-hook.hook.sh +++ b/tests/build-hook.hook.sh @@ -6,11 +6,11 @@ drv=$4 echo "HOOK for $drv" >&2 -outPath=$(sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv) +outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv` echo "output path is $outPath" >&2 -if $(echo $outPath | grep -q input-1); then +if `echo $outPath | grep -q input-1`; then echo "# accept" >&2 read x echo "got $x"