Merge remote-tracking branch 'origin/master' into flakes

This commit is contained in:
Eelco Dolstra 2019-09-13 19:52:03 +02:00
commit a25c022af3
7 changed files with 80 additions and 25 deletions

View file

@ -1 +1 @@
2.3 2.4

View file

@ -448,7 +448,7 @@ stdenv.mkDerivation { … }
<example> <example>
<title>Fetching an arbitrary ref</title> <title>Fetching an arbitrary ref</title>
<programlisting>builtins.fetchGit { <programlisting>builtins.fetchGit {
url = "https://gitub.com/NixOS/nix.git"; url = "https://github.com/NixOS/nix.git";
ref = "refs/heads/0.5-release"; ref = "refs/heads/0.5-release";
}</programlisting> }</programlisting>
</example> </example>
@ -499,11 +499,8 @@ stdenv.mkDerivation { … }
<title>Fetching a tag</title> <title>Fetching a tag</title>
<programlisting>builtins.fetchGit { <programlisting>builtins.fetchGit {
url = "https://github.com/nixos/nix.git"; url = "https://github.com/nixos/nix.git";
ref = "tags/1.9"; ref = "refs/tags/1.9";
}</programlisting> }</programlisting>
<note><para>Due to a bug (<link
xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>),
only non-annotated tags can be fetched.</para></note>
</example> </example>
<example> <example>

View file

@ -13,9 +13,8 @@ incompatible changes:</para>
<listitem> <listitem>
<para>Nix now uses BSD file locks instead of POSIX file <para>Nix now uses BSD file locks instead of POSIX file
locks. Since previous releases used POSIX file locks, you should locks. Because of this, you should not use Nix 2.3 and previous
not use Nix 2.2 and previous releases at the same time on a Nix releases at the same time on a Nix store.</para>
store.</para>
</listitem> </listitem>
</itemizedlist> </itemizedlist>
@ -47,9 +46,9 @@ incompatible changes:</para>
</listitem> </listitem>
<listitem> <listitem>
<para><command>nix</command>: Add <para>The <command>nix</command> command has a new
<option>--print-build-logs</option> (<option>-L</option>) flag to <option>--print-build-logs</option> (<option>-L</option>) flag to
print build log output to stderr rather than showing the last log print build log output to stderr, rather than showing the last log
line in the progress bar. To distinguish between concurrent line in the progress bar. To distinguish between concurrent
builds, log lines are prefixed by the name of the package. builds, log lines are prefixed by the name of the package.
</para> </para>
@ -57,7 +56,7 @@ incompatible changes:</para>
<listitem> <listitem>
<para>Builds are now executed in a pseudo-terminal, and the <para>Builds are now executed in a pseudo-terminal, and the
<envar>TERM</envar> evnironment variable is set to <envar>TERM</envar> environment variable is set to
<literal>xterm-256color</literal>. This allows many programs <literal>xterm-256color</literal>. This allows many programs
(e.g. <command>gcc</command>, <command>clang</command>, (e.g. <command>gcc</command>, <command>clang</command>,
<command>cmake</command>) to print colorized log output.</para> <command>cmake</command>) to print colorized log output.</para>

View file

@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n";
File::Path::make_path($releasesDir); File::Path::make_path($releasesDir);
if (system("mountpoint -q $releasesDir") != 0) { if (system("mountpoint -q $releasesDir") != 0) {
system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
} }
my $releaseDir = "$releasesDir/nix/$releaseName"; my $releaseDir = "$releasesDir/nix/$releaseName";

View file

@ -10,10 +10,13 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "json.hh" #include "json.hh"
#include "thread-pool.hh"
#include <chrono> #include <chrono>
#include <future> #include <future>
#include <regex>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the /* Optionally write a JSON file containing a listing of the
contents of the NAR. */ contents of the NAR. */
if (writeNARListing) { if (writeNARListing) {
@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
JSONObject jsonRoot(jsonOut); JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1); jsonRoot.attr("version", 1);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
{ {
auto res = jsonRoot.placeholder("root"); auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true); listNar(res, narAccessor, "", true);
@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
} }
else {
if (accessor_)
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
}
/* Compress the NAR. */ /* Compress the NAR. */
narInfo->compression = compression; narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration); % duration);
/* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" : + (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" : compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" : compression == "br" ? ".br" :
""); "");
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
if (writeDebugInfo) {
std::string buildIdDir = "/lib/debug/.build-id";
if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
ThreadPool threadPool(25);
auto doFile = [&](std::string member, std::string key, std::string target) {
checkInterrupt();
nlohmann::json json;
json["archive"] = target;
json["member"] = member;
// FIXME: or should we overwrite? The previous link may point
// to a GC'ed file, so overwriting might be useful...
if (fileExists(key)) return;
printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
upsertFile(key, json.dump(), "application/json");
};
std::regex regex1("^[0-9a-f]{2}$");
std::regex regex2("^[0-9a-f]{38}\\.debug$");
for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + "/" + s1;
if (narAccessor->stat(dir).type != FSAccessor::tDirectory
|| !std::regex_match(s1, regex1))
continue;
for (auto & s2 : narAccessor->readDirectory(dir)) {
auto debugPath = dir + "/" + s2;
if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
|| !std::regex_match(s2, regex2))
continue;
auto buildId = s1 + s2;
std::string key = "debuginfo/" + buildId;
std::string target = "../" + narInfo->url;
threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
}
}
threadPool.process();
}
}
/* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");

View file

@ -17,6 +17,7 @@ public:
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression", const Setting<bool> parallelCompression{this, false, "parallel-compression",

View file

@ -63,6 +63,8 @@ protected:
void LocalBinaryCacheStore::init() void LocalBinaryCacheStore::init()
{ {
createDirs(binaryCacheDir + "/nar"); createDirs(binaryCacheDir + "/nar");
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init(); BinaryCacheStore::init();
} }