From 252c78b288426bae2f379e8320d396b7eb40253b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 15:59:07 +0200 Subject: [PATCH 1/7] Tweak release notes --- doc/manual/release-notes/rl-2.3.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 291b56448..80fbd6396 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -13,9 +13,8 @@ incompatible changes: Nix now uses BSD file locks instead of POSIX file - locks. Since previous releases used POSIX file locks, you should - not use Nix 2.2 and previous releases at the same time on a Nix - store. + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store. @@ -47,9 +46,9 @@ incompatible changes: - nix: Add + The nix command has a new () flag to - print build log output to stderr rather than showing the last log + print build log output to stderr, rather than showing the last log line in the progress bar. To distinguish between concurrent builds, log lines are prefixed by the name of the package. @@ -57,7 +56,7 @@ incompatible changes: Builds are now executed in a pseudo-terminal, and the - TERM evnironment variable is set to + TERM environment variable is set to xterm-256color. This allows many programs (e.g. gcc, clang, cmake) to print colorized log output. From d20f814cdef55fd9d7cecf464b32fb8b5efecb1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 15:59:33 +0200 Subject: [PATCH 2/7] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index c0943d3e9..7208c2182 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3 \ No newline at end of file +2.4 \ No newline at end of file From 5dafde28dbec379678da6d033cdc5c48856babf5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 19:24:35 +0200 Subject: [PATCH 3/7] BinaryCacheStore: Add index-debug-info option This integrates the functionality of the index-debuginfo program in nixos-channel-scripts to maintain an index of DWARF debuginfo files in a format usable by dwarffs. Thus the debug info index is updated by Hydra rather than by the channel mirroring script. Example usage: $ nix copy --to 'file:///tmp/binary-cache?index-debug-info=true' /nix/store/vr9mhcch3fljzzkjld3kvkggvpq38cva-nix-2.2.2-debug $ cat /tmp/binary-cache/debuginfo/036b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug {"archive":"../nar/0313h2kdhk4v73xna9ysiksp2v8xrsk5xsw79mmwr3rg7byb4ka8.nar.xz","member":"lib/debug/.build-id/03/6b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug"} Fixes #3083. --- src/libstore/binary-cache-store.cc | 80 ++++++++++++++++++++---- src/libstore/binary-cache-store.hh | 1 + src/libstore/local-binary-cache-store.cc | 2 + 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 10cde8704..8e6f1f55d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,10 +10,13 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "thread-pool.hh" #include - #include +#include + +#include namespace nix { @@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref(accessor); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(info.path, *nar, narAccessor); + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refaddToCache(info.path, *nar, narAccessor); - { auto res = jsonRoot.placeholder("root"); listNar(res, narAccessor, "", true); @@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refaddToCache(info.path, *nar, makeNarAccessor(nar)); - } - /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); @@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refsize() / nar->size()) * 100.0) % duration); - /* Atomically write the NAR file. */ narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : ""); + + /* Optionally maintain an index of DWARF debug info files + consisting of JSON files named 'debuginfo/' that + specify the NAR file and member containing the debug info. */ + if (writeDebugInfo) { + + std::string buildIdDir = "/lib/debug/.build-id"; + + if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + + ThreadPool threadPool(25); + + auto doFile = [&](std::string member, std::string key, std::string target) { + checkInterrupt(); + + nlohmann::json json; + json["archive"] = target; + json["member"] = member; + + // FIXME: or should we overwrite? The previous link may point + // to a GC'ed file, so overwriting might be useful... + if (fileExists(key)) return; + + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + + upsertFile(key, json.dump(), "application/json"); + }; + + std::regex regex1("^[0-9a-f]{2}$"); + std::regex regex2("^[0-9a-f]{38}\\.debug$"); + + for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + "/" + s1; + + if (narAccessor->stat(dir).type != FSAccessor::tDirectory + || !std::regex_match(s1, regex1)) + continue; + + for (auto & s2 : narAccessor->readDirectory(dir)) { + auto debugPath = dir + "/" + s2; + + if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + || !std::regex_match(s2, regex2)) + continue; + + auto buildId = s1 + s2; + + std::string key = "debuginfo/" + buildId; + std::string target = "../" + narInfo->url; + + threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + } + } + + threadPool.process(); + } + } + + /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index af108880c..c77292294 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,6 +17,7 @@ public: const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting parallelCompression{this, false, "parallel-compression", diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b7001795b..9f99ee76d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -63,6 +63,8 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + if (writeDebugInfo) + createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); } From b9d6c6f000d2ba3efd7c124e46982425962f4fbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 21:44:40 +0200 Subject: [PATCH 4/7] upload-release.pl: Fix sshfs call --- maintainers/upload-release.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 1cdf5ed16..77534babb 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n"; File::Path::make_path($releasesDir); if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; + system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; } my $releaseDir = "$releasesDir/nix/$releaseName"; From a56b51a0ba7b0d6fdff7fd0127a118185b146f4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 14:14:03 +0200 Subject: [PATCH 5/7] Disable OpenSSL lock callback on OpenSSL >= 1.1.1 --- src/libmain/shared.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0afddfb78..d3dbfbc44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -80,6 +80,7 @@ string getArg(const string & opt, } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif static void sigHandler(int signo) { } @@ -105,9 +107,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif loadConfFile(); From 92ede15dd902f7c1d2771c194b8bb73fe406840f Mon Sep 17 00:00:00 2001 From: Julien Tanguy Date: Wed, 11 Sep 2019 14:11:37 +0200 Subject: [PATCH 6/7] docs: Fix a typo in github in an example --- doc/manual/expressions/builtins.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 4c1d618e9..39407c9ac 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -448,7 +448,7 @@ stdenv.mkDerivation { … } Fetching an arbitrary ref builtins.fetchGit { - url = "https://gitub.com/NixOS/nix.git"; + url = "https://github.com/NixOS/nix.git"; ref = "refs/heads/0.5-release"; } From ae244af242ca3621e5a3b9196f27d9fcbf297266 Mon Sep 17 00:00:00 2001 From: Julien Tanguy Date: Wed, 11 Sep 2019 14:18:47 +0200 Subject: [PATCH 7/7] docs: Use the explicit ref for fetchGit with a tag With the merge of #2582, the syntax "tags/1.9" for refs does not work anymore. However, the new syntax "refs/tags/1.9" seems to support annotated tags, such as "refs/tags/2.0". Closes #2385. --- doc/manual/expressions/builtins.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 39407c9ac..191fa7810 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -499,11 +499,8 @@ stdenv.mkDerivation { … } Fetching a tag builtins.fetchGit { url = "https://github.com/nixos/nix.git"; - ref = "tags/1.9"; + ref = "refs/tags/1.9"; } - Due to a bug (#2385), - only non-annotated tags can be fetched.