From 9f9080e2c019f188ba679a7a89284d7eaf629710 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 May 2014 17:19:36 +0200 Subject: [PATCH] nix-store -l: Fetch build logs from the Internet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a build log is not available locally, then ‘nix-store -l’ will now try to download it from the servers listed in the ‘log-servers’ option in nix.conf. For instance, if you have: log-servers = http://hydra.nixos.org/log then it will try to get logs from http://hydra.nixos.org/log/. So you can do things like: $ nix-store -l $(which xterm) and get a log even if xterm wasn't built locally. --- Makefile.config.in | 1 + doc/manual/conf-file.xml | 14 +++ doc/manual/nix-store.xml | 183 +++++++++++++++++++------------------ src/libstore/globals.cc | 1 + src/libstore/globals.hh | 3 + src/libutil/util.cc | 2 +- src/libutil/util.hh | 2 + src/nix-store/local.mk | 2 + src/nix-store/nix-store.cc | 29 +++++- 9 files changed, 144 insertions(+), 93 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 093748332..53bdbbf61 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -10,6 +10,7 @@ PACKAGE_VERSION = @PACKAGE_VERSION@ bash = @bash@ bindir = @bindir@ bsddiff_compat_include = @bsddiff_compat_include@ +curl = @curl@ datadir = @datadir@ datarootdir = @datarootdir@ dblatex = @dblatex@ diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 327d22c4a..29f7f9c51 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -465,6 +465,20 @@ flag, e.g. --option gc-keep-outputs false. + log-servers + + + + A list of URL prefixes (such as + http://hydra.nixos.org/log) from which + nix-store -l will try to fetch build logs if + they’re not available locally. + + + + + + diff --git a/doc/manual/nix-store.xml b/doc/manual/nix-store.xml index 87cbc307a..c9a912ff0 100644 --- a/doc/manual/nix-store.xml +++ b/doc/manual/nix-store.xml @@ -111,7 +111,7 @@ lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134 - + @@ -120,7 +120,7 @@ lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134 - + @@ -141,7 +141,7 @@ lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134 Description - + The operation essentially “builds” the specified store paths. Realisation is a somewhat overloaded term: @@ -196,14 +196,14 @@ printed.) - + Examples This operation is typically used to build store derivations produced by nix-instantiate: - + $ nix-store -r $(nix-instantiate ./test.nix) /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 @@ -216,7 +216,7 @@ linkend="sec-nix-build">nix-build does. - + @@ -239,7 +239,7 @@ linkend="sec-nix-build">nix-build does. Description - + Without additional flags, the operation performs a garbage collection on the Nix store. That is, all paths in the Nix store not reachable via file system references from a set of @@ -250,40 +250,40 @@ the Nix store not reachable via file system references from a set of - + This operation prints on standard output the set of roots used by the garbage collector. What constitutes a root is described in . - + - + This operation prints on standard output the set of “live” store paths, which are all the store paths reachable from the roots. Live paths should never be deleted, since that would break consistency — it would become possible that applications are installed that reference things that are no longer present in the store. - + - + This operation prints out on standard output the set of “dead” store paths, which is just the opposite of the set of live paths: any path in the store that is not live (with respect to the roots) is dead. - + - + This operation performs an actual garbage collection. All dead paths are removed from the store. This is the default. - + @@ -294,7 +294,7 @@ options control what gets deleted and in what order: bytes - + Keep deleting paths until at least bytes bytes have been deleted, then stop. The argument bytes can be @@ -302,7 +302,7 @@ options control what gets deleted and in what order: M, G or T, denoting KiB, MiB, GiB or TiB units. - + @@ -326,7 +326,7 @@ would be freed. Examples To delete all unreachable paths, just do: - + $ nix-store --gc deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' @@ -348,7 +348,7 @@ $ nix-store --gc --max-freed $((100 * 1024 * 1024)) - + Operation <option>--delete</option> @@ -433,7 +433,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' Description - + The operation displays various bits of information about the store paths . The queries are described below. At most one query can be specified. The default query is @@ -453,16 +453,16 @@ query is applied to the target of the symlink. - + For each argument to the query that is a store derivation, apply the query to the output path of the derivation instead. - + - + Realise each argument to the query first (see nix-store --realise). @@ -470,12 +470,12 @@ query is applied to the target of the symlink. - + - + Queries - + @@ -485,7 +485,7 @@ query is applied to the target of the symlink. derivations paths. These are the paths that will be produced when the derivation is built. - + @@ -500,10 +500,10 @@ query is applied to the target of the symlink. - + Also include the output path of store derivations, and their closures. - + @@ -517,13 +517,13 @@ query is applied to the target of the symlink. including binaries of build-time-only dependencies) is obtained by distributing the closure of a store derivation and specifying the option . - + - + Prints the set of references of the store paths paths, that is, their immediate @@ -531,9 +531,9 @@ query is applied to the target of the symlink. .) - + - + Prints the set of referrers of the store paths paths, that is, the store paths currently existing in the Nix store that refer to one @@ -542,9 +542,9 @@ query is applied to the target of the symlink. store paths are added or removed. - + - + Prints the closure of the set of store paths paths under the referrers relation; that is, all store paths that directly or indirectly refer to one of @@ -555,7 +555,7 @@ query is applied to the target of the symlink. - + Prints the deriver of the store paths paths. If the path has no deriver @@ -566,7 +566,7 @@ query is applied to the target of the symlink. - + Prints the references graph of the store paths paths in the format of the dot tool of AT&T's - + Prints the references graph of the store paths paths as a nested ASCII tree. References are ordered by descending closure size; this tends to @@ -591,7 +591,7 @@ query is applied to the target of the symlink. name - + Prints the value of the attribute name (i.e., environment variable) of the store derivations paths. It is an @@ -601,7 +601,7 @@ query is applied to the target of the symlink. - + Prints the SHA-256 hash of the contents of the store paths paths (that is, the hash of the output of nix-store --dump on the given @@ -611,7 +611,7 @@ query is applied to the target of the symlink. - + Prints the size in bytes of the contents of the store paths paths — to be precise, the size of the output of nix-store --dump on the @@ -622,7 +622,7 @@ query is applied to the target of the symlink. - + Prints the garbage collector roots that point, directly or indirectly, at the store paths paths. @@ -638,7 +638,7 @@ query is applied to the target of the symlink. Print the closure (runtime dependencies) of the svn program in the current user environment: - + $ nix-store -qR $(which svn) /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 @@ -723,7 +723,7 @@ $ nix-store -q --roots $(which svn) - + @@ -739,11 +739,11 @@ $ nix-store -q --roots $(which svn) Description - + TODO - + --> @@ -798,7 +798,7 @@ $ nix-store --add ./foo.c Description - + The operation verifies the internal consistency of the Nix database, and the consistency between the Nix database and the Nix store. Any inconsistencies encountered are @@ -811,32 +811,32 @@ in Nix itself. - + Checks that the contents of every valid store path has not been altered by computing a SHA-256 hash of the contents and comparing it with the hash stored in the Nix database at build time. Paths that have been modified are printed out. For large stores, is obviously quite slow. - + - + - + If any valid path is missing from the store, or (if is given) the contents of a valid path has been modified, then try to repair the path by redownloading it. See nix-store --repair-path for details. - + - + - + @@ -855,7 +855,7 @@ in Nix itself. Description - + The operation compares the contents of the given store paths to their cryptographic hashes stored in Nix’s database. For every changed path, it prints a warning @@ -863,7 +863,7 @@ message. The exit status is 0 if no path has changed, and 1 otherwise. - + Example To verify the integrity of the svn command and all its dependencies: @@ -875,7 +875,7 @@ $ nix-store --verify-path $(nix-store -qR $(which svn)) - + @@ -893,7 +893,7 @@ $ nix-store --verify-path $(nix-store -qR $(which svn)) Description - + The operation attempts to “repair” the specified paths by redownloading them using the available substituters. If no substitutes are available, then repair is not @@ -906,7 +906,7 @@ system may be left in a broken state (e.g., if the path contains a critical system component like the GNU C Library). - + Example @@ -921,7 +921,7 @@ fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... - + @@ -939,7 +939,7 @@ fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... Description - + The operation produces a NAR (Nix ARchive) file containing the contents of the file system tree rooted at path. The archive is written to @@ -970,7 +970,7 @@ links, but not other types of files (such as device nodes). --restore. - + @@ -989,13 +989,13 @@ links, but not other types of files (such as device nodes). Description - + The operation unpacks a NAR archive to path, which must not already exist. The archive is read from standard input. - + @@ -1014,7 +1014,7 @@ archive is read from standard input. Description - + The operation writes a serialisation of the specified store paths to standard output in a format that can be imported into another Nix store with nix-copy-closure command. - + @@ -1059,7 +1059,7 @@ command. Description - + The operation reads a serialisation of a set of store paths produced by nix-store --export from @@ -1069,7 +1069,7 @@ another path that doesn’t exist in the Nix store, the import fails. - + @@ -1087,7 +1087,7 @@ fails. Description - + The operation reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by @@ -1104,7 +1104,7 @@ on the achieved savings is printed on standard error. progress indication. - + Example @@ -1138,7 +1138,7 @@ there are 114486 files with equal contents out of 215894 files in total Description - + The operation prints the build log of the specified store paths on standard output. The build log is whatever the builder of a derivation wrote to standard output and @@ -1147,12 +1147,17 @@ the store path is used. Build logs are kept in /nix/var/log/nix/drvs. However, there is no -guarantee that a build log is available for any particular store -path. For instance, if the path was downloaded as a pre-built binary -through a substitute, then the log is unavailable. +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through +a substitute, then the log is unavailable. If the log is not available +locally, then nix-store will try to download the +log from the servers specified in the Nix option +. For example, if it’s set to +http://hydra.nixos.org/log, then Nix will check +http://hydra.nixos.org/log/base-name. - + Example @@ -1184,14 +1189,14 @@ ktorrent-2.2.1/NEWS Description - + The operation writes a dump of the Nix database to standard output. It can be loaded into an empty Nix store using . This is useful for making backups and when migrating to different database schemas. - + @@ -1208,13 +1213,13 @@ backups and when migrating to different database schemas. Description - + The operation reads a dump of the Nix database created by from standard input and loads it into the Nix database. - + @@ -1232,14 +1237,14 @@ loads it into the Nix database. Description - + The operation prints out the environment of a derivation in a format that can be evaluated by a shell. The command line arguments of the builder are placed in the variable _args. - + Example @@ -1252,7 +1257,7 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil - + @@ -1269,14 +1274,14 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil Description - + If build failure caching is enabled through the build-cache-failures configuration option, the operation will print out all store paths that have failed to build. - + Example @@ -1288,7 +1293,7 @@ $ nix-store --query-failed-paths - + @@ -1306,7 +1311,7 @@ $ nix-store --query-failed-paths Description - + If build failure caching is enabled through the build-cache-failures configuration option, the operation clears the “failed” @@ -1319,7 +1324,7 @@ You can provide the argument * to clear all store paths. - + Example @@ -1328,7 +1333,7 @@ $ nix-store --clear-failed-paths * - + @@ -1341,6 +1346,6 @@ $ nix-store --clear-failed-paths * - + diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 739199d48..180344e33 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -147,6 +147,7 @@ void Settings::update() get(envKeepDerivations, "env-keep-derivations"); get(sshSubstituterHosts, "ssh-substituter-hosts"); get(useSshSubstituter, "use-ssh-substituter"); + get(logServers, "log-servers"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 711c36529..65a6c388b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -197,6 +197,9 @@ struct Settings { /* Whether to show a stack trace if Nix evaluation fails. */ bool showTrace; + /* A list of URL prefixes that can return Nix build logs. */ + Strings logServers; + private: SettingsMap settings, overrides; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 846674a29..8fc78b146 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -901,7 +901,7 @@ string runProgram(Path program, bool searchPath, const Strings & args) /* Wait for the child to finish. */ int status = pid.wait(true); if (!statusOk(status)) - throw Error(format("program `%1%' %2%") + throw ExecError(format("program `%1%' %2%") % program % statusToString(status)); return result; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ce2d77c19..1e9ffcf51 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -257,6 +257,8 @@ void killUser(uid_t uid); string runProgram(Path program, bool searchPath = false, const Strings & args = Strings()); +MakeError(ExecError, Error) + /* Close all file descriptors except stdin, stdout, stderr, and those listed in the given set. Good practice in child processes. */ void closeMostFDs(const set & exceptions); diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk index 7f93e4c19..dc049f348 100644 --- a/src/nix-store/local.mk +++ b/src/nix-store/local.mk @@ -7,3 +7,5 @@ nix-store_SOURCES := $(wildcard $(d)/*.cc) nix-store_LIBS = libmain libstore libutil libformat nix-store_LDFLAGS = -lbz2 + +nix-store_CXXFLAGS = -DCURL=\"$(curl)\" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5da401c77..4fee7258c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -467,10 +467,11 @@ static void opReadLog(Strings opFlags, Strings opArgs) foreach (Strings::iterator, i, opArgs) { Path path = useDeriver(followLinksToStorePath(*i)); - for (int j = 0; j <= 2; j++) { - if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path); + string baseName = baseNameOf(path); + bool found = false; + + for (int j = 0; j < 2; j++) { - string baseName = baseNameOf(path); Path logPath = j == 0 ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() @@ -481,6 +482,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) /* !!! Make this run in O(1) memory. */ string log = readFile(logPath); writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); + found = true; break; } @@ -500,9 +502,30 @@ static void opReadLog(Strings opFlags, Strings opArgs) writeFull(STDOUT_FILENO, buf, n); } while (err != BZ_STREAM_END); BZ2_bzReadClose(&err, bz); + found = true; break; } } + + if (!found) { + for (auto & i : settings.logServers) { + string prefix = i; + if (!prefix.empty() && prefix.back() != '/') prefix += '/'; + string url = prefix + baseName; + try { + string log = runProgram(CURL, true, {"--fail", "--location", "--silent", "--", url}); + std::cout << "(using build log from " << url << ")" << std::endl; + std::cout << log; + found = true; + break; + } catch (ExecError & e) { + /* Ignore errors from curl. FIXME: actually, might be + nice to print a warning on HTTP status != 404. */ + } + } + } + + if (!found) throw Error(format("build log of derivation `%1%' is not available") % path); } }