From f6ddf48882a068dbf1b1bfff44d487316972d6e9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Mar 2020 17:23:56 +0100 Subject: [PATCH] Get rid of downloadCached() Everything uses the generic caching system now. --- src/libexpr/common-eval-args.cc | 8 +- src/libexpr/parser.y | 6 +- src/libexpr/primops.cc | 67 -------------- src/libexpr/primops/fetchTree.cc | 77 ++++++++++++++++ src/libstore/download.cc | 146 ------------------------------ src/libstore/download.hh | 34 +------ src/libstore/fetchers/fetchers.hh | 1 + src/libstore/fetchers/tarball.cc | 17 +++- src/nix-channel/nix-channel.cc | 17 ++-- 9 files changed, 113 insertions(+), 260 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 59a2b9c89..9ebb6a835 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -4,7 +4,9 @@ #include "util.hh" #include "eval.hh" #include "fetchers/registry.hh" +#include "fetchers/fetchers.hh" #include "flake/flakeref.hh" +#include "store-api.hh" namespace nix { @@ -68,9 +70,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) { - CachedDownloadRequest request(s); - request.unpack = true; - return getDownloader()->downloadCached(state.store, request).path; + return state.store->toRealPath( + fetchers::downloadTarball( + state.store, resolveUri(s), "source", false).storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..dd5436c04 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -545,6 +545,7 @@ formal #include "eval.hh" #include "download.hh" +#include "fetchers/fetchers.hh" #include "store-api.hh" @@ -687,9 +688,8 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - CachedDownloadRequest request(elem.second); - request.unpack = true; - res = { true, getDownloader()->downloadCached(store, request).path }; + res = { true, store->toRealPath(fetchers::downloadTarball( + store, resolveUri(elem.second), "source", false).storePath) }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index b0f8b4414..fb7d5497b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,6 +1,5 @@ #include "archive.hh" #include "derivations.hh" -#include "download.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -2046,68 +2045,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args } -/************************************************************* - * Networking - *************************************************************/ - - -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, const std::string & defaultName) -{ - CachedDownloadRequest request(""); - request.unpack = unpack; - request.name = defaultName; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") - request.name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); - } - - if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); - - } else - request.uri = state.forceStringNoCtx(*args[0], pos); - - state.checkURI(request.uri); - - if (evalSettings.pureEval && !request.expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - - auto res = getDownloader()->downloadCached(state.store, request); - - if (state.allowedPaths) - state.allowedPaths->insert(res.path); - - mkString(v, res.storePath, PathSet({res.storePath})); -} - - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchurl", false, ""); -} - - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true, "source"); -} - - /************************************************************* * Primop registration *************************************************************/ @@ -2290,10 +2227,6 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); addPrimOp("placeholder", 1, prim_placeholder); - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 2b3f1b76d..8e8b48fc8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "fetchers/fetchers.hh" #include "fetchers/registry.hh" +#include "download.hh" #include #include @@ -89,4 +90,80 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional url; + std::optional expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to '%s', at %s", + attr.name, who, attr.pos); + } + + if (!url) + throw EvalError("'url' argument required, at %s", pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(), hash.to_string()); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8e0d7d42a..35a35cd78 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -35,10 +35,6 @@ DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); -CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) - : uri(uri), ttl(settings.tarballTtl) -{ } - std::string resolveUri(const std::string & uri) { if (uri.compare(0, 8, "channel:") == 0) @@ -801,148 +797,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached( - ref store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - std::optional expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(*expectedStorePath)) { - CachedDownloadResult result; - result.storePath = store->printStorePath(*expectedStorePath); - result.path = store->toRealPath(result.storePath); - assert(!request.getLastModified); // FIXME - return result; - } - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - - std::optional storePath; - - string expectedETag; - - bool skip = false; - - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = store->parseStorePath(readLink(fileLink)); - // FIXME - store->addTempRoot(*storePath); - if (store->isValidPath(*storePath)) { - auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath.reset(); - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path.clone(); - } - - assert(storePath); - replaceSymlink(store->printStorePath(*storePath), fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (!storePath) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; - } - } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - std::optional unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); - store->addTempRoot(*unpackedStorePath); - if (!store->isValidPath(*unpackedStorePath)) - unpackedStorePath.reset(); - else - result.lastModified = lstat(unpackedLink).st_mtime; - } - if (!unpackedStorePath) { - printInfo("unpacking '%s'...", url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(*storePath), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - result.lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - // Store the last-modified date of the tarball in the symlink - // mtime. This saves us from having to store it somewhere - // else. - replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified); - storePath = std::move(*unpackedStorePath); - } - - if (expectedStorePath && *storePath != *expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(*storePath)).first - : hashFile(request.expectedHash.type, store->toRealPath(*storePath)); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); - } - - if (request.gcRoot) - store->addIndirectRoot(fileLink); - - result.storePath = store->printStorePath(*storePath); - result.path = store->toRealPath(result.storePath); - return result; -} - bool isUri(const string & s) { diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 487036833..28c8a9162 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -65,31 +65,6 @@ struct DownloadResult uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl; - bool gcRoot = false; - bool getLastModified = false; - - CachedDownloadRequest(const std::string & uri); - CachedDownloadRequest() = delete; -}; - -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional etag; - std::string effectiveUri; - std::optional lastModified; -}; - class Store; struct Downloader @@ -111,12 +86,6 @@ struct Downloader invoked on the thread of the caller. */ void download(DownloadRequest && request, Sink & sink); - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; @@ -138,4 +107,7 @@ public: bool isUri(const string & s); +/* Resolve deprecated 'channel:' URLs. */ +std::string resolveUri(const std::string & uri); + } diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 9c931dfa1..deb1c9c59 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -97,6 +97,7 @@ struct DownloadFileResult { StorePath storePath; std::string etag; + std::string effectiveUrl; }; DownloadFileResult downloadFile( diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 53971ec2b..54b59babc 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -28,7 +28,8 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return { .storePath = std::move(cached->storePath), - .etag = getStrAttr(cached->infoAttrs, "etag") + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") }; DownloadRequest request(url); @@ -40,6 +41,7 @@ DownloadFileResult downloadFile( Attrs infoAttrs({ {"etag", res.etag}, + {"url", res.effectiveUri}, }); std::optional storePath; @@ -67,9 +69,22 @@ DownloadFileResult downloadFile( *storePath, immutable); + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + return { .storePath = std::move(*storePath), .etag = res.etag, + .effectiveUrl = res.effectiveUri, }; } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 1b337a712..8b0937c53 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -3,6 +3,7 @@ #include "download.hh" #include "store-api.hh" #include "legacy.hh" +#include "fetchers/fetchers.hh" #include #include @@ -86,12 +87,9 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - CachedDownloadRequest request(url); - request.ttl = 0; - auto dl = getDownloader(); - auto result = dl->downloadCached(store, request); - auto filename = result.path; - url = chomp(result.effectiveUri); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto filename = store->toRealPath(result.storePath); + url = result.effectiveUrl; // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -114,11 +112,10 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); } catch (DownloadError & e) { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } - chomp(filename); } // Regardless of where it came from, add the expression representing this channel to accumulated expression @@ -185,6 +182,8 @@ static int _main(int argc, char ** argv) } else if (*arg == "--rollback") { cmd = cRollback; } else { + if (hasPrefix(*arg, "-")) + throw UsageError("unsupported argument '%s'", *arg); args.push_back(std::move(*arg)); } return true;