From 529add316c5356a8060c35f987643b7bf5c796dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2019 23:20:50 +0800 Subject: [PATCH] downloadCached: Return ETag This allows fetchFlake() to return the Git revision of a GitHub archive. --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/flake.cc | 49 +++++++++++++++++++++++---------- src/libstore/download.cc | 17 +++++++++--- src/libstore/download.hh | 12 ++++++-- src/nix-channel/nix-channel.cc | 6 ++-- 7 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..37c74a94b 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); + return getDownloader()->downloadCached(state.store, s, true).path; else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index cbd576d7d..0f1ac05d6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -657,7 +657,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f787ad96b..7e3dd23ed 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2112,7 +2112,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index a788e935e..f9a1a3d6a 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -87,7 +87,13 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, return flakeRef; } -static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) +struct FlakeSourceInfo +{ + Path storePath; + std::optional rev; +}; + +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) { assert(flakeRef.isDirect()); @@ -99,25 +105,36 @@ static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) // FIXME: support passing auth tokens for private repos. - auto storePath = getDownloader()->downloadCached(state.store, - fmt("https://api.github.com/repos/%s/%s/tarball/%s", - refData->owner, refData->repo, - refData->rev - ? refData->rev->to_string(Base16, false) - : refData->ref - ? *refData->ref - : "master"), - true, "source"); + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + refData->rev + ? refData->rev->to_string(Base16, false) + : refData->ref + ? *refData->ref + : "master"); - // FIXME: extract revision hash from ETag. + auto result = getDownloader()->downloadCached(state.store, url, true, "source"); - return storePath; + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + FlakeSourceInfo info; + info.storePath = result.path; + info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + + return info; } else if (auto refData = std::get_if(&flakeRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, refData->ref, refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); - return gitInfo.storePath; + FlakeSourceInfo info; + info.storePath = gitInfo.storePath; + info.rev = Hash(gitInfo.rev, htSHA1); + return info; } else abort(); @@ -138,7 +155,11 @@ struct Flake static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { - auto flakePath = fetchFlake(state, flakeRef); + auto sourceInfo = fetchFlake(state, flakeRef); + debug("got flake source '%s' with revision %s", + sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + + auto flakePath = sourceInfo.storePath; state.store->assertStorePath(flakePath); if (state.allowedPaths) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bb..360d48b09 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -771,7 +771,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); @@ -783,8 +783,11 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa Path expectedStorePath; if (expectedHash) { expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -803,6 +806,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -814,6 +819,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa skip = true; if (effectiveUrl) *effectiveUrl = url_; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -831,6 +837,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa auto res = download(request); if (effectiveUrl) *effectiveUrl = res.effectiveUrl; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; @@ -852,6 +859,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa } catch (DownloadError & e) { if (storePath.empty()) throw; printError(format("warning: %1%; using cached result") % e.msg()); + result.etag = expectedETag; } } @@ -885,7 +893,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa url, expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..8acfe4e1a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -41,6 +41,12 @@ struct DownloadResult uint64_t bodySize = 0; }; +struct CachedDownloadResult +{ + Path path; + std::optional etag; +}; + class Store; struct Downloader @@ -64,8 +70,10 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached( + ref store, const string & uri, bool unpack, string name = "", + const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, + int ttl = settings.tarballTtl); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e3..7b23088a2 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -88,7 +88,7 @@ static void update(const StringSet & channelNames) // definition from a consistent location if the redirect changes mid-download. std::string effectiveUrl; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); + auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; url = chomp(std::move(effectiveUrl)); // If the URL contains a version number, append it to the name @@ -123,10 +123,10 @@ static void update(const StringSet & channelNames) // Download the channel tarball. auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } catch (DownloadError & e) { fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } chomp(filename); }