downloadCached: Return ETag

This allows fetchFlake() to return the Git revision of a GitHub
archive.

src/libexpr/common-eval-args.cc

@@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s))
-        return getDownloader()->downloadCached(state.store, s, true);
+        return getDownloader()->downloadCached(state.store, s, true).path;
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/parser.y

@@ -657,7 +657,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            res = { true, getDownloader()->downloadCached(store, elem.second, true) };
+            res = { true, getDownloader()->downloadCached(store, elem.second, true).path };
         } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };

src/libexpr/primops.cc

@@ -2112,7 +2112,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
-    Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
+    Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path;
 
     if (state.allowedPaths)
         state.allowedPaths->insert(res);

src/libexpr/primops/flake.cc

@@ -87,7 +87,13 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef,
         return flakeRef;
 }
 
-static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef)
+struct FlakeSourceInfo
+{
+    Path storePath;
+    std::optional<Hash> rev;
+};
+
+static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
 {
     assert(flakeRef.isDirect());
 
@@ -99,25 +105,36 @@ static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef)
 
         // FIXME: support passing auth tokens for private repos.
 
-        auto storePath = getDownloader()->downloadCached(state.store,
+        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
-            fmt("https://api.github.com/repos/%s/%s/tarball/%s",
+            refData->owner, refData->repo,
-                refData->owner, refData->repo,
+            refData->rev
-                refData->rev
+                ? refData->rev->to_string(Base16, false)
-                  ? refData->rev->to_string(Base16, false)
+                : refData->ref
-                  : refData->ref
+                  ? *refData->ref
-                    ? *refData->ref
+                  : "master");
-                    : "master"),
-            true, "source");
 
-        // FIXME: extract revision hash from ETag.
+        auto result = getDownloader()->downloadCached(state.store, url, true, "source");
 
-        return storePath;
+        if (!result.etag)
+            throw Error("did not receive an ETag header from '%s'", url);
+
+        if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
+            throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
+
+        FlakeSourceInfo info;
+        info.storePath = result.path;
+        info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
+
+        return info;
     }
 
     else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) {
         auto gitInfo = exportGit(state.store, refData->uri, refData->ref,
             refData->rev ? refData->rev->to_string(Base16, false) : "", "source");
-        return gitInfo.storePath;
+        FlakeSourceInfo info;
+        info.storePath = gitInfo.storePath;
+        info.rev = Hash(gitInfo.rev, htSHA1);
+        return info;
     }
 
     else abort();
@@ -138,7 +155,11 @@ struct Flake
 
 static Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
 {
-    auto flakePath = fetchFlake(state, flakeRef);
+    auto sourceInfo = fetchFlake(state, flakeRef);
+    debug("got flake source '%s' with revision %s",
+        sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
+
+    auto flakePath = sourceInfo.storePath;
     state.store->assertStorePath(flakePath);
 
     if (state.allowedPaths)

src/libstore/download.cc

@@ -771,7 +771,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
+CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
 {
     auto url = resolveUri(url_);
 
@@ -783,8 +783,11 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     Path expectedStorePath;
     if (expectedHash) {
         expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
-        if (store->isValidPath(expectedStorePath))
+        if (store->isValidPath(expectedStorePath)) {
-            return store->toRealPath(expectedStorePath);
+            CachedDownloadResult result;
+            result.path = store->toRealPath(expectedStorePath);
+            return result;
+        }
     }
 
     Path cacheDir = getCacheDir() + "/nix/tarballs";
@@ -803,6 +806,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     bool skip = false;
 
+    CachedDownloadResult result;
+
     if (pathExists(fileLink) && pathExists(dataFile)) {
         storePath = readLink(fileLink);
         store->addTempRoot(storePath);
@@ -814,6 +819,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                     skip = true;
                     if (effectiveUrl)
                         *effectiveUrl = url_;
+                    result.etag = ss[1];
                 } else if (!ss[1].empty()) {
                     debug(format("verifying previous ETag '%1%'") % ss[1]);
                     expectedETag = ss[1];
@@ -831,6 +837,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             auto res = download(request);
             if (effectiveUrl)
                 *effectiveUrl = res.effectiveUrl;
+            result.etag = res.etag;
 
             if (!res.cached) {
                 ValidPathInfo info;
@@ -852,6 +859,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         } catch (DownloadError & e) {
             if (storePath.empty()) throw;
             printError(format("warning: %1%; using cached result") % e.msg());
+            result.etag = expectedETag;
         }
     }
 
@@ -885,7 +893,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             url, expectedHash.to_string(), gotHash.to_string());
     }
 
-    return store->toRealPath(storePath);
+    result.path = store->toRealPath(storePath);
+    return result;
 }
 
 

src/libstore/download.hh

@@ -41,6 +41,12 @@ struct DownloadResult
     uint64_t bodySize = 0;
 };
 
+struct CachedDownloadResult
+{
+    Path path;
+    std::optional<std::string> etag;
+};
+
 class Store;
 
 struct Downloader
@@ -64,8 +70,10 @@ struct Downloader
        and is more recent than ‘tarball-ttl’ seconds. Otherwise,
        use the recorded ETag to verify if the server has a more
        recent version, and if so, download it to the Nix store. */
-    Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "",
+    CachedDownloadResult downloadCached(
-        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl);
+        ref<Store> store, const string & uri, bool unpack, string name = "",
+        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr,
+        int ttl = settings.tarballTtl);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };

src/nix-channel/nix-channel.cc

@@ -88,7 +88,7 @@ static void update(const StringSet & channelNames)
         // definition from a consistent location if the redirect changes mid-download.
         std::string effectiveUrl;
         auto dl = getDownloader();
-        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0);
+        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path;
         url = chomp(std::move(effectiveUrl));
 
         // If the URL contains a version number, append it to the name
@@ -123,10 +123,10 @@ static void update(const StringSet & channelNames)
             // Download the channel tarball.
             auto fullURL = url + "/nixexprs.tar.xz";
             try {
-                filename = dl->downloadCached(store, fullURL, false);
+                filename = dl->downloadCached(store, fullURL, false).path;
             } catch (DownloadError & e) {
                 fullURL = url + "/nixexprs.tar.bz2";
-                filename = dl->downloadCached(store, fullURL, false);
+                filename = dl->downloadCached(store, fullURL, false).path;
             }
             chomp(filename);
         }