Merge pull request #2898 from NixOS/last-modified

Expose lastModified attribute

release.nix

@@ -19,7 +19,8 @@ let
       releaseTools.sourceTarball {
         name = "nix-tarball";
         version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount or 0}_${nix.shortRev or "0000000"}";
+        versionSuffix = if officialRelease then "" else
+          "pre${if nix ? lastModified then builtins.substring 0 8 nix.lastModified else toString nix.revCount or 0}_${nix.shortRev or "0000000"}";
         src = nix;
         inherit officialRelease;
 

src/libexpr/primops/fetchGit.cc

@@ -69,6 +69,9 @@ GitInfo exportGit(ref<Store> store, std::string uri,
 
             gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
             gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" }));
+            // FIXME: maybe we should use the timestamp of the last
+            // modified dirty file?
+            gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" }));
 
             return gitInfo;
         }
@@ -85,8 +88,9 @@ GitInfo exportGit(ref<Store> store, std::string uri,
     }
 
     deletePath(getCacheDir() + "/nix/git");
+    deletePath(getCacheDir() + "/nix/gitv2");
 
-    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
+    Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false);
     Path repoDir;
 
     if (isLocal) {
@@ -181,6 +185,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
         if (store->isValidPath(storePath)) {
             gitInfo.storePath = storePath;
             gitInfo.revCount = json["revCount"];
+            gitInfo.lastModified = json["lastModified"];
             return gitInfo;
         }
 
@@ -200,6 +205,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
     gitInfo.storePath = store->addToStore(name, tmpDir);
 
     gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() }));
+    gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() }));
 
     nlohmann::json json;
     json["storePath"] = gitInfo.storePath;
@@ -207,6 +213,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
     json["name"] = name;
     json["rev"] = gitInfo.rev.gitRev();
     json["revCount"] = gitInfo.revCount;
+    json["lastModified"] = gitInfo.lastModified;
 
     writeFile(storeLink, json.dump());
 

src/libexpr/primops/fetchGit.hh

@@ -12,6 +12,7 @@ struct GitInfo
     std::string ref;
     Hash rev{htSHA1};
     uint64_t revCount;
+    time_t lastModified;
 };
 
 GitInfo exportGit(ref<Store> store, std::string uri,

src/libexpr/primops/flake.cc

@@ -8,6 +8,8 @@
 #include <iostream>
 #include <queue>
 #include <regex>
+#include <ctime>
+#include <iomanip>
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -232,6 +234,18 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
     if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable())
         throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef);
 
+    auto doGit = [&](const GitInfo & gitInfo) {
+        FlakeRef ref(resolvedRef.baseRef());
+        ref.ref = gitInfo.ref;
+        ref.rev = gitInfo.rev;
+        SourceInfo info(ref);
+        info.storePath = gitInfo.storePath;
+        info.revCount = gitInfo.revCount;
+        info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
+        info.lastModified = gitInfo.lastModified;
+        return info;
+    };
+
     // This only downloads only one revision of the repo, not the entire history.
     if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
 
@@ -251,6 +265,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
         request.unpack = true;
         request.name = "source";
         request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl;
+        request.getLastModified = true;
         auto result = getDownloader()->downloadCached(state.store, request);
 
         if (!result.etag)
@@ -264,35 +279,20 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
         SourceInfo info(ref);
         info.storePath = result.storePath;
         info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
+        info.lastModified = result.lastModified;
 
         return info;
     }
 
     // This downloads the entire git history
     else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) {
-        auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source");
+        return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"));
-        FlakeRef ref(resolvedRef.baseRef());
-        ref.ref = gitInfo.ref;
-        ref.rev = gitInfo.rev;
-        SourceInfo info(ref);
-        info.storePath = gitInfo.storePath;
-        info.revCount = gitInfo.revCount;
-        info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
-        return info;
     }
 
     else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) {
         if (!pathExists(refData->path + "/.git"))
             throw Error("flake '%s' does not reference a Git repository", refData->path);
-        auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source");
+        return doGit(exportGit(state.store, refData->path, {}, {}, "source"));
-        FlakeRef ref(resolvedRef.baseRef());
-        ref.ref = gitInfo.ref;
-        ref.rev = gitInfo.rev;
-        SourceInfo info(ref);
-        info.storePath = gitInfo.storePath;
-        info.revCount = gitInfo.revCount;
-        info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
-        return info;
     }
 
     else abort();
@@ -529,6 +529,11 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo
 
     if (sourceInfo.revCount)
         mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount);
+
+    if (sourceInfo.lastModified)
+        mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")),
+            fmt("%s",
+                std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S")));
 }
 
 void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)

src/libexpr/primops/flake.hh

@@ -81,10 +81,22 @@ void writeRegistry(const FlakeRegistry &, const Path &);
 
 struct SourceInfo
 {
+    // Immutable flakeref that this source tree was obtained from.
     FlakeRef resolvedRef;
+
     Path storePath;
+
+    // Number of ancestors of the most recent commit.
     std::optional<uint64_t> revCount;
-    Hash narHash; // store path hash
+
+    // NAR hash of the store path.
+    Hash narHash;
+
+    // A stable timestamp of this source tree. For Git and GitHub
+    // flakes, the commit date (not author date!) of the most recent
+    // commit.
+    std::optional<time_t> lastModified;
+
     SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {};
 };
 

src/libstore/download.cc

@@ -808,6 +808,7 @@ CachedDownloadResult Downloader::downloadCached(
             CachedDownloadResult result;
             result.storePath = expectedStorePath;
             result.path = store->toRealPath(expectedStorePath);
+            assert(!request.getLastModified); // FIXME
             return result;
         }
     }
@@ -892,16 +893,26 @@ CachedDownloadResult Downloader::downloadCached(
             store->addTempRoot(unpackedStorePath);
             if (!store->isValidPath(unpackedStorePath))
                 unpackedStorePath = "";
+            else
+                result.lastModified = lstat(unpackedLink).st_mtime;
         }
         if (unpackedStorePath.empty()) {
             printInfo(format("unpacking '%1%'...") % url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
             // FIXME: this requires GNU tar for decompression.
-            runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
+            runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir});
-            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
+            auto members = readDirectory(tmpDir);
+            if (members.size() != 1)
+                throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
+            auto topDir = tmpDir + "/" + members.begin()->name;
+            result.lastModified = lstat(topDir).st_mtime;
+            unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
-        replaceSymlink(unpackedStorePath, unpackedLink);
+        // Store the last-modified date of the tarball in the symlink
+        // mtime. This saves us from having to store it somewhere
+        // else.
+        replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified);
         storePath = unpackedStorePath;
     }
 

src/libstore/download.hh

@@ -49,6 +49,7 @@ struct CachedDownloadRequest
     Hash expectedHash;
     unsigned int ttl = settings.tarballTtl;
     bool gcRoot = false;
+    bool getLastModified = false;
 
     CachedDownloadRequest(const std::string & uri)
         : uri(uri) { }
@@ -62,6 +63,7 @@ struct CachedDownloadResult
     Path path;
     std::optional<std::string> etag;
     std::string effectiveUri;
+    std::optional<time_t> lastModified;
 };
 
 class Store;

src/libutil/util.cc

@@ -22,6 +22,7 @@
 #include <sys/ioctl.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <sys/time.h>
 #include <unistd.h>
 
 #ifdef __APPLE__
@@ -552,20 +553,31 @@ Paths createDirs(const Path & path)
 }
 
 
-void createSymlink(const Path & target, const Path & link)
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
 {
     if (symlink(target.c_str(), link.c_str()))
         throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target);
+    if (mtime) {
+        struct timeval times[2];
+        times[0].tv_sec = *mtime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = *mtime;
+        times[1].tv_usec = 0;
+        if (lutimes(link.c_str(), times))
+            throw SysError("setting time of symlink '%s'", link);
+    }
 }
 
 
-void replaceSymlink(const Path & target, const Path & link)
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
 {
     for (unsigned int n = 0; true; n++) {
         Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
 
         try {
-            createSymlink(target, tmp);
+            createSymlink(target, tmp, mtime);
         } catch (SysError & e) {
             if (e.errNo == EEXIST) continue;
             throw;

src/libutil/util.hh

@@ -142,10 +142,12 @@ Path getDataDir();
 Paths createDirs(const Path & path);
 
 /* Create a symlink. */
-void createSymlink(const Path & target, const Path & link);
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime = {});
 
 /* Atomically create or replace a symlink. */
-void replaceSymlink(const Path & target, const Path & link);
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime = {});
 
 
 /* Wrappers arount read()/write() that read/write exactly the

src/nix/flake.cc

@@ -7,6 +7,7 @@
 
 #include <nlohmann/json.hpp>
 #include <queue>
+#include <iomanip>
 
 using namespace nix;
 
@@ -78,7 +79,10 @@ static void printSourceInfo(const SourceInfo & sourceInfo)
     if (sourceInfo.resolvedRef.rev)
         std::cout << fmt("Revision:      %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false));
     if (sourceInfo.revCount)
-        std::cout << fmt("Revcount:    %s\n", *sourceInfo.revCount);
+        std::cout << fmt("Revisions:     %s\n", *sourceInfo.revCount);
+    if (sourceInfo.lastModified)
+        std::cout << fmt("Last modified: %s\n",
+            std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T"));
     std::cout << fmt("Path:          %s\n", sourceInfo.storePath);
 }
 
@@ -91,6 +95,8 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j)
         j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false);
     if (sourceInfo.revCount)
         j["revCount"] = *sourceInfo.revCount;
+    if (sourceInfo.lastModified)
+        j["lastModified"] = *sourceInfo.lastModified;
     j["path"] = sourceInfo.storePath;
 }
 

tests/flakes.sh

@@ -124,6 +124,7 @@ nix flake info --flake-registry $registry $flake1Dir | grep -q 'ID: *flake1'
 json=$(nix flake info --flake-registry $registry flake1 --json | jq .)
 [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
 [[ -d $(echo "$json" | jq -r .path) ]]
+[[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]]
 
 # Test 'nix build' on a flake.
 nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo