From 0f840483c731f48983832f7f627909f8463f05f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 May 2019 20:34:02 +0200 Subject: [PATCH 1/2] Add date of last commit to SourceInfo This is primarily useful for version string generation, where we need a monotonically increasing number. The revcount is the preferred thing to use, but isn't available for GitHub flakes (since it requires fetching the entire history). The last commit timestamp OTOH can be extracted from GitHub tarballs. --- release.nix | 3 ++- src/libexpr/primops/fetchGit.cc | 9 +++++++- src/libexpr/primops/fetchGit.hh | 1 + src/libexpr/primops/flake.cc | 39 ++++++++++++++++++--------------- src/libexpr/primops/flake.hh | 14 +++++++++++- src/nix/flake.cc | 24 ++++++++++++-------- tests/flakes.sh | 1 + 7 files changed, 61 insertions(+), 30 deletions(-) diff --git a/release.nix b/release.nix index f98e6d6ed..d28c44910 100644 --- a/release.nix +++ b/release.nix @@ -19,7 +19,8 @@ let releaseTools.sourceTarball { name = "nix-tarball"; version = builtins.readFile ./.version; - versionSuffix = if officialRelease then "" else "pre${toString nix.revCount or 0}_${nix.shortRev or "0000000"}"; + versionSuffix = if officialRelease then "" else + "pre${if nix ? lastModified then builtins.substring 0 8 nix.lastModified else toString nix.revCount or 0}_${nix.shortRev or "0000000"}"; src = nix; inherit officialRelease; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index f6b096c4a..10f6b6f72 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -69,6 +69,9 @@ GitInfo exportGit(ref store, std::string uri, gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })); + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" })); return gitInfo; } @@ -85,8 +88,9 @@ GitInfo exportGit(ref store, std::string uri, } deletePath(getCacheDir() + "/nix/git"); + deletePath(getCacheDir() + "/nix/gitv2"); - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false); Path repoDir; if (isLocal) { @@ -181,6 +185,7 @@ GitInfo exportGit(ref store, std::string uri, if (store->isValidPath(storePath)) { gitInfo.storePath = storePath; gitInfo.revCount = json["revCount"]; + gitInfo.lastModified = json["lastModified"]; return gitInfo; } @@ -200,6 +205,7 @@ GitInfo exportGit(ref store, std::string uri, gitInfo.storePath = store->addToStore(name, tmpDir); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); + gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() })); nlohmann::json json; json["storePath"] = gitInfo.storePath; @@ -207,6 +213,7 @@ GitInfo exportGit(ref store, std::string uri, json["name"] = name; json["rev"] = gitInfo.rev.gitRev(); json["revCount"] = gitInfo.revCount; + json["lastModified"] = gitInfo.lastModified; writeFile(storeLink, json.dump()); diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 2ad6a5e5c..006fa8b5f 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -12,6 +12,7 @@ struct GitInfo std::string ref; Hash rev{htSHA1}; uint64_t revCount; + time_t lastModified; }; GitInfo exportGit(ref store, std::string uri, diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 162e5c915..7cbbf9e99 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include namespace nix { @@ -232,6 +234,18 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + auto doGit = [&](const GitInfo & gitInfo) { + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; + SourceInfo info(ref); + info.storePath = gitInfo.storePath; + info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = gitInfo.lastModified; + return info; + }; + // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&resolvedRef.data)) { @@ -270,29 +284,13 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool // This downloads the entire git history else if (auto refData = std::get_if(&resolvedRef.data)) { - auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); - FlakeRef ref(resolvedRef.baseRef()); - ref.ref = gitInfo.ref; - ref.rev = gitInfo.rev; - SourceInfo info(ref); - info.storePath = gitInfo.storePath; - info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - return info; + return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); } else if (auto refData = std::get_if(&resolvedRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); - auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); - FlakeRef ref(resolvedRef.baseRef()); - ref.ref = gitInfo.ref; - ref.rev = gitInfo.rev; - SourceInfo info(ref); - info.storePath = gitInfo.storePath; - info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - return info; + return doGit(exportGit(state.store, refData->path, {}, {}, "source")); } else abort(); @@ -529,6 +527,11 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo if (sourceInfo.revCount) mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); + + if (sourceInfo.lastModified) + mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), + fmt("%s", + std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); } void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index a26103736..0e2706e32 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -81,10 +81,22 @@ void writeRegistry(const FlakeRegistry &, const Path &); struct SourceInfo { + // Immutable flakeref that this source tree was obtained from. FlakeRef resolvedRef; + Path storePath; + + // Number of ancestors of the most recent commit. std::optional revCount; - Hash narHash; // store path hash + + // NAR hash of the store path. + Hash narHash; + + // A stable timestamp of this source tree. For Git and GitHub + // flakes, the commit date (not author date!) of the most recent + // commit. + std::optional lastModified; + SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d8c422d3d..7836f0cfe 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include #include +#include using namespace nix; @@ -72,14 +73,17 @@ struct CmdFlakeList : EvalCommand static void printSourceInfo(const SourceInfo & sourceInfo) { - std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); + std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); if (sourceInfo.resolvedRef.ref) - std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); + std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); if (sourceInfo.resolvedRef.rev) - std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); + std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); if (sourceInfo.revCount) - std::cout << fmt("Revcount: %s\n", *sourceInfo.revCount); - std::cout << fmt("Path: %s\n", sourceInfo.storePath); + std::cout << fmt("Revisions: %s\n", *sourceInfo.revCount); + if (sourceInfo.lastModified) + std::cout << fmt("Last modified: %s\n", + std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T")); + std::cout << fmt("Path: %s\n", sourceInfo.storePath); } static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) @@ -91,14 +95,16 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false); if (sourceInfo.revCount) j["revCount"] = *sourceInfo.revCount; + if (sourceInfo.lastModified) + j["lastModified"] = *sourceInfo.lastModified; j["path"] = sourceInfo.storePath; } static void printFlakeInfo(const Flake & flake) { - std::cout << fmt("ID: %s\n", flake.id); - std::cout << fmt("Description: %s\n", flake.description); - std::cout << fmt("Epoch: %s\n", flake.epoch); + std::cout << fmt("ID: %s\n", flake.id); + std::cout << fmt("Description: %s\n", flake.description); + std::cout << fmt("Epoch: %s\n", flake.epoch); printSourceInfo(flake.sourceInfo); } @@ -114,7 +120,7 @@ static nlohmann::json flakeToJson(const Flake & flake) static void printNonFlakeInfo(const NonFlake & nonFlake) { - std::cout << fmt("ID: %s\n", nonFlake.alias); + std::cout << fmt("ID: %s\n", nonFlake.alias); printSourceInfo(nonFlake.sourceInfo); } diff --git a/tests/flakes.sh b/tests/flakes.sh index 6081e8939..d95d34c76 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -124,6 +124,7 @@ nix flake info --flake-registry $registry $flake1Dir | grep -q 'ID: *flake1' json=$(nix flake info --flake-registry $registry flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] [[ -d $(echo "$json" | jq -r .path) ]] +[[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] # Test 'nix build' on a flake. nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo From ae7b56cd9a5ed8810828736fbb930a7c14ea44ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 May 2019 22:35:41 +0200 Subject: [PATCH 2/2] Get last commit time of github flakes --- src/libexpr/primops/flake.cc | 2 ++ src/libstore/download.cc | 17 ++++++++++++++--- src/libstore/download.hh | 2 ++ src/libutil/util.cc | 18 +++++++++++++++--- src/libutil/util.hh | 6 ++++-- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 7cbbf9e99..257b81887 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -265,6 +265,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool request.unpack = true; request.name = "source"; request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; auto result = getDownloader()->downloadCached(state.store, request); if (!result.etag) @@ -278,6 +279,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool SourceInfo info(ref); info.storePath = result.storePath; info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = result.lastModified; return info; } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0d1974d3b..0338727c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -808,6 +808,7 @@ CachedDownloadResult Downloader::downloadCached( CachedDownloadResult result; result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); + assert(!request.getLastModified); // FIXME return result; } } @@ -892,16 +893,26 @@ CachedDownloadResult Downloader::downloadCached( store->addTempRoot(unpackedStorePath); if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = ""; + else + result.lastModified = lstat(unpackedLink).st_mtime; } if (unpackedStorePath.empty()) { printInfo(format("unpacking '%1%'...") % url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir}); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + result.lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } - replaceSymlink(unpackedStorePath, unpackedLink); + // Store the last-modified date of the tarball in the symlink + // mtime. This saves us from having to store it somewhere + // else. + replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified); storePath = unpackedStorePath; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 404e51195..43b1c5c09 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -49,6 +49,7 @@ struct CachedDownloadRequest Hash expectedHash; unsigned int ttl = settings.tarballTtl; bool gcRoot = false; + bool getLastModified = false; CachedDownloadRequest(const std::string & uri) : uri(uri) { } @@ -62,6 +63,7 @@ struct CachedDownloadResult Path path; std::optional etag; std::string effectiveUri; + std::optional lastModified; }; class Store; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f82f902fc..92c8957ff 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #ifdef __APPLE__ @@ -552,20 +553,31 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const Path & target, const Path & link, + std::optional mtime) { if (symlink(target.c_str(), link.c_str())) throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp); + createSymlink(target, tmp, mtime); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 35f9169f6..e05ef1e7d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -142,10 +142,12 @@ Path getDataDir(); Paths createDirs(const Path & path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path & target, const Path & link, + std::optional mtime = {}); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime = {}); /* Wrappers arount read()/write() that read/write exactly the