Merge pull request #2898 from NixOS/last-modified

Expose lastModified attribute
This commit is contained in:
Eelco Dolstra 2019-05-29 10:14:40 +02:00 committed by GitHub
commit 315f1980ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 98 additions and 38 deletions

View file

@ -19,7 +19,8 @@ let
releaseTools.sourceTarball { releaseTools.sourceTarball {
name = "nix-tarball"; name = "nix-tarball";
version = builtins.readFile ./.version; version = builtins.readFile ./.version;
versionSuffix = if officialRelease then "" else "pre${toString nix.revCount or 0}_${nix.shortRev or "0000000"}"; versionSuffix = if officialRelease then "" else
"pre${if nix ? lastModified then builtins.substring 0 8 nix.lastModified else toString nix.revCount or 0}_${nix.shortRev or "0000000"}";
src = nix; src = nix;
inherit officialRelease; inherit officialRelease;

View file

@ -69,6 +69,9 @@ GitInfo exportGit(ref<Store> store, std::string uri,
gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" }));
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" }));
return gitInfo; return gitInfo;
} }
@ -85,8 +88,9 @@ GitInfo exportGit(ref<Store> store, std::string uri,
} }
deletePath(getCacheDir() + "/nix/git"); deletePath(getCacheDir() + "/nix/git");
deletePath(getCacheDir() + "/nix/gitv2");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false);
Path repoDir; Path repoDir;
if (isLocal) { if (isLocal) {
@ -181,6 +185,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
if (store->isValidPath(storePath)) { if (store->isValidPath(storePath)) {
gitInfo.storePath = storePath; gitInfo.storePath = storePath;
gitInfo.revCount = json["revCount"]; gitInfo.revCount = json["revCount"];
gitInfo.lastModified = json["lastModified"];
return gitInfo; return gitInfo;
} }
@ -200,6 +205,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
gitInfo.storePath = store->addToStore(name, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() }));
gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() }));
nlohmann::json json; nlohmann::json json;
json["storePath"] = gitInfo.storePath; json["storePath"] = gitInfo.storePath;
@ -207,6 +213,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
json["name"] = name; json["name"] = name;
json["rev"] = gitInfo.rev.gitRev(); json["rev"] = gitInfo.rev.gitRev();
json["revCount"] = gitInfo.revCount; json["revCount"] = gitInfo.revCount;
json["lastModified"] = gitInfo.lastModified;
writeFile(storeLink, json.dump()); writeFile(storeLink, json.dump());

View file

@ -12,6 +12,7 @@ struct GitInfo
std::string ref; std::string ref;
Hash rev{htSHA1}; Hash rev{htSHA1};
uint64_t revCount; uint64_t revCount;
time_t lastModified;
}; };
GitInfo exportGit(ref<Store> store, std::string uri, GitInfo exportGit(ref<Store> store, std::string uri,

View file

@ -8,6 +8,8 @@
#include <iostream> #include <iostream>
#include <queue> #include <queue>
#include <regex> #include <regex>
#include <ctime>
#include <iomanip>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
namespace nix { namespace nix {
@ -232,6 +234,18 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable())
throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef);
auto doGit = [&](const GitInfo & gitInfo) {
FlakeRef ref(resolvedRef.baseRef());
ref.ref = gitInfo.ref;
ref.rev = gitInfo.rev;
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
info.lastModified = gitInfo.lastModified;
return info;
};
// This only downloads only one revision of the repo, not the entire history. // This only downloads only one revision of the repo, not the entire history.
if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) { if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
@ -251,6 +265,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
request.unpack = true; request.unpack = true;
request.name = "source"; request.name = "source";
request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl;
request.getLastModified = true;
auto result = getDownloader()->downloadCached(state.store, request); auto result = getDownloader()->downloadCached(state.store, request);
if (!result.etag) if (!result.etag)
@ -264,35 +279,20 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
SourceInfo info(ref); SourceInfo info(ref);
info.storePath = result.storePath; info.storePath = result.storePath;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash; info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
info.lastModified = result.lastModified;
return info; return info;
} }
// This downloads the entire git history // This downloads the entire git history
else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) { else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) {
auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"));
FlakeRef ref(resolvedRef.baseRef());
ref.ref = gitInfo.ref;
ref.rev = gitInfo.rev;
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
} }
else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) { else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) {
if (!pathExists(refData->path + "/.git")) if (!pathExists(refData->path + "/.git"))
throw Error("flake '%s' does not reference a Git repository", refData->path); throw Error("flake '%s' does not reference a Git repository", refData->path);
auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); return doGit(exportGit(state.store, refData->path, {}, {}, "source"));
FlakeRef ref(resolvedRef.baseRef());
ref.ref = gitInfo.ref;
ref.rev = gitInfo.rev;
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
} }
else abort(); else abort();
@ -529,6 +529,11 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo
if (sourceInfo.revCount) if (sourceInfo.revCount)
mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount);
if (sourceInfo.lastModified)
mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")),
fmt("%s",
std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S")));
} }
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)

View file

@ -81,10 +81,22 @@ void writeRegistry(const FlakeRegistry &, const Path &);
struct SourceInfo struct SourceInfo
{ {
// Immutable flakeref that this source tree was obtained from.
FlakeRef resolvedRef; FlakeRef resolvedRef;
Path storePath; Path storePath;
// Number of ancestors of the most recent commit.
std::optional<uint64_t> revCount; std::optional<uint64_t> revCount;
Hash narHash; // store path hash
// NAR hash of the store path.
Hash narHash;
// A stable timestamp of this source tree. For Git and GitHub
// flakes, the commit date (not author date!) of the most recent
// commit.
std::optional<time_t> lastModified;
SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {};
}; };

View file

@ -808,6 +808,7 @@ CachedDownloadResult Downloader::downloadCached(
CachedDownloadResult result; CachedDownloadResult result;
result.storePath = expectedStorePath; result.storePath = expectedStorePath;
result.path = store->toRealPath(expectedStorePath); result.path = store->toRealPath(expectedStorePath);
assert(!request.getLastModified); // FIXME
return result; return result;
} }
} }
@ -892,16 +893,26 @@ CachedDownloadResult Downloader::downloadCached(
store->addTempRoot(unpackedStorePath); store->addTempRoot(unpackedStorePath);
if (!store->isValidPath(unpackedStorePath)) if (!store->isValidPath(unpackedStorePath))
unpackedStorePath = ""; unpackedStorePath = "";
else
result.lastModified = lstat(unpackedLink).st_mtime;
} }
if (unpackedStorePath.empty()) { if (unpackedStorePath.empty()) {
printInfo(format("unpacking '%1%'...") % url); printInfo(format("unpacking '%1%'...") % url);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true); AutoDelete autoDelete(tmpDir, true);
// FIXME: this requires GNU tar for decompression. // FIXME: this requires GNU tar for decompression.
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir});
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
result.lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
} }
replaceSymlink(unpackedStorePath, unpackedLink); // Store the last-modified date of the tarball in the symlink
// mtime. This saves us from having to store it somewhere
// else.
replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified);
storePath = unpackedStorePath; storePath = unpackedStorePath;
} }

View file

@ -49,6 +49,7 @@ struct CachedDownloadRequest
Hash expectedHash; Hash expectedHash;
unsigned int ttl = settings.tarballTtl; unsigned int ttl = settings.tarballTtl;
bool gcRoot = false; bool gcRoot = false;
bool getLastModified = false;
CachedDownloadRequest(const std::string & uri) CachedDownloadRequest(const std::string & uri)
: uri(uri) { } : uri(uri) { }
@ -62,6 +63,7 @@ struct CachedDownloadResult
Path path; Path path;
std::optional<std::string> etag; std::optional<std::string> etag;
std::string effectiveUri; std::string effectiveUri;
std::optional<time_t> lastModified;
}; };
class Store; class Store;

View file

@ -22,6 +22,7 @@
#include <sys/ioctl.h> #include <sys/ioctl.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/wait.h> #include <sys/wait.h>
#include <sys/time.h>
#include <unistd.h> #include <unistd.h>
#ifdef __APPLE__ #ifdef __APPLE__
@ -552,20 +553,31 @@ Paths createDirs(const Path & path)
} }
void createSymlink(const Path & target, const Path & link) void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{ {
if (symlink(target.c_str(), link.c_str())) if (symlink(target.c_str(), link.c_str()))
throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target);
if (mtime) {
struct timeval times[2];
times[0].tv_sec = *mtime;
times[0].tv_usec = 0;
times[1].tv_sec = *mtime;
times[1].tv_usec = 0;
if (lutimes(link.c_str(), times))
throw SysError("setting time of symlink '%s'", link);
}
} }
void replaceSymlink(const Path & target, const Path & link) void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{ {
for (unsigned int n = 0; true; n++) { for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try { try {
createSymlink(target, tmp); createSymlink(target, tmp, mtime);
} catch (SysError & e) { } catch (SysError & e) {
if (e.errNo == EEXIST) continue; if (e.errNo == EEXIST) continue;
throw; throw;

View file

@ -142,10 +142,12 @@ Path getDataDir();
Paths createDirs(const Path & path); Paths createDirs(const Path & path);
/* Create a symlink. */ /* Create a symlink. */
void createSymlink(const Path & target, const Path & link); void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
/* Atomically create or replace a symlink. */ /* Atomically create or replace a symlink. */
void replaceSymlink(const Path & target, const Path & link); void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
/* Wrappers arount read()/write() that read/write exactly the /* Wrappers arount read()/write() that read/write exactly the

View file

@ -7,6 +7,7 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <queue> #include <queue>
#include <iomanip>
using namespace nix; using namespace nix;
@ -78,7 +79,10 @@ static void printSourceInfo(const SourceInfo & sourceInfo)
if (sourceInfo.resolvedRef.rev) if (sourceInfo.resolvedRef.rev)
std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false));
if (sourceInfo.revCount) if (sourceInfo.revCount)
std::cout << fmt("Revcount: %s\n", *sourceInfo.revCount); std::cout << fmt("Revisions: %s\n", *sourceInfo.revCount);
if (sourceInfo.lastModified)
std::cout << fmt("Last modified: %s\n",
std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T"));
std::cout << fmt("Path: %s\n", sourceInfo.storePath); std::cout << fmt("Path: %s\n", sourceInfo.storePath);
} }
@ -91,6 +95,8 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j)
j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false); j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false);
if (sourceInfo.revCount) if (sourceInfo.revCount)
j["revCount"] = *sourceInfo.revCount; j["revCount"] = *sourceInfo.revCount;
if (sourceInfo.lastModified)
j["lastModified"] = *sourceInfo.lastModified;
j["path"] = sourceInfo.storePath; j["path"] = sourceInfo.storePath;
} }

View file

@ -124,6 +124,7 @@ nix flake info --flake-registry $registry $flake1Dir | grep -q 'ID: *flake1'
json=$(nix flake info --flake-registry $registry flake1 --json | jq .) json=$(nix flake info --flake-registry $registry flake1 --json | jq .)
[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
[[ -d $(echo "$json" | jq -r .path) ]] [[ -d $(echo "$json" | jq -r .path) ]]
[[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]]
# Test 'nix build' on a flake. # Test 'nix build' on a flake.
nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo