lix-project/lix
20
55
Fork 31
Code Issues 344 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge pull request #2898 from NixOS/last-modified

Browse source 
Expose lastModified attribute
This commit is contained in:
Eelco Dolstra 2019-05-29 10:14:40 +02:00 committed by GitHub
parent 479757dc15 ae7b56cd9a
commit 315f1980ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 98 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
release.nix
View file

@ -19,7 +19,8 @@ let
releaseTools.sourceTarball {
name = "nix-tarball";
version = builtins.readFile ./.version;
versionSuffix = if officialRelease then "" else "pre${toString nix.revCount or 0}_${nix.shortRev or "0000000"}";
versionSuffix = if officialRelease then "" else
"pre${if nix ? lastModified then builtins.substring 0 8 nix.lastModified else toString nix.revCount or 0}_${nix.shortRev or "0000000"}";
src = nix;
inherit officialRelease;

9
src/libexpr/primops/fetchGit.cc
View file

@ -69,6 +69,9 @@ GitInfo exportGit(ref<Store> store, std::string uri,
gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" }));
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" }));
return gitInfo;
}
@ -85,8 +88,9 @@ GitInfo exportGit(ref<Store> store, std::string uri,
}
deletePath(getCacheDir() + "/nix/git");
deletePath(getCacheDir() + "/nix/gitv2");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false);
Path repoDir;
if (isLocal) {
@ -181,6 +185,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
if (store->isValidPath(storePath)) {
gitInfo.storePath = storePath;
gitInfo.revCount = json["revCount"];
gitInfo.lastModified = json["lastModified"];
return gitInfo;
}
@ -200,6 +205,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
gitInfo.storePath = store->addToStore(name, tmpDir);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() }));
gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() }));
nlohmann::json json;
json["storePath"] = gitInfo.storePath;
@ -207,6 +213,7 @@ GitInfo exportGit(ref<Store> store, std::string uri,
json["name"] = name;
json["rev"] = gitInfo.rev.gitRev();
json["revCount"] = gitInfo.revCount;
json["lastModified"] = gitInfo.lastModified;
writeFile(storeLink, json.dump());

1
src/libexpr/primops/fetchGit.hh
View file

@ -12,6 +12,7 @@ struct GitInfo
std::string ref;
Hash rev{htSHA1};
uint64_t revCount;
time_t lastModified;
};
GitInfo exportGit(ref<Store> store, std::string uri,

41
src/libexpr/primops/flake.cc
View file

@ -8,6 +8,8 @@
#include <iostream>
#include <queue>
#include <regex>
#include <ctime>
#include <iomanip>
#include <nlohmann/json.hpp>
namespace nix {
@ -232,6 +234,18 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable())
throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef);
auto doGit = [&](const GitInfo & gitInfo) {
FlakeRef ref(resolvedRef.baseRef());
ref.ref = gitInfo.ref;
ref.rev = gitInfo.rev;
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
info.lastModified = gitInfo.lastModified;
return info;
};
// This only downloads only one revision of the repo, not the entire history.
if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
@ -251,6 +265,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
request.unpack = true;
request.name = "source";
request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl;
request.getLastModified = true;
auto result = getDownloader()->downloadCached(state.store, request);
if (!result.etag)
@ -264,35 +279,20 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
SourceInfo info(ref);
info.storePath = result.storePath;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
info.lastModified = result.lastModified;
return info;
}
// This downloads the entire git history
else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) {
auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source");
FlakeRef ref(resolvedRef.baseRef());
ref.ref = gitInfo.ref;
ref.rev = gitInfo.rev;
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"));
}
else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) {
if (!pathExists(refData->path + "/.git"))
throw Error("flake '%s' does not reference a Git repository", refData->path);
auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source");
FlakeRef ref(resolvedRef.baseRef());
ref.ref = gitInfo.ref;
ref.rev = gitInfo.rev;
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
return doGit(exportGit(state.store, refData->path, {}, {}, "source"));
}
else abort();
@ -529,6 +529,11 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo
if (sourceInfo.revCount)
mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount);
if (sourceInfo.lastModified)
mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")),
fmt("%s",
std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S")));
}
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)

14
src/libexpr/primops/flake.hh
View file

@ -81,10 +81,22 @@ void writeRegistry(const FlakeRegistry &, const Path &);
struct SourceInfo
{
// Immutable flakeref that this source tree was obtained from.
FlakeRef resolvedRef;
Path storePath;
// Number of ancestors of the most recent commit.
std::optional<uint64_t> revCount;
Hash narHash; // store path hash
// NAR hash of the store path.
Hash narHash;
// A stable timestamp of this source tree. For Git and GitHub
// flakes, the commit date (not author date!) of the most recent
// commit.
std::optional<time_t> lastModified;
SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {};
};

17
src/libstore/download.cc
View file

@ -808,6 +808,7 @@ CachedDownloadResult Downloader::downloadCached(
CachedDownloadResult result;
result.storePath = expectedStorePath;
result.path = store->toRealPath(expectedStorePath);
assert(!request.getLastModified); // FIXME
return result;
}
}
@ -892,16 +893,26 @@ CachedDownloadResult Downloader::downloadCached(
store->addTempRoot(unpackedStorePath);
if (!store->isValidPath(unpackedStorePath))
unpackedStorePath = "";
else
result.lastModified = lstat(unpackedLink).st_mtime;
}
if (unpackedStorePath.empty()) {
printInfo(format("unpacking '%1%'...") % url);
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
// FIXME: this requires GNU tar for decompression.
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir});
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
result.lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
}
replaceSymlink(unpackedStorePath, unpackedLink);
// Store the last-modified date of the tarball in the symlink
// mtime. This saves us from having to store it somewhere
// else.
replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified);
storePath = unpackedStorePath;
}

2
src/libstore/download.hh
View file

@ -49,6 +49,7 @@ struct CachedDownloadRequest
Hash expectedHash;
unsigned int ttl = settings.tarballTtl;
bool gcRoot = false;
bool getLastModified = false;
CachedDownloadRequest(const std::string & uri)
: uri(uri) { }
@ -62,6 +63,7 @@ struct CachedDownloadResult
Path path;
std::optional<std::string> etag;
std::string effectiveUri;
std::optional<time_t> lastModified;
};
class Store;

18
src/libutil/util.cc
View file

@ -22,6 +22,7 @@
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <unistd.h>
#ifdef __APPLE__
@ -552,20 +553,31 @@ Paths createDirs(const Path & path)
}
void createSymlink(const Path & target, const Path & link)
void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{
if (symlink(target.c_str(), link.c_str()))
throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target);
if (mtime) {
struct timeval times[2];
times[0].tv_sec = *mtime;
times[0].tv_usec = 0;
times[1].tv_sec = *mtime;
times[1].tv_usec = 0;
if (lutimes(link.c_str(), times))
throw SysError("setting time of symlink '%s'", link);
}
}
void replaceSymlink(const Path & target, const Path & link)
void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{
for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try {
createSymlink(target, tmp);
createSymlink(target, tmp, mtime);
} catch (SysError & e) {
if (e.errNo == EEXIST) continue;
throw;

6
src/libutil/util.hh
View file

@ -142,10 +142,12 @@ Path getDataDir();
Paths createDirs(const Path & path);
/* Create a symlink. */
void createSymlink(const Path & target, const Path & link);
void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
/* Atomically create or replace a symlink. */
void replaceSymlink(const Path & target, const Path & link);
void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
/* Wrappers arount read()/write() that read/write exactly the

24
src/nix/flake.cc
View file

@ -7,6 +7,7 @@
#include <nlohmann/json.hpp>
#include <queue>
#include <iomanip>
using namespace nix;
@ -72,14 +73,17 @@ struct CmdFlakeList : EvalCommand
static void printSourceInfo(const SourceInfo & sourceInfo)
{
std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string());
std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string());
if (sourceInfo.resolvedRef.ref)
std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref);
std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref);
if (sourceInfo.resolvedRef.rev)
std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false));
std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false));
if (sourceInfo.revCount)
std::cout << fmt("Revcount: %s\n", *sourceInfo.revCount);
std::cout << fmt("Path: %s\n", sourceInfo.storePath);
std::cout << fmt("Revisions: %s\n", *sourceInfo.revCount);
if (sourceInfo.lastModified)
std::cout << fmt("Last modified: %s\n",
std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T"));
std::cout << fmt("Path: %s\n", sourceInfo.storePath);
}
static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j)
@ -91,14 +95,16 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j)
j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false);
if (sourceInfo.revCount)
j["revCount"] = *sourceInfo.revCount;
if (sourceInfo.lastModified)
j["lastModified"] = *sourceInfo.lastModified;
j["path"] = sourceInfo.storePath;
}
static void printFlakeInfo(const Flake & flake)
{
std::cout << fmt("ID: %s\n", flake.id);
std::cout << fmt("Description: %s\n", flake.description);
std::cout << fmt("Epoch: %s\n", flake.epoch);
std::cout << fmt("ID: %s\n", flake.id);
std::cout << fmt("Description: %s\n", flake.description);
std::cout << fmt("Epoch: %s\n", flake.epoch);
printSourceInfo(flake.sourceInfo);
}
@ -114,7 +120,7 @@ static nlohmann::json flakeToJson(const Flake & flake)
static void printNonFlakeInfo(const NonFlake & nonFlake)
{
std::cout << fmt("ID: %s\n", nonFlake.alias);
std::cout << fmt("ID: %s\n", nonFlake.alias);
printSourceInfo(nonFlake.sourceInfo);
}

1
tests/flakes.sh
View file

@ -124,6 +124,7 @@ nix flake info --flake-registry $registry $flake1Dir | grep -q 'ID: *flake1'
json=$(nix flake info --flake-registry $registry flake1 --json | jq .)
[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
[[ -d $(echo "$json" | jq -r .path) ]]
[[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]]
# Test 'nix build' on a flake.
nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo