From c1f04fae350acf9d72c56ef4f83037b479f25ab0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Apr 2015 11:55:36 +0200 Subject: [PATCH] Implement a TTL on cached fetchurl/fetchTarball results This is because we don't want to do HTTP requests on every evaluation, even though we can prevent a full redownload via the cached ETag. The default is one hour. --- src/libexpr/primops.cc | 37 ++++++++++++++++++++++++------------- src/libstore/globals.cc | 8 ++++++++ src/libstore/globals.hh | 2 ++ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 77ca42042..03a3c8286 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1631,14 +1631,22 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, string expectedETag; + int ttl = settings.get("tarball-ttl", 60 * 60); + bool skip = false; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); if (store->isValidPath(storePath)) { auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 2 && ss[0] == url) { - printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]); - expectedETag = ss[1]; + if (ss.size() >= 3 && ss[0] == url) { + time_t lastChecked; + if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) + skip = true; + else if (!ss[1].empty()) { + printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]); + expectedETag = ss[1]; + } } } else storePath = ""; @@ -1648,19 +1656,22 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); - if (expectedETag.empty()) - printMsg(lvlInfo, format("downloading ‘%1%’...") % url); - else - printMsg(lvlInfo, format("checking ‘%1%’...") % url); - Curl curl; + if (!skip) { - if (curl.fetch(url, expectedETag)) - storePath = store->addTextToStore(name, curl.data, PathSet(), state.repair); + if (expectedETag.empty()) + printMsg(lvlInfo, format("downloading ‘%1%’...") % url); + else + printMsg(lvlInfo, format("checking ‘%1%’...") % url); + Curl curl; - assert(!storePath.empty()); - replaceSymlink(storePath, fileLink); + if (curl.fetch(url, expectedETag)) + storePath = store->addTextToStore(name, curl.data, PathSet(), state.repair); - writeFile(dataFile, url + "\n" + curl.etag + "\n"); + assert(!storePath.empty()); + replaceSymlink(storePath, fileLink); + + writeFile(dataFile, url + "\n" + curl.etag + "\n" + int2String(time(0)) + "\n"); + } if (unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e382b3aac..f900fb290 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -143,6 +143,14 @@ bool Settings::get(const string & name, bool def) } +int Settings::get(const string & name, int def) +{ + int res = def; + _get(res, name); + return res; +} + + void Settings::update() { _get(tryFallback, "build-fallback"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0230a540e..0a1072e36 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -27,6 +27,8 @@ struct Settings { bool get(const string & name, bool def); + int get(const string & name, int def); + void update(); string pack();