HttpBinaryCacheStore: Fix caching of WantMassQuery

Also, test HttpBinaryCacheStore in addition to LocalBinaryCacheStore.
This commit is contained in:
Eelco Dolstra 2016-06-01 15:15:21 +02:00
parent 7850d3d279
commit cf198952d0
6 changed files with 60 additions and 39 deletions

View file

@ -39,7 +39,7 @@ public:
void init() override void init() override
{ {
// FIXME: do this lazily? // FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri)) { if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
try { try {
BinaryCacheStore::init(); BinaryCacheStore::init();
} catch (UploadToHTTP &) { } catch (UploadToHTTP &) {
@ -95,7 +95,9 @@ static RegisterStoreImplementation regStore([](
-> std::shared_ptr<Store> -> std::shared_ptr<Store>
{ {
if (std::string(uri, 0, 7) != "http://" && if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://") return 0; std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
) return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init(); store->init();
return store; return store;

View file

@ -17,9 +17,6 @@ public:
: BinaryCacheStore(params) : BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir) , binaryCacheDir(binaryCacheDir)
{ {
/* For testing the NAR info cache. */
if (getEnv("_NIX_CACHE_FILE_URLS") == "1")
diskCache = getNarInfoDiskCache();
} }
void init() override; void init() override;
@ -57,9 +54,6 @@ void LocalBinaryCacheStore::init()
{ {
createDirs(binaryCacheDir + "/nar"); createDirs(binaryCacheDir + "/nar");
BinaryCacheStore::init(); BinaryCacheStore::init();
if (diskCache && !diskCache->cacheExists(getUri()))
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
} }
static void atomicWrite(const Path & path, const std::string & s) static void atomicWrite(const Path & path, const std::string & s)
@ -96,7 +90,9 @@ static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params) const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store> -> std::shared_ptr<Store>
{ {
if (std::string(uri, 0, 7) != "file://") return 0; if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
std::string(uri, 0, 7) != "file://")
return 0;
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
store->init(); store->init();
return store; return store;

View file

@ -57,6 +57,8 @@ public:
{ {
int id; int id;
Path storeDir; Path storeDir;
bool wantMassQuery;
int priority;
}; };
struct State struct State
@ -126,24 +128,28 @@ public:
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
assert(sqlite3_changes(state->db) == 1); assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir}; state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
} }
bool cacheExists(const std::string & uri) override bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) override
{ {
auto state(_state.lock()); auto state(_state.lock());
auto i = state->caches.find(uri); auto i = state->caches.find(uri);
if (i != state->caches.end()) return true; if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri));
auto queryCache(state->queryCache.use()(uri)); if (!queryCache.next()) return false;
state->caches.emplace(uri,
if (queryCache.next()) { Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2), (int) queryCache.getInt(3)});
state->caches[uri] = Cache{(int) queryCache.getInt(0), queryCache.getStr(1)};
return true;
} }
return false; auto & cache(getCache(*state, uri));
wantMassQuery = cache.wantMassQuery;
priority = cache.priority;
return true;
} }
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(

View file

@ -13,7 +13,8 @@ public:
virtual void createCache(const std::string & uri, const Path & storeDir, virtual void createCache(const std::string & uri, const Path & storeDir,
bool wantMassQuery, int priority) = 0; bool wantMassQuery, int priority) = 0;
virtual bool cacheExists(const std::string & uri) = 0; virtual bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) = 0; const std::string & uri, const std::string & hashPart) = 0;

View file

@ -71,7 +71,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
void init() override void init() override
{ {
if (!diskCache->cacheExists(getUri())) { if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
/* Create the bucket if it doesn't already exists. */ /* Create the bucket if it doesn't already exists. */
// FIXME: HeadBucket would be more appropriate, but doesn't return // FIXME: HeadBucket would be more appropriate, but doesn't return

View file

@ -9,34 +9,50 @@ outPath=$(nix-build dependencies.nix --no-out-link)
nix-push --dest $cacheDir $outPath nix-push --dest $cacheDir $outPath
# By default, a binary cache doesn't support "nix-env -qas", but does basicTests() {
# support installation.
clearStore
clearCacheCache
export _NIX_CACHE_FILE_URLS=1 # By default, a binary cache doesn't support "nix-env -qas", but does
# support installation.
clearStore
clearCacheCache
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
nix-store --option binary-caches "file://$cacheDir" -r $outPath nix-store --option binary-caches "file://$cacheDir" -r $outPath
[ -x $outPath/program ] [ -x $outPath/program ]
# But with the right configuration, "nix-env -qas" should also work. # But with the right configuration, "nix-env -qas" should also work.
clearStore clearStore
clearCacheCache clearCacheCache
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
[ -z "$x" ] [ -z "$x" ]
nix-store --option binary-caches "file://$cacheDir" -r $outPath nix-store --option binary-caches "file://$cacheDir" -r $outPath
nix-store --check-validity $outPath nix-store --check-validity $outPath
nix-store -qR $outPath | grep input-2 nix-store -qR $outPath | grep input-2
echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info
}
# Test LocalBinaryCacheStore.
basicTests
# Test HttpBinaryCacheStore.
export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1
basicTests
unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE
# Test whether Nix notices if the NAR doesn't match the hash in the NAR info. # Test whether Nix notices if the NAR doesn't match the hash in the NAR info.