HttpBinaryCacheStore: Fix caching of WantMassQuery

Also, test HttpBinaryCacheStore in addition to LocalBinaryCacheStore.
This commit is contained in:
Eelco Dolstra 2016-06-01 15:15:21 +02:00
parent 7850d3d279
commit cf198952d0
6 changed files with 60 additions and 39 deletions

View file

@ -39,7 +39,7 @@ public:
void init() override
{
// FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri)) {
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP &) {
@ -95,7 +95,9 @@ static RegisterStoreImplementation regStore([](
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://") return 0;
std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
) return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init();
return store;

View file

@ -17,9 +17,6 @@ public:
: BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir)
{
/* For testing the NAR info cache. */
if (getEnv("_NIX_CACHE_FILE_URLS") == "1")
diskCache = getNarInfoDiskCache();
}
void init() override;
@ -57,9 +54,6 @@ void LocalBinaryCacheStore::init()
{
createDirs(binaryCacheDir + "/nar");
BinaryCacheStore::init();
if (diskCache && !diskCache->cacheExists(getUri()))
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
}
static void atomicWrite(const Path & path, const std::string & s)
@ -96,7 +90,9 @@ static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, 7) != "file://") return 0;
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
std::string(uri, 0, 7) != "file://")
return 0;
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
store->init();
return store;

View file

@ -57,6 +57,8 @@ public:
{
int id;
Path storeDir;
bool wantMassQuery;
int priority;
};
struct State
@ -126,24 +128,28 @@ public:
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir};
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
}
bool cacheExists(const std::string & uri) override
bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) override
{
auto state(_state.lock());
auto i = state->caches.find(uri);
if (i != state->caches.end()) return true;
auto queryCache(state->queryCache.use()(uri));
if (queryCache.next()) {
state->caches[uri] = Cache{(int) queryCache.getInt(0), queryCache.getStr(1)};
return true;
if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri));
if (!queryCache.next()) return false;
state->caches.emplace(uri,
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2), (int) queryCache.getInt(3)});
}
return false;
auto & cache(getCache(*state, uri));
wantMassQuery = cache.wantMassQuery;
priority = cache.priority;
return true;
}
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(

View file

@ -13,7 +13,8 @@ public:
virtual void createCache(const std::string & uri, const Path & storeDir,
bool wantMassQuery, int priority) = 0;
virtual bool cacheExists(const std::string & uri) = 0;
virtual bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) = 0;

View file

@ -71,7 +71,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
void init() override
{
if (!diskCache->cacheExists(getUri())) {
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
/* Create the bucket if it doesn't already exists. */
// FIXME: HeadBucket would be more appropriate, but doesn't return

View file

@ -9,34 +9,50 @@ outPath=$(nix-build dependencies.nix --no-out-link)
nix-push --dest $cacheDir $outPath
# By default, a binary cache doesn't support "nix-env -qas", but does
# support installation.
clearStore
clearCacheCache
basicTests() {
export _NIX_CACHE_FILE_URLS=1
# By default, a binary cache doesn't support "nix-env -qas", but does
# support installation.
clearStore
clearCacheCache
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
nix-store --option binary-caches "file://$cacheDir" -r $outPath
nix-store --option binary-caches "file://$cacheDir" -r $outPath
[ -x $outPath/program ]
[ -x $outPath/program ]
# But with the right configuration, "nix-env -qas" should also work.
clearStore
clearCacheCache
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
# But with the right configuration, "nix-env -qas" should also work.
clearStore
clearCacheCache
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
[ -z "$x" ]
x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
[ -z "$x" ]
nix-store --option binary-caches "file://$cacheDir" -r $outPath
nix-store --option binary-caches "file://$cacheDir" -r $outPath
nix-store --check-validity $outPath
nix-store -qR $outPath | grep input-2
nix-store --check-validity $outPath
nix-store -qR $outPath | grep input-2
echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info
}
# Test LocalBinaryCacheStore.
basicTests
# Test HttpBinaryCacheStore.
export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1
basicTests
unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE
# Test whether Nix notices if the NAR doesn't match the hash in the NAR info.