Add function queryPathFromHashPart()

To implement binary caches efficiently, Hydra needs to be able to map
the hash part of a store path (e.g. "gbg...zr7") to the full store
path (e.g. "/nix/store/gbg...kzr7-subversion-1.7.5").  (The binary
cache mechanism uses hash parts as a key for looking up store paths to
ensure privacy.)  However, doing a search in the Nix store for
/nix/store/<hash>* is expensive since it requires reading the entire
directory.  queryPathFromHashPart() prevents this by doing a cheap
database lookup.
This commit is contained in:
Eelco Dolstra 2012-07-17 18:55:39 -04:00
parent 220818f758
commit ccc52adfb2
9 changed files with 67 additions and 0 deletions

View file

@ -14,6 +14,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
our @EXPORT = qw( our @EXPORT = qw(
isValidPath queryReferences queryPathInfo queryDeriver queryPathHash isValidPath queryReferences queryPathInfo queryDeriver queryPathHash
queryPathFromHashPart
topoSortPaths computeFSClosure followLinksToStorePath exportPaths topoSortPaths computeFSClosure followLinksToStorePath exportPaths
hashPath hashFile hashString hashPath hashFile hashString
addToStore makeFixedOutputPath addToStore makeFixedOutputPath

View file

@ -108,6 +108,17 @@ SV * queryPathInfo(char * path, int base32)
} }
SV * queryPathFromHashPart(char * hashPart)
PPCODE:
try {
doInit();
Path path = store->queryPathFromHashPart(hashPart);
XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
} catch (Error & e) {
croak(e.what());
}
SV * computeFSClosure(int flipDirection, int includeOutputs, ...) SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
PPCODE: PPCODE:
try { try {

View file

@ -405,6 +405,10 @@ void LocalStore::openDB(bool create)
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
stmtQueryDerivationOutputs.create(db, stmtQueryDerivationOutputs.create(db,
"select id, path from DerivationOutputs where drv = ?;"); "select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup.
stmtQueryPathFromHashPart.create(db,
"select path from ValidPaths where path >= ? limit 1;");
} }
@ -865,6 +869,26 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
} }
Path LocalStore::queryPathFromHashPart(const string & hashPart)
{
if (hashPart.size() != 32) throw Error("invalid hash part");
SQLiteTxn txn(db);
Path prefix = nixStore + "/" + hashPart;
SQLiteStmtUse use(stmtQueryPathFromHashPart);
stmtQueryPathFromHashPart.bind(prefix);
int res = sqlite3_step(stmtQueryPathFromHashPart);
if (res == SQLITE_DONE) return "";
if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
}
void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
{ {
if (run.pid != -1) return; if (run.pid != -1) return;

View file

@ -121,6 +121,8 @@ public:
StringSet queryDerivationOutputNames(const Path & path); StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
PathSet querySubstitutablePaths(); PathSet querySubstitutablePaths();
bool hasSubstitutes(const Path & path); bool hasSubstitutes(const Path & path);
@ -217,6 +219,7 @@ private:
SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs; SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
int getSchema(); int getSchema();

View file

@ -341,6 +341,18 @@ PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
} }
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
{
openConnection();
writeInt(wopQueryPathFromHashPart, to);
writeString(hashPart, to);
processStderr();
Path path = readString(from);
if (!path.empty()) assertStorePath(path);
return path;
}
Path RemoteStore::addToStore(const Path & _srcPath, Path RemoteStore::addToStore(const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter) bool recursive, HashType hashAlgo, PathFilter & filter)
{ {

View file

@ -43,6 +43,8 @@ public:
StringSet queryDerivationOutputNames(const Path & path); StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
bool hasSubstitutes(const Path & path); bool hasSubstitutes(const Path & path);
bool querySubstitutablePathInfo(const Path & path, bool querySubstitutablePathInfo(const Path & path,

View file

@ -139,6 +139,10 @@ public:
/* Query the output names of the derivation denoted by `path'. */ /* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path) = 0; virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
/* Query whether a path has substitutes. */ /* Query whether a path has substitutes. */
virtual bool hasSubstitutes(const Path & path) = 0; virtual bool hasSubstitutes(const Path & path) = 0;

View file

@ -40,6 +40,7 @@ typedef enum {
wopQueryPathInfo = 26, wopQueryPathInfo = 26,
wopImportPaths = 27, wopImportPaths = 27,
wopQueryDerivationOutputNames = 28, wopQueryDerivationOutputNames = 28,
wopQueryPathFromHashPart = 29,
} WorkerOp; } WorkerOp;

View file

@ -350,6 +350,15 @@ static void performOp(unsigned int clientVersion,
break; break;
} }
case wopQueryPathFromHashPart: {
string hashPart = readString(from);
startWork();
Path path = store->queryPathFromHashPart(hashPart);
stopWork();
writeString(path, to);
break;
}
case wopAddToStore: { case wopAddToStore: {
string baseName = readString(from); string baseName = readString(from);
bool fixed = readInt(from) == 1; /* obsolete */ bool fixed = readInt(from) == 1; /* obsolete */