2017-07-06 12:53:52 +00:00
|
|
|
|
/* A local disk cache for fast lookups of NAR index files in a binary
|
|
|
|
|
cache. */
|
|
|
|
|
|
|
|
|
|
#include "binary-cache-store.hh"
|
|
|
|
|
#include "fs-accessor.hh"
|
|
|
|
|
#include "sqlite.hh"
|
|
|
|
|
#include "sync.hh"
|
|
|
|
|
|
|
|
|
|
#include <sqlite3.h>
|
|
|
|
|
#include <nlohmann/json.hpp>
|
|
|
|
|
|
|
|
|
|
using namespace nix;
|
|
|
|
|
|
|
|
|
|
MakeError(BadJSON, Error);
|
|
|
|
|
|
|
|
|
|
class FileCache
|
|
|
|
|
{
|
|
|
|
|
struct State
|
|
|
|
|
{
|
|
|
|
|
SQLite db;
|
|
|
|
|
SQLiteStmt queryPath, insertPath, queryFiles, insertFile;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
Sync<State> state_;
|
|
|
|
|
|
2017-07-07 14:45:45 +00:00
|
|
|
|
struct Stat : FSAccessor::Stat
|
|
|
|
|
{
|
|
|
|
|
std::string target;
|
|
|
|
|
};
|
|
|
|
|
|
2017-07-06 12:53:52 +00:00
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
FileCache(const Path & path)
|
|
|
|
|
{
|
|
|
|
|
auto state(state_.lock());
|
|
|
|
|
|
|
|
|
|
static std::string cacheSchema = R"sql(
|
|
|
|
|
|
|
|
|
|
create table if not exists StorePaths (
|
|
|
|
|
id integer primary key autoincrement not null,
|
|
|
|
|
path text unique not null
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
create table if not exists StorePathContents (
|
|
|
|
|
storePath integer not null,
|
|
|
|
|
subPath text not null,
|
|
|
|
|
type integer not null,
|
|
|
|
|
fileSize integer,
|
|
|
|
|
isExecutable integer,
|
2017-07-07 14:45:45 +00:00
|
|
|
|
target text,
|
2017-07-06 12:53:52 +00:00
|
|
|
|
primary key (storePath, subPath),
|
|
|
|
|
foreign key (storePath) references StorePaths(id) on delete cascade
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
)sql";
|
|
|
|
|
|
|
|
|
|
state->db = SQLite(path);
|
|
|
|
|
state->db.exec("pragma foreign_keys = 1");
|
|
|
|
|
state->db.exec(cacheSchema);
|
|
|
|
|
|
|
|
|
|
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
|
2022-09-08 12:21:15 +00:00
|
|
|
|
SQLiteError::throw_(state->db, "setting timeout");
|
2017-07-06 12:53:52 +00:00
|
|
|
|
|
|
|
|
|
state->queryPath.create(state->db,
|
|
|
|
|
"select id from StorePaths where path = ?");
|
|
|
|
|
state->insertPath.create(state->db,
|
|
|
|
|
"insert or ignore into StorePaths(path) values (?)");
|
|
|
|
|
state->queryFiles.create(state->db,
|
2017-07-07 14:45:45 +00:00
|
|
|
|
"select subPath, type, fileSize, isExecutable, target from StorePathContents where storePath = ?");
|
2017-07-06 12:53:52 +00:00
|
|
|
|
state->insertFile.create(state->db,
|
2017-07-07 14:45:45 +00:00
|
|
|
|
"insert into StorePathContents(storePath, subPath, type, fileSize, isExecutable, target) values (?, ?, ?, ?, ?, ?)");
|
2017-07-06 12:53:52 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Return the files in a store path, using a SQLite database to
|
|
|
|
|
cache the results. */
|
2017-07-07 14:45:45 +00:00
|
|
|
|
std::map<std::string, Stat>
|
2017-07-06 12:53:52 +00:00
|
|
|
|
getFiles(ref<BinaryCacheStore> binaryCache, const Path & storePath)
|
|
|
|
|
{
|
2017-07-07 14:45:45 +00:00
|
|
|
|
std::map<std::string, Stat> files;
|
2017-07-06 12:53:52 +00:00
|
|
|
|
|
|
|
|
|
/* Look up the path in the SQLite cache. */
|
|
|
|
|
{
|
|
|
|
|
auto state(state_.lock());
|
|
|
|
|
auto useQueryPath(state->queryPath.use()(storePath));
|
|
|
|
|
if (useQueryPath.next()) {
|
|
|
|
|
auto id = useQueryPath.getInt(0);
|
|
|
|
|
auto useQueryFiles(state->queryFiles.use()(id));
|
|
|
|
|
while (useQueryFiles.next()) {
|
2017-07-07 14:45:45 +00:00
|
|
|
|
Stat st;
|
|
|
|
|
st.type = (FSAccessor::Type) useQueryFiles.getInt(1);
|
|
|
|
|
st.fileSize = (uint64_t) useQueryFiles.getInt(2);
|
|
|
|
|
st.isExecutable = useQueryFiles.getInt(3) != 0;
|
|
|
|
|
if (!useQueryFiles.isNull(4))
|
|
|
|
|
st.target = useQueryFiles.getStr(4);
|
|
|
|
|
files.emplace(useQueryFiles.getStr(0), st);
|
2017-07-06 12:53:52 +00:00
|
|
|
|
}
|
|
|
|
|
return files;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
using json = nlohmann::json;
|
|
|
|
|
|
|
|
|
|
std::function<void(const std::string &, json &)> recurse;
|
|
|
|
|
|
|
|
|
|
recurse = [&](const std::string & relPath, json & v) {
|
2017-07-07 14:45:45 +00:00
|
|
|
|
Stat st;
|
2017-07-06 12:53:52 +00:00
|
|
|
|
|
|
|
|
|
std::string type = v["type"];
|
|
|
|
|
|
|
|
|
|
if (type == "directory") {
|
|
|
|
|
st.type = FSAccessor::Type::tDirectory;
|
|
|
|
|
for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
|
|
|
|
|
std::string name = i.key();
|
|
|
|
|
recurse(relPath.empty() ? name : relPath + "/" + name, i.value());
|
|
|
|
|
}
|
|
|
|
|
} else if (type == "regular") {
|
|
|
|
|
st.type = FSAccessor::Type::tRegular;
|
|
|
|
|
st.fileSize = v["size"];
|
|
|
|
|
st.isExecutable = v.value("executable", false);
|
|
|
|
|
} else if (type == "symlink") {
|
|
|
|
|
st.type = FSAccessor::Type::tSymlink;
|
2017-07-07 14:45:45 +00:00
|
|
|
|
st.target = v.value("target", "");
|
2017-07-06 12:53:52 +00:00
|
|
|
|
} else return;
|
|
|
|
|
|
|
|
|
|
files[relPath] = st;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/* It's not in the cache, so get the .ls.xz file (which
|
|
|
|
|
contains a JSON serialisation of the listing of the NAR
|
|
|
|
|
contents) from the binary cache. */
|
|
|
|
|
auto now1 = std::chrono::steady_clock::now();
|
2020-07-08 11:25:30 +00:00
|
|
|
|
auto s = binaryCache->getFile(std::string(baseNameOf(storePath).substr(0, StorePath::HashLen)) + ".ls");
|
2017-07-06 12:53:52 +00:00
|
|
|
|
if (!s)
|
|
|
|
|
printInfo("warning: no listing of %s in binary cache", storePath);
|
|
|
|
|
else {
|
|
|
|
|
try {
|
|
|
|
|
json ls = json::parse(*s);
|
|
|
|
|
|
|
|
|
|
if (ls.value("version", 0) != 1)
|
|
|
|
|
throw Error("NAR index for ‘%s’ has an unsupported version", storePath);
|
|
|
|
|
|
|
|
|
|
recurse("", ls.at("root"));
|
2019-01-31 01:56:40 +00:00
|
|
|
|
} catch (json::parse_error & e) {
|
2017-07-06 12:53:52 +00:00
|
|
|
|
// FIXME: some filenames have non-UTF8 characters in them,
|
|
|
|
|
// which is not supported by nlohmann::json. So we have to
|
|
|
|
|
// skip the entire package.
|
|
|
|
|
throw BadJSON(e.what());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Insert the store path into the database. */
|
|
|
|
|
{
|
|
|
|
|
auto state(state_.lock());
|
|
|
|
|
SQLiteTxn txn(state->db);
|
|
|
|
|
|
|
|
|
|
if (state->queryPath.use()(storePath).next()) return files;
|
|
|
|
|
state->insertPath.use()(storePath).exec();
|
|
|
|
|
uint64_t id = sqlite3_last_insert_rowid(state->db);
|
|
|
|
|
|
|
|
|
|
for (auto & x : files) {
|
|
|
|
|
state->insertFile.use()
|
|
|
|
|
(id)
|
|
|
|
|
(x.first)
|
|
|
|
|
(x.second.type)
|
|
|
|
|
(x.second.fileSize, x.second.type == FSAccessor::Type::tRegular)
|
|
|
|
|
(x.second.isExecutable, x.second.type == FSAccessor::Type::tRegular)
|
2017-07-07 14:45:45 +00:00
|
|
|
|
(x.second.target, x.second.type == FSAccessor::Type::tSymlink)
|
2017-07-06 12:53:52 +00:00
|
|
|
|
.exec();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
txn.commit();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto now2 = std::chrono::steady_clock::now();
|
|
|
|
|
printInfo("processed %s in %d ms", storePath,
|
|
|
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
|
|
|
|
|
|
|
|
|
|
return files;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|