Misc changes from the flakes branch

This commit is contained in:
Eelco Dolstra 2020-03-24 14:26:13 +01:00
parent c85097da7c
commit 0a10854f85
22 changed files with 110 additions and 60 deletions

2
.gitignore vendored
View file

@ -47,7 +47,7 @@ perl/Makefile.config
/src/libexpr/nix.tbl
# /src/libstore/
/src/libstore/*.gen.hh
*.gen.*
/src/nix/nix

View file

@ -37,6 +37,7 @@ prefix = @prefix@
sandbox_shell = @sandbox_shell@
storedir = @storedir@
sysconfdir = @sysconfdir@
system = @system@
doc_generate = @doc_generate@
xmllint = @xmllint@
xsltproc = @xsltproc@

View file

@ -368,7 +368,7 @@ struct EvalSettings : Config
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
};
extern EvalSettings evalSettings;

View file

@ -1398,7 +1398,7 @@ void DerivationGoal::tryToBuild()
few seconds and then retry this goal. */
PathSet lockFiles;
for (auto & outPath : drv->outputPaths())
lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath)));
lockFiles.insert(worker.store.Store::toRealPath(outPath));
if (!outputLocks.lockPaths(lockFiles, "", false)) {
worker.waitForAWhile(shared_from_this());
@ -1429,7 +1429,7 @@ void DerivationGoal::tryToBuild()
for (auto & i : drv->outputs) {
if (worker.store.isValidPath(i.second.path)) continue;
debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path)));
deletePath(worker.store.Store::toRealPath(i.second.path));
}
/* Don't do a remote build if the derivation has the attribute
@ -1686,7 +1686,7 @@ void DerivationGoal::buildDone()
/* Delete unused redirected outputs (when doing hash rewriting). */
for (auto & i : redirectedOutputs)
deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second)));
deletePath(worker.store.Store::toRealPath(i.second));
/* Delete the chroot (if we were using one). */
autoDelChroot.reset(); /* this runs the destructor */
@ -2072,7 +2072,7 @@ void DerivationGoal::startBuilder()
environment using bind-mounts. We put it in the Nix store
to ensure that we can create hard-links to non-directory
inputs in the fake Nix store in the chroot (see below). */
chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot";
chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
deletePath(chrootRootDir);
/* Clean up the chroot directory automatically. */
@ -2551,7 +2551,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
void DerivationGoal::writeStructuredAttrs()
{
auto & structuredAttrs = parsedDrv->getStructuredAttrs();
auto structuredAttrs = parsedDrv->getStructuredAttrs();
if (!structuredAttrs) return;
auto json = *structuredAttrs;
@ -2917,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path)
#if __linux__
Path source = worker.store.toRealPath(worker.store.printStorePath(path));
Path source = worker.store.Store::toRealPath(path);
Path target = chrootRootDir + worker.store.printStorePath(path);
debug("bind-mounting %s -> %s", target, source);
@ -3579,7 +3579,7 @@ void DerivationGoal::registerOutputs()
if (needsHashRewrite()) {
auto r = redirectedOutputs.find(i.second.path);
if (r != redirectedOutputs.end()) {
auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second));
auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair
&& redirectedBadOutputs.count(i.second.path)
&& pathExists(redirected))
@ -3672,7 +3672,7 @@ void DerivationGoal::registerOutputs()
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest));
Path actualDest = worker.store.Store::toRealPath(dest);
if (worker.store.isValidPath(dest))
std::rethrow_exception(delayedException);

View file

@ -156,7 +156,7 @@ void buildProfile(const Path & out, Packages && pkgs)
addPkg(pkgDir, priorityCounter++);
}
printError("created %d symlinks in user environment", state.symlinks);
debug("created %d symlinks in user environment", state.symlinks);
}
void builtinBuildenv(const BasicDerivation & drv)

View file

@ -163,10 +163,11 @@ static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
) return 0;
(!forceHttp || std::string(uri, 0, 7) != "file://"))
return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init();
return store;

View file

@ -298,9 +298,7 @@ void LocalStore::openDB(State & state, bool create)
/* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
if (sqlite3_open_v2(dbPath.c_str(), &db.db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open Nix database '%1%'") % dbPath);
state.db = SQLite(dbPath, create);
#ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls
@ -312,11 +310,6 @@ void LocalStore::openDB(State & state, bool create)
SetDllDirectoryW(L"");
#endif
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
db.exec("pragma foreign_keys = 1");
/* !!! check whether sqlite has been built with foreign key
support */
@ -350,7 +343,7 @@ void LocalStore::openDB(State & state, bool create)
/* Initialise the database schema, if necessary. */
if (create) {
const char * schema =
static const char schema[] =
#include "schema.sql.gen.hh"
;
db.exec(schema);
@ -1275,7 +1268,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
dumpPath(toRealPath(printStorePath(i)), *hashSink);
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
if (info->narHash != nullHash && info->narHash != current.first) {

View file

@ -78,12 +78,7 @@ public:
state->db = SQLite(dbPath);
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(state->db, "setting timeout");
// We can always reproduce the cache.
state->db.exec("pragma synchronous = off");
state->db.exec("pragma main.journal_mode = truncate");
state->db.isCache();
state->db.exec(schema);

View file

@ -10,7 +10,7 @@ class NarInfoDiskCache
public:
typedef enum { oValid, oInvalid, oUnknown } Outcome;
virtual ~NarInfoDiskCache() { };
virtual ~NarInfoDiskCache() { }
virtual void createCache(const std::string & uri, const Path & storeDir,
bool wantMassQuery, int priority) = 0;

View file

@ -1,5 +1,7 @@
#include "parsed-derivations.hh"
#include <nlohmann/json.hpp>
namespace nix {
ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
auto jsonAttr = drv.env.find("__json");
if (jsonAttr != drv.env.end()) {
try {
structuredAttrs = nlohmann::json::parse(jsonAttr->second);
structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second));
} catch (std::exception & e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
}
}
}
ParsedDerivation::~ParsedDerivation() { }
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
{
if (structuredAttrs) {

View file

@ -1,6 +1,6 @@
#include "derivations.hh"
#include <nlohmann/json.hpp>
#include <nlohmann/json_fwd.hpp>
namespace nix {
@ -8,15 +8,17 @@ class ParsedDerivation
{
StorePath drvPath;
BasicDerivation & drv;
std::optional<nlohmann::json> structuredAttrs;
std::unique_ptr<nlohmann::json> structuredAttrs;
public:
ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
const std::optional<nlohmann::json> & getStructuredAttrs() const
~ParsedDerivation();
const nlohmann::json * getStructuredAttrs() const
{
return structuredAttrs;
return structuredAttrs.get();
}
std::optional<std::string> getStringAttr(const std::string & name) const;

View file

@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile)
}
Path getDefaultProfile()
{
Path profileLink = getHome() + "/.nix-profile";
try {
if (!pathExists(profileLink)) {
replaceSymlink(
getuid() == 0
? settings.nixStateDir + "/profiles/default"
: fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
profileLink);
}
return absPath(readLink(profileLink), dirOf(profileLink));
} catch (Error &) {
return profileLink;
}
}
}

View file

@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile);
rebuilt. */
string optimisticLockProfile(const Path & profile);
/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
it. */
Path getDefaultProfile();
}

View file

@ -25,11 +25,16 @@ namespace nix {
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
}
SQLite::SQLite(const Path & path)
SQLite::SQLite(const Path & path, bool create)
{
if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path);
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
exec("pragma foreign_keys = 1");
}
SQLite::~SQLite()
@ -42,6 +47,12 @@ SQLite::~SQLite()
}
}
void SQLite::isCache()
{
exec("pragma synchronous = off");
exec("pragma main.journal_mode = truncate");
}
void SQLite::exec(const std::string & stmt)
{
retrySQLite<void>([&]() {
@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
{
if (notNull) {
if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{
if (notNull) {

View file

@ -5,8 +5,8 @@
#include "types.hh"
class sqlite3;
class sqlite3_stmt;
struct sqlite3;
struct sqlite3_stmt;
namespace nix {
@ -15,13 +15,16 @@ struct SQLite
{
sqlite3 * db = 0;
SQLite() { }
SQLite(const Path & path);
SQLite(const Path & path, bool create = true);
SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
~SQLite();
operator sqlite3 * () { return db; }
/* Disable synchronous mode, set truncate journal mode. */
void isCache();
void exec(const std::string & stmt);
};
@ -52,6 +55,7 @@ struct SQLiteStmt
/* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null

View file

@ -677,6 +677,11 @@ public:
return storePath;
}
Path toRealPath(const StorePath & storePath)
{
return toRealPath(printStorePath(storePath));
}
virtual void createUser(const std::string & userName, uid_t userId)
{ }

View file

@ -80,6 +80,18 @@ struct Hash
or base-64. By default, this is prefixed by the hash type
(e.g. "sha256:"). */
std::string to_string(Base base = Base32, bool includeType = true) const;
std::string gitRev() const
{
assert(type == htSHA1);
return to_string(Base16, false);
}
std::string gitShortRev() const
{
assert(type == htSHA1);
return std::string(to_string(Base16, false), 0, 7);
}
};

View file

@ -476,10 +476,11 @@ string base64Decode(const string & s);
/* Get a value for the specified key from an associate container, or a
default value if the key doesn't exist. */
template <class T>
std::optional<std::string> get(const T & map, const std::string & key)
std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
{
auto i = map.find(key);
return i == map.end() ? std::optional<std::string>() : i->second;
if (i == map.end()) return {};
return std::optional<typename T::mapped_type>(i->second);
}

View file

@ -1428,21 +1428,8 @@ static int _main(int argc, char * * argv)
if (globals.profile == "")
globals.profile = getEnv("NIX_PROFILE").value_or("");
if (globals.profile == "") {
Path profileLink = getHome() + "/.nix-profile";
try {
if (!pathExists(profileLink)) {
replaceSymlink(
getuid() == 0
? settings.nixStateDir + "/profiles/default"
: fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
profileLink);
}
globals.profile = absPath(readLink(profileLink), dirOf(profileLink));
} catch (Error &) {
globals.profile = profileLink;
}
}
if (globals.profile == "")
globals.profile = getDefaultProfile();
op(globals, opFlags, opArgs);

View file

@ -48,7 +48,7 @@ basicTests
# Test HttpBinaryCacheStore.
export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1
export _NIX_FORCE_HTTP=1
basicTests
@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
otherKey="$(cat $TEST_ROOT/pk3)"
_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
# Downloading should fail if we don't provide a key.

View file

@ -3,7 +3,7 @@ rec {
path = "@coreutils@";
system = builtins.currentSystem;
system = "@system@";
shared = builtins.getEnv "_NIX_TEST_SHARED";

View file

@ -9,6 +9,8 @@ clearStore
repo=$TEST_ROOT/git
export _NIX_FORCE_HTTP=1
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
git init $repo