lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Merge remote-tracking branch 'origin/master' into flakes

Browse source
This commit is contained in:
Eelco Dolstra 2019-12-18 14:25:25 +01:00
parent 31c240ee8b f8abbdd456
commit 87873d0d65
25 changed files with 128 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
nix-rust/src/c.rs
View file

@ -34,7 +34,8 @@ pub extern "C" fn ffi_StorePath_new(
path: &str,
store_dir: &str,
) -> Result<StorePath, error::CppException> {
StorePath::new(std::path::Path::new(path), store_dir).map_err(|err| err.into())
StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir))
.map_err(|err| err.into())
}
#[no_mangle]

4
nix-rust/src/error.rs
View file

@ -4,6 +4,7 @@ use std::fmt;
pub enum Error {
InvalidPath(crate::store::StorePath),
BadStorePath(std::path::PathBuf),
NotInStore(std::path::PathBuf),
BadNarInfo,
BadBase32,
StorePathNameEmpty,
@ -46,6 +47,9 @@ impl fmt::Display for Error {
Error::InvalidPath(_) => write!(f, "invalid path"),
Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
Error::NotInStore(path) => {
write!(f, "path '{}' is not in the Nix store", path.display())
}
Error::BadBase32 => write!(f, "invalid base32 string"),
Error::StorePathNameEmpty => write!(f, "store path name is empty"),
Error::StorePathNameTooLong => {

6
nix-rust/src/store/path.rs
View file

@ -13,8 +13,10 @@ pub const STORE_PATH_HASH_BYTES: usize = 20;
pub const STORE_PATH_HASH_CHARS: usize = 32;
impl StorePath {
pub fn new(path: &Path, _store_dir: &str) -> Result<Self, Error> {
// FIXME: check store_dir
pub fn new(path: &Path, store_dir: &Path) -> Result<Self, Error> {
if path.parent() != Some(store_dir) {
return Err(Error::NotInStore(path.into()));
}
Self::new_from_base_name(
path.file_name()
.ok_or(Error::BadStorePath(path.into()))?

2
src/libexpr/get-drvs.cc
View file

@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("")
{
auto [drvPath, selectedOutputs] = store->parseDrvPathWithOutputs(drvPathWithOutputs);
auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
this->drvPath = store->printStorePath(drvPath);

4
src/libstore/binary-cache-store.cc
View file

@ -49,9 +49,9 @@ void BinaryCacheStore::init()
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
% getUri() % value % storeDir);
} else if (name == "WantMassQuery") {
wantMassQuery_ = value == "1";
wantMassQuery.setDefault(value == "1" ? "true" : "false");
} else if (name == "Priority") {
string2Int(value, priority);
priority.setDefault(fmt("%d", std::stoi(value)));
}
}
}

9
src/libstore/binary-cache-store.hh
View file

@ -52,11 +52,6 @@ public:
std::shared_ptr<std::string> getFile(const std::string & path);
protected:
bool wantMassQuery_ = false;
int priority = 50;
public:
virtual void init();
@ -79,8 +74,6 @@ public:
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
bool wantMassQuery() override { return wantMassQuery_; }
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
@ -107,8 +100,6 @@ public:
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
int getPriority() override { return priority; }
};
MakeError(NoSuchBinaryCacheFile, Error);

4
src/libstore/daemon.cc
View file

@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopBuildPaths: {
std::vector<StorePathWithOutputs> drvs;
for (auto & s : readStrings<Strings>(from))
drvs.push_back(store->parseDrvPathWithOutputs(s));
drvs.push_back(store->parsePathWithOutputs(s));
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
@ -721,7 +721,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryMissing: {
std::vector<StorePathWithOutputs> targets;
for (auto & s : readStrings<Strings>(from))
targets.push_back(store->parseDrvPathWithOutputs(s));
targets.push_back(store->parsePathWithOutputs(s));
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;

10
src/libstore/derivations.cc
View file

@ -375,16 +375,6 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
}
StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s)
{
size_t n = s.find("!");
return n == s.npos
? StorePathWithOutputs{parseStorePath(s), std::set<string>()}
: StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)),
tokenizeString<std::set<string>>(string(s, n + 1), ",")};
}
std::string StorePathWithOutputs::to_string(const Store & store) const
{
return outputs.empty()

2
src/libstore/globals.cc
View file

@ -127,7 +127,7 @@ template<> void BaseSetting<SandboxMode>::set(const std::string & str)
else throw UsageError("option '%s' has invalid value '%s'", name, str);
}
template<> std::string BaseSetting<SandboxMode>::to_string()
template<> std::string BaseSetting<SandboxMode>::to_string() const
{
if (value == smEnabled) return "true";
else if (value == smRelaxed) return "relaxed";

7
src/libstore/http-binary-cache-store.cc
View file

@ -42,13 +42,16 @@ public:
void init() override
{
// FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
priority.setDefault(fmt("%d", cacheInfo->priority));
} else {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", cacheUri);
}
diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
}
}

2
src/libstore/local-store.cc
View file

@ -833,7 +833,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
for (auto & sub : getDefaultSubstituters()) {
if (remaining.empty()) break;
if (sub->storeDir != storeDir) continue;
if (!sub->wantMassQuery()) continue;
if (!sub->wantMassQuery) continue;
auto valid = sub->queryValidPaths(remaining);

16
src/libstore/nar-info-disk-cache.cc
View file

@ -142,26 +142,26 @@ public:
});
}
bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) override
std::optional<CacheInfo> cacheExists(const std::string & uri) override
{
return retrySQLite<bool>([&]() {
return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
auto state(_state.lock());
auto i = state->caches.find(uri);
if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri));
if (!queryCache.next()) return false;
if (!queryCache.next())
return std::nullopt;
state->caches.emplace(uri,
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
}
auto & cache(getCache(*state, uri));
wantMassQuery = cache.wantMassQuery;
priority = cache.priority;
return true;
return CacheInfo {
.wantMassQuery = cache.wantMassQuery,
.priority = cache.priority
};
});
}

9
src/libstore/nar-info-disk-cache.hh
View file

@ -15,8 +15,13 @@ public:
virtual void createCache(const std::string & uri, const Path & storeDir,
bool wantMassQuery, int priority) = 0;
virtual bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) = 0;
struct CacheInfo
{
bool wantMassQuery;
int priority;
};
virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) = 0;

15
src/libstore/path.cc
View file

@ -96,4 +96,19 @@ StorePathSet singleton(const StorePath & path)
return res;
}
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
}
StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
{
auto [path, outputs] = nix::parsePathWithOutputs(s);
return {parseStorePath(path), std::move(outputs)};
}
}

24
src/libstore/path.hh
View file

@ -7,6 +7,8 @@ namespace nix {
/* See path.rs. */
struct StorePath;
struct Store;
extern "C" {
void ffi_StorePath_drop(void *);
bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
@ -67,6 +69,28 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
: path(path.clone()), outputs(outputs)
{ }
StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs)
: path(std::move(path)), outputs(std::move(outputs))
{ }
StorePathWithOutputs(const StorePathWithOutputs & other)
: path(other.path.clone()), outputs(other.outputs)
{ }
std::string to_string(const Store & store) const;
};
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
}
namespace std {

9
src/libstore/s3-binary-cache-store.cc
View file

@ -205,11 +205,12 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
void init() override
{
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
if (auto cacheInfo = diskCache->cacheExists(getUri())) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
priority.setDefault(fmt("%d", cacheInfo->priority));
} else {
BinaryCacheStore::init();
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
diskCache->createCache(getUri(), storeDir, wantMassQuery, priority);
}
}

4
src/libstore/sqlite.hh
View file

@ -103,8 +103,8 @@ void handleSQLiteBusy(const SQLiteBusy & e);
/* Convenience function for retrying a SQLite transaction when the
database is busy. */
template<typename T>
T retrySQLite(std::function<T()> fun)
template<typename T, typename F>
T retrySQLite(F && fun)
{
while (true) {
try {

15
src/libstore/store-api.cc
View file

@ -39,9 +39,9 @@ Path Store::toStorePath(const Path & path) const
}
Path Store::followLinksToStore(const Path & _path) const
Path Store::followLinksToStore(std::string_view _path) const
{
Path path = absPath(_path);
Path path = absPath(std::string(_path));
while (!isInStore(path)) {
if (!isLink(path)) break;
string target = readLink(path);
@ -53,12 +53,19 @@ Path Store::followLinksToStore(const Path & _path) const
}
StorePath Store::followLinksToStorePath(const Path & path) const
StorePath Store::followLinksToStorePath(std::string_view path) const
{
return parseStorePath(toStorePath(followLinksToStore(path)));
}
StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
{
auto [path2, outputs] = nix::parsePathWithOutputs(path);
return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs));
}
string storePathToHash(const Path & path)
{
auto base = baseNameOf(path);
@ -963,7 +970,7 @@ std::list<ref<Store>> getDefaultSubstituters()
addStore(uri);
stores.sort([](ref<Store> & a, ref<Store> & b) {
return a->getPriority() < b->getPriority();
return a->priority < b->priority;
});
return stores;

36
src/libstore/store-api.hh
View file

@ -242,23 +242,6 @@ struct BuildResult
};
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
: path(path.clone()), outputs(outputs)
{ }
StorePathWithOutputs(const StorePathWithOutputs & other)
: path(other.path.clone()), outputs(other.outputs)
{ }
std::string to_string(const Store & store) const;
};
class Store : public std::enable_shared_from_this<Store>, public Config
{
public:
@ -273,6 +256,10 @@ public:
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
protected:
struct State
@ -305,7 +292,7 @@ public:
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parseDrvPathWithOutputs(const string & s);
StorePathWithOutputs parsePathWithOutputs(const string & s);
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
@ -324,11 +311,13 @@ public:
Path toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(const Path & path) const;
Path followLinksToStore(std::string_view path) const;
/* Same as followLinksToStore(), but apply toStorePath() to the
result. */
StorePath followLinksToStorePath(const Path & path) const;
StorePath followLinksToStorePath(std::string_view path) const;
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */
StorePath makeStorePath(const string & type,
@ -438,8 +427,6 @@ public:
virtual void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) { return; };
virtual bool wantMassQuery() { return false; }
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
@ -664,11 +651,6 @@ public:
return 0;
};
/* Get the priority of the store, used to order substituters. In
particular, binary caches can specify a priority field in their
"nix-cache-info" file. Lower value means higher priority. */
virtual int getPriority() { return 0; }
virtual Path toRealPath(const Path & storePath)
{
return storePath;

15
src/libutil/config.cc
View file

@ -154,6 +154,11 @@ AbstractSetting::AbstractSetting(
{
}
void AbstractSetting::setDefault(const std::string & str)
{
if (!overriden) set(str);
}
void AbstractSetting::toJSON(JSONPlaceholder & out)
{
out.write(to_string());
@ -185,7 +190,7 @@ template<> void BaseSetting<std::string>::set(const std::string & str)
value = str;
}
template<> std::string BaseSetting<std::string>::to_string()
template<> std::string BaseSetting<std::string>::to_string() const
{
return value;
}
@ -199,7 +204,7 @@ void BaseSetting<T>::set(const std::string & str)
}
template<typename T>
std::string BaseSetting<T>::to_string()
std::string BaseSetting<T>::to_string() const
{
static_assert(std::is_integral<T>::value, "Integer required.");
return std::to_string(value);
@ -215,7 +220,7 @@ template<> void BaseSetting<bool>::set(const std::string & str)
throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
}
template<> std::string BaseSetting<bool>::to_string()
template<> std::string BaseSetting<bool>::to_string() const
{
return value ? "true" : "false";
}
@ -239,7 +244,7 @@ template<> void BaseSetting<Strings>::set(const std::string & str)
value = tokenizeString<Strings>(str);
}
template<> std::string BaseSetting<Strings>::to_string()
template<> std::string BaseSetting<Strings>::to_string() const
{
return concatStringsSep(" ", value);
}
@ -256,7 +261,7 @@ template<> void BaseSetting<StringSet>::set(const std::string & str)
value = tokenizeString<StringSet>(str);
}
template<> std::string BaseSetting<StringSet>::to_string()
template<> std::string BaseSetting<StringSet>::to_string() const
{
return concatStringsSep(" ", value);
}

8
src/libutil/config.hh
View file

@ -115,6 +115,8 @@ public:
bool overriden = false;
void setDefault(const std::string & str);
protected:
AbstractSetting(
@ -131,13 +133,13 @@ protected:
virtual void set(const std::string & value) = 0;
virtual std::string to_string() = 0;
virtual std::string to_string() const = 0;
virtual void toJSON(JSONPlaceholder & out);
virtual void convertToArg(Args & args, const std::string & category);
bool isOverriden() { return overriden; }
bool isOverriden() const { return overriden; }
};
/* A setting of type T. */
@ -174,7 +176,7 @@ public:
value = v;
}
std::string to_string() override;
std::string to_string() const override;
void convertToArg(Args & args, const std::string & category) override;

8
src/libutil/util.cc
View file

@ -1213,7 +1213,7 @@ void _interrupted()
//////////////////////////////////////////////////////////////////////
template<class C> C tokenizeString(const string & s, const string & separators)
template<class C> C tokenizeString(std::string_view s, const string & separators)
{
C result;
string::size_type pos = s.find_first_not_of(separators, 0);
@ -1227,9 +1227,9 @@ template<class C> C tokenizeString(const string & s, const string & separators)
return result;
}
template Strings tokenizeString(const string & s, const string & separators);
template StringSet tokenizeString(const string & s, const string & separators);
template vector<string> tokenizeString(const string & s, const string & separators);
template Strings tokenizeString(std::string_view s, const string & separators);
template StringSet tokenizeString(std::string_view s, const string & separators);
template vector<string> tokenizeString(std::string_view s, const string & separators);
string chomp(const string & s)

2
src/libutil/util.hh
View file

@ -346,7 +346,7 @@ MakeError(FormatError, Error);
/* String tokenizer. */
template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r");
template<class C> C tokenizeString(std::string_view s, const string & separators = " \t\n\r");
/* Concatenate the given strings with a separator between the

4
src/nix-store/nix-store.cc
View file

@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
std::vector<StorePathWithOutputs> paths;
for (auto & i : opArgs)
paths.push_back(store->parseDrvPathWithOutputs(i));
paths.push_back(store->followLinksToStorePathWithOutputs(i));
unsigned long long downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
@ -893,7 +893,7 @@ static void opServe(Strings opFlags, Strings opArgs)
std::vector<StorePathWithOutputs> paths;
for (auto & s : readStrings<Strings>(in))
paths.emplace_back(store->parseDrvPathWithOutputs(s));
paths.emplace_back(store->parsePathWithOutputs(s));
getBuildSettings();

3
tests/nix-build.sh
View file

@ -23,3 +23,6 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link)
outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
[[ $outPath = $outPath2 ]]
outPath2=$(nix-store -r $(nix-instantiate --indirect --add-root $TEST_ROOT/indirect dependencies.nix)!out)
[[ $outPath = $outPath2 ]]