Make system-features
a store setting
This seems more correct. It also means one can specify the features a store should support with --store and remote-store=..., which is useful. I use this to clean up the build remotes test.
This commit is contained in:
parent
574bf60b4d
commit
4720853129
|
@ -38,9 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
|
||||||
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
|
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
|
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
|
||||||
for (auto & feature : requiredFeatures)
|
for (auto & feature : requiredFeatures)
|
||||||
if (!settings.systemFeatures.get().count(feature)) return false;
|
if (!store.systemFeatures.get().count(feature)) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,7 +106,7 @@ static int _main(int argc, char * * argv)
|
||||||
auto canBuildLocally = amWilling
|
auto canBuildLocally = amWilling
|
||||||
&& ( neededSystem == settings.thisSystem
|
&& ( neededSystem == settings.thisSystem
|
||||||
|| settings.extraPlatforms.get().count(neededSystem) > 0)
|
|| settings.extraPlatforms.get().count(neededSystem) > 0)
|
||||||
&& allSupportedLocally(requiredFeatures);
|
&& allSupportedLocally(*store, requiredFeatures);
|
||||||
|
|
||||||
/* Error ignored here, will be caught later */
|
/* Error ignored here, will be caught later */
|
||||||
mkdir(currentLoad.c_str(), 0777);
|
mkdir(currentLoad.c_str(), 0777);
|
||||||
|
@ -224,15 +224,7 @@ static int _main(int argc, char * * argv)
|
||||||
|
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
|
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
|
||||||
|
|
||||||
Store::Params storeParams;
|
sshStore = bestMachine->openStore();
|
||||||
if (hasPrefix(bestMachine->storeUri, "ssh://")) {
|
|
||||||
storeParams["max-connections"] = "1";
|
|
||||||
storeParams["log-fd"] = "4";
|
|
||||||
if (bestMachine->sshKey != "")
|
|
||||||
storeParams["ssh-key"] = bestMachine->sshKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
sshStore = openStore(bestMachine->storeUri, storeParams);
|
|
||||||
sshStore->connect();
|
sshStore->connect();
|
||||||
storeUri = bestMachine->storeUri;
|
storeUri = bestMachine->storeUri;
|
||||||
|
|
||||||
|
|
|
@ -1475,7 +1475,7 @@ void DerivationGoal::tryToBuild()
|
||||||
/* Don't do a remote build if the derivation has the attribute
|
/* Don't do a remote build if the derivation has the attribute
|
||||||
`preferLocalBuild' set. Also, check and repair modes are only
|
`preferLocalBuild' set. Also, check and repair modes are only
|
||||||
supported for local builds. */
|
supported for local builds. */
|
||||||
bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
|
bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
|
||||||
|
|
||||||
/* Is the build hook willing to accept this job? */
|
/* Is the build hook willing to accept this job? */
|
||||||
if (!buildLocally) {
|
if (!buildLocally) {
|
||||||
|
@ -1964,13 +1964,13 @@ void linkOrCopy(const Path & from, const Path & to)
|
||||||
void DerivationGoal::startBuilder()
|
void DerivationGoal::startBuilder()
|
||||||
{
|
{
|
||||||
/* Right platform? */
|
/* Right platform? */
|
||||||
if (!parsedDrv->canBuildLocally())
|
if (!parsedDrv->canBuildLocally(worker.store))
|
||||||
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
|
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
|
||||||
drv->platform,
|
drv->platform,
|
||||||
concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
|
concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
|
||||||
worker.store.printStorePath(drvPath),
|
worker.store.printStorePath(drvPath),
|
||||||
settings.thisSystem,
|
settings.thisSystem,
|
||||||
concatStringsSep<StringSet>(", ", settings.systemFeatures));
|
concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
|
||||||
|
|
||||||
if (drv->isBuiltin())
|
if (drv->isBuiltin())
|
||||||
preloadNSS();
|
preloadNSS();
|
||||||
|
@ -3179,7 +3179,7 @@ void DerivationGoal::runChild()
|
||||||
createDirs(chrootRootDir + "/dev/shm");
|
createDirs(chrootRootDir + "/dev/shm");
|
||||||
createDirs(chrootRootDir + "/dev/pts");
|
createDirs(chrootRootDir + "/dev/pts");
|
||||||
ss.push_back("/dev/full");
|
ss.push_back("/dev/full");
|
||||||
if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
|
if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
|
||||||
ss.push_back("/dev/kvm");
|
ss.push_back("/dev/kvm");
|
||||||
ss.push_back("/dev/null");
|
ss.push_back("/dev/null");
|
||||||
ss.push_back("/dev/random");
|
ss.push_back("/dev/random");
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#include "machines.hh"
|
#include "machines.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
|
||||||
|
@ -48,6 +49,29 @@ bool Machine::mandatoryMet(const std::set<string> & features) const {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ref<Store> Machine::openStore() const {
|
||||||
|
Store::Params storeParams;
|
||||||
|
if (hasPrefix(storeUri, "ssh://")) {
|
||||||
|
storeParams["max-connections"] = "1";
|
||||||
|
storeParams["log-fd"] = "4";
|
||||||
|
if (sshKey != "")
|
||||||
|
storeParams["ssh-key"] = sshKey;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
auto & fs = storeParams["system-features"];
|
||||||
|
auto append = [&](auto feats) {
|
||||||
|
for (auto & f : feats) {
|
||||||
|
if (fs.size() > 0) fs += ' ';
|
||||||
|
fs += f;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
append(supportedFeatures);
|
||||||
|
append(mandatoryFeatures);
|
||||||
|
}
|
||||||
|
|
||||||
|
return nix::openStore(storeUri, storeParams);
|
||||||
|
}
|
||||||
|
|
||||||
void parseMachines(const std::string & s, Machines & machines)
|
void parseMachines(const std::string & s, Machines & machines)
|
||||||
{
|
{
|
||||||
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
|
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
|
||||||
|
|
|
@ -4,6 +4,8 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
class Store;
|
||||||
|
|
||||||
struct Machine {
|
struct Machine {
|
||||||
|
|
||||||
const string storeUri;
|
const string storeUri;
|
||||||
|
@ -28,6 +30,8 @@ struct Machine {
|
||||||
decltype(supportedFeatures) supportedFeatures,
|
decltype(supportedFeatures) supportedFeatures,
|
||||||
decltype(mandatoryFeatures) mandatoryFeatures,
|
decltype(mandatoryFeatures) mandatoryFeatures,
|
||||||
decltype(sshPublicHostKey) sshPublicHostKey);
|
decltype(sshPublicHostKey) sshPublicHostKey);
|
||||||
|
|
||||||
|
ref<Store> openStore() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef std::vector<Machine> Machines;
|
typedef std::vector<Machine> Machines;
|
||||||
|
|
|
@ -94,7 +94,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::canBuildLocally() const
|
bool ParsedDerivation::canBuildLocally(Store & localStore) const
|
||||||
{
|
{
|
||||||
if (drv.platform != settings.thisSystem.get()
|
if (drv.platform != settings.thisSystem.get()
|
||||||
&& !settings.extraPlatforms.get().count(drv.platform)
|
&& !settings.extraPlatforms.get().count(drv.platform)
|
||||||
|
@ -102,14 +102,14 @@ bool ParsedDerivation::canBuildLocally() const
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
for (auto & feature : getRequiredSystemFeatures())
|
for (auto & feature : getRequiredSystemFeatures())
|
||||||
if (!settings.systemFeatures.get().count(feature)) return false;
|
if (!localStore.systemFeatures.get().count(feature)) return false;
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::willBuildLocally() const
|
bool ParsedDerivation::willBuildLocally(Store & localStore) const
|
||||||
{
|
{
|
||||||
return getBoolAttr("preferLocalBuild") && canBuildLocally();
|
return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::substitutesAllowed() const
|
bool ParsedDerivation::substitutesAllowed() const
|
||||||
|
|
|
@ -29,9 +29,9 @@ public:
|
||||||
|
|
||||||
StringSet getRequiredSystemFeatures() const;
|
StringSet getRequiredSystemFeatures() const;
|
||||||
|
|
||||||
bool canBuildLocally() const;
|
bool canBuildLocally(Store & localStore) const;
|
||||||
|
|
||||||
bool willBuildLocally() const;
|
bool willBuildLocally(Store & localStore) const;
|
||||||
|
|
||||||
bool substitutesAllowed() const;
|
bool substitutesAllowed() const;
|
||||||
};
|
};
|
||||||
|
|
|
@ -164,6 +164,10 @@ public:
|
||||||
|
|
||||||
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
|
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
|
||||||
|
|
||||||
|
Setting<StringSet> systemFeatures{this, settings.systemFeatures,
|
||||||
|
"system-features",
|
||||||
|
"Optional features that the system this store builds on implements (like \"kvm\")."};
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
struct PathInfoCacheValue {
|
struct PathInfoCacheValue {
|
||||||
|
|
|
@ -23,6 +23,7 @@ let
|
||||||
shell = busybox;
|
shell = busybox;
|
||||||
name = "build-remote-input-2";
|
name = "build-remote-input-2";
|
||||||
buildCommand = "echo BAR > $out";
|
buildCommand = "echo BAR > $out";
|
||||||
|
requiredSystemFeatures = ["bar"];
|
||||||
};
|
};
|
||||||
|
|
||||||
in
|
in
|
||||||
|
@ -34,6 +35,6 @@ in
|
||||||
''
|
''
|
||||||
read x < ${input1}
|
read x < ${input1}
|
||||||
read y < ${input2}
|
read y < ${input2}
|
||||||
echo $x$y > $out
|
echo "$x $y" > $out
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,31 +1,36 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
clearStore
|
|
||||||
|
|
||||||
if ! canUseSandbox; then exit; fi
|
if ! canUseSandbox; then exit; fi
|
||||||
if ! [[ $busybox =~ busybox ]]; then exit; fi
|
if ! [[ $busybox =~ busybox ]]; then exit; fi
|
||||||
|
|
||||||
chmod -R u+w $TEST_ROOT/machine0 || true
|
|
||||||
chmod -R u+w $TEST_ROOT/machine1 || true
|
|
||||||
chmod -R u+w $TEST_ROOT/machine2 || true
|
|
||||||
rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2
|
|
||||||
rm -f $TEST_ROOT/result
|
|
||||||
|
|
||||||
unset NIX_STORE_DIR
|
unset NIX_STORE_DIR
|
||||||
unset NIX_STATE_DIR
|
unset NIX_STATE_DIR
|
||||||
|
|
||||||
|
function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
|
||||||
|
|
||||||
|
builders=(
|
||||||
|
# system-features will automatically be added to the outer URL, but not inner
|
||||||
|
# remote-store URL.
|
||||||
|
"ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
|
||||||
|
"$TEST_ROOT/machine2 - - 1 1 bar"
|
||||||
|
)
|
||||||
|
|
||||||
# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
|
# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
|
||||||
# child process. This allows us to test LegacySSHStore::buildDerivation().
|
# child process. This allows us to test LegacySSHStore::buildDerivation().
|
||||||
|
# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().
|
||||||
nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
|
nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
|
||||||
--arg busybox $busybox \
|
--arg busybox $busybox \
|
||||||
--store $TEST_ROOT/machine0 \
|
--store $TEST_ROOT/machine0 \
|
||||||
--builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \
|
--builders "$(join_by '; ' "${builders[@]}")"
|
||||||
--system-features foo
|
|
||||||
|
|
||||||
outPath=$(readlink -f $TEST_ROOT/result)
|
outPath=$(readlink -f $TEST_ROOT/result)
|
||||||
|
|
||||||
cat $TEST_ROOT/machine0/$outPath | grep FOOBAR
|
grep 'FOO BAR' $TEST_ROOT/machine0/$outPath
|
||||||
|
|
||||||
# Ensure that input1 was built on store2 due to the required feature.
|
# Ensure that input1 was built on store1 due to the required feature.
|
||||||
(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh)
|
(! nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh)
|
||||||
nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh
|
nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh
|
||||||
|
|
||||||
|
# Ensure that input2 was built on store2 due to the required feature.
|
||||||
|
(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-2.sh)
|
||||||
|
nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-2.sh
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
nix_tests = \
|
nix_tests = \
|
||||||
init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
||||||
config.sh \
|
config.sh \
|
||||||
gc.sh \
|
gc.sh \
|
||||||
gc-concurrent.sh \
|
gc-concurrent.sh \
|
||||||
|
|
Loading…
Reference in a new issue