delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #1188 from DeterminateSystems/nix-2.7

Browse source 
Nix 2.7
This commit is contained in:
Graham Christensen 2022-03-29 15:49:37 -04:00 committed by GitHub
parent 78ef4ae9a5 089da272c7
commit bbb0998699
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 74 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
flake.lock
View file

@ -3,16 +3,15 @@
"lowdown-src": { "lowdown-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"lastModified": 1617481909, "lastModified": 1633514407,
"narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=", "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
"owner": "kristapsdz", "owner": "kristapsdz",
"repo": "lowdown", "repo": "lowdown",
"rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d", "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "kristapsdz", "owner": "kristapsdz",
"ref": "VERSION_0_8_4",
"repo": "lowdown", "repo": "lowdown",
"type": "github" "type": "github"
} }
@ -20,28 +19,31 @@
"nix": { "nix": {
"inputs": { "inputs": {
"lowdown-src": "lowdown-src", "lowdown-src": "lowdown-src",
"nixpkgs": "nixpkgs" "nixpkgs": "nixpkgs",
"nixpkgs-regression": "nixpkgs-regression"
}, },
"locked": { "locked": {
"lastModified": 1628586117, "lastModified": 1646680282,
"narHash": "sha256-8hS4xy7fq3z9XZIMYm4sQi9SzhcYqEJfdbwgDePoWuc=", "narHash": "sha256-m8tqCS6uHveDon5GSro5yZor9H+sHeh+v/veF1IGw24=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nix", "repo": "nix",
"rev": "a6ba313a0aac3b6e2fef434cb42d190a0849238e", "rev": "ffe155abd36366a870482625543f9bf924a58281",
"type": "github" "type": "github"
}, },
"original": { "original": {
"id": "nix", "owner": "NixOS",
"type": "indirect" "ref": "2.7.0",
"repo": "nix",
"type": "github"
} }
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1624862269, "lastModified": 1632864508,
"narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=", "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "f77036342e2b690c61c97202bf48f2ce13acc022", "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -50,6 +52,21 @@
"type": "indirect" "type": "indirect"
} }
}, },
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
"narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "github"
},
"original": {
"id": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "indirect"
}
},
"root": { "root": {
"inputs": { "inputs": {
"nix": "nix", "nix": "nix",

1
flake.nix
View file

@ -2,6 +2,7 @@
description = "A Nix-based continuous build system"; description = "A Nix-based continuous build system";
inputs.nixpkgs.follows = "nix/nixpkgs"; inputs.nixpkgs.follows = "nix/nixpkgs";
inputs.nix.url = github:NixOS/nix/2.7.0;
outputs = { self, nixpkgs, nix }: outputs = { self, nixpkgs, nix }:
let let

16
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -63,13 +63,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
static MyArgs myArgs; static MyArgs myArgs;
static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute)
{ {
Strings res; Strings res;
std::function<void(Value & v)> rec; std::function<void(Value & v)> rec;
rec = [&](Value & v) { rec = [&](Value & v) {
state.forceValue(v); state.forceValue(v, noPos);
if (v.type() == nString) if (v.type() == nString)
res.push_back(v.string.s); res.push_back(v.string.s);
else if (v.isList()) else if (v.isList())
@ -78,7 +78,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const stri
else if (v.type() == nAttrs) { else if (v.type() == nAttrs) {
auto a = v.attrs->find(state.symbols.create(subAttribute)); auto a = v.attrs->find(state.symbols.create(subAttribute));
if (a != v.attrs->end()) if (a != v.attrs->end())
res.push_back(state.forceString(*a->value)); res.push_back(std::string(state.forceString(*a->value)));
} }
}; };
@ -113,7 +113,7 @@ static void worker(
callFlake(state, lockedFlake, *vFlake); callFlake(state, lockedFlake, *vFlake);
auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
state.forceValue(*vOutputs); state.forceValue(*vOutputs, noPos);
auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
if (!aHydraJobs) if (!aHydraJobs)
@ -157,7 +157,7 @@ static void worker(
if (drv->querySystem() == "unknown") if (drv->querySystem() == "unknown")
throw EvalError("derivation must have a 'system' attribute"); throw EvalError("derivation must have a 'system' attribute");
auto drvPath = drv->queryDrvPath(); auto drvPath = state.store->printStorePath(drv->requireDrvPath());
nlohmann::json job; nlohmann::json job;
@ -186,13 +186,13 @@ static void worker(
for (auto & i : context) for (auto & i : context)
if (i.at(0) == '!') { if (i.at(0) == '!') {
size_t index = i.find("!", 1); size_t index = i.find("!", 1);
job["constituents"].push_back(string(i, index + 1)); job["constituents"].push_back(std::string(i, index + 1));
} }
state.forceList(*a->value, *a->pos); state.forceList(*a->value, *a->pos);
for (unsigned int n = 0; n < a->value->listSize(); ++n) { for (unsigned int n = 0; n < a->value->listSize(); ++n) {
auto v = a->value->listElems()[n]; auto v = a->value->listElems()[n];
state.forceValue(*v); state.forceValue(*v, noPos);
if (v->type() == nString) if (v->type() == nString)
job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); job["namedConstituents"].push_back(state.forceStringNoCtx(*v));
} }
@ -210,7 +210,7 @@ static void worker(
nlohmann::json out; nlohmann::json out;
for (auto & j : outputs) for (auto & j : outputs)
out[j.first] = j.second; out[j.first] = state.store->printStorePath(j.second);
job["outputs"] = std::move(out); job["outputs"] = std::move(out);
reply["job"] = std::move(job); reply["job"] = std::move(job);

2
src/hydra-queue-runner/Makefile.am
View file

@ -2,7 +2,7 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \ builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh state.hh db.hh \ hydra-build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh nar-extractor.cc nar-extractor.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

17
src/hydra-queue-runner/build-remote.cc
View file

@ -5,6 +5,7 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <fcntl.h> #include <fcntl.h>
#include "build-result.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
#include "state.hh" #include "state.hh"
#include "util.hh" #include "util.hh"
@ -49,7 +50,7 @@ static Strings extraStoreArgs(std::string & machine)
static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child) static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
{ {
string pgmName; std::string pgmName;
Pipe to, from; Pipe to, from;
to.create(); to.create();
from.create(); from.create();
@ -81,7 +82,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
if (machine->sshPublicHostKey != "") { if (machine->sshPublicHostKey != "") {
Path fileName = tmpDir + "/host-key"; Path fileName = tmpDir + "/host-key";
auto p = machine->sshName.find("@"); auto p = machine->sshName.find("@");
string host = p != string::npos ? string(machine->sshName, p + 1) : machine->sshName; std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName;
writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n"); writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n");
append(argv, {"-oUserKnownHostsFile=" + fileName}); append(argv, {"-oUserKnownHostsFile=" + fileName});
} }
@ -185,8 +186,8 @@ void State::buildRemote(ref<Store> destStore,
{ {
assert(BuildResult::TimedOut == 8); assert(BuildResult::TimedOut == 8);
string base(step->drvPath.to_string()); std::string base(step->drvPath.to_string());
result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); result.logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2);
AutoDelete autoDelete(result.logFile, false); AutoDelete autoDelete(result.logFile, false);
createDirs(dirOf(result.logFile)); createDirs(dirOf(result.logFile));
@ -249,7 +250,7 @@ void State::buildRemote(ref<Store> destStore,
} catch (EndOfFile & e) { } catch (EndOfFile & e) {
child.pid.wait(); child.pid.wait();
string s = chomp(readFile(result.logFile)); std::string s = chomp(readFile(result.logFile));
throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
} }
@ -287,9 +288,9 @@ void State::buildRemote(ref<Store> destStore,
this will copy the inputs to the binary cache from the local this will copy the inputs to the binary cache from the local
store. */ store. */
if (localStore != std::shared_ptr<Store>(destStore)) { if (localStore != std::shared_ptr<Store>(destStore)) {
StorePathSet closure; copyClosure(*localStore, *destStore,
localStore->computeFSClosure(step->drv->inputSrcs, closure); step->drv->inputSrcs,
copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute); NoRepair, NoCheckSigs, NoSubstitute);
} }
{ {

4
src/hydra-queue-runner/build-result.cc
View file

@ -1,4 +1,4 @@
#include "build-result.hh" #include "hydra-build-result.hh"
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
@ -78,7 +78,7 @@ BuildOutput getBuildOutput(
product.type = match[1]; product.type = match[1];
product.subtype = match[2]; product.subtype = match[2];
std::string s(match[3]); std::string s(match[3]);
product.path = s[0] == '"' ? string(s, 1, s.size() - 2) : s; product.path = s[0] == '"' ? std::string(s, 1, s.size() - 2) : s;
product.defaultPath = match[5]; product.defaultPath = match[5];
/* Ensure that the path exists and points into the Nix /* Ensure that the path exists and points into the Nix

2
src/hydra-queue-runner/builder.cc
View file

@ -1,7 +1,7 @@
#include <cmath> #include <cmath>
#include "state.hh" #include "state.hh"
#include "build-result.hh" #include "hydra-build-result.hh"
#include "finally.hh" #include "finally.hh"
#include "binary-cache-store.hh" #include "binary-cache-store.hh"

0
src/hydra-queue-runner/build-result.hh → src/hydra-queue-runner/hydra-build-result.hh
View file

18
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -7,7 +7,7 @@
#include <fcntl.h> #include <fcntl.h>
#include "state.hh" #include "state.hh"
#include "build-result.hh" #include "hydra-build-result.hh"
#include "store-api.hh" #include "store-api.hh"
#include "remote-store.hh" #include "remote-store.hh"
@ -87,7 +87,7 @@ void State::parseMachines(const std::string & contents)
} }
for (auto line : tokenizeString<Strings>(contents, "\n")) { for (auto line : tokenizeString<Strings>(contents, "\n")) {
line = trim(string(line, 0, line.find('#'))); line = trim(std::string(line, 0, line.find('#')));
auto tokens = tokenizeString<std::vector<std::string>>(line); auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.size() < 3) continue; if (tokens.size() < 3) continue;
tokens.resize(8); tokens.resize(8);
@ -95,7 +95,7 @@ void State::parseMachines(const std::string & contents)
auto machine = std::make_shared<Machine>(); auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0]; machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ","); machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2] == "-" ? string("") : tokens[2]; machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
if (tokens[3] != "") if (tokens[3] != "")
machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value(); machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
else else
@ -149,7 +149,7 @@ void State::parseMachines(const std::string & contents)
void State::monitorMachinesFile() void State::monitorMachinesFile()
{ {
string defaultMachinesFile = "/etc/nix/machines"; std::string defaultMachinesFile = "/etc/nix/machines";
auto machinesFiles = tokenizeString<std::vector<Path>>( auto machinesFiles = tokenizeString<std::vector<Path>>(
getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":");
@ -191,7 +191,7 @@ void State::monitorMachinesFile()
debug("reloading machines files"); debug("reloading machines files");
string contents; std::string contents;
for (auto & machinesFile : machinesFiles) { for (auto & machinesFile : machinesFiles) {
try { try {
contents += readFile(machinesFile); contents += readFile(machinesFile);
@ -308,7 +308,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath) Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath)
{ {
restart: restart:
auto stepNr = allocBuildStep(txn, build->id); auto stepNr = allocBuildStep(txn, build->id);
@ -683,14 +683,14 @@ void State::showStatus()
auto conn(dbPool.get()); auto conn(dbPool.get());
receiver statusDumped(*conn, "status_dumped"); receiver statusDumped(*conn, "status_dumped");
string status; std::string status;
bool barf = false; bool barf = false;
/* Get the last JSON status dump from the database. */ /* Get the last JSON status dump from the database. */
{ {
pqxx::work txn(*conn); pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'"); auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>(); if (res.size()) status = res[0][0].as<std::string>();
} }
if (status != "") { if (status != "") {
@ -710,7 +710,7 @@ void State::showStatus()
{ {
pqxx::work txn(*conn); pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'"); auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>(); if (res.size()) status = res[0][0].as<std::string>();
} }
} }

2
src/hydra-queue-runner/nar-extractor.cc
View file

@ -64,7 +64,7 @@ struct Extractor : ParseSink
} }
} }
void createSymlink(const Path & path, const string & target) override void createSymlink(const Path & path, const std::string & target) override
{ {
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink }); members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
} }

18
src/hydra-queue-runner/queue-monitor.cc
View file

@ -1,5 +1,5 @@
#include "state.hh" #include "state.hh"
#include "build-result.hh" #include "hydra-build-result.hh"
#include "globals.hh" #include "globals.hh"
#include <cstring> #include <cstring>
@ -111,12 +111,12 @@ bool State::getQueuedBuilds(Connection & conn,
if (builds_->count(id)) continue; if (builds_->count(id)) continue;
auto build = std::make_shared<Build>( auto build = std::make_shared<Build>(
localStore->parseStorePath(row["drvPath"].as<string>())); localStore->parseStorePath(row["drvPath"].as<std::string>()));
build->id = id; build->id = id;
build->jobsetId = row["jobset_id"].as<JobsetID>(); build->jobsetId = row["jobset_id"].as<JobsetID>();
build->projectName = row["project"].as<string>(); build->projectName = row["project"].as<std::string>();
build->jobsetName = row["jobset"].as<string>(); build->jobsetName = row["jobset"].as<std::string>();
build->jobName = row["job"].as<string>(); build->jobName = row["job"].as<std::string>();
build->maxSilentTime = row["maxsilent"].as<int>(); build->maxSilentTime = row["maxsilent"].as<int>();
build->buildTimeout = row["timeout"].as<int>(); build->buildTimeout = row["timeout"].as<int>();
build->timestamp = row["timestamp"].as<time_t>(); build->timestamp = row["timestamp"].as<time_t>();
@ -513,9 +513,9 @@ Step::ptr State::createStep(ref<Store> destStore,
// FIXME: should copy directly from substituter to destStore. // FIXME: should copy directly from substituter to destStore.
} }
StorePathSet closure; copyClosure(*localStore, *destStore,
localStore->computeFSClosure({*path}, closure); StorePathSet { *path },
copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute); NoRepair, CheckSigs, NoSubstitute);
time_t stopTime = time(0); time_t stopTime = time(0);
@ -620,7 +620,7 @@ void State::processJobsetSharesChange(Connection & conn)
auto res = txn.exec("select project, name, schedulingShares from Jobsets"); auto res = txn.exec("select project, name, schedulingShares from Jobsets");
for (auto const & row : res) { for (auto const & row : res) {
auto jobsets_(jobsets.lock()); auto jobsets_(jobsets.lock());
auto i = jobsets_->find(std::make_pair(row["project"].as<string>(), row["name"].as<string>())); auto i = jobsets_->find(std::make_pair(row["project"].as<std::string>(), row["name"].as<std::string>()));
if (i == jobsets_->end()) continue; if (i == jobsets_->end()) continue;
i->second->setShares(row["schedulingShares"].as<unsigned int>()); i->second->setShares(row["schedulingShares"].as<unsigned int>());
} }

1
src/hydra-queue-runner/state.hh
View file

@ -12,6 +12,7 @@
#include "parsed-derivations.hh" #include "parsed-derivations.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "pool.hh" #include "pool.hh"
#include "build-result.hh"
#include "store-api.hh" #include "store-api.hh"
#include "sync.hh" #include "sync.hh"
#include "nar-extractor.hh" #include "nar-extractor.hh"

2
src/libhydra/db.hh
View file

@ -18,7 +18,7 @@ struct Connection : pqxx::connection
std::string upper_prefix = "DBI:Pg:"; std::string upper_prefix = "DBI:Pg:";
if (hasPrefix(s, lower_prefix) || hasPrefix(s, upper_prefix)) { if (hasPrefix(s, lower_prefix) || hasPrefix(s, upper_prefix)) {
return concatStringsSep(" ", tokenizeString<Strings>(string(s, lower_prefix.size()), ";")); return concatStringsSep(" ", tokenizeString<Strings>(std::string(s, lower_prefix.size()), ";"));
} }
throw Error("$HYDRA_DBI does not denote a PostgreSQL database"); throw Error("$HYDRA_DBI does not denote a PostgreSQL database");

2
src/libhydra/hydra-config.hh
View file

@ -17,7 +17,7 @@ struct HydraConfig
if (hydraConfigFile && pathExists(*hydraConfigFile)) { if (hydraConfigFile && pathExists(*hydraConfigFile)) {
for (auto line : tokenizeString<Strings>(readFile(*hydraConfigFile), "\n")) { for (auto line : tokenizeString<Strings>(readFile(*hydraConfigFile), "\n")) {
line = trim(string(line, 0, line.find('#'))); line = trim(std::string(line, 0, line.find('#')));
auto eq = line.find('='); auto eq = line.find('=');
if (eq == std::string::npos) continue; if (eq == std::string::npos) continue;