delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #1188 from DeterminateSystems/nix-2.7

Browse source 
Nix 2.7
This commit is contained in:
Graham Christensen 2022-03-29 15:49:37 -04:00 committed by GitHub
parent 78ef4ae9a5 089da272c7
commit bbb0998699
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 74 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
flake.lock
View file

@ -3,16 +3,15 @@
"lowdown-src": {
"flake": false,
"locked": {
"lastModified": 1617481909,
"narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
"lastModified": 1633514407,
"narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
"owner": "kristapsdz",
"repo": "lowdown",
"rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
"rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
"type": "github"
},
"original": {
"owner": "kristapsdz",
"ref": "VERSION_0_8_4",
"repo": "lowdown",
"type": "github"
}
@ -20,28 +19,31 @@
"nix": {
"inputs": {
"lowdown-src": "lowdown-src",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
"lastModified": 1628586117,
"narHash": "sha256-8hS4xy7fq3z9XZIMYm4sQi9SzhcYqEJfdbwgDePoWuc=",
"lastModified": 1646680282,
"narHash": "sha256-m8tqCS6uHveDon5GSro5yZor9H+sHeh+v/veF1IGw24=",
"owner": "NixOS",
"repo": "nix",
"rev": "a6ba313a0aac3b6e2fef434cb42d190a0849238e",
"rev": "ffe155abd36366a870482625543f9bf924a58281",
"type": "github"
},
"original": {
"id": "nix",
"type": "indirect"
"owner": "NixOS",
"ref": "2.7.0",
"repo": "nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1624862269,
"narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
"lastModified": 1632864508,
"narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
"rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
"type": "github"
},
"original": {
@ -50,6 +52,21 @@
"type": "indirect"
}
},
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
"narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "github"
},
"original": {
"id": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "indirect"
}
},
"root": {
"inputs": {
"nix": "nix",

1
flake.nix
View file

@ -2,6 +2,7 @@
description = "A Nix-based continuous build system";
inputs.nixpkgs.follows = "nix/nixpkgs";
inputs.nix.url = github:NixOS/nix/2.7.0;
outputs = { self, nixpkgs, nix }:
let

16
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -63,13 +63,13 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
static MyArgs myArgs;
static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute)
static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute)
{
Strings res;
std::function<void(Value & v)> rec;
rec = [&](Value & v) {
state.forceValue(v);
state.forceValue(v, noPos);
if (v.type() == nString)
res.push_back(v.string.s);
else if (v.isList())
@ -78,7 +78,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const stri
else if (v.type() == nAttrs) {
auto a = v.attrs->find(state.symbols.create(subAttribute));
if (a != v.attrs->end())
res.push_back(state.forceString(*a->value));
res.push_back(std::string(state.forceString(*a->value)));
}
};
@ -113,7 +113,7 @@ static void worker(
callFlake(state, lockedFlake, *vFlake);
auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
state.forceValue(*vOutputs);
state.forceValue(*vOutputs, noPos);
auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
if (!aHydraJobs)
@ -157,7 +157,7 @@ static void worker(
if (drv->querySystem() == "unknown")
throw EvalError("derivation must have a 'system' attribute");
auto drvPath = drv->queryDrvPath();
auto drvPath = state.store->printStorePath(drv->requireDrvPath());
nlohmann::json job;
@ -186,13 +186,13 @@ static void worker(
for (auto & i : context)
if (i.at(0) == '!') {
size_t index = i.find("!", 1);
job["constituents"].push_back(string(i, index + 1));
job["constituents"].push_back(std::string(i, index + 1));
}
state.forceList(*a->value, *a->pos);
for (unsigned int n = 0; n < a->value->listSize(); ++n) {
auto v = a->value->listElems()[n];
state.forceValue(*v);
state.forceValue(*v, noPos);
if (v->type() == nString)
job["namedConstituents"].push_back(state.forceStringNoCtx(*v));
}
@ -210,7 +210,7 @@ static void worker(
nlohmann::json out;
for (auto & j : outputs)
out[j.first] = j.second;
out[j.first] = state.store->printStorePath(j.second);
job["outputs"] = std::move(out);
reply["job"] = std::move(job);

2
src/hydra-queue-runner/Makefile.am
View file

@ -2,7 +2,7 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh state.hh db.hh \
hydra-build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

17
src/hydra-queue-runner/build-remote.cc
View file

@ -5,6 +5,7 @@
#include <sys/stat.h>
#include <fcntl.h>
#include "build-result.hh"
#include "serve-protocol.hh"
#include "state.hh"
#include "util.hh"
@ -49,7 +50,7 @@ static Strings extraStoreArgs(std::string & machine)
static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
{
string pgmName;
std::string pgmName;
Pipe to, from;
to.create();
from.create();
@ -81,7 +82,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
if (machine->sshPublicHostKey != "") {
Path fileName = tmpDir + "/host-key";
auto p = machine->sshName.find("@");
string host = p != string::npos ? string(machine->sshName, p + 1) : machine->sshName;
std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName;
writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n");
append(argv, {"-oUserKnownHostsFile=" + fileName});
}
@ -185,8 +186,8 @@ void State::buildRemote(ref<Store> destStore,
{
assert(BuildResult::TimedOut == 8);
string base(step->drvPath.to_string());
result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
std::string base(step->drvPath.to_string());
result.logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2);
AutoDelete autoDelete(result.logFile, false);
createDirs(dirOf(result.logFile));
@ -249,7 +250,7 @@ void State::buildRemote(ref<Store> destStore,
} catch (EndOfFile & e) {
child.pid.wait();
string s = chomp(readFile(result.logFile));
std::string s = chomp(readFile(result.logFile));
throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
}
@ -287,9 +288,9 @@ void State::buildRemote(ref<Store> destStore,
this will copy the inputs to the binary cache from the local
store. */
if (localStore != std::shared_ptr<Store>(destStore)) {
StorePathSet closure;
localStore->computeFSClosure(step->drv->inputSrcs, closure);
copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
copyClosure(*localStore, *destStore,
step->drv->inputSrcs,
NoRepair, NoCheckSigs, NoSubstitute);
}
{

4
src/hydra-queue-runner/build-result.cc
View file

@ -1,4 +1,4 @@
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
#include "util.hh"
#include "fs-accessor.hh"
@ -78,7 +78,7 @@ BuildOutput getBuildOutput(
product.type = match[1];
product.subtype = match[2];
std::string s(match[3]);
product.path = s[0] == '"' ? string(s, 1, s.size() - 2) : s;
product.path = s[0] == '"' ? std::string(s, 1, s.size() - 2) : s;
product.defaultPath = match[5];
/* Ensure that the path exists and points into the Nix

2
src/hydra-queue-runner/builder.cc
View file

@ -1,7 +1,7 @@
#include <cmath>
#include "state.hh"
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "finally.hh"
#include "binary-cache-store.hh"

0
src/hydra-queue-runner/build-result.hh → src/hydra-queue-runner/hydra-build-result.hh
View file

18
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -7,7 +7,7 @@
#include <fcntl.h>
#include "state.hh"
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
#include "remote-store.hh"
@ -87,7 +87,7 @@ void State::parseMachines(const std::string & contents)
}
for (auto line : tokenizeString<Strings>(contents, "\n")) {
line = trim(string(line, 0, line.find('#')));
line = trim(std::string(line, 0, line.find('#')));
auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.size() < 3) continue;
tokens.resize(8);
@ -95,7 +95,7 @@ void State::parseMachines(const std::string & contents)
auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2] == "-" ? string("") : tokens[2];
machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
if (tokens[3] != "")
machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
else
@ -149,7 +149,7 @@ void State::parseMachines(const std::string & contents)
void State::monitorMachinesFile()
{
string defaultMachinesFile = "/etc/nix/machines";
std::string defaultMachinesFile = "/etc/nix/machines";
auto machinesFiles = tokenizeString<std::vector<Path>>(
getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":");
@ -191,7 +191,7 @@ void State::monitorMachinesFile()
debug("reloading machines files");
string contents;
std::string contents;
for (auto & machinesFile : machinesFiles) {
try {
contents += readFile(machinesFile);
@ -308,7 +308,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath)
Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath)
{
restart:
auto stepNr = allocBuildStep(txn, build->id);
@ -683,14 +683,14 @@ void State::showStatus()
auto conn(dbPool.get());
receiver statusDumped(*conn, "status_dumped");
string status;
std::string status;
bool barf = false;
/* Get the last JSON status dump from the database. */
{
pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>();
if (res.size()) status = res[0][0].as<std::string>();
}
if (status != "") {
@ -710,7 +710,7 @@ void State::showStatus()
{
pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>();
if (res.size()) status = res[0][0].as<std::string>();
}
}

2
src/hydra-queue-runner/nar-extractor.cc
View file

@ -64,7 +64,7 @@ struct Extractor : ParseSink
}
}
void createSymlink(const Path & path, const string & target) override
void createSymlink(const Path & path, const std::string & target) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
}

18
src/hydra-queue-runner/queue-monitor.cc
View file

@ -1,5 +1,5 @@
#include "state.hh"
#include "build-result.hh"
#include "hydra-build-result.hh"
#include "globals.hh"
#include <cstring>
@ -111,12 +111,12 @@ bool State::getQueuedBuilds(Connection & conn,
if (builds_->count(id)) continue;
auto build = std::make_shared<Build>(
localStore->parseStorePath(row["drvPath"].as<string>()));
localStore->parseStorePath(row["drvPath"].as<std::string>()));
build->id = id;
build->jobsetId = row["jobset_id"].as<JobsetID>();
build->projectName = row["project"].as<string>();
build->jobsetName = row["jobset"].as<string>();
build->jobName = row["job"].as<string>();
build->projectName = row["project"].as<std::string>();
build->jobsetName = row["jobset"].as<std::string>();
build->jobName = row["job"].as<std::string>();
build->maxSilentTime = row["maxsilent"].as<int>();
build->buildTimeout = row["timeout"].as<int>();
build->timestamp = row["timestamp"].as<time_t>();
@ -513,9 +513,9 @@ Step::ptr State::createStep(ref<Store> destStore,
// FIXME: should copy directly from substituter to destStore.
}
StorePathSet closure;
localStore->computeFSClosure({*path}, closure);
copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute);
copyClosure(*localStore, *destStore,
StorePathSet { *path },
NoRepair, CheckSigs, NoSubstitute);
time_t stopTime = time(0);
@ -620,7 +620,7 @@ void State::processJobsetSharesChange(Connection & conn)
auto res = txn.exec("select project, name, schedulingShares from Jobsets");
for (auto const & row : res) {
auto jobsets_(jobsets.lock());
auto i = jobsets_->find(std::make_pair(row["project"].as<string>(), row["name"].as<string>()));
auto i = jobsets_->find(std::make_pair(row["project"].as<std::string>(), row["name"].as<std::string>()));
if (i == jobsets_->end()) continue;
i->second->setShares(row["schedulingShares"].as<unsigned int>());
}

1
src/hydra-queue-runner/state.hh
View file

@ -12,6 +12,7 @@
#include "parsed-derivations.hh"
#include "pathlocks.hh"
#include "pool.hh"
#include "build-result.hh"
#include "store-api.hh"
#include "sync.hh"
#include "nar-extractor.hh"

2
src/libhydra/db.hh
View file

@ -18,7 +18,7 @@ struct Connection : pqxx::connection
std::string upper_prefix = "DBI:Pg:";
if (hasPrefix(s, lower_prefix) || hasPrefix(s, upper_prefix)) {
return concatStringsSep(" ", tokenizeString<Strings>(string(s, lower_prefix.size()), ";"));
return concatStringsSep(" ", tokenizeString<Strings>(std::string(s, lower_prefix.size()), ";"));
}
throw Error("$HYDRA_DBI does not denote a PostgreSQL database");

2
src/libhydra/hydra-config.hh
View file

@ -17,7 +17,7 @@ struct HydraConfig
if (hydraConfigFile && pathExists(*hydraConfigFile)) {
for (auto line : tokenizeString<Strings>(readFile(*hydraConfigFile), "\n")) {
line = trim(string(line, 0, line.find('#')));
line = trim(std::string(line, 0, line.find('#')));
auto eq = line.find('=');
if (eq == std::string::npos) continue;