From e9da80fff6234fab2458173272ee0bedbe8935c3 Mon Sep 17 00:00:00 2001 From: chayleaf Date: Tue, 21 Nov 2023 18:41:52 +0700 Subject: [PATCH 1/2] support nix 2.18 --- flake.lock | 8 ++++---- flake.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 7 ++++--- src/hydra-queue-runner/build-remote.cc | 6 +++--- src/hydra-queue-runner/queue-monitor.cc | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/flake.lock b/flake.lock index 4b18fbb4..af913ea3 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1690219894, - "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=", + "lastModified": 1696259154, + "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=", "owner": "NixOS", "repo": "nix", - "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4", + "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.17.0", + "ref": "2.18.1", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 7e7d50e2..2f2abe62 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.17.0"; + inputs.nix.url = "github:NixOS/nix/2.18.1"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 79523944..30ab9740 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -7,6 +7,7 @@ #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "util.hh" #include "get-drvs.hh" #include "globals.hh" @@ -208,13 +209,13 @@ static void worker( for (auto & c : context) std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - job["constituents"].push_back(state.store->printStorePath(b.drvPath)); + job["constituents"].push_back(b.drvPath->to_string(*state.store)); }, [&](const NixStringContextElem::Opaque & o) { }, [&](const NixStringContextElem::DrvDeep & d) { }, - }, c.raw()); + }, c.raw); state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute"); for (unsigned int n = 0; n < a->value->listSize(); ++n) { @@ -516,7 +517,7 @@ int main(int argc, char * * argv) auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); auto drv2 = store->readDerivation(drvPath2); job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; + drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first}; } if (brokenJobs.empty()) { diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 56ce1ccf..92438349 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -276,9 +276,9 @@ void State::buildRemote(ref destStore, for (auto & p : step->drv->inputSrcs) inputs.insert(p); - for (auto & input : step->drv->inputDrvs) { - auto drv2 = localStore->readDerivation(input.first); - for (auto & name : input.second) { + for (auto & [drvPath, node] : step->drv->inputDrvs.map) { + auto drv2 = localStore->readDerivation(drvPath); + for (auto & name : node.value) { if (auto i = get(drv2.outputs, name)) { auto outPath = i->path(*localStore, drv2.name, name); inputs.insert(*outPath); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 0bb167a2..6c339af6 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -315,7 +315,7 @@ bool State::getQueuedBuilds(Connection & conn, if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) { prom.queue_checks_early_exits.Increment(); break; - } + } } prom.queue_checks_finished.Increment(); @@ -561,7 +561,7 @@ Step::ptr State::createStep(ref destStore, printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath)); /* Create steps for the dependencies. */ - for (auto & i : step->drv->inputDrvs) { + for (auto & i : step->drv->inputDrvs.map) { auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); From c922e73c11449759c8e1945ed01fb4cd120ef57d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Nov 2023 14:38:26 -0500 Subject: [PATCH 2/2] Update to Nix 2.19 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/f5f4de6a550327b4b1a06123c2e450f1b92c73b6' (2023-10-02) → 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-11-27) --- flake.lock | 8 ++-- flake.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 8 ++-- src/hydra-evaluator/hydra-evaluator.cc | 1 + src/hydra-queue-runner/build-remote.cc | 43 +++++++++++++------- src/hydra-queue-runner/build-result.cc | 12 +++--- src/hydra-queue-runner/hydra-queue-runner.cc | 3 +- src/hydra-queue-runner/nar-extractor.cc | 12 ++++-- src/hydra-queue-runner/nar-extractor.hh | 4 +- src/libhydra/db.hh | 1 + src/libhydra/hydra-config.hh | 1 + 11 files changed, 61 insertions(+), 34 deletions(-) diff --git a/flake.lock b/flake.lock index af913ea3..9a9046c8 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1696259154, - "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=", + "lastModified": 1701122567, + "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", "owner": "NixOS", "repo": "nix", - "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6", + "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.18.1", + "ref": "2.19.2", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 2f2abe62..4e8f6dd0 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.18.1"; + inputs.nix.url = "github:NixOS/nix/2.19.2"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 30ab9740..2fe2c80f 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -8,6 +8,8 @@ #include "eval.hh" #include "eval-inline.hh" #include "eval-settings.hh" +#include "signals.hh" +#include "terminal.hh" #include "util.hh" #include "get-drvs.hh" #include "globals.hh" @@ -54,7 +56,7 @@ using namespace nix; static Path gcRootsDir; static size_t maxMemorySize; -struct MyArgs : MixEvalArgs, MixCommonArgs +struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs { Path releaseExpr; bool flake = false; @@ -95,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std: rec = [&](Value & v) { state.forceValue(v, noPos); if (v.type() == nString) - res.push_back(v.string.s); + res.emplace_back(v.string_view()); else if (v.isList()) for (unsigned int n = 0; n < v.listSize(); ++n) rec(*v.listElems()[n]); @@ -222,7 +224,7 @@ static void worker( auto v = a->value->listElems()[n]; state.forceValue(*v, noPos); if (v->type() == nString) - job["namedConstituents"].push_back(v->str()); + job["namedConstituents"].push_back(v->string_view()); } } diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index a1ccf047..75506ff8 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -2,6 +2,7 @@ #include "hydra-config.hh" #include "pool.hh" #include "shared.hh" +#include "signals.hh" #include #include diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 92438349..5552e91a 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -9,9 +9,11 @@ #include "path.hh" #include "serve-protocol.hh" #include "state.hh" +#include "current-process.hh" +#include "processes.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "finally.hh" #include "url.hh" @@ -106,26 +108,32 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, - FdSource & from, FdSink & to, const StorePathSet & paths, + FdSource & from, FdSink & to, ServeProto::Version remoteVersion, const StorePathSet & paths, bool useSubstitutes = false) { StorePathSet closure; destStore.computeFSClosure(paths, closure); - WorkerProto::WriteConn wconn { .to = to }; - WorkerProto::ReadConn rconn { .from = from }; + ServeProto::WriteConn wconn { + .to = to, + .version = remoteVersion, + }; + ServeProto::ReadConn rconn { + .from = from, + .version = remoteVersion, + }; /* Send the "query valid paths" command with the "lock" option enabled. This prevents a race where the remote host garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; - WorkerProto::write(destStore, wconn, closure); + ServeProto::write(destStore, wconn, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = WorkerProto::Serialise::read(destStore, rconn); + auto present = ServeProto::Serialise::read(destStore, rconn); if (present.size() == closure.size()) return; @@ -227,9 +235,7 @@ void State::buildRemote(ref destStore, }); FdSource from(child.from.get()); - WorkerProto::ReadConn rconn { .from = from }; FdSink to(child.to.get()); - WorkerProto::WriteConn wconn { .to = to }; Finally updateStats([&]() { bytesReceived += from.read; @@ -237,7 +243,7 @@ void State::buildRemote(ref destStore, }); /* Handshake. */ - unsigned int remoteVersion; + ServeProto::Version remoteVersion; try { to << SERVE_MAGIC_1 << 0x206; @@ -258,6 +264,15 @@ void State::buildRemote(ref destStore, throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } + ServeProto::ReadConn rconn { + .from = from, + .version = remoteVersion, + }; + ServeProto::WriteConn wconn { + .to = to, + .version = remoteVersion, + }; + { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; @@ -313,7 +328,7 @@ void State::buildRemote(ref destStore, destStore->computeFSClosure(inputs, closure); copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); } else { - copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true); + copyClosureTo(machine->state->sendLock, *destStore, from, to, remoteVersion, inputs, true); } auto now2 = std::chrono::steady_clock::now(); @@ -373,7 +388,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - WorkerProto::Serialise::read(*localStore, rconn); + ServeProto::Serialise::read(*localStore, rconn); } switch ((BuildResult::Status) res) { case BuildResult::Built: @@ -450,13 +465,13 @@ void State::buildRemote(ref destStore, std::map infos; size_t totalNarSize = 0; to << ServeProto::Command::QueryPathInfos; - WorkerProto::write(*localStore, wconn, outputs); + ServeProto::write(*localStore, wconn, outputs); to.flush(); while (true) { auto storePathS = readString(from); if (storePathS == "") break; auto deriver = readString(from); // deriver - auto references = WorkerProto::Serialise::read(*localStore, rconn); + auto references = ServeProto::Serialise::read(*localStore, rconn); readLongLong(from); // download size auto narSize = readLongLong(from); auto narHash = Hash::parseAny(readString(from), htSHA256); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index ea8b4a6a..691c1f19 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -1,7 +1,7 @@ #include "hydra-build-result.hh" #include "store-api.hh" #include "util.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include @@ -63,7 +63,7 @@ BuildOutput getBuildOutput( auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products"); if (productsFile == narMembers.end() || - productsFile->second.type != FSAccessor::Type::tRegular) + productsFile->second.type != SourceAccessor::Type::tRegular) continue; assert(productsFile->second.contents); @@ -94,7 +94,7 @@ BuildOutput getBuildOutput( product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); - if (file->second.type == FSAccessor::Type::tRegular) { + if (file->second.type == SourceAccessor::Type::tRegular) { product.isRegular = true; product.fileSize = file->second.fileSize.value(); product.sha256hash = file->second.sha256.value(); @@ -117,7 +117,7 @@ BuildOutput getBuildOutput( auto file = narMembers.find(product.path); assert(file != narMembers.end()); - if (file->second.type == FSAccessor::Type::tDirectory) + if (file->second.type == SourceAccessor::Type::tDirectory) res.products.push_back(product); } } @@ -126,7 +126,7 @@ BuildOutput getBuildOutput( for (auto & output : outputs) { auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name"); if (file == narMembers.end() || - file->second.type != FSAccessor::Type::tRegular) + file->second.type != SourceAccessor::Type::tRegular) continue; res.releaseName = trim(file->second.contents.value()); // FIXME: validate release name @@ -136,7 +136,7 @@ BuildOutput getBuildOutput( for (auto & output : outputs) { auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics"); if (file == narMembers.end() || - file->second.type != FSAccessor::Type::tRegular) + file->second.type != SourceAccessor::Type::tRegular) continue; for (auto & line : tokenizeString(file->second.contents.value(), "\n")) { auto fields = tokenizeString>(line); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 91779288..1d54bb93 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -10,6 +10,7 @@ #include +#include "signals.hh" #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" @@ -467,7 +468,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, product.type, product.subtype, product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt, - product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt, + product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt, product.path, product.name, product.defaultPath); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 9f0eb431..3c6857bf 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -24,13 +24,13 @@ struct Extractor : ParseSink void createDirectory(const Path & path) override { - members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory }); + members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory }); } void createRegularFile(const Path & path) override { curMember = &members.insert_or_assign(prefix + path, NarMemberData { - .type = FSAccessor::Type::tRegular, + .type = SourceAccessor::Type::tRegular, .fileSize = 0, .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, }).first->second; @@ -66,8 +66,14 @@ struct Extractor : ParseSink void createSymlink(const Path & path, const std::string & target) override { - members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink }); + members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink }); } + + void isExecutable() override + { } + + void closeRegularFile() override + { } }; diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh index 45b2706c..2634135b 100644 --- a/src/hydra-queue-runner/nar-extractor.hh +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -1,13 +1,13 @@ #pragma once -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "types.hh" #include "serialise.hh" #include "hash.hh" struct NarMemberData { - nix::FSAccessor::Type type; + nix::SourceAccessor::Type type; std::optional fileSize; std::optional contents; std::optional sha256; diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh index 00e8f406..1e927573 100644 --- a/src/libhydra/db.hh +++ b/src/libhydra/db.hh @@ -2,6 +2,7 @@ #include +#include "environment-variables.hh" #include "util.hh" diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh index 1688c278..b1275896 100644 --- a/src/libhydra/hydra-config.hh +++ b/src/libhydra/hydra-config.hh @@ -2,6 +2,7 @@ #include +#include "file-system.hh" #include "util.hh" struct HydraConfig