From b23431a657d8a9b2f478c95dd81034780751a262 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Fri, 4 Aug 2023 15:53:06 +0200 Subject: [PATCH 1/3] Support Nix 2.17 --- flake.lock | 8 ++++---- flake.nix | 2 +- src/hydra-queue-runner/build-remote.cc | 25 +++++++++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/flake.lock b/flake.lock index ee85f6fa..4b18fbb4 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1686048923, - "narHash": "sha256-/XCWa2osNFIpPC5MkxlX6qTZf/DaTLwS3LWN0SRFiuU=", + "lastModified": 1690219894, + "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=", "owner": "NixOS", "repo": "nix", - "rev": "84050709ea18f3285a85d729f40c8f8eddf5008e", + "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.16.1", + "ref": "2.17.0", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 6bbec9b0..7e7d50e2 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.16.1"; + inputs.nix.url = "github:NixOS/nix/2.17.0"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 6baff7df..6bbd22e2 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -6,6 +6,7 @@ #include #include "build-result.hh" +#include "path.hh" #include "serve-protocol.hh" #include "state.hh" #include "util.hh" @@ -110,18 +111,20 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, StorePathSet closure; destStore.computeFSClosure(paths, closure); + WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { .from = from }; /* Send the "query valid paths" command with the "lock" option enabled. This prevents a race where the remote host garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - to << cmdQueryValidPaths << 1 << useSubstitutes; - workerProtoWrite(destStore, to, closure); + to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes; + WorkerProto::write(destStore, wconn, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = WorkerProto::read(destStore, from); + auto present = WorkerProto::Serialise::read(destStore, rconn); if (present.size() == closure.size()) return; @@ -136,7 +139,7 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore, std::unique_lock sendLock(sendMutex, std::chrono::seconds(600)); - to << cmdImportPaths; + to << ServeProto::Command::ImportPaths; destStore.exportPaths(missing, to); to.flush(); @@ -223,7 +226,9 @@ void State::buildRemote(ref destStore, }); FdSource from(child.from.get()); + WorkerProto::ReadConn rconn { .from = from }; FdSink to(child.to.get()); + WorkerProto::WriteConn wconn { .to = to }; Finally updateStats([&]() { bytesReceived += from.read; @@ -334,7 +339,7 @@ void State::buildRemote(ref destStore, updateStep(ssBuilding); - to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); + to << ServeProto::Command::BuildDerivation << localStore->printStorePath(step->drvPath); writeDerivation(to, *localStore, basicDrv); to << maxSilentTime << buildTimeout; if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) @@ -367,7 +372,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - WorkerProto::read(*localStore, from); + WorkerProto::Serialise::read(*localStore, rconn); } switch ((BuildResult::Status) res) { case BuildResult::Built: @@ -443,14 +448,14 @@ void State::buildRemote(ref destStore, /* Get info about each output path. */ std::map infos; size_t totalNarSize = 0; - to << cmdQueryPathInfos; - workerProtoWrite(*localStore, to, outputs); + to << ServeProto::Command::QueryPathInfos; + WorkerProto::write(*localStore, wconn, outputs); to.flush(); while (true) { auto storePathS = readString(from); if (storePathS == "") break; auto deriver = readString(from); // deriver - auto references = WorkerProto::read(*localStore, from); + auto references = WorkerProto::Serialise::read(*localStore, rconn); readLongLong(from); // download size auto narSize = readLongLong(from); auto narHash = Hash::parseAny(readString(from), htSHA256); @@ -494,7 +499,7 @@ void State::buildRemote(ref destStore, lambda function only gets executed if someone tries to read from source2, we will send the command from here rather than outside the lambda. */ - to << cmdDumpStorePath << localStore->printStorePath(path); + to << ServeProto::Command::DumpStorePath << localStore->printStorePath(path); to.flush(); TeeSource tee(from, sink); From 9f0427385fa9f306c4cd651dc97377624b90997c Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sun, 20 Aug 2023 14:55:56 +0200 Subject: [PATCH 2/3] Apply LTO fix suggested by Ericson2314 --- src/hydra-queue-runner/build-remote.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 6bbd22e2..46c94e60 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -11,6 +11,7 @@ #include "state.hh" #include "util.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "finally.hh" #include "url.hh" From 35ccc9ebb22796270aafe1b567420e13529e3be5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Aug 2023 17:04:45 +0200 Subject: [PATCH 3/3] Fix indentation Co-authored-by: John Ericson --- src/hydra-queue-runner/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 46c94e60..56ce1ccf 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -373,7 +373,7 @@ void State::buildRemote(ref destStore, } } if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) { - WorkerProto::Serialise::read(*localStore, rconn); + WorkerProto::Serialise::read(*localStore, rconn); } switch ((BuildResult::Status) res) { case BuildResult::Built: