From 62d29dd05220c0db75ec8fe6951023f6e6f6b239 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 25 Sep 2024 13:20:56 +0200 Subject: [PATCH 1/5] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'lix': 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=02eb07cfd539c34c080cb1baf042e5e780c1fcc2' (2024-09-01) → 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=31954b51367737defbae87fba053b068897416fb' (2024-09-26) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/6e99f2a27d600612004fbd2c3282d614bfee6421' (2024-08-30) → 'github:NixOS/nixpkgs/759537f06e6999e141588ff1c9be7f3a5c060106' (2024-09-25) --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 092378c7..6a730f32 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1725228396, - "narHash": "sha256-QBXwqyPuHUKBiuyzHBxqH/MpjPY9DQiY2M81P2t6b/0=", + "lastModified": 1727334821, + "narHash": "sha256-80CJtKmLvPOUnqz9O6rLYUoTHtVXI3KCQ/ahPgpDACA=", "ref": "refs/heads/main", - "rev": "02eb07cfd539c34c080cb1baf042e5e780c1fcc2", - "revCount": 16214, + "rev": "31954b51367737defbae87fba053b068897416fb", + "revCount": 16258, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1725001927, - "narHash": "sha256-eV+63gK0Mp7ygCR0Oy4yIYSNcum2VQwnZamHxYTNi+M=", + "lastModified": 1727264057, + "narHash": "sha256-KQPI8CTTnB9CrJ7LrmLC4VWbKZfljEPBXOFGZFRpxao=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421", + "rev": "759537f06e6999e141588ff1c9be7f3a5c060106", "type": "github" }, "original": { -- 2.44.1 From e92ac734e619a6e92747a05e6440c98c06a6e27d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 26 Sep 2024 09:31:43 +0200 Subject: [PATCH 2/5] Fix build with latest Lix Since ca1dc3f70bf98e2424b7b2666ee2180675b67451, the NAR parser has moved the preallocate & receive steps into the file handle class to remove the assumption that only one file can be handled at a time. --- .envrc | 2 +- src/hydra-queue-runner/nar-extractor.cc | 65 +++++++++++++------------ 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/.envrc b/.envrc index 3550a30f..4359d1f6 100644 --- a/.envrc +++ b/.envrc @@ -1 +1 @@ -use flake +use flake .#clang diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index b3350193..31c0db65 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -7,8 +7,38 @@ using namespace nix; -struct Extractor : ParseSink +struct Extractor : NARParseVisitor { + class MyFileHandle : public FileHandle + { + NarMemberData * memberData; + uint64_t expectedSize; + std::unique_ptr hashSink; + + public: + MyFileHandle(NarMemberData * memberData, uint64_t size) : memberData(memberData) + { + expectedSize = size; + hashSink = std::make_unique(HashType::SHA256); + } + + void receiveContents(std::string_view data) override + { + *memberData->fileSize += data.size(); + (*hashSink)(data); + if (memberData->contents) { + memberData->contents->append(data); + } + assert(memberData->fileSize <= expectedSize); + if (memberData->fileSize == expectedSize) { + auto [hash, len] = hashSink->finish(); + assert(memberData->fileSize == len); + memberData->sha256 = hash; + hashSink.reset(); + } + } + }; + std::unordered_set filesToKeep { "/nix-support/hydra-build-products", "/nix-support/hydra-release-name", @@ -16,7 +46,6 @@ struct Extractor : ParseSink }; NarMemberDatas & members; - NarMemberData * curMember = nullptr; Path prefix; Extractor(NarMemberDatas & members, Path prefix) @@ -28,41 +57,15 @@ struct Extractor : ParseSink members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory }); } - void createRegularFile(const Path & path) override + std::unique_ptr createRegularFile(const Path & path, uint64_t size, bool executable) override { - curMember = &members.insert_or_assign(prefix + path, NarMemberData { + auto memberData = &members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tRegular, .fileSize = 0, .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, }).first->second; - } - std::optional expectedSize; - std::unique_ptr hashSink; - - void preallocateContents(uint64_t size) override - { - expectedSize = size; - hashSink = std::make_unique(HashType::SHA256); - } - - void receiveContents(std::string_view data) override - { - assert(expectedSize); - assert(curMember); - assert(hashSink); - *curMember->fileSize += data.size(); - (*hashSink)(data); - if (curMember->contents) { - curMember->contents->append(data); - } - assert(curMember->fileSize <= expectedSize); - if (curMember->fileSize == expectedSize) { - auto [hash, len] = hashSink->finish(); - assert(curMember->fileSize == len); - curMember->sha256 = hash; - hashSink.reset(); - } + return std::unique_ptr(new MyFileHandle{memberData, size}); } void createSymlink(const Path & path, const std::string & target) override -- 2.44.1 From 06a30ec8df139cd4c7fb21ba9beb1dc54a4fc3fa Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 6 Oct 2024 19:43:34 +0200 Subject: [PATCH 3/5] hydra-queue-runner/nar-extractor: address review comments * Turn `NarMemberData` in `MyFileHandle` into a reference. * `expectedSize` now uses an initializer. * Used `std::make_unique` instead of `std::unique_ptr(new ...)` as suggested by clangd. --- src/hydra-queue-runner/nar-extractor.cc | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 31c0db65..7619b2ee 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -11,29 +11,28 @@ struct Extractor : NARParseVisitor { class MyFileHandle : public FileHandle { - NarMemberData * memberData; + NarMemberData & memberData; uint64_t expectedSize; std::unique_ptr hashSink; public: - MyFileHandle(NarMemberData * memberData, uint64_t size) : memberData(memberData) + MyFileHandle(NarMemberData & memberData, uint64_t size) : memberData(memberData), expectedSize(size) { - expectedSize = size; hashSink = std::make_unique(HashType::SHA256); } void receiveContents(std::string_view data) override { - *memberData->fileSize += data.size(); + *memberData.fileSize += data.size(); (*hashSink)(data); - if (memberData->contents) { - memberData->contents->append(data); + if (memberData.contents) { + memberData.contents->append(data); } - assert(memberData->fileSize <= expectedSize); - if (memberData->fileSize == expectedSize) { + assert(memberData.fileSize <= expectedSize); + if (memberData.fileSize == expectedSize) { auto [hash, len] = hashSink->finish(); - assert(memberData->fileSize == len); - memberData->sha256 = hash; + assert(memberData.fileSize == len); + memberData.sha256 = hash; hashSink.reset(); } } @@ -65,7 +64,7 @@ struct Extractor : NARParseVisitor .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt, }).first->second; - return std::unique_ptr(new MyFileHandle{memberData, size}); + return std::make_unique(*memberData, size); } void createSymlink(const Path & path, const std::string & target) override -- 2.44.1 From 4c1077ff3658e6a5a5fadf424fc1ea0222507d65 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 7 Oct 2024 18:03:59 +0200 Subject: [PATCH 4/5] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'lix': 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=31954b51367737defbae87fba053b068897416fb' (2024-09-26) → 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6' (2024-10-05) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/759537f06e6999e141588ff1c9be7f3a5c060106' (2024-09-25) → 'github:NixOS/nixpkgs/ecbc1ca8ffd6aea8372ad16be9ebbb39889e55b6' (2024-10-06) --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 6a730f32..9fa7feed 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1727334821, - "narHash": "sha256-80CJtKmLvPOUnqz9O6rLYUoTHtVXI3KCQ/ahPgpDACA=", + "lastModified": 1728163191, + "narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=", "ref": "refs/heads/main", - "rev": "31954b51367737defbae87fba053b068897416fb", - "revCount": 16258, + "rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6", + "revCount": 16323, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1727264057, - "narHash": "sha256-KQPI8CTTnB9CrJ7LrmLC4VWbKZfljEPBXOFGZFRpxao=", + "lastModified": 1728193676, + "narHash": "sha256-PbDWAIjKJdlVg+qQRhzdSor04bAPApDqIv2DofTyynk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "759537f06e6999e141588ff1c9be7f3a5c060106", + "rev": "ecbc1ca8ffd6aea8372ad16be9ebbb39889e55b6", "type": "github" }, "original": { -- 2.44.1 From 995e1922d726d50bf87be602043bf14caa9f2d24 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 7 Oct 2024 18:06:43 +0200 Subject: [PATCH 5/5] `ignoreException` has been split into two The Finally part is a destructor, so using `ignoreExceptionInDestructor` seems to be the correct choice here. --- src/hydra-queue-runner/builder.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 3f340c72..15bf8f10 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -1,5 +1,6 @@ #include +#include "error.hh" #include "state.hh" #include "hydra-build-result.hh" #include "finally.hh" @@ -185,7 +186,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, unlink(result.logFile.c_str()); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } }); -- 2.44.1