diff --git a/flake.lock b/flake.lock index cf2b2d72..854d695d 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1721091462, - "narHash": "sha256-0cmEeoOiB91BviTJHzIyxkY+Gxv3O8ZnnExVAoXEFGI=", + "lastModified": 1723331518, + "narHash": "sha256-JVnQ3OLbXQAlkOluFc3gWhZMbhared1Rg5YvNEc92m0=", "ref": "refs/heads/main", - "rev": "6b4d46e9e0e1dd80e0977684ab20d14bcd1a6bc3", - "revCount": 15967, + "rev": "5137cea99044d54337e439510a647743110b2d7d", + "revCount": 16128, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -111,11 +111,11 @@ "nix2container": { "flake": false, "locked": { - "lastModified": 1712990762, - "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "lastModified": 1720642556, + "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=", "owner": "nlewo", "repo": "nix2container", - "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4", "type": "github" }, "original": { @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1720691131, - "narHash": "sha256-CWT+KN8aTPyMIx8P303gsVxUnkinIz0a/Cmasz1jyIM=", + "lastModified": 1723282977, + "narHash": "sha256-oTK91aOlA/4IsjNAZGMEBz7Sq1zBS0Ltu4/nIQdYDOg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a046c1202e11b62cbede5385ba64908feb7bfac4", + "rev": "a781ff33ae258bbcfd4ed6e673860c3e923bf2cc", "type": "github" }, "original": { @@ -159,11 +159,11 @@ "pre-commit-hooks": { "flake": false, "locked": { - "lastModified": 1712055707, - "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 122b32ce..fcab7b54 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -368,7 +368,7 @@ static std::map queryPathInfos( auto references = ServeProto::Serialise::read(localStore, conn); readLongLong(conn.from); // download size auto narSize = readLongLong(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashType::SHA256); auto ca = ContentAddress::parseOpt(readString(conn.from)); readStrings(conn.from); // sigs ValidPathInfo info(localStore.parseStorePath(storePathS), narHash); @@ -397,8 +397,7 @@ static void copyPathFromRemote( /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the NAR that getBuildOutput() needs. */ - auto source2 = sinkToSource([&](Sink & sink) - { + auto coro = [&]() -> WireFormatGenerator { /* Note: we should only send the command to dump the store path to the remote if the NAR is actually going to get read by the destination store, which won't happen if this path @@ -409,11 +408,11 @@ static void copyPathFromRemote( conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path); conn.to.flush(); - TeeSource tee(conn.from, sink); - extractNarData(tee, localStore.printStorePath(info.path), narMembers); - }); + co_yield extractNarDataFilter(conn.from, localStore.printStorePath(info.path), narMembers); + }; + GeneratorSource source2{coro()}; - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); + destStore.addToStore(info, source2, NoRepair, NoCheckSigs); } static void copyPathsFromRemote( diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 93851369..d0b99939 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -34,11 +34,8 @@ BuildOutput getBuildOutput( auto outputS = store->printStorePath(output); if (!narMembers.count(outputS)) { printInfo("fetching NAR contents of '%s'...", outputS); - auto source = sinkToSource([&](Sink & sink) - { - sink << store->narFromPath(output); - }); - extractNarData(*source, outputS, narMembers); + GeneratorSource source{store->narFromPath(output)}; + extractNarData(source, outputS, narMembers); } } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5b679d18..7cd50877 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -534,7 +534,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, product.type, product.subtype, product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt, - product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt, + product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base::Base16, false)) : std::nullopt, product.path, product.name, product.defaultPath); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 9f0eb431..7a4dd996 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -42,7 +42,7 @@ struct Extractor : ParseSink void preallocateContents(uint64_t size) override { expectedSize = size; - hashSink = std::make_unique(htSHA256); + hashSink = std::make_unique(HashType::SHA256); } void receiveContents(std::string_view data) override @@ -76,7 +76,19 @@ void extractNarData( const Path & prefix, NarMemberDatas & members) { - Extractor extractor(members, prefix); - parseDump(extractor, source); - // Note: this point may not be reached if we're in a coroutine. + auto parser = extractNarDataFilter(source, prefix, members); + while (parser.next()) { + // ignore raw data + } +} + +nix::WireFormatGenerator extractNarDataFilter( + Source & source, + const Path & prefix, + NarMemberDatas & members) +{ + return [](Source & source, const Path & prefix, NarMemberDatas & members) -> WireFormatGenerator { + Extractor extractor(members, prefix); + co_yield parseAndCopyDump(extractor, source); + }(source, prefix, members); } diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh index 45b2706c..e6ba91b0 100644 --- a/src/hydra-queue-runner/nar-extractor.hh +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -21,3 +21,8 @@ void extractNarData( nix::Source & source, const nix::Path & prefix, NarMemberDatas & members); + +nix::WireFormatGenerator extractNarDataFilter( + nix::Source & source, + const nix::Path & prefix, + NarMemberDatas & members); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index bc58a8d0..9cc4357e 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -727,7 +727,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref product.fileSize = row[2].as(); } if (!row[3].is_null()) - product.sha256hash = Hash::parseAny(row[3].as(), htSHA256); + product.sha256hash = Hash::parseAny(row[3].as(), HashType::SHA256); if (!row[4].is_null()) product.path = row[4].as(); product.name = row[5].as();