Merge remote-tracking branch 'upstream/master' into split-buildRemote

flake.lock

@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1690219894,
+        "lastModified": 1701122567,
-        "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=",
+        "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4",
+        "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.17.0",
+        "ref": "2.19.2",
         "repo": "nix",
         "type": "github"
       }

flake.nix

@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.17.0";
+  inputs.nix.url = "github:NixOS/nix/2.19.2";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -7,6 +7,9 @@
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
+#include "signals.hh"
+#include "terminal.hh"
 #include "util.hh"
 #include "get-drvs.hh"
 #include "globals.hh"
@@ -53,7 +56,7 @@ using namespace nix;
 static Path gcRootsDir;
 static size_t maxMemorySize;
 
-struct MyArgs : MixEvalArgs, MixCommonArgs
+struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
 {
     Path releaseExpr;
     bool flake = false;
@@ -94,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std:
     rec = [&](Value & v) {
         state.forceValue(v, noPos);
         if (v.type() == nString)
-            res.push_back(v.string.s);
+            res.emplace_back(v.string_view());
         else if (v.isList())
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 rec(*v.listElems()[n]);
@@ -208,20 +211,20 @@ static void worker(
                     for (auto & c : context)
                         std::visit(overloaded {
                             [&](const NixStringContextElem::Built & b) {
-                                job["constituents"].push_back(state.store->printStorePath(b.drvPath));
+                                job["constituents"].push_back(b.drvPath->to_string(*state.store));
                             },
                             [&](const NixStringContextElem::Opaque & o) {
                             },
                             [&](const NixStringContextElem::DrvDeep & d) {
                             },
-                        }, c.raw());
+                        }, c.raw);
 
                     state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
                     for (unsigned int n = 0; n < a->value->listSize(); ++n) {
                         auto v = a->value->listElems()[n];
                         state.forceValue(*v, noPos);
                         if (v->type() == nString)
-                            job["namedConstituents"].push_back(v->str());
+                            job["namedConstituents"].push_back(v->string_view());
                     }
                 }
 
@@ -516,7 +519,7 @@ int main(int argc, char * * argv)
                     auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
                     auto drv2 = store->readDerivation(drvPath2);
                     job["constituents"].push_back(store->printStorePath(drvPath2));
-                    drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
+                    drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
                 }
 
                 if (brokenJobs.empty()) {

src/hydra-evaluator/hydra-evaluator.cc

@@ -2,6 +2,7 @@
 #include "hydra-config.hh"
 #include "pool.hh"
 #include "shared.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <thread>

src/hydra-queue-runner/build-remote.cc

@@ -9,6 +9,8 @@
 #include "path.hh"
 #include "serve-protocol.hh"
 #include "state.hh"
+#include "current-process.hh"
+#include "processes.hh"
 #include "util.hh"
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
@@ -221,9 +223,9 @@ static BasicDerivation sendInputs(
 {
     BasicDerivation basicDrv(*step.drv);
 
-    for (const auto & input : step.drv->inputDrvs) {
+    for (auto & [drvPath, node] : step.drv->inputDrvs.map) {
-        auto drv2 = localStore.readDerivation(input.first);
+        auto drv2 = localStore.readDerivation(drvPath);
-        for (auto & name : input.second) {
+        for (auto & name : node.value) {
             if (auto i = get(drv2.outputs, name)) {
                 auto outPath = i->path(localStore, drv2.name, name);
                 basicDrv.inputSrcs.insert(*outPath);

src/hydra-queue-runner/build-result.cc

@@ -1,7 +1,7 @@
 #include "hydra-build-result.hh"
 #include "store-api.hh"
 #include "util.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 
 #include <regex>
 
@@ -63,7 +63,7 @@ BuildOutput getBuildOutput(
 
         auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
         if (productsFile == narMembers.end() ||
-            productsFile->second.type != FSAccessor::Type::tRegular)
+            productsFile->second.type != SourceAccessor::Type::tRegular)
             continue;
         assert(productsFile->second.contents);
 
@@ -94,7 +94,7 @@ BuildOutput getBuildOutput(
 
             product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
 
-            if (file->second.type == FSAccessor::Type::tRegular) {
+            if (file->second.type == SourceAccessor::Type::tRegular) {
                 product.isRegular = true;
                 product.fileSize = file->second.fileSize.value();
                 product.sha256hash = file->second.sha256.value();
@@ -117,7 +117,7 @@ BuildOutput getBuildOutput(
 
             auto file = narMembers.find(product.path);
             assert(file != narMembers.end());
-            if (file->second.type == FSAccessor::Type::tDirectory)
+            if (file->second.type == SourceAccessor::Type::tDirectory)
                 res.products.push_back(product);
         }
     }
@@ -126,7 +126,7 @@ BuildOutput getBuildOutput(
     for (auto & output : outputs) {
         auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
         if (file == narMembers.end() ||
-            file->second.type != FSAccessor::Type::tRegular)
+            file->second.type != SourceAccessor::Type::tRegular)
             continue;
         res.releaseName = trim(file->second.contents.value());
         // FIXME: validate release name
@@ -136,7 +136,7 @@ BuildOutput getBuildOutput(
     for (auto & output : outputs) {
         auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
         if (file == narMembers.end() ||
-            file->second.type != FSAccessor::Type::tRegular)
+            file->second.type != SourceAccessor::Type::tRegular)
             continue;
         for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
             auto fields = tokenizeString<std::vector<std::string>>(line);

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -10,6 +10,7 @@
 
 #include <nlohmann/json.hpp>
 
+#include "signals.hh"
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "store-api.hh"
@@ -467,7 +468,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
              product.type,
              product.subtype,
              product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
-             product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
+             product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt,
              product.path,
              product.name,
              product.defaultPath);

src/hydra-queue-runner/nar-extractor.cc

@@ -24,13 +24,13 @@ struct Extractor : ParseSink
 
     void createDirectory(const Path & path) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
+        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
     void createRegularFile(const Path & path) override
     {
         curMember = &members.insert_or_assign(prefix + path, NarMemberData {
-            .type = FSAccessor::Type::tRegular,
+            .type = SourceAccessor::Type::tRegular,
             .fileSize = 0,
             .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
         }).first->second;
@@ -66,8 +66,14 @@ struct Extractor : ParseSink
 
     void createSymlink(const Path & path, const std::string & target) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
+        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
     }
+
+    void isExecutable() override
+    { }
+
+    void closeRegularFile() override
+    { }
 };
 
 

src/hydra-queue-runner/nar-extractor.hh

@@ -1,13 +1,13 @@
 #pragma once
 
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "types.hh"
 #include "serialise.hh"
 #include "hash.hh"
 
 struct NarMemberData
 {
-    nix::FSAccessor::Type type;
+    nix::SourceAccessor::Type type;
     std::optional<uint64_t> fileSize;
     std::optional<std::string> contents;
     std::optional<nix::Hash> sha256;

src/hydra-queue-runner/queue-monitor.cc

@@ -315,7 +315,7 @@ bool State::getQueuedBuilds(Connection & conn,
         if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
             prom.queue_checks_early_exits.Increment();
             break;
-        } 
+        }
     }
 
     prom.queue_checks_finished.Increment();
@@ -561,7 +561,7 @@ Step::ptr State::createStep(ref<Store> destStore,
     printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath));
 
     /* Create steps for the dependencies. */
-    for (auto & i : step->drv->inputDrvs) {
+    for (auto & i : step->drv->inputDrvs.map) {
         auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
         if (dep) {
             auto step_(step->state.lock());

src/hydra-queue-runner/state.hh

@@ -305,19 +305,25 @@ struct Machine
     struct Connection {
         nix::FdSink to;
         nix::FdSource from;
-        unsigned int remoteVersion;
+        nix::ServeProto::Version remoteVersion;
 
         // Backpointer to the machine
         ptr machine;
 
         operator nix::ServeProto::ReadConn ()
         {
-            return { .from = from };
+            return {
+                .from = from,
+                .version = remoteVersion,
+            };
         }
 
         operator nix::ServeProto::WriteConn ()
         {
-            return { .to = to };
+            return {
+                .to = to,
+                .version = remoteVersion,
+            };
         }
     };
 };

src/libhydra/db.hh

@@ -2,6 +2,7 @@
 
 #include <pqxx/pqxx>
 
+#include "environment-variables.hh"
 #include "util.hh"
 
 

src/libhydra/hydra-config.hh

@@ -2,6 +2,7 @@
 
 #include <map>
 
+#include "file-system.hh"
 #include "util.hh"
 
 struct HydraConfig