ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Update to Nix 2.19

Browse source 
Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/f5f4de6a550327b4b1a06123c2e450f1b92c73b6' (2023-10-02)
  → 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-11-27)
This commit is contained in:
John Ericson 2023-11-30 14:38:26 -05:00
parent e9da80fff6
commit c922e73c11
11 changed files with 61 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
flake.lock
View file

@ -42,16 +42,16 @@
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
"lastModified": 1696259154,
"narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=",
"lastModified": 1701122567,
"narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
"owner": "NixOS",
"repo": "nix",
"rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6",
"rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "2.18.1",
"ref": "2.19.2",
"repo": "nix",
"type": "github"
}

2
flake.nix
View file

@ -2,7 +2,7 @@
description = "A Nix-based continuous build system";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
inputs.nix.url = "github:NixOS/nix/2.18.1";
inputs.nix.url = "github:NixOS/nix/2.19.2";
inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
outputs = { self, nixpkgs, nix }:

8
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -8,6 +8,8 @@
#include "eval.hh"
#include "eval-inline.hh"
#include "eval-settings.hh"
#include "signals.hh"
#include "terminal.hh"
#include "util.hh"
#include "get-drvs.hh"
#include "globals.hh"
@ -54,7 +56,7 @@ using namespace nix;
static Path gcRootsDir;
static size_t maxMemorySize;
struct MyArgs : MixEvalArgs, MixCommonArgs
struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
{
Path releaseExpr;
bool flake = false;
@ -95,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std:
rec = [&](Value & v) {
state.forceValue(v, noPos);
if (v.type() == nString)
res.push_back(v.string.s);
res.emplace_back(v.string_view());
else if (v.isList())
for (unsigned int n = 0; n < v.listSize(); ++n)
rec(*v.listElems()[n]);
@ -222,7 +224,7 @@ static void worker(
auto v = a->value->listElems()[n];
state.forceValue(*v, noPos);
if (v->type() == nString)
job["namedConstituents"].push_back(v->str());
job["namedConstituents"].push_back(v->string_view());
}
}

1
src/hydra-evaluator/hydra-evaluator.cc
View file

@ -2,6 +2,7 @@
#include "hydra-config.hh"
#include "pool.hh"
#include "shared.hh"
#include "signals.hh"
#include <algorithm>
#include <thread>

43
src/hydra-queue-runner/build-remote.cc
View file

@ -9,9 +9,11 @@
#include "path.hh"
#include "serve-protocol.hh"
#include "state.hh"
#include "current-process.hh"
#include "processes.hh"
#include "util.hh"
#include "worker-protocol.hh"
#include "worker-protocol-impl.hh"
#include "serve-protocol.hh"
#include "serve-protocol-impl.hh"
#include "finally.hh"
#include "url.hh"
@ -106,26 +108,32 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
FdSource & from, FdSink & to, const StorePathSet & paths,
FdSource & from, FdSink & to, ServeProto::Version remoteVersion, const StorePathSet & paths,
bool useSubstitutes = false)
{
StorePathSet closure;
destStore.computeFSClosure(paths, closure);
WorkerProto::WriteConn wconn { .to = to };
WorkerProto::ReadConn rconn { .from = from };
ServeProto::WriteConn wconn {
.to = to,
.version = remoteVersion,
};
ServeProto::ReadConn rconn {
.from = from,
.version = remoteVersion,
};
/* Send the "query valid paths" command with the "lock" option
enabled. This prevents a race where the remote host
garbage-collect paths that are already there. Optionally, ask
the remote host to substitute missing paths. */
// FIXME: substitute output pollutes our build log
to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
WorkerProto::write(destStore, wconn, closure);
ServeProto::write(destStore, wconn, closure);
to.flush();
/* Get back the set of paths that are already valid on the remote
host. */
auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, rconn);
auto present = ServeProto::Serialise<StorePathSet>::read(destStore, rconn);
if (present.size() == closure.size()) return;
@ -227,9 +235,7 @@ void State::buildRemote(ref<Store> destStore,
});
FdSource from(child.from.get());
WorkerProto::ReadConn rconn { .from = from };
FdSink to(child.to.get());
WorkerProto::WriteConn wconn { .to = to };
Finally updateStats([&]() {
bytesReceived += from.read;
@ -237,7 +243,7 @@ void State::buildRemote(ref<Store> destStore,
});
/* Handshake. */
unsigned int remoteVersion;
ServeProto::Version remoteVersion;
try {
to << SERVE_MAGIC_1 << 0x206;
@ -258,6 +264,15 @@ void State::buildRemote(ref<Store> destStore,
throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
}
ServeProto::ReadConn rconn {
.from = from,
.version = remoteVersion,
};
ServeProto::WriteConn wconn {
.to = to,
.version = remoteVersion,
};
{
auto info(machine->state->connectInfo.lock());
info->consecutiveFailures = 0;
@ -313,7 +328,7 @@ void State::buildRemote(ref<Store> destStore,
destStore->computeFSClosure(inputs, closure);
copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
} else {
copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true);
copyClosureTo(machine->state->sendLock, *destStore, from, to, remoteVersion, inputs, true);
}
auto now2 = std::chrono::steady_clock::now();
@ -373,7 +388,7 @@ void State::buildRemote(ref<Store> destStore,
}
}
if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
WorkerProto::Serialise<DrvOutputs>::read(*localStore, rconn);
ServeProto::Serialise<DrvOutputs>::read(*localStore, rconn);
}
switch ((BuildResult::Status) res) {
case BuildResult::Built:
@ -450,13 +465,13 @@ void State::buildRemote(ref<Store> destStore,
std::map<StorePath, ValidPathInfo> infos;
size_t totalNarSize = 0;
to << ServeProto::Command::QueryPathInfos;
WorkerProto::write(*localStore, wconn, outputs);
ServeProto::write(*localStore, wconn, outputs);
to.flush();
while (true) {
auto storePathS = readString(from);
if (storePathS == "") break;
auto deriver = readString(from); // deriver
auto references = WorkerProto::Serialise<StorePathSet>::read(*localStore, rconn);
auto references = ServeProto::Serialise<StorePathSet>::read(*localStore, rconn);
readLongLong(from); // download size
auto narSize = readLongLong(from);
auto narHash = Hash::parseAny(readString(from), htSHA256);

12
src/hydra-queue-runner/build-result.cc
View file

@ -1,7 +1,7 @@
#include "hydra-build-result.hh"
#include "store-api.hh"
#include "util.hh"
#include "fs-accessor.hh"
#include "source-accessor.hh"
#include <regex>
@ -63,7 +63,7 @@ BuildOutput getBuildOutput(
auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
if (productsFile == narMembers.end() ||
productsFile->second.type != FSAccessor::Type::tRegular)
productsFile->second.type != SourceAccessor::Type::tRegular)
continue;
assert(productsFile->second.contents);
@ -94,7 +94,7 @@ BuildOutput getBuildOutput(
product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
if (file->second.type == FSAccessor::Type::tRegular) {
if (file->second.type == SourceAccessor::Type::tRegular) {
product.isRegular = true;
product.fileSize = file->second.fileSize.value();
product.sha256hash = file->second.sha256.value();
@ -117,7 +117,7 @@ BuildOutput getBuildOutput(
auto file = narMembers.find(product.path);
assert(file != narMembers.end());
if (file->second.type == FSAccessor::Type::tDirectory)
if (file->second.type == SourceAccessor::Type::tDirectory)
res.products.push_back(product);
}
}
@ -126,7 +126,7 @@ BuildOutput getBuildOutput(
for (auto & output : outputs) {
auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
file->second.type != SourceAccessor::Type::tRegular)
continue;
res.releaseName = trim(file->second.contents.value());
// FIXME: validate release name
@ -136,7 +136,7 @@ BuildOutput getBuildOutput(
for (auto & output : outputs) {
auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
file->second.type != SourceAccessor::Type::tRegular)
continue;
for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
auto fields = tokenizeString<std::vector<std::string>>(line);

3
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -10,6 +10,7 @@
#include <nlohmann/json.hpp>
#include "signals.hh"
#include "state.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
@ -467,7 +468,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
product.type,
product.subtype,
product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt,
product.path,
product.name,
product.defaultPath);

12
src/hydra-queue-runner/nar-extractor.cc
View file

@ -24,13 +24,13 @@ struct Extractor : ParseSink
void createDirectory(const Path & path) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
}
void createRegularFile(const Path & path) override
{
curMember = &members.insert_or_assign(prefix + path, NarMemberData {
.type = FSAccessor::Type::tRegular,
.type = SourceAccessor::Type::tRegular,
.fileSize = 0,
.contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
}).first->second;
@ -66,8 +66,14 @@ struct Extractor : ParseSink
void createSymlink(const Path & path, const std::string & target) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
}
void isExecutable() override
{ }
void closeRegularFile() override
{ }
};

4
src/hydra-queue-runner/nar-extractor.hh
View file

@ -1,13 +1,13 @@
#pragma once
#include "fs-accessor.hh"
#include "source-accessor.hh"
#include "types.hh"
#include "serialise.hh"
#include "hash.hh"
struct NarMemberData
{
nix::FSAccessor::Type type;
nix::SourceAccessor::Type type;
std::optional<uint64_t> fileSize;
std::optional<std::string> contents;
std::optional<nix::Hash> sha256;

1
src/libhydra/db.hh
View file

@ -2,6 +2,7 @@
#include <pqxx/pqxx>
#include "environment-variables.hh"
#include "util.hh"

1
src/libhydra/hydra-config.hh
View file

@ -2,6 +2,7 @@
#include <map>
#include "file-system.hh"
#include "util.hh"
struct HydraConfig