ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Merge remote-tracking branch 'upstream/master' into split-buildRemote

Browse source
This commit is contained in:
John Ericson 2023-12-04 09:12:58 -05:00
parent 622c25e3c4 f216bce0e6
commit 67eeabd518
13 changed files with 52 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
flake.lock
View file

@ -42,16 +42,16 @@
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
"lastModified": 1690219894,
"narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=",
"lastModified": 1701122567,
"narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
"owner": "NixOS",
"repo": "nix",
"rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4",
"rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "2.17.0",
"ref": "2.19.2",
"repo": "nix",
"type": "github"
}

2
flake.nix
View file

@ -2,7 +2,7 @@
description = "A Nix-based continuous build system";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
inputs.nix.url = "github:NixOS/nix/2.17.0";
inputs.nix.url = "github:NixOS/nix/2.19.2";
inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
outputs = { self, nixpkgs, nix }:

15
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -7,6 +7,9 @@
#include "store-api.hh"
#include "eval.hh"
#include "eval-inline.hh"
#include "eval-settings.hh"
#include "signals.hh"
#include "terminal.hh"
#include "util.hh"
#include "get-drvs.hh"
#include "globals.hh"
@ -53,7 +56,7 @@ using namespace nix;
static Path gcRootsDir;
static size_t maxMemorySize;
struct MyArgs : MixEvalArgs, MixCommonArgs
struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
{
Path releaseExpr;
bool flake = false;
@ -94,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std:
rec = [&](Value & v) {
state.forceValue(v, noPos);
if (v.type() == nString)
res.push_back(v.string.s);
res.emplace_back(v.string_view());
else if (v.isList())
for (unsigned int n = 0; n < v.listSize(); ++n)
rec(*v.listElems()[n]);
@ -208,20 +211,20 @@ static void worker(
for (auto & c : context)
std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
job["constituents"].push_back(state.store->printStorePath(b.drvPath));
job["constituents"].push_back(b.drvPath->to_string(*state.store));
},
[&](const NixStringContextElem::Opaque & o) {
},
[&](const NixStringContextElem::DrvDeep & d) {
},
}, c.raw());
}, c.raw);
state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
for (unsigned int n = 0; n < a->value->listSize(); ++n) {
auto v = a->value->listElems()[n];
state.forceValue(*v, noPos);
if (v->type() == nString)
job["namedConstituents"].push_back(v->str());
job["namedConstituents"].push_back(v->string_view());
}
}
@ -516,7 +519,7 @@ int main(int argc, char * * argv)
auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
auto drv2 = store->readDerivation(drvPath2);
job["constituents"].push_back(store->printStorePath(drvPath2));
drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
}
if (brokenJobs.empty()) {

1
src/hydra-evaluator/hydra-evaluator.cc
View file

@ -2,6 +2,7 @@
#include "hydra-config.hh"
#include "pool.hh"
#include "shared.hh"
#include "signals.hh"
#include <algorithm>
#include <thread>

8
src/hydra-queue-runner/build-remote.cc
View file

@ -9,6 +9,8 @@
#include "path.hh"
#include "serve-protocol.hh"
#include "state.hh"
#include "current-process.hh"
#include "processes.hh"
#include "util.hh"
#include "serve-protocol.hh"
#include "serve-protocol-impl.hh"
@ -221,9 +223,9 @@ static BasicDerivation sendInputs(
{
BasicDerivation basicDrv(*step.drv);
for (const auto & input : step.drv->inputDrvs) {
auto drv2 = localStore.readDerivation(input.first);
for (auto & name : input.second) {
for (auto & [drvPath, node] : step.drv->inputDrvs.map) {
auto drv2 = localStore.readDerivation(drvPath);
for (auto & name : node.value) {
if (auto i = get(drv2.outputs, name)) {
auto outPath = i->path(localStore, drv2.name, name);
basicDrv.inputSrcs.insert(*outPath);

12
src/hydra-queue-runner/build-result.cc
View file

@ -1,7 +1,7 @@
#include "hydra-build-result.hh"
#include "store-api.hh"
#include "util.hh"
#include "fs-accessor.hh"
#include "source-accessor.hh"
#include <regex>
@ -63,7 +63,7 @@ BuildOutput getBuildOutput(
auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
if (productsFile == narMembers.end() ||
productsFile->second.type != FSAccessor::Type::tRegular)
productsFile->second.type != SourceAccessor::Type::tRegular)
continue;
assert(productsFile->second.contents);
@ -94,7 +94,7 @@ BuildOutput getBuildOutput(
product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
if (file->second.type == FSAccessor::Type::tRegular) {
if (file->second.type == SourceAccessor::Type::tRegular) {
product.isRegular = true;
product.fileSize = file->second.fileSize.value();
product.sha256hash = file->second.sha256.value();
@ -117,7 +117,7 @@ BuildOutput getBuildOutput(
auto file = narMembers.find(product.path);
assert(file != narMembers.end());
if (file->second.type == FSAccessor::Type::tDirectory)
if (file->second.type == SourceAccessor::Type::tDirectory)
res.products.push_back(product);
}
}
@ -126,7 +126,7 @@ BuildOutput getBuildOutput(
for (auto & output : outputs) {
auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
file->second.type != SourceAccessor::Type::tRegular)
continue;
res.releaseName = trim(file->second.contents.value());
// FIXME: validate release name
@ -136,7 +136,7 @@ BuildOutput getBuildOutput(
for (auto & output : outputs) {
auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
file->second.type != SourceAccessor::Type::tRegular)
continue;
for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
auto fields = tokenizeString<std::vector<std::string>>(line);

3
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -10,6 +10,7 @@
#include <nlohmann/json.hpp>
#include "signals.hh"
#include "state.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
@ -467,7 +468,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
product.type,
product.subtype,
product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt,
product.path,
product.name,
product.defaultPath);

12
src/hydra-queue-runner/nar-extractor.cc
View file

@ -24,13 +24,13 @@ struct Extractor : ParseSink
void createDirectory(const Path & path) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
}
void createRegularFile(const Path & path) override
{
curMember = &members.insert_or_assign(prefix + path, NarMemberData {
.type = FSAccessor::Type::tRegular,
.type = SourceAccessor::Type::tRegular,
.fileSize = 0,
.contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
}).first->second;
@ -66,8 +66,14 @@ struct Extractor : ParseSink
void createSymlink(const Path & path, const std::string & target) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
}
void isExecutable() override
{ }
void closeRegularFile() override
{ }
};

4
src/hydra-queue-runner/nar-extractor.hh
View file

@ -1,13 +1,13 @@
#pragma once
#include "fs-accessor.hh"
#include "source-accessor.hh"
#include "types.hh"
#include "serialise.hh"
#include "hash.hh"
struct NarMemberData
{
nix::FSAccessor::Type type;
nix::SourceAccessor::Type type;
std::optional<uint64_t> fileSize;
std::optional<std::string> contents;
std::optional<nix::Hash> sha256;

4
src/hydra-queue-runner/queue-monitor.cc
View file

@ -315,7 +315,7 @@ bool State::getQueuedBuilds(Connection & conn,
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
prom.queue_checks_early_exits.Increment();
break;
}
}
}
prom.queue_checks_finished.Increment();
@ -561,7 +561,7 @@ Step::ptr State::createStep(ref<Store> destStore,
printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath));
/* Create steps for the dependencies. */
for (auto & i : step->drv->inputDrvs) {
for (auto & i : step->drv->inputDrvs.map) {
auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
if (dep) {
auto step_(step->state.lock());

12
src/hydra-queue-runner/state.hh
View file

@ -305,19 +305,25 @@ struct Machine
struct Connection {
nix::FdSink to;
nix::FdSource from;
unsigned int remoteVersion;
nix::ServeProto::Version remoteVersion;
// Backpointer to the machine
ptr machine;
operator nix::ServeProto::ReadConn ()
{
return { .from = from };
return {
.from = from,
.version = remoteVersion,
};
}
operator nix::ServeProto::WriteConn ()
{
return { .to = to };
return {
.to = to,
.version = remoteVersion,
};
}
};
};

1
src/libhydra/db.hh
View file

@ -2,6 +2,7 @@
#include <pqxx/pqxx>
#include "environment-variables.hh"
#include "util.hh"

1
src/libhydra/hydra-config.hh
View file

@ -2,6 +2,7 @@
#include <map>
#include "file-system.hh"
#include "util.hh"
struct HydraConfig