Compare commits

...

4 commits

Author SHA1 Message Date
Maximilian Bosch 1ed8a623a8
flake.lock: Update
Flake lock file updates:

• Updated input 'nix-eval-jobs':
    'git+https://git.lix.systems/ma27/nix-eval-jobs?ref=bring-back-constituents&rev=5ce92e2c4d318b30edaafa4d8664b9e15e941f32' (2024-10-13)
  → 'git+https://git.lix.systems/ma27/nix-eval-jobs?ref=bring-back-constituents&rev=7e02a188ffc474e443f1004eedb0b8e9a72597d1' (2024-10-26)
• Updated input 'nix-eval-jobs/flake-parts':
    'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01)
  → 'github:hercules-ci/flake-parts/3d04084d54bedc3d6b8b736c70ef449225c361b1' (2024-10-01)
• Updated input 'nix-eval-jobs/nix-github-actions':
    'github:nix-community/nix-github-actions/622f829f5fe69310a866c8a6cd07e747c44ef820' (2024-07-04)
  → 'github:nix-community/nix-github-actions/e04df33f62cdcf93d73e9a04142464753a16db67' (2024-10-24)
• Updated input 'nix-eval-jobs/treefmt-nix':
    'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12)
  → 'github:numtide/treefmt-nix/aac86347fb5063960eccb19493e0cadcdb4205ca' (2024-10-22)
2024-10-26 21:35:39 +02:00
Maximilian Bosch cdfc5c81e8
Bring back constituents
This partially reverts commit 370a4bf138.
2024-10-13 23:18:48 +02:00
Maximilian Bosch 838df090a1
Allow specifying jobset inputs for flake builds for Hydra plugins
It seems to be a common pattern to configure Hydra plugins on a
per-jobset basis with jobset inputs. Since these are not needed anymore
for flakes, it's also not possible anymore to use plugins for flake
jobsets.

This patch changes this and adds a warning to avoid confusion. In the
future, we may want to restrict jobset inputs for flakes to string
values only.
2024-10-13 13:52:32 +02:00
Maximilian Bosch 71c0909d11
Add 'private' flag to projects to display project & all associated things for authenticated users only 2024-10-13 13:52:32 +02:00
32 changed files with 655 additions and 112 deletions

View file

@ -312,6 +312,21 @@ Declarative Projects
see this [chapter](./plugins/declarative-projects.md)
Private Projects
----------------
By checking the `Private` checkbox in the project creation form, a project
and everything related to it (jobsets, evals, builds, etc.) can only be accessed
if a user is authenticated. Otherwise, a 404 will be returned by the API and Web
UI. This is the main difference to "hidden" projects where everything can
be obtained if the URLs are known.
Please note that the store paths that are realized in evaluations that belong to
private projects aren't protected! It is assumed that the hashes are unknown
and thus inaccessible. For a real protection of the binary cache it's recommended
to either use `nix.sshServe` instead or to protect the routes `/nar/*` and `*.narinfo`
with a reverse proxy.
Email Notifications
-------------------

View file

@ -24,11 +24,11 @@
]
},
"locked": {
"lastModified": 1722555600,
"narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
"lastModified": 1727826117,
"narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
"rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
"type": "github"
},
"original": {
@ -74,17 +74,18 @@
"treefmt-nix": "treefmt-nix"
},
"locked": {
"lastModified": 1723579251,
"narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=",
"ref": "refs/heads/main",
"rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08",
"revCount": 609,
"lastModified": 1729971268,
"narHash": "sha256-AsoeDSO98rPZ2xIoxkmzIdwQ7nQGjfebTYhV3SLxUpc=",
"ref": "bring-back-constituents",
"rev": "7e02a188ffc474e443f1004eedb0b8e9a72597d1",
"revCount": 611,
"type": "git",
"url": "https://git.lix.systems/lix-project/nix-eval-jobs"
"url": "https://git.lix.systems/ma27/nix-eval-jobs"
},
"original": {
"ref": "bring-back-constituents",
"type": "git",
"url": "https://git.lix.systems/lix-project/nix-eval-jobs"
"url": "https://git.lix.systems/ma27/nix-eval-jobs"
}
},
"nix-github-actions": {
@ -95,11 +96,11 @@
]
},
"locked": {
"lastModified": 1720066371,
"narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=",
"lastModified": 1729742964,
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "622f829f5fe69310a866c8a6cd07e747c44ef820",
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
"type": "github"
},
"original": {
@ -187,11 +188,11 @@
]
},
"locked": {
"lastModified": 1723454642,
"narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
"lastModified": 1729613947,
"narHash": "sha256-XGOvuIPW1XRfPgHtGYXd5MAmJzZtOuwlfKDgxX5KT3s=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "349de7bc435bdff37785c2466f054ed1766173be",
"rev": "aac86347fb5063960eccb19493e0cadcdb4205ca",
"type": "github"
},
"original": {

View file

@ -6,7 +6,7 @@
inputs.lix.url = "git+https://git.lix.systems/lix-project/lix";
inputs.lix.inputs.nixpkgs.follows = "nixpkgs";
inputs.nix-eval-jobs.url = "git+https://git.lix.systems/lix-project/nix-eval-jobs";
inputs.nix-eval-jobs.url = "git+https://git.lix.systems/ma27/nix-eval-jobs?ref=bring-back-constituents";
inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
inputs.nix-eval-jobs.inputs.lix.follows = "lix";

View file

@ -184,6 +184,9 @@ paths:
visible:
description: when set to true the project is displayed in the web interface
type: boolean
private:
description: when set to true the project and all related objects are only accessible to authenticated users
type: boolean
declarative:
description: declarative input configured for this project
type: object
@ -625,6 +628,9 @@ components:
hidden:
description: when set to true the project is not displayed in the web interface
type: boolean
private:
description: when set to true the project and all related objects are only accessible to authenticated users
type: boolean
enabled:
description: when set to true the project gets scheduled for evaluation
type: boolean

View file

@ -98,6 +98,7 @@ let
FileLibMagic
FileSlurper
FileWhich
HTMLTreeBuilderXPath
IOCompress
IPCRun
IPCRun3

View file

@ -1,21 +0,0 @@
# IMPORTANT: if you delete this file your app will not work as
# expected. you have been warned
use strict;
use warnings;
use inc::Module::Install;
name 'Hydra';
all_from 'lib/Hydra.pm';
requires 'Catalyst::Runtime' => '5.7015';
requires 'Catalyst::Plugin::ConfigLoader';
requires 'Catalyst::Plugin::Static::Simple';
requires 'Catalyst::Action::RenderView';
requires 'parent';
requires 'Config::General'; # This should reflect the config file format you've chosen
# See Catalyst::Plugin::ConfigLoader for supported formats
catalyst;
install_script glob('script/*.pl');
auto_install;
WriteAll;

View file

@ -23,23 +23,37 @@ sub all : Chained('get_builds') PathPart {
$c->stash->{total} = $c->stash->{allBuilds}->search({finished => 1})->count
unless defined $c->stash->{total};
$c->stash->{builds} = [ $c->stash->{allBuilds}->search(
{ finished => 1 },
{ order_by => "stoptime DESC"
my $extra = {
order_by => "stoptime DESC"
, columns => [@buildListColumns]
, rows => $resultsPerPage
, page => $page }) ];
, page => $page };
my $criteria = { finished => 1 };
unless ($c->user_exists) {
$extra->{join} = {"jobset" => "project"};
$criteria->{"project.private"} = 0;
}
$c->stash->{builds} = [ $c->stash->{allBuilds}->search(
$criteria,
$extra
) ];
}
sub nix : Chained('get_builds') PathPart('channel/latest') CaptureArgs(0) {
my ($self, $c) = @_;
my $private = $c->user_exists ? [1,0] : [0];
$c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest";
$c->stash->{channelBuilds} = $c->stash->{latestSucceeded}
->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
->search({}, { columns => [@buildListColumns, 'drvpath', 'description', 'homepage']
, join => ["buildoutputs"]
->search({"project.private" => {-in => $private}},
{ columns => [@buildListColumns, 'drvpath', 'description', 'homepage']
, join => ["buildoutputs", {"jobset" => "project"}]
, order_by => ["me.id", "buildoutputs.name"]
, '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] });
}

View file

@ -49,6 +49,8 @@ sub latestbuilds : Chained('api') PathPart('latestbuilds') Args(0) {
error($c, "Parameter not defined!") if !defined $nr;
my $project = $c->request->params->{project};
checkProjectVisibleForGuest($c, $c->stash->{project});
my $jobset = $c->request->params->{jobset};
my $job = $c->request->params->{job};
my $system = $c->request->params->{system};
@ -106,6 +108,8 @@ sub jobsets : Chained('api') PathPart('jobsets') Args(0) {
my $project = $c->model('DB::Projects')->find($projectName)
or notFound($c, "Project $projectName doesn't exist.");
checkProjectVisibleForGuest($c, $project);
my @jobsets = jobsetOverview($c, $project);
my @list;
@ -124,7 +128,17 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
my $nr = $c->request->params->{nr};
error($c, "Parameter not defined!") if !defined $nr;
my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["priority DESC", "id"]});
my $criteria = {finished => 0};
my $extra = {
rows => $nr,
order_by => ["priority DESC", "id"]
};
unless ($c->user_exists) {
$criteria->{"project.private"} = 0;
$extra->{join} = {"jobset" => "project"};
}
my @builds = $c->model('DB::Builds')->search($criteria, $extra);
my @list;
push @list, buildToHash($_) foreach @builds;
@ -198,6 +212,16 @@ sub scmdiff : Path('/api/scmdiff') Args(0) {
my $rev1 = $c->request->params->{rev1};
my $rev2 = $c->request->params->{rev2};
unless ($c->user_exists) {
my $search = $c->model('DB::JobsetEvalInputs')->search(
{ "project.private" => 0, "me.uri" => $uri },
{ join => { "eval" => { jobset => "project" } } }
);
if ($search == 0) {
die("invalid revisions: [$rev1] [$rev2]")
}
}
die("invalid revisions: [$rev1] [$rev2]") if $rev1 !~ m/^[a-zA-Z0-9_.]+$/ || $rev2 !~ m/^[a-zA-Z0-9_.]+$/;
# FIXME: injection danger.

View file

@ -39,6 +39,9 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
$c->stash->{project} = $c->stash->{build}->project;
$c->stash->{jobset} = $c->stash->{build}->jobset;
$c->stash->{job} = $c->stash->{build}->job;
checkProjectVisibleForGuest($c, $c->stash->{project});
$c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})];
$c->stash->{runcommandlogProblem} = undef;

View file

@ -3,6 +3,7 @@ package Hydra::Controller::Channel;
use strict;
use warnings;
use base 'Hydra::Base::Controller::REST';
use Hydra::Helper::CatalystUtils;
sub channel : Chained('/') PathPart('channel/custom') CaptureArgs(3) {
@ -10,6 +11,8 @@ sub channel : Chained('/') PathPart('channel/custom') CaptureArgs(3) {
$c->stash->{project} = $c->model('DB::Projects')->find($projectName);
checkProjectVisibleForGuest($c, $c->stash->{project});
notFound($c, "Project $projectName doesn't exist.")
if !$c->stash->{project};

View file

@ -27,6 +27,8 @@ sub job : Chained('/') PathPart('job') CaptureArgs(3) {
$c->stash->{job} = $jobName;
$c->stash->{project} = $c->stash->{jobset}->project;
checkProjectVisibleForGuest($c, $c->stash->{project});
}
sub shield :Chained('job') PathPart('shield') Args(0) {

View file

@ -17,6 +17,8 @@ sub jobsetChain :Chained('/') :PathPart('jobset') :CaptureArgs(2) {
$c->stash->{project} = $project;
checkProjectVisibleForGuest($c, $c->stash->{project});
$c->stash->{jobset} = $project->jobsets->find({ name => $jobsetName });
if (!$c->stash->{jobset} && !($c->action->name eq "jobset" and $c->request->method eq "PUT")) {
@ -294,26 +296,24 @@ sub updateJobset {
# Set the inputs of this jobset.
$jobset->jobsetinputs->delete;
if ($type == 0) {
foreach my $name (keys %{$c->stash->{params}->{inputs}}) {
my $inputData = $c->stash->{params}->{inputs}->{$name};
my $type = $inputData->{type};
my $value = $inputData->{value};
my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0;
my $types = knownInputTypes($c);
foreach my $name (keys %{$c->stash->{params}->{inputs}}) {
my $inputData = $c->stash->{params}->{inputs}->{$name};
my $type = $inputData->{type};
my $value = $inputData->{value};
my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0;
my $types = knownInputTypes($c);
badRequest($c, "Invalid input name $name.") unless $name =~ /^[[:alpha:]][\w-]*$/;
badRequest($c, "Invalid input type $type; valid types: $types.") unless defined $c->stash->{inputTypes}->{$type};
badRequest($c, "Invalid input name $name.") unless $name =~ /^[[:alpha:]][\w-]*$/;
badRequest($c, "Invalid input type $type; valid types: $types.") unless defined $c->stash->{inputTypes}->{$type};
my $input = $jobset->jobsetinputs->create(
{ name => $name,
type => $type,
emailresponsible => $emailresponsible
});
my $input = $jobset->jobsetinputs->create(
{ name => $name,
type => $type,
emailresponsible => $emailresponsible
});
$value = checkInputValue($c, $name, $type, $value);
$input->jobsetinputalts->create({altnr => 0, value => $value});
}
$value = checkInputValue($c, $name, $type, $value);
$input->jobsetinputalts->create({altnr => 0, value => $value});
}
}

View file

@ -19,6 +19,8 @@ sub evalChain : Chained('/') PathPart('eval') CaptureArgs(1) {
$c->stash->{eval} = $eval;
$c->stash->{jobset} = $eval->jobset;
$c->stash->{project} = $eval->jobset->project;
checkProjectVisibleForGuest($c, $c->stash->{project});
}

View file

@ -16,6 +16,8 @@ sub projectChain :Chained('/') :PathPart('project') :CaptureArgs(1) {
$c->stash->{project} = $c->model('DB::Projects')->find($projectName);
checkProjectVisibleForGuest($c, $c->stash->{project});
$c->stash->{isProjectOwner} = !$isCreate && isProjectOwner($c, $c->stash->{project});
notFound($c, "Project $projectName doesn't exist.")
@ -161,6 +163,7 @@ sub updateProject {
, homepage => trim($c->stash->{params}->{homepage})
, enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, private => defined $c->stash->{params}->{private} ? 1 : 0
, owner => $owner
, enable_dynamic_run_command => $enable_dynamic_run_command
, declfile => trim($c->stash->{params}->{declarative}->{file})

View file

@ -110,7 +110,13 @@ sub deserialize :ActionClass('Deserialize') { }
sub index :Path :Args(0) {
my ($self, $c) = @_;
$c->stash->{template} = 'overview.tt';
$c->stash->{projects} = [$c->model('DB::Projects')->search({}, {order_by => ['enabled DESC', 'name']})];
my $includePrivate = $c->user_exists ? [1,0] : [0];
$c->stash->{projects} = [$c->model('DB::Projects')->search(
{private => {-in => $includePrivate}},
{order_by => ['enabled DESC', 'name']}
)];
$c->stash->{newsItems} = [$c->model('DB::NewsItems')->search({}, { order_by => ['createtime DESC'], rows => 5 })];
$self->status_ok($c,
entity => $c->stash->{projects}
@ -122,15 +128,23 @@ sub queue :Local :Args(0) :ActionClass('REST') { }
sub queue_GET {
my ($self, $c) = @_;
my $criteria = {finished => 0};
my $extra = {
columns => [@buildListColumns],
order_by => ["priority DESC", "id"]
};
unless ($c->user_exists) {
$criteria->{"project.private"} = 0;
$extra->{join} = {"jobset" => "project"};
}
$c->stash->{template} = 'queue.tt';
$c->stash->{flashMsg} //= $c->flash->{buildMsg};
$self->status_ok(
$c,
entity => [$c->model('DB::Builds')->search(
{ finished => 0 },
{ order_by => ["globalpriority desc", "id"],
, columns => [@buildListColumns]
})]
$criteria,
$extra
)]
);
}
@ -139,10 +153,15 @@ sub queue_summary :Local :Path('queue-summary') :Args(0) {
my ($self, $c) = @_;
$c->stash->{template} = 'queue-summary.tt';
my $extra = " where ";
unless ($c->user_exists) {
$extra = "inner join Projects p on p.name = project where p.private = 0 and ";
}
$c->stash->{queued} = dbh($c)->selectall_arrayref(
"select jobsets.project as project, jobsets.name as jobset, count(*) as queued, min(timestamp) as oldest, max(timestamp) as newest from Builds " .
"join Jobsets jobsets on jobsets.id = builds.jobset_id " .
"where finished = 0 group by jobsets.project, jobsets.name order by queued desc",
"$extra finished = 0 group by jobsets.project, jobsets.name order by queued desc",
{ Slice => {} });
$c->stash->{systems} = dbh($c)->selectall_arrayref(
@ -155,12 +174,19 @@ sub status :Local :Args(0) :ActionClass('REST') { }
sub status_GET {
my ($self, $c) = @_;
my $criteria = { "buildsteps.busy" => { '!=', 0 } };
my $join = ["buildsteps"];
unless ($c->user_exists) {
$criteria->{"project.private"} = 0;
push @{$join}, {"jobset" => "project"};
}
$self->status_ok(
$c,
entity => [$c->model('DB::Builds')->search(
{ "buildsteps.busy" => { '!=', 0 } },
$criteria,
{ order_by => ["globalpriority DESC", "id"],
join => "buildsteps",
join => $join,
columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
})]
);
@ -202,13 +228,18 @@ sub machines :Local Args(0) {
}
}
my $extra = "where";
unless ($c->user_exists) {
$extra = "inner join Projects p on p.name = jobsets.project where p.private = 0 and ";
}
$c->stash->{machines} = $machines;
$c->stash->{steps} = dbh($c)->selectall_arrayref(
"select build, stepnr, s.system as system, s.drvpath as drvpath, machine, s.starttime as starttime, jobsets.project as project, jobsets.name as jobset, job, s.busy as busy " .
"from BuildSteps s " .
"join Builds b on s.build = b.id " .
"join Jobsets jobsets on jobsets.id = b.jobset_id " .
"where busy != 0 order by machine, stepnr",
"$extra busy != 0 order by machine, stepnr",
{ Slice => {} });
$c->stash->{template} = 'machine-status.tt';
$self->status_ok($c, entity => $c->stash->{machines});
@ -450,16 +481,28 @@ sub steps :Local Args(0) {
my $resultsPerPage = 20;
my $criteria = {
"me.starttime" => { '!=', undef },
"me.stoptime" => { '!=', undef }
};
my $extra = {
order_by => [ "me.stoptime desc" ],
rows => $resultsPerPage,
offset => ($page - 1) * $resultsPerPage,
};
unless ($c->user_exists) {
$criteria->{"project.private"} = 0;
$extra->{join} = [{"build" => {"jobset" => "project"}}];
}
$c->stash->{page} = $page;
$c->stash->{resultsPerPage} = $resultsPerPage;
$c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
{ starttime => { '!=', undef },
stoptime => { '!=', undef }
},
{ order_by => [ "stoptime desc" ],
rows => $resultsPerPage,
offset => ($page - 1) * $resultsPerPage
}) ];
$criteria,
$extra
) ];
$c->stash->{total} = approxTableSize($c, "IndexBuildStepsOnStopTime");
}
@ -480,28 +523,58 @@ sub search :Local Args(0) {
$c->model('DB')->schema->txn_do(sub {
$c->model('DB')->schema->storage->dbh->do("SET LOCAL statement_timeout = 20000");
$c->stash->{projects} = [ $c->model('DB::Projects')->search(
{ -and =>
my $projectCriteria = {
-and =>
[ { -or => [ name => { ilike => "%$query%" }, displayName => { ilike => "%$query%" }, description => { ilike => "%$query%" } ] }
, { hidden => 0 }
]
},
{ order_by => ["name"] } ) ];
};
$c->stash->{jobsets} = [ $c->model('DB::Jobsets')->search(
{ -and =>
my $jobsetCriteria = {
-and =>
[ { -or => [ "me.name" => { ilike => "%$query%" }, "me.description" => { ilike => "%$query%" } ] }
, { "project.hidden" => 0, "me.hidden" => 0 }
]
},
{ order_by => ["project", "name"], join => ["project"] } ) ];
};
$c->stash->{jobs} = [ $c->model('DB::Builds')->search(
{ "job" => { ilike => "%$query%" }
my $buildCriteria = {
"job" => { ilike => "%$query%" }
, "project.hidden" => 0
, "jobset.hidden" => 0
, iscurrent => 1
},
};
my $buildSearchExtra = {
order_by => ["id desc"]
, rows => $c->stash->{limit}, join => []
};
my $outCriteria = {
"buildoutputs.path" => { ilike => "%$query%" }
};
my $drvCriteria = { "drvpath" => { ilike => "%$query%" } };
unless ($c->user_exists) {
$projectCriteria->{private} = 0;
$jobsetCriteria->{"project.private"} = 0;
$buildCriteria->{"project.private"} = 0;
push @{$buildSearchExtra->{join}}, {"jobset" => "project"};
$outCriteria->{"project.private"} = 0;
$drvCriteria->{"project.private"} = 0;
}
$c->stash->{projects} = [ $c->model('DB::Projects')->search(
$projectCriteria,
{ order_by => ["name"] } ) ];
$c->stash->{jobsets} = [ $c->model('DB::Jobsets')->search(
$jobsetCriteria,
{ order_by => ["project", "name"], join => ["project"] } ) ];
$c->stash->{jobs} = [ $c->model('DB::Builds')->search(
$buildCriteria,
{
order_by => ["jobset.project", "jobset.name", "job"],
join => { "jobset" => "project" },
@ -510,17 +583,16 @@ sub search :Local Args(0) {
];
# Perform build search in separate queries to prevent seq scan on buildoutputs table.
my $outExtra = $buildSearchExtra;
push @{$outExtra->{join}}, "buildoutputs";
$c->stash->{builds} = [ $c->model('DB::Builds')->search(
{ "buildoutputs.path" => { ilike => "%$query%" } },
{ order_by => ["id desc"], join => ["buildoutputs"]
, rows => $c->stash->{limit}
} ) ];
$outCriteria,
$outExtra
) ];
$c->stash->{buildsdrv} = [ $c->model('DB::Builds')->search(
{ "drvpath" => { ilike => "%$query%" } },
{ order_by => ["id desc"]
, rows => $c->stash->{limit}
} ) ];
$drvCriteria,
$buildSearchExtra ) ];
$c->stash->{resource} = { projects => $c->stash->{projects},
jobsets => $c->stash->{jobsets},

View file

@ -29,6 +29,7 @@ our @EXPORT = qw(
approxTableSize
requireLocalStore
dbh
checkProjectVisibleForGuest
);
@ -256,6 +257,14 @@ sub requireProjectOwner {
unless isProjectOwner($c, $project);
}
sub checkProjectVisibleForGuest {
my ($c, $project) = @_;
if (defined $project && $project->private == 1 && !$c->user_exists) {
my $projectName = $project->name;
notFound($c, "Project $projectName not found!");
}
}
sub isAdmin {
my ($c) = @_;

View file

@ -182,17 +182,34 @@ sub findLog {
my ($c, $drvPath, @outPaths) = @_;
if (defined $drvPath) {
unless ($c->user_exists) {
my $existsForGuest = $c->model('DB::BuildSteps')->search(
{"me.drvpath" => $drvPath, "project.private" => 0},
{join => {build => {"jobset" => "project"}}}
);
if ($existsForGuest == 0) {
notFound($c, "Resource not found");
}
}
my $logPath = getDrvLogPath($drvPath);
return $logPath if defined $logPath;
}
return undef if scalar @outPaths == 0;
my $join = ["buildstepoutputs"];
my $criteria = { path => { -in => [@outPaths] } };
unless ($c->user_exists) {
push @{$join}, {"build" => {jobset => "project"}};
$criteria->{"project.private"} = 0;
}
my @steps = $c->model('DB::BuildSteps')->search(
{ path => { -in => [@outPaths] } },
$criteria,
{ select => ["drvpath"]
, distinct => 1
, join => "buildstepoutputs"
, join => $join
});
foreach my $step (@steps) {
@ -285,9 +302,19 @@ sub getEvals {
my $me = $evals_result_set->current_source_alias;
my @evals = $evals_result_set->search(
{ hasnewbuilds => 1 },
{ order_by => "$me.id DESC", rows => $rows, offset => $offset });
my $criteria = { hasnewbuilds => 1 };
my $extra = {
order_by => "$me.id DESC",
rows => $rows,
offset => $offset
};
unless ($c->user_exists) {
$extra->{join} = {"jobset" => "project"};
$criteria->{"project.private"} = 0;
}
my @evals = $evals_result_set->search($criteria, $extra);
my @res = ();
my $cache = {};

View file

@ -62,6 +62,12 @@ __PACKAGE__->table("projects");
default_value: 0
is_nullable: 0
=head2 private
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 owner
data_type: 'text'
@ -107,6 +113,8 @@ __PACKAGE__->add_columns(
{ data_type => "integer", default_value => 1, is_nullable => 0 },
"hidden",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"private",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"owner",
{ data_type => "text", is_foreign_key => 1, is_nullable => 0 },
"homepage",
@ -236,8 +244,8 @@ Composing rels: L</projectmembers> -> username
__PACKAGE__->many_to_many("usernames", "projectmembers", "username");
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-11-22 12:51:02
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ppyLpFU2fZASFANhD7vUgg
use JSON::MaybeXS;
@ -267,6 +275,7 @@ sub as_json {
"enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"private" => $self->get_column("private") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"jobsets" => [ map { $_->name } $self->jobsets ]
);

View file

@ -42,7 +42,15 @@
[% END %]
[% BLOCK renderJobsetInputs %]
<table class="table table-striped table-condensed show-on-legacy">
<div class="card show-on-flake border-danger">
<div class="text-danger card-body">
<h5 class="card-title">Jobset Inputs don't take any effect for flakes</h5>
<p class="card-text">
These are only available to configure Hydra plugins.
</p>
</div>
</div>
<table class="table table-striped table-condensed">
<thead>
<tr><th></th><th>Input name</th><th>Type</th><th style="width: 50%">Value</th><th>Notify committers</th></tr>
</thead>

View file

@ -17,6 +17,13 @@
</div>
</div>
<div class="form-group row">
<label class="col-sm-3" for="editprojectprivate">Private</label>
<div class="col-sm-9">
<input type="checkbox" id="editprojectprivate" name="private" [% IF project.private %] checked="checked" [% END %]/>
</div>
</div>
<div class="form-group row">
<label class="col-sm-3" for="editprojectidentifier">Identifier</label>
<div class="col-sm-9">

View file

@ -54,7 +54,7 @@
<tbody>
[% FOREACH p IN projects %]
<tr class="project [% IF !p.enabled %]disabled-project[% END %]">
<td><span class="[% IF !p.enabled %]disabled-project[% END %] [%+ IF p.hidden %]hidden-project[% END %]">[% INCLUDE renderProjectName project=p.name inRow=1 %]</span></td>
<td>[% IF p.private %]&#128274;[% END %] <span class="[% IF !p.enabled %]disabled-project[% END %] [%+ IF p.hidden %]hidden-project[% END %]">[% INCLUDE renderProjectName project=p.name inRow=1 %]</span></td>
<td>[% HTML.escape(p.displayname) %]</td>
<td>[% WRAPPER maybeLink uri=p.homepage %][% HTML.escape(p.description) %][% END %]</td>
</tr>

View file

@ -1,4 +1,9 @@
[% WRAPPER layout.tt title="Project $project.name" %]
[% IF project.private %]
[% lock = ' &#128274;' %]
[% ELSE %]
[% lock = '' %]
[% END %]
[% WRAPPER layout.tt titleHTML="Project $project.name$lock" title="Project $project.name" %]
[% PROCESS common.tt %]
<ul class="nav nav-tabs">

View file

@ -371,6 +371,7 @@ sub evalJobs {
push @cmd, ("--gc-roots-dir", getGCRootsDir);
push @cmd, ("--max-jobs", 1);
push @cmd, "--meta";
push @cmd, "--constituents";
push @cmd, "--force-recurse";
push @cmd, ("--option", "allow-import-from-derivation", "false") if $config->{allow_import_from_derivation} // "true" ne "true";
push @cmd, ("--workers", $config->{evaluator_workers} // 1);
@ -778,6 +779,8 @@ sub checkJobsetWrapped {
, nixexprpath => $jobset->nixexprpath
});
my @jobsWithConstituents;
while (defined(my $job = $jobsIter->())) {
if ($jobsetsJobset) {
die "The .jobsets jobset must only have a single job named 'jobsets'"
@ -790,6 +793,10 @@ sub checkJobsetWrapped {
checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins)
unless defined $job->{error};
if (defined $job->{constituents}) {
push @jobsWithConstituents, $job;
}
}
# Have any builds been added or removed since last time?
@ -812,6 +819,39 @@ sub checkJobsetWrapped {
$ev->jobsetevalmembers->create({ build => $id, isnew => $x->{new} });
}
# Create AggregateConstituents mappings. Since there can
# be jobs that alias each other, if there are multiple
# builds for the same derivation, pick the one with the
# shortest name.
my %drvPathToId;
foreach my $id (keys %buildMap) {
my $x = $buildMap{$id};
my $y = $drvPathToId{$x->{drvPath}};
if (defined $y) {
next if length $x->{attr} > length $y->{attr};
next if length $x->{attr} == length $y->{attr} && $x->{attr} ge $y->{attr};
}
$drvPathToId{$x->{drvPath}} = $x;
}
foreach my $job (values @jobsWithConstituents) {
next unless defined $job->{constituents};
if (defined $job->{error}) {
die "aggregate job $job->{attr} failed with the error: $job->{error}\n";
}
my $x = $drvPathToId{$job->{drvPath}} or
die "aggregate job $job->{attr} has no corresponding build record.\n";
foreach my $drvPath (@{$job->{constituents}}) {
my $constituent = $drvPathToId{$drvPath};
if (defined $constituent) {
$db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}});
} else {
warn "aggregate job $job->{attr} has a constituent $drvPath that doesn't correspond to a Hydra build\n";
}
}
}
foreach my $name (keys %{$inputInfo}) {
for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) {
my $input = $inputInfo->{$name}->[$n];

View file

@ -44,6 +44,7 @@ create table Projects (
description text,
enabled integer not null default 1,
hidden integer not null default 0,
private integer not null default 0,
owner text not null,
homepage text, -- URL for the project
declfile text, -- File containing declarative jobset specification

View file

@ -1,5 +1,3 @@
-- Records of RunCommand executions
--
-- The intended flow is:

View file

@ -78,4 +78,28 @@ subtest "validating the JSON representation of a build" => sub {
}, "The build's JSON matches our API.");
};
subtest "accessing the constituents API" => sub {
my $url = $build_url . "/constituents";
my $constituents = request(GET $url,
Accept => 'application/json',
);
ok($constituents->is_success, "Getting the constituent builds");
my $data;
my $valid_json = lives { $data = decode_json($constituents->content); };
ok($valid_json, "We get back valid JSON.");
if (!$valid_json) {
use Data::Dumper;
print STDERR Dumper $constituents->content;
}
my ($buildA) = grep { $_->{nixname} eq "empty-dir-a" } @$data;
my ($buildB) = grep { $_->{nixname} eq "empty-dir-b" } @$data;
is($buildA->{job}, "a");
is($buildB->{job}, "b");
};
done_testing;

View file

@ -51,7 +51,8 @@ subtest "Read project 'tests'" => sub {
homepage => "",
jobsets => [],
name => "tests",
owner => "root"
owner => "root",
"private" => JSON::MaybeXS::false
});
};
@ -96,7 +97,8 @@ subtest "Transitioning from declarative project to normal" => sub {
file => "bogus",
type => "boolean",
value => "false"
}
},
"private" => JSON::MaybeXS::false
});
};
@ -135,7 +137,8 @@ subtest "Transitioning from declarative project to normal" => sub {
homepage => "",
jobsets => [],
name => "tests",
owner => "root"
owner => "root",
"private" => JSON::MaybeXS::false
});
};
};

View file

@ -0,0 +1,32 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
use Hydra::Helper::Exec;
my $ctx = test_context();
my $jobsetCtx = $ctx->makeJobset(
expression => 'constituents-broken.nix',
);
my $jobset = $jobsetCtx->{"jobset"};
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
);
isnt($res, 0, "hydra-eval-jobset exits non-zero");
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
like(
$stderr,
qr/aggregate job mixed_aggregate failed with the error: "constituentA": does not exist/,
"The stderr record includes a relevant error message"
);
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
like(
$jobset->errormsg,
qr/aggregate job mixed_aggregate failed with the error: "constituentA": does not exist/,
"The jobset records a relevant error message"
);
done_testing;

View file

@ -0,0 +1,20 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
my $ctx = test_context();
my $builds = $ctx->makeAndEvaluateJobset(
expression => 'constituents.nix',
);
my $constituentA = $builds->{"constituentA"};
my $directAggregate = $builds->{"direct_aggregate"};
my $indirectAggregate = $builds->{"indirect_aggregate"};
is(system('nix-store', '--delete', $constituentA->drvpath), 256, "Deleting a constituent derivation fails");
is(system('nix-store', '--delete', $directAggregate->drvpath), 256, "Deleting the direct aggregate derivation fails");
is(system('nix-store', '--delete', $indirectAggregate->drvpath), 256, "Deleting the indirect aggregate derivation fails");
done_testing;

168
t/private-projects.t Normal file
View file

@ -0,0 +1,168 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
use HTTP::Request::Common;
use HTML::TreeBuilder::XPath;
use JSON::MaybeXS;
my %ctx = test_init(
use_external_destination_store => 0
);
require Hydra::Schema;
require Hydra::Model::DB;
require Catalyst::Test;
Catalyst::Test->import('Hydra');
my $db = Hydra::Model::DB->new;
hydra_setup($db);
my $scratch = "$ctx{tmpdir}/scratch";
mkdir $scratch;
my $uri = "file://$scratch/git-repo";
my $jobset = createJobsetWithOneInput('gitea', 'git-input.nix', 'src', 'git', $uri, $ctx{jobsdir});
ok(request('/project/tests')->is_success, "Project 'tests' exists");
my $project = $db->resultset('Projects')->find({name => "tests"})->update({private => 1});
ok(
!request('/project/tests')->is_success,
"Project 'tests' is private now and should be unreachable"
);
my $user = $db->resultset('Users')->create({
username => "testing",
emailaddress => 'testing@invalid.org',
password => ''
});
$user->setPassword('foobar');
my $auth = request(
POST(
'/login',
{username => 'testing', 'password' => 'foobar'},
Origin => 'http://localhost', Accept => 'application/json'
),
{host => 'localhost'}
);
ok(
$auth->code == 302,
"Successfully logged in"
);
my $cookie = (split /;/, $auth->header('set_cookie'))[0];
ok(
request(GET(
'/project/tests',
Cookie => $cookie
))->is_success,
"Project visible for authenticated user."
);
updateRepository('gitea', "$ctx{testdir}/jobs/git-update.sh", $scratch);
ok(evalSucceeds($jobset), "Evaluating nix expression");
is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand.nix should result in 1 build1");
ok(
request('/eval/1')->code == 404,
'Eval of private project not available for unauthenticated user.'
);
ok(
request(GET '/eval/1', Cookie => $cookie)->is_success,
'Eval available for authenticated User'
);
ok(
request(GET '/jobset/tests/gitea', Cookie => $cookie)->is_success,
'Jobset available for user'
);
ok(
request(GET '/jobset/tests/gitea')->code == 404,
'Jobset unavailable for guest'
);
ok(
request('/build/1')->code == 404,
'Build of private project not available for unauthenticated user.'
);
ok(
request(GET '/build/1', Cookie => $cookie)->is_success,
'Build available for authenticated User'
);
(my $build) = queuedBuildsForJobset($jobset);
ok(runBuild($build), "Build should succeed with exit code 0");
ok(
request(GET '/jobset/tests/gitea/channel/latest', Cookie => $cookie)->is_success,
'Channel available for authenticated user'
);
ok(
request(GET '/jobset/tests/gitea/channel/latest')->code == 404,
'Channel unavailable for guest'
);
updateRepository('gitea', "$ctx{testdir}/jobs/git-update.sh", $scratch);
ok(evalSucceeds($jobset), "Evaluating nix expression");
my $latest_builds_unauth = request(GET "/all");
my $tree = HTML::TreeBuilder::XPath->new;
$tree->parse($latest_builds_unauth->content);
ok(!$tree->exists('/html//tbody/tr'), "No builds available");
my $latest_builds = request(GET "/all", Cookie => $cookie);
$tree = HTML::TreeBuilder::XPath->new;
$tree->parse($latest_builds->content);
ok($tree->exists('/html//tbody/tr'), "Builds available");
my $p2 = $db->resultset("Projects")->create({name => "public", displayname => "public", owner => "root"});
my $jobset2 = $p2->jobsets->create({
name => "public", nixexprpath => 'basic.nix', nixexprinput => "jobs", emailoverride => ""
});
my $jobsetinput = $jobset2->jobsetinputs->create({name => "jobs", type => "path"});
$jobsetinput->jobsetinputalts->create({altnr => 0, value => $ctx{jobsdir}});
updateRepository('gitea', "$ctx{testdir}/jobs/git-update.sh", $scratch);
ok(evalSucceeds($jobset2), "Evaluating nix expression");
is(
nrQueuedBuildsForJobset($jobset2),
3,
"Evaluating jobs/runcommand.nix should result in 3 builds"
);
(my $b1, my $b2, my $b3) = queuedBuildsForJobset($jobset2);
ok(runBuild($b1), "Build should succeed with exit code 0");
ok(runBuild($b2), "Build should succeed with exit code 0");
ok(runBuild($b3), "Build should succeed with exit code 0");
my $latest_builds_unauth2 = request(GET "/all");
$tree = HTML::TreeBuilder::XPath->new;
$tree->parse($latest_builds_unauth2->content);
is(
scalar $tree->findvalues('/html//tbody/tr'),
3,
"Three builds available"
);
my $latest_builds2 = request(GET "/all", Cookie => $cookie);
$tree = HTML::TreeBuilder::XPath->new;
$tree->parse($latest_builds2->content);
is(
scalar $tree->findvalues('/html//tbody/tr'),
4,
"Three builds available"
);
done_testing;

View file

@ -0,0 +1,32 @@
use feature 'unicode_strings';
use strict;
use warnings;
use Setup;
my %ctx = test_init();
require Hydra::Schema;
require Hydra::Model::DB;
use Test2::V0;
my $db = Hydra::Model::DB->new;
hydra_setup($db);
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
my $jobset = createBaseJobset("broken-constituent", "broken-constituent.nix", $ctx{jobsdir});
ok(evalSucceeds($jobset), "Evaluating jobs/broken-constituent.nix should exit with return code 0");
is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix should not queue any builds");
like(
$jobset->errormsg,
qr/^"does-not-exist": does not exist$/m,
"Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
like(
$jobset->errormsg,
qr/^"does-not-evaluate": "error: assertion 'false' failed/m,
"Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
done_testing;

View file

@ -0,0 +1,35 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
my $ctx = test_context();
my $builds = $ctx->makeAndEvaluateJobset(
expression => 'constituents.nix',
);
my $constituentBuildA = $builds->{"constituentA"};
my $constituentBuildB = $builds->{"constituentB"};
my $eval = $constituentBuildA->jobsetevals->first();
is($eval->evaluationerror->has_error, 0);
subtest "Verifying the direct aggregate" => sub {
my $aggBuild = $builds->{"direct_aggregate"};
is($aggBuild->constituents->first()->id, $constituentBuildA->id, "The ID of the constituent is correct");
};
subtest "Verifying the indirect aggregate" => sub {
my $indirectBuild = $builds->{"indirect_aggregate"};
is($indirectBuild->constituents->first()->id, $constituentBuildA->id, "The ID of the constituent is correct");
};
subtest "Verifying a mix of direct and indirect aggregate references" => sub {
my $mixedBuild = $builds->{"mixed_aggregate"};
my ($constituentA, $constituentB) = $mixedBuild->constituents()->search({}, {order_by => { -asc => "job"} });
is($constituentA->id, $constituentBuildA->id, "The ID of the constituent is correct");
is($constituentB->id, $constituentBuildB->id, "The ID of the constituent is correct");
};
done_testing;