diff --git a/release.nix b/release.nix index e757dfe9..0dd984cc 100644 --- a/release.nix +++ b/release.nix @@ -6,7 +6,7 @@ let pkgs = import {}; - genAttrs' = pkgs.lib.genAttrs [ "x86_64-linux" "i686-linux" ]; + genAttrs' = pkgs.lib.genAttrs [ "x86_64-linux" /* "i686-linux" */ ]; hydraServer = hydraPkg: { config, pkgs, ... }: @@ -50,6 +50,13 @@ in rec { addToSearchPath PERL5LIB $(pwd)/src/lib ''; + postUnpack = '' + # Clean up when building from a working tree. + if [ -z "$IN_NIX_SHELL" ]; then + (cd $sourceRoot && (git ls-files -o --directory | xargs -r rm -rfv)) || true + fi + ''; + configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; @@ -167,7 +174,7 @@ in rec { tests.install = genAttrs' (system: with import { inherit system; }; simpleTest { - machine = hydraServer (builtins.getAttr system build); # build.${system} + machine = hydraServer build.${system}; testScript = '' $machine->waitForJob("hydra-init"); @@ -182,7 +189,7 @@ in rec { tests.api = genAttrs' (system: with import { inherit system; }; simpleTest { - machine = hydraServer (builtins.getAttr system build); # build.${system} + machine = hydraServer build.${system}; testScript = let dbi = "dbi:Pg:dbname=hydra;user=root;"; in '' @@ -190,7 +197,7 @@ in rec { # Create an admin account and some other state. $machine->succeed - ( "su hydra -c \"hydra-create-user root --email-address 'e.dolstra\@tudelft.nl' --password foobar --role admin\"" + ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\"" , "mkdir /run/jobset /tmp/nix" , "chmod 755 /run/jobset /tmp/nix" , "cp ${./tests/api-test.nix} /run/jobset/default.nix" @@ -200,18 +207,18 @@ in rec { # Start the web interface with some weird settings. $machine->succeed("systemctl stop hydra-server hydra-evaluator hydra-queue-runner"); - $machine->mustSucceed("su hydra -c 'NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix DBIC_TRACE=1 hydra-server -d' >&2 &"); + $machine->mustSucceed("su - hydra -c 'NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix NIX_REMOTE= DBIC_TRACE=1 hydra-server -d' >&2 &"); $machine->waitForOpenPort("3000"); # Run the API tests. - $machine->mustSucceed("su hydra -c 'perl ${./tests/api-test.pl}' >&2"); + $machine->mustSucceed("su - hydra -c 'perl ${./tests/api-test.pl}' >&2"); ''; }); /* tests.s3backup = genAttrs' (system: with import { inherit system; }; - let hydra = builtins.getAttr system build; in # build."${system}" + let hydra = build.${system} simpleTest { machine = { config, pkgs, ... }: diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots index 4c252eb0..7b6fb806 100755 --- a/src/script/hydra-update-gc-roots +++ b/src/script/hydra-update-gc-roots @@ -22,20 +22,27 @@ sub addRoot { } -my @columns = ( "id", "project", "jobset", "job", "system", "finished", "drvpath", "timestamp", "buildstatus" ); +my @columns = + ( "id", "project", "jobset", "job", "system", "finished", "drvpath", "timestamp", "buildstatus" + , { "outpaths" => \ "(select string_agg(path, ' ') from BuildOutputs where build = me.id)" } + ); + +my %seenBuilds; sub keepBuild { my ($build, $keepFailedDrvs) = @_; + return if defined $seenBuilds{$build->id}; + $seenBuilds{$build->id} = 1; print STDERR " keeping ", ($build->finished ? "" : "scheduled "), "build ", $build->id, " (", $build->get_column('project'), ":", $build->get_column('jobset'), ":", $build->get_column('job'), "; ", $build->system, "; ", strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n"; if ($build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6)) { - foreach my $out ($build->buildoutputs->all) { - if (isValidPath($out->path)) { - addRoot $out->path; + foreach my $path (split / /, $build->get_column('outpaths')) { + if (isValidPath($path)) { + addRoot $path; } else { - print STDERR " warning: output ", $out->path, " has disappeared\n" if $build->finished; + print STDERR " warning: output ", $path, " has disappeared\n" if $build->finished; } } } @@ -108,10 +115,33 @@ foreach my $project ($db->resultset('Projects')->search({}, { order_by => ["name , order_by => "id desc", rows => $keepnr }); } + # Note: we also keep the derivations of failed builds so that + # they can be restarted. keepBuild($_, 1) foreach $jobset->builds->search( { id => { -in => $db->resultset('JobsetEvalMembers')->search({ eval => { -in => [@evals] } }, { select => "build" })->as_query } + , finished => 1 }, { order_by => ["job", "id"], columns => [ @columns ] }); + + print STDERR "*** looking for the most recent successful builds of current jobs in ", + $project->name, ":", $jobset->name, "\n"; + + # Keep the most recently succeeded build of a current job. Oh + # I really need to stop using DBIx::Class. + keepBuild($_, 1) foreach $jobset->builds->search( + { id => { -in => $jobset->builds->search( + { finished => 1 + , buildstatus => [0, 6] + , job => { -in => $jobset->builds->search( + { eval => { -in => [@evals] } }, + { select => "job", distinct => 1, join => "jobsetevalmembers" } + )->as_query } + }, + { group_by => 'job' + , select => [ { max => 'id', -as => 'm' } ] + })->as_query } + }, + { columns => [ @columns ] }); } } diff --git a/tests/api-test.pl b/tests/api-test.pl index faf23ee1..4ee2dd7a 100644 --- a/tests/api-test.pl +++ b/tests/api-test.pl @@ -49,7 +49,7 @@ ok(exists $jobset->{jobsetinputs}->{"my-src"}, "The new jobset has a 'my-src' in ok($jobset->{jobsetinputs}->{"my-src"}->{jobsetinputalts}->[0] eq "/run/jobset", "The 'my-src' input is in /run/jobset"); -system("NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix hydra-evaluator sample default"); +system("NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix NIX_REMOTE= hydra-evaluator sample default"); $result = request_json({ uri => '/jobset/sample/default/evals' }); ok($result->code() == 200, "Can get evals of a jobset"); my $evals = decode_json($result->content())->{evals};