Merge branch 'master' into build-ng

release.nix

@@ -6,7 +6,7 @@ let
 
   pkgs = import <nixpkgs> {};
 
-  genAttrs' = pkgs.lib.genAttrs [ "x86_64-linux" "i686-linux" ];
+  genAttrs' = pkgs.lib.genAttrs [ "x86_64-linux" /* "i686-linux" */ ];
 
   hydraServer = hydraPkg:
     { config, pkgs, ... }:
@@ -50,6 +50,13 @@ in rec {
         addToSearchPath PERL5LIB $(pwd)/src/lib
       '';
 
+      postUnpack = ''
+        # Clean up when building from a working tree.
+        if [ -z "$IN_NIX_SHELL" ]; then
+          (cd $sourceRoot && (git ls-files -o --directory | xargs -r rm -rfv)) || true
+        fi
+      '';
+
       configureFlags =
         [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
 
@@ -167,7 +174,7 @@ in rec {
   tests.install = genAttrs' (system:
     with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
     simpleTest {
-      machine = hydraServer (builtins.getAttr system build); # build.${system}
+      machine = hydraServer build.${system};
       testScript =
         ''
           $machine->waitForJob("hydra-init");
@@ -182,7 +189,7 @@ in rec {
   tests.api = genAttrs' (system:
     with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
     simpleTest {
-      machine = hydraServer (builtins.getAttr system build); # build.${system}
+      machine = hydraServer build.${system};
       testScript =
         let dbi = "dbi:Pg:dbname=hydra;user=root;"; in
         ''
@@ -190,7 +197,7 @@ in rec {
 
           # Create an admin account and some other state.
           $machine->succeed
-              ( "su hydra -c \"hydra-create-user root --email-address 'e.dolstra\@tudelft.nl' --password foobar --role admin\""
+              ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\""
               , "mkdir /run/jobset /tmp/nix"
               , "chmod 755 /run/jobset /tmp/nix"
               , "cp ${./tests/api-test.nix} /run/jobset/default.nix"
@@ -200,18 +207,18 @@ in rec {
 
           # Start the web interface with some weird settings.
           $machine->succeed("systemctl stop hydra-server hydra-evaluator hydra-queue-runner");
-          $machine->mustSucceed("su hydra -c 'NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix DBIC_TRACE=1 hydra-server -d' >&2 &");
+          $machine->mustSucceed("su - hydra -c 'NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix NIX_REMOTE= DBIC_TRACE=1 hydra-server -d' >&2 &");
           $machine->waitForOpenPort("3000");
 
           # Run the API tests.
-          $machine->mustSucceed("su hydra -c 'perl ${./tests/api-test.pl}' >&2");
+          $machine->mustSucceed("su - hydra -c 'perl ${./tests/api-test.pl}' >&2");
         '';
   });
 
   /*
   tests.s3backup = genAttrs' (system:
     with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
-    let hydra = builtins.getAttr system build; in # build."${system}"
+    let hydra = build.${system}
     simpleTest {
       machine =
         { config, pkgs, ... }:

src/script/hydra-update-gc-roots

@@ -22,20 +22,27 @@ sub addRoot {
 }
 
 
-my @columns = ( "id", "project", "jobset", "job", "system", "finished", "drvpath", "timestamp", "buildstatus" );
+my @columns =
+    ( "id", "project", "jobset", "job", "system", "finished", "drvpath", "timestamp", "buildstatus"
+    , { "outpaths" => \ "(select string_agg(path, ' ') from BuildOutputs where build = me.id)" }
+    );
+
+my %seenBuilds;
 
 sub keepBuild {
     my ($build, $keepFailedDrvs) = @_;
+    return if defined $seenBuilds{$build->id};
+    $seenBuilds{$build->id} = 1;
     print STDERR "  keeping ", ($build->finished ? "" : "scheduled "), "build ", $build->id, " (",
         $build->get_column('project'), ":", $build->get_column('jobset'), ":", $build->get_column('job'), "; ",
         $build->system, "; ",
         strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n";
     if ($build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6)) {
-        foreach my $out ($build->buildoutputs->all) {
+        foreach my $path (split / /, $build->get_column('outpaths')) {
-            if (isValidPath($out->path)) {
+            if (isValidPath($path)) {
-                addRoot $out->path;
+                addRoot $path;
             } else {
-                print STDERR "    warning: output ", $out->path, " has disappeared\n" if $build->finished;
+                print STDERR "    warning: output ", $path, " has disappeared\n" if $build->finished;
             }
         }
     }
@@ -108,10 +115,33 @@ foreach my $project ($db->resultset('Projects')->search({}, { order_by => ["name
                 , order_by => "id desc", rows => $keepnr });
         }
 
+        # Note: we also keep the derivations of failed builds so that
+        # they can be restarted.
         keepBuild($_, 1) foreach $jobset->builds->search(
             { id => { -in => $db->resultset('JobsetEvalMembers')->search({ eval => { -in => [@evals] } }, { select => "build" })->as_query }
+            , finished => 1
             },
             { order_by => ["job", "id"], columns => [ @columns ] });
+
+        print STDERR "*** looking for the most recent successful builds of current jobs in ",
+            $project->name, ":", $jobset->name, "\n";
+
+        # Keep the most recently succeeded build of a current job. Oh
+        # I really need to stop using DBIx::Class.
+        keepBuild($_, 1) foreach $jobset->builds->search(
+            { id => { -in => $jobset->builds->search(
+                { finished => 1
+                , buildstatus => [0, 6]
+                , job => { -in => $jobset->builds->search(
+                                { eval => { -in => [@evals] } },
+                                { select => "job", distinct => 1, join => "jobsetevalmembers" }
+                               )->as_query }
+                },
+                { group_by => 'job'
+                , select => [ { max => 'id', -as => 'm' } ]
+                })->as_query }
+            },
+            { columns => [ @columns ] });
     }
 }
 

tests/api-test.pl

@@ -49,7 +49,7 @@ ok(exists $jobset->{jobsetinputs}->{"my-src"}, "The new jobset has a 'my-src' in
 
 ok($jobset->{jobsetinputs}->{"my-src"}->{jobsetinputalts}->[0] eq "/run/jobset", "The 'my-src' input is in /run/jobset");
 
-system("NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix hydra-evaluator sample default");
+system("NIX_STORE_DIR=/tmp/nix/store NIX_LOG_DIR=/tmp/nix/var/log/nix NIX_STATE_DIR=/tmp/nix/var/nix NIX_REMOTE= hydra-evaluator sample default");
 $result = request_json({ uri => '/jobset/sample/default/evals' });
 ok($result->code() == 200, "Can get evals of a jobset");
 my $evals = decode_json($result->content())->{evals};