* Store jobset evaluations in the database explicitly. This includes

recording the builds that are part of a jobset evaluation.  We need
  this to be able to answer queries such as "return the latest NixOS
  ISO for which the installation test succeeded".  This wasn't previously
  possible because the database didn't record which builds of (say)
  the `isoMinimal' job and the `tests.installer.simple' job came from
  the same evaluation of the nixos:trunk jobset.

  Keeping a record of evaluations is also useful for logging purposes.

doc/dev-notes.txt

@@ -46,6 +46,10 @@
 
   update buildschedulinginfo set priority = 200 where id = <ID>;
 
+* Changing the priority of all builds for a jobset:
+
+  update buildschedulinginfo set priority = 20 where id in (select id from builds where finished = 0 and project = 'nixpkgs' and jobset = 'trunk');
+  
 
 * Steps to install:
 
@@ -103,6 +107,10 @@
   alter table Builds add column nixExprInput text;
   alter table Builds add column nixExprPath text;
 
+  # Adding JobsetEvals.
+  drop table JobsetInputHashes;
+  (add JobsetEvals, JobsetEvalMembers)
+
   
 * Job selection:
   
@@ -138,7 +146,7 @@
 
 * Installing deps.nix in a profile for testing:
 
-  $ nix-env -p /nix/var/nix/profiles/per-user/eelco/hydra-deps -f deps.nix -i \* --arg pkgs 'import /home/eelco/Dev/nixpkgs {}'
+  $ nix-env -p $NIX_USER_PROFILE_DIR/hydra-deps -f deps.nix -i \* --arg pkgs 'import /etc/nixos/nixpkgs {}'
   
 
 * select x.project, x.jobset, x.job, x.system, x.id, x.timestamp, r.buildstatus, b.id, b.timestamp
@@ -154,3 +162,8 @@
 * Using PostgreSQL:
 
   $ HYDRA_DBI="dbi:Pg:dbname=hydra;" hydra_server.pl
+
+
+* Find the builds with the highest number of build steps:
+
+  select id, (select count(*) from buildsteps where build = x.id) as n from builds x order by n desc;

src/lib/Hydra/Helper/AddBuilds.pm

@@ -74,7 +74,7 @@ sub fetchInputPath {
 
         # Some simple caching: don't check a path more than once every N seconds.
         (my $cachedInput) = $db->resultset('CachedPathInputs')->search(
-            {srcpath => $uri, lastseen => {">", $timestamp - 60}},
+            {srcpath => $uri, lastseen => {">", $timestamp - 30}},
             {rows => 1, order_by => "lastseen DESC"});
 
         if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
@@ -505,7 +505,7 @@ sub checkBuild {
         my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1});
         if (scalar(@previousBuilds) > 0) {
             print STDERR "already scheduled/built\n";
-            $currentBuilds->{$_->id} = 1 foreach @previousBuilds;
+            $currentBuilds->{$_->id} = 0 foreach @previousBuilds;
             return;
         }
         

src/lib/Hydra/Schema/BuildSchedulingInfo.pm

@@ -44,7 +44,7 @@ __PACKAGE__->table("BuildSchedulingInfo");
 =head2 locker
 
   data_type: text
-  default_value: (empty string)
+  default_value: ''
   is_nullable: 0
   size: undef
 
@@ -85,7 +85,7 @@ __PACKAGE__->add_columns(
   "busy",
   { data_type => "integer", default_value => 0, is_nullable => 0, size => undef },
   "locker",
-  { data_type => "text", default_value => "", is_nullable => 0, size => undef },
+  { data_type => "text", default_value => "''", is_nullable => 0, size => undef },
   "logfile",
   {
     data_type => "text",
@@ -118,8 +118,8 @@ Related object: L<Hydra::Schema::Builds>
 __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }, {});
 
 
-# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yEhHeANRynKf72dp5URvZA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qOU/YGv3fgPynBXovV6gfg
 
 # You can replace this text with custom content, and it will be preserved on regeneration
 1;

src/lib/Hydra/Schema/Builds.pm

@@ -420,9 +420,23 @@ __PACKAGE__->has_many(
   { "foreign.build" => "self.id" },
 );
 
+=head2 jobsetevalmembers
 
-# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 11:19:24
+Type: has_many
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oCkX9bughWPZg6JKaOxDJA
+
+Related object: L<Hydra::Schema::JobsetEvalMembers>
+
+=cut
+
+__PACKAGE__->has_many(
+  "jobsetevalmembers",
+  "Hydra::Schema::JobsetEvalMembers",
+  { "foreign.build" => "self.id" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sE2/zTcfETC8Eahh6NQDZA
 
 use Hydra::Helper::Nix;
 

src/lib/Hydra/Schema/JobsetEvalMembers.pm

@@ -0,0 +1,102 @@
+package Hydra::Schema::JobsetEvalMembers;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+
+=head1 NAME
+
+Hydra::Schema::JobsetEvalMembers
+
+=cut
+
+__PACKAGE__->table("JobsetEvalMembers");
+
+=head1 ACCESSORS
+
+=head2 eval
+
+  data_type: integer
+  default_value: undef
+  is_foreign_key: 1
+  is_nullable: 0
+  size: undef
+
+=head2 build
+
+  data_type: integer
+  default_value: undef
+  is_foreign_key: 1
+  is_nullable: 0
+  size: undef
+
+=head2 isnew
+
+  data_type: integer
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=cut
+
+__PACKAGE__->add_columns(
+  "eval",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_foreign_key => 1,
+    is_nullable => 0,
+    size => undef,
+  },
+  "build",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_foreign_key => 1,
+    is_nullable => 0,
+    size => undef,
+  },
+  "isnew",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+);
+__PACKAGE__->set_primary_key("eval", "build");
+
+=head1 RELATIONS
+
+=head2 eval
+
+Type: belongs_to
+
+Related object: L<Hydra::Schema::JobsetEvals>
+
+=cut
+
+__PACKAGE__->belongs_to("eval", "Hydra::Schema::JobsetEvals", { id => "eval" }, {});
+
+=head2 build
+
+Type: belongs_to
+
+Related object: L<Hydra::Schema::Builds>
+
+=cut
+
+__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {});
+
+
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vwefi8q3HolhFCkB9aEVWw
+
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/lib/Hydra/Schema/JobsetEvals.pm

@@ -0,0 +1,200 @@
+package Hydra::Schema::JobsetEvals;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+
+=head1 NAME
+
+Hydra::Schema::JobsetEvals
+
+=cut
+
+__PACKAGE__->table("JobsetEvals");
+
+=head1 ACCESSORS
+
+=head2 id
+
+  data_type: integer
+  default_value: undef
+  is_auto_increment: 1
+  is_nullable: 0
+  size: undef
+
+=head2 project
+
+  data_type: text
+  default_value: undef
+  is_foreign_key: 1
+  is_nullable: 0
+  size: undef
+
+=head2 jobset
+
+  data_type: text
+  default_value: undef
+  is_foreign_key: 1
+  is_nullable: 0
+  size: undef
+
+=head2 timestamp
+
+  data_type: integer
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 checkouttime
+
+  data_type: integer
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 evaltime
+
+  data_type: integer
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 hasnewbuilds
+
+  data_type: integer
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 hash
+
+  data_type: text
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=cut
+
+__PACKAGE__->add_columns(
+  "id",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_auto_increment => 1,
+    is_nullable => 0,
+    size => undef,
+  },
+  "project",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_foreign_key => 1,
+    is_nullable => 0,
+    size => undef,
+  },
+  "jobset",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_foreign_key => 1,
+    is_nullable => 0,
+    size => undef,
+  },
+  "timestamp",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "checkouttime",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "evaltime",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "hasnewbuilds",
+  {
+    data_type => "integer",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "hash",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+);
+__PACKAGE__->set_primary_key("id");
+
+=head1 RELATIONS
+
+=head2 project
+
+Type: belongs_to
+
+Related object: L<Hydra::Schema::Projects>
+
+=cut
+
+__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" }, {});
+
+=head2 jobset
+
+Type: belongs_to
+
+Related object: L<Hydra::Schema::Jobsets>
+
+=cut
+
+__PACKAGE__->belongs_to(
+  "jobset",
+  "Hydra::Schema::Jobsets",
+  { name => "jobset", project => "project" },
+  {},
+);
+
+=head2 jobsetevalmembers
+
+Type: has_many
+
+Related object: L<Hydra::Schema::JobsetEvalMembers>
+
+=cut
+
+__PACKAGE__->has_many(
+  "jobsetevalmembers",
+  "Hydra::Schema::JobsetEvalMembers",
+  { "foreign.eval" => "self.id" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:33:51
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QD7ZMOLp9HpK0mAYkk0d/Q
+
+use Hydra::Helper::Nix;
+
+# !!! Ugly, should be generated.
+my $hydradbi = getHydraDBPath;
+if ($hydradbi =~ m/^dbi:Pg/) {
+    __PACKAGE__->sequence('jobsetevals_id_seq');
+}
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/lib/Hydra/Schema/JobsetInputHashes.pm

@@ -1,119 +0,0 @@
-package Hydra::Schema::JobsetInputHashes;
-
-# Created by DBIx::Class::Schema::Loader
-# DO NOT MODIFY THE FIRST PART OF THIS FILE
-
-use strict;
-use warnings;
-
-use base 'DBIx::Class::Core';
-
-
-=head1 NAME
-
-Hydra::Schema::JobsetInputHashes
-
-=cut
-
-__PACKAGE__->table("JobsetInputHashes");
-
-=head1 ACCESSORS
-
-=head2 project
-
-  data_type: text
-  default_value: undef
-  is_foreign_key: 1
-  is_nullable: 0
-  size: undef
-
-=head2 jobset
-
-  data_type: text
-  default_value: undef
-  is_foreign_key: 1
-  is_nullable: 0
-  size: undef
-
-=head2 hash
-
-  data_type: text
-  default_value: undef
-  is_nullable: 0
-  size: undef
-
-=head2 timestamp
-
-  data_type: integer
-  default_value: undef
-  is_nullable: 0
-  size: undef
-
-=cut
-
-__PACKAGE__->add_columns(
-  "project",
-  {
-    data_type => "text",
-    default_value => undef,
-    is_foreign_key => 1,
-    is_nullable => 0,
-    size => undef,
-  },
-  "jobset",
-  {
-    data_type => "text",
-    default_value => undef,
-    is_foreign_key => 1,
-    is_nullable => 0,
-    size => undef,
-  },
-  "hash",
-  {
-    data_type => "text",
-    default_value => undef,
-    is_nullable => 0,
-    size => undef,
-  },
-  "timestamp",
-  {
-    data_type => "integer",
-    default_value => undef,
-    is_nullable => 0,
-    size => undef,
-  },
-);
-__PACKAGE__->set_primary_key("project", "jobset", "hash");
-
-=head1 RELATIONS
-
-=head2 project
-
-Type: belongs_to
-
-Related object: L<Hydra::Schema::Projects>
-
-=cut
-
-__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" }, {});
-
-=head2 jobset
-
-Type: belongs_to
-
-Related object: L<Hydra::Schema::Jobsets>
-
-=cut
-
-__PACKAGE__->belongs_to(
-  "jobset",
-  "Hydra::Schema::Jobsets",
-  { name => "jobset", project => "project" },
-  {},
-);
-
-
-# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dK9vFHXInejDW/rl1i/kFA
-
-1;

src/lib/Hydra/Schema/Jobsets.pm

@@ -253,17 +253,17 @@ __PACKAGE__->has_many(
   },
 );
 
-=head2 jobsetinputhashes
+=head2 jobsetevals
 
 Type: has_many
 
-Related object: L<Hydra::Schema::JobsetInputHashes>
+Related object: L<Hydra::Schema::JobsetEvals>
 
 =cut
 
 __PACKAGE__->has_many(
-  "jobsetinputhashes",
+  "jobsetevals",
-  "Hydra::Schema::JobsetInputHashes",
+  "Hydra::Schema::JobsetEvals",
   {
     "foreign.jobset"  => "self.name",
     "foreign.project" => "self.project",
@@ -271,7 +271,7 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ORCZ73BJrscvmyf/4ds0UQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Z0HutYxnzYVuQc3W51mq5Q
 
 1;

src/lib/Hydra/Schema/Projects.pm

@@ -216,22 +216,22 @@ __PACKAGE__->has_many(
   { "foreign.project" => "self.name" },
 );
 
-=head2 jobsetinputhashes
+=head2 jobsetevals
 
 Type: has_many
 
-Related object: L<Hydra::Schema::JobsetInputHashes>
+Related object: L<Hydra::Schema::JobsetEvals>
 
 =cut
 
 __PACKAGE__->has_many(
-  "jobsetinputhashes",
+  "jobsetevals",
-  "Hydra::Schema::JobsetInputHashes",
+  "Hydra::Schema::JobsetEvals",
   { "foreign.project" => "self.name" },
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:45
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yH/9hz6FH09kgusRNWrqPg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SXJ+FzgNDad87OKSBH2qrg
 
 1;

src/script/hydra_scheduler.pl

@@ -20,6 +20,7 @@ STDOUT->autoflush();
 my $db = openHydraDB;
 my %config = new Config::General($ENV{"HYDRA_CONFIG"})->getall;
 
+
 sub fetchInputs {
     my ($project, $jobset, $inputInfo) = @_;
     foreach my $input ($jobset->jobsetinputs->all) {
@@ -99,7 +100,9 @@ sub checkJobset {
     my $inputInfo = {};
     
     # Fetch all values for all inputs.
+    my $checkoutStart = time;
     fetchInputs($project, $jobset, $inputInfo);
+    my $checkoutStop = time;
 
     # Hash the arguments to hydra_eval_jobs and check the
     # JobsetInputHashes to see if we've already evaluated this set of
@@ -107,7 +110,7 @@ sub checkJobset {
     my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo));
     my $argsHash = sha256_hex("@args");
 
-    if ($jobset->jobsetinputhashes->find({hash => $argsHash})) {
+    if ($jobset->jobsetevals->find({hash => $argsHash})) {
         print "  already evaluated, skipping\n";
         txn_do($db, sub {
             $jobset->update({lastcheckedtime => time});
@@ -116,18 +119,20 @@ sub checkJobset {
     }
     
     # Evaluate the job expression.
+    my $evalStart = time;
     my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath);
-
+    my $evalStop = time;
-    # Schedule each successfully evaluated job.
-    my %currentBuilds;
-    foreach my $job (permute @{$jobs->{job}}) {
-        next if $job->{jobName} eq "";
-        print "considering job " . $job->{jobName} . "\n";
-        checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds);
-    }
 
     txn_do($db, sub {
         
+        # Schedule each successfully evaluated job.
+        my %currentBuilds;
+        foreach my $job (permute @{$jobs->{job}}) {
+            next if $job->{jobName} eq "";
+            print "considering job " . $job->{jobName} . "\n";
+            checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds);
+        }
+
         # Update the last checked times and error messages for each
         # job.
         my %failedJobNames;
@@ -149,8 +154,24 @@ sub checkJobset {
             $build->update({iscurrent => 0}) unless $currentBuilds{$build->id};
         }
 
-        $jobset->jobsetinputhashes->create({hash => $argsHash, timestamp => time});
+        my $hasNewBuilds = 0;
+        while (my ($id, $new) = each %currentBuilds) {
+            $hasNewBuilds = 1 if $new;
+        }
 
+        my $ev = $jobset->jobsetevals->create(
+            { hash => $argsHash
+            , timestamp => time
+            , checkouttime => abs($checkoutStop - $checkoutStart)
+            , evaltime => abs($evalStop - $evalStart)
+            , hasnewbuilds => $hasNewBuilds
+            });
+
+        if ($hasNewBuilds) {
+            while (my ($id, $new) = each %currentBuilds) {
+                $ev->jobsetevalmembers->create({ build => $id, isnew => $new });
+            }
+        }
     });
        
     # Store the errors messages for jobs that failed to evaluate.

src/sql/hydra.sql

@@ -403,24 +403,47 @@ create table ReleaseMembers (
 );
 
 
--- This table is used to prevent repeated Nix expression evaluation
+create table JobsetEvals (
--- for the same set of inputs for a jobset.  In the scheduler, after
+#ifdef POSTGRESQL
--- obtaining the current inputs for a jobset, we hash the inputs
+    id            serial primary key not null,
--- together, and if the resulting hash already appears in this table,
+#else
--- we can skip the jobset.  Otherwise it's added to the table, and the
+    id            integer primary key autoincrement not null,
--- Nix expression for the jobset is evaluated.  The hash is computed
+#endif
--- over the command-line arguments to hydra_eval_jobs.
+
-create table JobsetInputHashes (
     project       text not null,
     jobset        text not null,
+    
+    timestamp     integer not null, -- when this entry was added
+    checkoutTime  integer not null, -- how long obtaining the inputs took (in seconds)
+    evalTime      integer not null, -- how long evaluation took (in seconds)
+
+    -- If 0, then the evaluation of this jobset did not cause any new
+    -- builds to be added to the database.  Otherwise, *all* the
+    -- builds resulting from the evaluation of the jobset (including
+    -- existing ones) can be found in the JobsetEvalMembers table.
+    hasNewBuilds  integer not null,
+
+    -- Used to prevent repeated Nix expression evaluation for the same
+    -- set of inputs for a jobset.  In the scheduler, after obtaining
+    -- the current inputs for a jobset, we hash the inputs together,
+    -- and if the resulting hash already appears in this table, we can
+    -- skip the jobset.  Otherwise we proceed.  The hash is computed
+    -- over the command-line arguments to hydra_eval_jobs.
     hash          text not null,
-    timestamp     integer not null,
+
-    primary key   (project, jobset, hash),
     foreign key   (project) references Projects(name) on delete cascade on update cascade,
     foreign key   (project, jobset) references Jobsets(project, name) on delete cascade on update cascade
 );
 
 
+create table JobsetEvalMembers (
+    eval          integer not null references JobsetEvals(id) on delete cascade,
+    build         integer not null references Builds(id) on delete cascade,
+    isNew         integer not null,
+    primary key   (eval, build)
+);
+
+
 create table UriRevMapper (
     baseuri       text not null,
     uri           text not null,