* Caching of "path" inputs, and fake a revision number for those.

src/Hydra/lib/Hydra/Schema.pm

@@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
 __PACKAGE__->load_classes;
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:scYtOwO4xnXnKWlNGWyjcQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Buildinputs.pm

@@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
 __PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Nv28F7I+7azffAhYPWYTvA
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Buildproducts.pm

@@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
 __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fj95oLQNEptO9LgSfclVGQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm

@@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
 __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CvVVHVHL2UEZ5XMkgf01OA
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Builds.pm

@@ -70,8 +70,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ty4vMjEqpl6yM81PrOtjqA
 
 __PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency');
 

src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm

@@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id");
 __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:81cloGb6HT/PTPgzB8v5Tg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Buildsteps.pm

@@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr");
 __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cNgqL+ZLxQirdHUa/i6mww
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm

@@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zZEUHrrOIOIg1JH0Pgi5xg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm

@@ -43,8 +43,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uzlil1U0VlHnTe1tvaRO4A
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Jobsets.pm

@@ -48,8 +48,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ednIpbErLs3gjqQd9w4KsA
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/lib/Hydra/Schema/Projects.pm

@@ -30,8 +30,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OGLaq1Y/rLPlz+whXTZnzw
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/Hydra/programs/Scheduler.pl

@@ -4,6 +4,7 @@ use strict;
 use XML::Simple;
 use Hydra::Schema;
 use IPC::Run;
+use POSIX qw(strftime);
 
 
 my $db = Hydra::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
@@ -44,17 +45,64 @@ sub fetchInput {
 
     if ($type eq "path") {
         my $uri = $alt->value;
-        print "copying input ", $input->name, " from $uri\n";
+
-        
+        my $timestamp = time;
-        my $storePath = `nix-store --add "$uri"`
+        my $sha256;
-            or die "cannot copy path $uri to the Nix store";
+        my $storePath;
-        chomp $storePath;
+
-        
+        # Some simple caching: don't check a path more than once every N seconds.
+        (my $cachedInput) = $db->resultset('Cachedpathinputs')->search(
+            {srcpath => $uri, lastseen => {">", $timestamp - 60}},
+            {rows => 1, order_by => "lastseen DESC"});
+
+        if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+            print "CACHED $uri $cachedInput ", $cachedInput->timestamp, " ", $cachedInput->timestamp, "\n";
+            $storePath = $cachedInput->storepath;
+            $sha256 = $cachedInput->sha256hash;
+            $timestamp = $cachedInput->timestamp;
+        } else {
+
+            print "copying input ", $input->name, " from $uri\n";
+            $storePath = `nix-store --add "$uri"`
+                or die "cannot copy path $uri to the Nix store";
+            chomp $storePath;
+
+            $sha256 = getStorePathHash $storePath;
+
+            ($cachedInput) = $db->resultset('Cachedpathinputs')->search(
+                {srcpath => $uri, sha256hash => $sha256});
+
+            # Path inputs don't have a natural notion of a "revision",
+            # so we simulate it by using the timestamp that we first
+            # saw this path have this SHA-256 hash.  So if the
+            # contents of the path changes, we get a new "revision",
+            # but if it doesn't change (or changes back), we don't get
+            # a new "revision".
+            if (!defined $cachedInput) {
+                $db->txn_do(sub {
+                    $db->resultset('Cachedpathinputs')->create(
+                        { srcpath => $uri
+                        , timestamp => $timestamp
+                        , lastseen => $timestamp
+                        , sha256hash => $sha256
+                        , storepath => $storePath
+                        });
+                });
+            } else {
+                $timestamp = $cachedInput->timestamp;
+                $db->txn_do(sub {
+                    $cachedInput->lastseen(time);
+                    $cachedInput->update;
+                });
+            }
+        }
+
         $$inputInfo{$input->name} =
             { type => $type
             , uri => $uri
             , storePath => $storePath
-            , sha256hash => getStorePathHash $storePath
+            , sha256hash => $sha256
+            , revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
             };
     }
 
@@ -132,8 +180,7 @@ sub checkJob {
                 , name => $inputName
                 , type => $input->{type}
                 , uri => $input->{uri}
-                #, revision => $input->{orig}->revision
+                , revision => $input->{revision}
-                #, tag => $input->{orig}->tag
                 , value => $input->{value}
                 , dependency => $input->{id}
                 , path => ($input->{storePath} or "") # !!! temporary hack
@@ -181,10 +228,12 @@ sub checkJobAlternatives {
         
         foreach my $alt ($input->jobsetinputalts) {
             #print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
-            fetchInput($input, $alt, $inputInfo); # !!! caching
+            fetchInput($input, $alt, $inputInfo);
             my @newArgs = @{$extraArgs};
             if (defined $inputInfo->{$argName}->{storePath}) {
-                push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}";
+                push @newArgs, "--arg", $argName,
+                    "{path = " . $inputInfo->{$argName}->{storePath} . ";" .
+                    " rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
             } elsif (defined $inputInfo->{$argName}->{value}) {
                 push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
             }

src/Hydra/root/build.tt

@@ -129,7 +129,7 @@
 
 <table class="tablesorter">
   <thead>
-    <tr><th>Name</th><th>Type</th><th>What</th><th>Store path</th></tr>
+    <tr><th>Name</th><th>Type</th><th>Value</th><th>Revision</th><th>Store path</th></tr>
   </thead>
   <tbody>
     [% FOREACH input IN build.inputs -%]
@@ -145,6 +145,7 @@
             <tt>[% input.uri %]</tt>
           [% END %]
         </td>
+        <td>[% IF input.revision %][% input.revision %][% END %]</td>
         <td><tt>[% input.path %]</tt></td>
       </tr>
     [% END -%]

src/hydra.sql

@@ -225,3 +225,17 @@ create table JobsetInputAlts (
     primary key   (project, jobset, input, altnr),
     foreign key   (project, jobset, input) references JobsetInputs(project, jobset, name) on delete cascade -- ignored by sqlite
 );
+
+
+-- Cache for inputs of type "path" (used for testing Hydra), storing
+-- the SHA-256 hash and store path for each source path.  Also stores
+-- the timestamp when we first saw the path have these contents, which
+-- may be used to generate release names.
+create table CachedPathInputs (
+    srcPath       text not null,
+    timestamp     integer not null, -- when we first saw this hash
+    lastSeen      integer not null, -- when we last saw this hash
+    sha256hash    text not null,
+    storePath     text not null,
+    primary key   (srcPath, sha256hash)
+);