diff --git a/src/Hydra/lib/Hydra/Schema.pm b/src/Hydra/lib/Hydra/Schema.pm index e192e083..3d625445 100644 --- a/src/Hydra/lib/Hydra/Schema.pm +++ b/src/Hydra/lib/Hydra/Schema.pm @@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_classes; -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:scYtOwO4xnXnKWlNGWyjcQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildinputs.pm b/src/Hydra/lib/Hydra/Schema/Buildinputs.pm index 0c5eb4e8..d98be3ff 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildinputs.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildinputs.pm @@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); __PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Nv28F7I+7azffAhYPWYTvA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildproducts.pm b/src/Hydra/lib/Hydra/Schema/Buildproducts.pm index 9b0fa38a..1d61e983 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildproducts.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildproducts.pm @@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr"); __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fj95oLQNEptO9LgSfclVGQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm b/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm index 6c994ef9..c881f35a 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm @@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CvVVHVHL2UEZ5XMkgf01OA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Builds.pm b/src/Hydra/lib/Hydra/Schema/Builds.pm index 68eca555..c603b079 100644 --- a/src/Hydra/lib/Hydra/Schema/Builds.pm +++ b/src/Hydra/lib/Hydra/Schema/Builds.pm @@ -70,8 +70,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ty4vMjEqpl6yM81PrOtjqA __PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency'); diff --git a/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm b/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm index 3e3d2ba4..41d2cec1 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm @@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:81cloGb6HT/PTPgzB8v5Tg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildsteps.pm b/src/Hydra/lib/Hydra/Schema/Buildsteps.pm index 73eae700..f1ec4341 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildsteps.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildsteps.pm @@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cNgqL+ZLxQirdHUa/i6mww # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm b/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm index 095b39bb..4f92ebb3 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm @@ -31,8 +31,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zZEUHrrOIOIg1JH0Pgi5xg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm b/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm index 6bcfc9be..b86c284b 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm @@ -43,8 +43,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uzlil1U0VlHnTe1tvaRO4A # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsets.pm b/src/Hydra/lib/Hydra/Schema/Jobsets.pm index f158e9f2..71a199da 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsets.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsets.pm @@ -48,8 +48,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ednIpbErLs3gjqQd9w4KsA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Projects.pm b/src/Hydra/lib/Hydra/Schema/Projects.pm index 2e00b952..db8d20b4 100644 --- a/src/Hydra/lib/Hydra/Schema/Projects.pm +++ b/src/Hydra/lib/Hydra/Schema/Projects.pm @@ -30,8 +30,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OGLaq1Y/rLPlz+whXTZnzw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/programs/Scheduler.pl b/src/Hydra/programs/Scheduler.pl index 8c8746ef..f69e41df 100644 --- a/src/Hydra/programs/Scheduler.pl +++ b/src/Hydra/programs/Scheduler.pl @@ -4,6 +4,7 @@ use strict; use XML::Simple; use Hydra::Schema; use IPC::Run; +use POSIX qw(strftime); my $db = Hydra::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {}); @@ -44,17 +45,64 @@ sub fetchInput { if ($type eq "path") { my $uri = $alt->value; - print "copying input ", $input->name, " from $uri\n"; - - my $storePath = `nix-store --add "$uri"` - or die "cannot copy path $uri to the Nix store"; - chomp $storePath; - + + my $timestamp = time; + my $sha256; + my $storePath; + + # Some simple caching: don't check a path more than once every N seconds. + (my $cachedInput) = $db->resultset('Cachedpathinputs')->search( + {srcpath => $uri, lastseen => {">", $timestamp - 60}}, + {rows => 1, order_by => "lastseen DESC"}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + print "CACHED $uri $cachedInput ", $cachedInput->timestamp, " ", $cachedInput->timestamp, "\n"; + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + $timestamp = $cachedInput->timestamp; + } else { + + print "copying input ", $input->name, " from $uri\n"; + $storePath = `nix-store --add "$uri"` + or die "cannot copy path $uri to the Nix store"; + chomp $storePath; + + $sha256 = getStorePathHash $storePath; + + ($cachedInput) = $db->resultset('Cachedpathinputs')->search( + {srcpath => $uri, sha256hash => $sha256}); + + # Path inputs don't have a natural notion of a "revision", + # so we simulate it by using the timestamp that we first + # saw this path have this SHA-256 hash. So if the + # contents of the path changes, we get a new "revision", + # but if it doesn't change (or changes back), we don't get + # a new "revision". + if (!defined $cachedInput) { + $db->txn_do(sub { + $db->resultset('Cachedpathinputs')->create( + { srcpath => $uri + , timestamp => $timestamp + , lastseen => $timestamp + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } else { + $timestamp = $cachedInput->timestamp; + $db->txn_do(sub { + $cachedInput->lastseen(time); + $cachedInput->update; + }); + } + } + $$inputInfo{$input->name} = { type => $type , uri => $uri , storePath => $storePath - , sha256hash => getStorePathHash $storePath + , sha256hash => $sha256 + , revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) }; } @@ -132,8 +180,7 @@ sub checkJob { , name => $inputName , type => $input->{type} , uri => $input->{uri} - #, revision => $input->{orig}->revision - #, tag => $input->{orig}->tag + , revision => $input->{revision} , value => $input->{value} , dependency => $input->{id} , path => ($input->{storePath} or "") # !!! temporary hack @@ -181,10 +228,12 @@ sub checkJobAlternatives { foreach my $alt ($input->jobsetinputalts) { #print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n"; - fetchInput($input, $alt, $inputInfo); # !!! caching + fetchInput($input, $alt, $inputInfo); my @newArgs = @{$extraArgs}; if (defined $inputInfo->{$argName}->{storePath}) { - push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}"; + push @newArgs, "--arg", $argName, + "{path = " . $inputInfo->{$argName}->{storePath} . ";" . + " rev = \"" . $inputInfo->{$argName}->{revision} . "\";}"; } elsif (defined $inputInfo->{$argName}->{value}) { push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value}; } diff --git a/src/Hydra/root/build.tt b/src/Hydra/root/build.tt index 9c054dba..36a34df6 100644 --- a/src/Hydra/root/build.tt +++ b/src/Hydra/root/build.tt @@ -129,7 +129,7 @@ - + [% FOREACH input IN build.inputs -%] @@ -145,6 +145,7 @@ [% input.uri %] [% END %] + [% END -%] diff --git a/src/hydra.sql b/src/hydra.sql index 61174fb3..e99dd158 100644 --- a/src/hydra.sql +++ b/src/hydra.sql @@ -225,3 +225,17 @@ create table JobsetInputAlts ( primary key (project, jobset, input, altnr), foreign key (project, jobset, input) references JobsetInputs(project, jobset, name) on delete cascade -- ignored by sqlite ); + + +-- Cache for inputs of type "path" (used for testing Hydra), storing +-- the SHA-256 hash and store path for each source path. Also stores +-- the timestamp when we first saw the path have these contents, which +-- may be used to generate release names. +create table CachedPathInputs ( + srcPath text not null, + timestamp integer not null, -- when we first saw this hash + lastSeen integer not null, -- when we last saw this hash + sha256hash text not null, + storePath text not null, + primary key (srcPath, sha256hash) +);
NameTypeWhatStore path
NameTypeValueRevisionStore path
[% IF input.revision %][% input.revision %][% END %] [% input.path %]