snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

* Caching of "path" inputs, and fake a revision number for those.

Browse source
This commit is contained in:
Eelco Dolstra 2008-11-25 14:59:08 +00:00
parent a8f748e547
commit 8fbb79efe6
14 changed files with 98 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Hydra/lib/Hydra/Schema.pm
View file

@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:scYtOwO4xnXnKWlNGWyjcQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Buildinputs.pm
View file

@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
__PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Nv28F7I+7azffAhYPWYTvA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Buildproducts.pm
View file

@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fj95oLQNEptO9LgSfclVGQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm
View file

@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CvVVHVHL2UEZ5XMkgf01OA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Builds.pm
View file

@ -70,8 +70,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ty4vMjEqpl6yM81PrOtjqA
__PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency');

4
src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm
View file

@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:81cloGb6HT/PTPgzB8v5Tg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Buildsteps.pm
View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cNgqL+ZLxQirdHUa/i6mww
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm
View file

@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zZEUHrrOIOIg1JH0Pgi5xg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm
View file

@ -43,8 +43,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uzlil1U0VlHnTe1tvaRO4A
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Jobsets.pm
View file

@ -48,8 +48,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ednIpbErLs3gjqQd9w4KsA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/Hydra/lib/Hydra/Schema/Projects.pm
View file

@ -30,8 +30,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OGLaq1Y/rLPlz+whXTZnzw
# You can replace this text with custom content, and it will be preserved on regeneration

71
src/Hydra/programs/Scheduler.pl
View file

@ -4,6 +4,7 @@ use strict;
use XML::Simple;
use Hydra::Schema;
use IPC::Run;
use POSIX qw(strftime);
my $db = Hydra::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
@ -44,17 +45,64 @@ sub fetchInput {
if ($type eq "path") {
my $uri = $alt->value;
print "copying input ", $input->name, " from $uri\n";
my $storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store";
chomp $storePath;
my $timestamp = time;
my $sha256;
my $storePath;
# Some simple caching: don't check a path more than once every N seconds.
(my $cachedInput) = $db->resultset('Cachedpathinputs')->search(
{srcpath => $uri, lastseen => {">", $timestamp - 60}},
{rows => 1, order_by => "lastseen DESC"});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
print "CACHED $uri $cachedInput ", $cachedInput->timestamp, " ", $cachedInput->timestamp, "\n";
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
} else {
print "copying input ", $input->name, " from $uri\n";
$storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store";
chomp $storePath;
$sha256 = getStorePathHash $storePath;
($cachedInput) = $db->resultset('Cachedpathinputs')->search(
{srcpath => $uri, sha256hash => $sha256});
# Path inputs don't have a natural notion of a "revision",
# so we simulate it by using the timestamp that we first
# saw this path have this SHA-256 hash. So if the
# contents of the path changes, we get a new "revision",
# but if it doesn't change (or changes back), we don't get
# a new "revision".
if (!defined $cachedInput) {
$db->txn_do(sub {
$db->resultset('Cachedpathinputs')->create(
{ srcpath => $uri
, timestamp => $timestamp
, lastseen => $timestamp
, sha256hash => $sha256
, storepath => $storePath
});
});
} else {
$timestamp = $cachedInput->timestamp;
$db->txn_do(sub {
$cachedInput->lastseen(time);
$cachedInput->update;
});
}
}
$$inputInfo{$input->name} =
{ type => $type
, uri => $uri
, storePath => $storePath
, sha256hash => getStorePathHash $storePath
, sha256hash => $sha256
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
};
}
@ -132,8 +180,7 @@ sub checkJob {
, name => $inputName
, type => $input->{type}
, uri => $input->{uri}
#, revision => $input->{orig}->revision
#, tag => $input->{orig}->tag
, revision => $input->{revision}
, value => $input->{value}
, dependency => $input->{id}
, path => ($input->{storePath} or "") # !!! temporary hack
@ -181,10 +228,12 @@ sub checkJobAlternatives {
foreach my $alt ($input->jobsetinputalts) {
#print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo); # !!! caching
fetchInput($input, $alt, $inputInfo);
my @newArgs = @{$extraArgs};
if (defined $inputInfo->{$argName}->{storePath}) {
push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}";
push @newArgs, "--arg", $argName,
"{path = " . $inputInfo->{$argName}->{storePath} . ";" .
" rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
} elsif (defined $inputInfo->{$argName}->{value}) {
push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
}

3
src/Hydra/root/build.tt
View file

@ -129,7 +129,7 @@
<table class="tablesorter">
<thead>
<tr><th>Name</th><th>Type</th><th>What</th><th>Store path</th></tr>
<tr><th>Name</th><th>Type</th><th>Value</th><th>Revision</th><th>Store path</th></tr>
</thead>
<tbody>
[% FOREACH input IN build.inputs -%]
@ -145,6 +145,7 @@
<tt>[% input.uri %]</tt>
[% END %]
</td>
<td>[% IF input.revision %][% input.revision %][% END %]</td>
<td><tt>[% input.path %]</tt></td>
</tr>
[% END -%]

14
src/hydra.sql
View file

@ -225,3 +225,17 @@ create table JobsetInputAlts (
primary key (project, jobset, input, altnr),
foreign key (project, jobset, input) references JobsetInputs(project, jobset, name) on delete cascade -- ignored by sqlite
);
-- Cache for inputs of type "path" (used for testing Hydra), storing
-- the SHA-256 hash and store path for each source path. Also stores
-- the timestamp when we first saw the path have these contents, which
-- may be used to generate release names.
create table CachedPathInputs (
srcPath text not null,
timestamp integer not null, -- when we first saw this hash
lastSeen integer not null, -- when we last saw this hash
sha256hash text not null,
storePath text not null,
primary key (srcPath, sha256hash)
);