* Caching of "path" inputs, and fake a revision number for those.
This commit is contained in:
parent
a8f748e547
commit
8fbb79efe6
|
@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
|
|||
__PACKAGE__->load_classes;
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:scYtOwO4xnXnKWlNGWyjcQ
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
|
|||
__PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Nv28F7I+7azffAhYPWYTvA
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
|
|||
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fj95oLQNEptO9LgSfclVGQ
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
|
|||
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CvVVHVHL2UEZ5XMkgf01OA
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -70,8 +70,8 @@ __PACKAGE__->has_many(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ty4vMjEqpl6yM81PrOtjqA
|
||||
|
||||
__PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency');
|
||||
|
||||
|
|
|
@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id");
|
|||
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:81cloGb6HT/PTPgzB8v5Tg
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr");
|
|||
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cNgqL+ZLxQirdHUa/i6mww
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zZEUHrrOIOIg1JH0Pgi5xg
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -43,8 +43,8 @@ __PACKAGE__->has_many(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uzlil1U0VlHnTe1tvaRO4A
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -48,8 +48,8 @@ __PACKAGE__->has_many(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ednIpbErLs3gjqQd9w4KsA
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -30,8 +30,8 @@ __PACKAGE__->has_many(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw
|
||||
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OGLaq1Y/rLPlz+whXTZnzw
|
||||
|
||||
|
||||
# You can replace this text with custom content, and it will be preserved on regeneration
|
||||
|
|
|
@ -4,6 +4,7 @@ use strict;
|
|||
use XML::Simple;
|
||||
use Hydra::Schema;
|
||||
use IPC::Run;
|
||||
use POSIX qw(strftime);
|
||||
|
||||
|
||||
my $db = Hydra::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
|
||||
|
@ -44,17 +45,64 @@ sub fetchInput {
|
|||
|
||||
if ($type eq "path") {
|
||||
my $uri = $alt->value;
|
||||
print "copying input ", $input->name, " from $uri\n";
|
||||
|
||||
my $storePath = `nix-store --add "$uri"`
|
||||
or die "cannot copy path $uri to the Nix store";
|
||||
chomp $storePath;
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
# Some simple caching: don't check a path more than once every N seconds.
|
||||
(my $cachedInput) = $db->resultset('Cachedpathinputs')->search(
|
||||
{srcpath => $uri, lastseen => {">", $timestamp - 60}},
|
||||
{rows => 1, order_by => "lastseen DESC"});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
print "CACHED $uri $cachedInput ", $cachedInput->timestamp, " ", $cachedInput->timestamp, "\n";
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
} else {
|
||||
|
||||
print "copying input ", $input->name, " from $uri\n";
|
||||
$storePath = `nix-store --add "$uri"`
|
||||
or die "cannot copy path $uri to the Nix store";
|
||||
chomp $storePath;
|
||||
|
||||
$sha256 = getStorePathHash $storePath;
|
||||
|
||||
($cachedInput) = $db->resultset('Cachedpathinputs')->search(
|
||||
{srcpath => $uri, sha256hash => $sha256});
|
||||
|
||||
# Path inputs don't have a natural notion of a "revision",
|
||||
# so we simulate it by using the timestamp that we first
|
||||
# saw this path have this SHA-256 hash. So if the
|
||||
# contents of the path changes, we get a new "revision",
|
||||
# but if it doesn't change (or changes back), we don't get
|
||||
# a new "revision".
|
||||
if (!defined $cachedInput) {
|
||||
$db->txn_do(sub {
|
||||
$db->resultset('Cachedpathinputs')->create(
|
||||
{ srcpath => $uri
|
||||
, timestamp => $timestamp
|
||||
, lastseen => $timestamp
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
} else {
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
$db->txn_do(sub {
|
||||
$cachedInput->lastseen(time);
|
||||
$cachedInput->update;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
$$inputInfo{$input->name} =
|
||||
{ type => $type
|
||||
, uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => getStorePathHash $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -132,8 +180,7 @@ sub checkJob {
|
|||
, name => $inputName
|
||||
, type => $input->{type}
|
||||
, uri => $input->{uri}
|
||||
#, revision => $input->{orig}->revision
|
||||
#, tag => $input->{orig}->tag
|
||||
, revision => $input->{revision}
|
||||
, value => $input->{value}
|
||||
, dependency => $input->{id}
|
||||
, path => ($input->{storePath} or "") # !!! temporary hack
|
||||
|
@ -181,10 +228,12 @@ sub checkJobAlternatives {
|
|||
|
||||
foreach my $alt ($input->jobsetinputalts) {
|
||||
#print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
|
||||
fetchInput($input, $alt, $inputInfo); # !!! caching
|
||||
fetchInput($input, $alt, $inputInfo);
|
||||
my @newArgs = @{$extraArgs};
|
||||
if (defined $inputInfo->{$argName}->{storePath}) {
|
||||
push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}";
|
||||
push @newArgs, "--arg", $argName,
|
||||
"{path = " . $inputInfo->{$argName}->{storePath} . ";" .
|
||||
" rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
|
||||
} elsif (defined $inputInfo->{$argName}->{value}) {
|
||||
push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
|
||||
}
|
||||
|
|
|
@ -129,7 +129,7 @@
|
|||
|
||||
<table class="tablesorter">
|
||||
<thead>
|
||||
<tr><th>Name</th><th>Type</th><th>What</th><th>Store path</th></tr>
|
||||
<tr><th>Name</th><th>Type</th><th>Value</th><th>Revision</th><th>Store path</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
[% FOREACH input IN build.inputs -%]
|
||||
|
@ -145,6 +145,7 @@
|
|||
<tt>[% input.uri %]</tt>
|
||||
[% END %]
|
||||
</td>
|
||||
<td>[% IF input.revision %][% input.revision %][% END %]</td>
|
||||
<td><tt>[% input.path %]</tt></td>
|
||||
</tr>
|
||||
[% END -%]
|
||||
|
|
|
@ -225,3 +225,17 @@ create table JobsetInputAlts (
|
|||
primary key (project, jobset, input, altnr),
|
||||
foreign key (project, jobset, input) references JobsetInputs(project, jobset, name) on delete cascade -- ignored by sqlite
|
||||
);
|
||||
|
||||
|
||||
-- Cache for inputs of type "path" (used for testing Hydra), storing
|
||||
-- the SHA-256 hash and store path for each source path. Also stores
|
||||
-- the timestamp when we first saw the path have these contents, which
|
||||
-- may be used to generate release names.
|
||||
create table CachedPathInputs (
|
||||
srcPath text not null,
|
||||
timestamp integer not null, -- when we first saw this hash
|
||||
lastSeen integer not null, -- when we last saw this hash
|
||||
sha256hash text not null,
|
||||
storePath text not null,
|
||||
primary key (srcPath, sha256hash)
|
||||
);
|
||||
|
|
Loading…
Reference in a new issue