lix-project/hydra
3
0
Fork 6
Code Issues 2 Pull requests 1 Packages Projects Releases Wiki Activity

add support for git as jobinput

Browse source
This commit is contained in:
Rob Vermaas 2009-11-17 15:16:41 +00:00
parent 7eda090e74
commit 2fb05b34bf
26 changed files with 159 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
release.nix
View file

@ -88,6 +88,7 @@ let
mv $out/libexec/hydra/script $out/bin
cp ${"${nixpkgs}/pkgs/build-support/fetchsvn/nix-prefetch-svn"} $out/bin/nix-prefetch-svn
cp ${"${nixpkgs}/pkgs/build-support/fetchgit/nix-prefetch-git"} $out/bin/nix-prefetch-git
make -C src/c NIX=${nix} ATERM=${aterm242fixes}
cp src/c/hydra_eval_jobs $out/bin

2
src/lib/Hydra/Controller/Jobset.pm
View file

@ -120,7 +120,7 @@ sub checkInput {
error($c, "Invalid input type: $inputType") unless
$inputType eq "svn" || $inputType eq "cvs" || $inputType eq "tarball" ||
$inputType eq "string" || $inputType eq "path" || $inputType eq "boolean" ||
$inputType eq "build";
$inputType eq "git" || $inputType eq "build";
return ($inputName, $inputType);
}

94
src/lib/Hydra/Helper/AddBuilds.pm
View file

@ -63,11 +63,9 @@ sub attrsToSQL {
return $query;
}
sub fetchInput {
sub fetchInputPath {
my ($db, $project, $jobset, $name, $type, $value) = @_;
if ($type eq "path") {
my $uri = $value;
my $timestamp = time;
@ -128,7 +126,9 @@ sub fetchInput {
};
}
elsif ($type eq "svn") {
sub fetchInputSVN {
my ($db, $project, $jobset, $name, $type, $value) = @_;
my $uri = $value;
my $sha256;
@ -180,7 +180,9 @@ sub fetchInput {
};
}
elsif ($type eq "build") {
sub fetchInputBuild {
my ($db, $project, $jobset, $name, $type, $value) = @_;
my ($projectName, $jobsetName, $jobName, $attrs) = parseJobName($value);
$projectName ||= $project->name;
$jobsetName ||= $jobset->name;
@ -213,6 +215,86 @@ sub fetchInput {
};
}
sub fetchInputGit {
my ($db, $project, $jobset, $name, $type, $value) = @_;
my $uri = $value;
my $timestamp = time;
my $sha256;
my $storePath;
# Some simple caching: don't check a path more than once every N seconds.
(my $cachedInput) = $db->resultset('CachedGitInputs')->search(
{uri => $uri, lastseen => {">", $timestamp - 3600}},
{rows => 1, order_by => "lastseen DESC"});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
} else {
# Then download this revision into the store.
print STDERR "checking out Git input from $uri";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
my $stdout; my $stderr;
(my $res, $stdout, $stderr) = captureStdoutStderr(
"nix-prefetch-git", $uri);
die "Cannot check out Git repository `$uri':\n$stderr" unless $res;
($sha256, $storePath) = split ' ', $stdout;
($cachedInput) = $db->resultset('CachedGitInputs')->search(
{uri => $uri, sha256hash => $sha256});
if (!defined $cachedInput) {
txn_do($db, sub {
$db->resultset('CachedGitInputs')->create(
{ uri => $uri
, timestamp => $timestamp
, lastseen => $timestamp
, sha256hash => $sha256
, storepath => $storePath
});
});
} else {
$timestamp = $cachedInput->timestamp;
txn_do($db, sub {
$cachedInput->update({lastseen => time});
});
}
}
return
{ type => $type
, uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
};
}
sub fetchInputCVS {
my ($db, $project, $jobset, $name, $type, $value) = @_;
}
sub fetchInput {
my ($db, $project, $jobset, $name, $type, $value) = @_;
if ($type eq "path") {
return fetchInputPath($db, $project, $jobset, $name, $type, $value);
}
elsif ($type eq "svn") {
return fetchInputSVN($db, $project, $jobset, $name, $type, $value);
}
elsif ($type eq "build") {
return fetchInputBuild($db, $project, $jobset, $name, $type, $value);
}
elsif ($type eq "git") {
return fetchInputGit($db, $project, $jobset, $name, $type, $value);
}
elsif ($type eq "string") {
die unless defined $value;
return {type => $type, value => $value};
@ -242,7 +324,7 @@ sub inputsToArgs {
when ("boolean") {
push @res, "--arg", $input, $alt->{value};
}
when (["svn", "path", "build"]) {
when (["svn", "path", "build", "git", "cvs"]) {
push @res, "--arg", $input, (
"{ outPath = builtins.storePath " . $alt->{storePath} . "" .
(defined $alt->{revision} ? "; rev = \"" . $alt->{revision} . "\"" : "") .

4
src/lib/Hydra/Schema.pm
View file

@ -11,8 +11,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:s5XNrT7kEVqbLED5g/J/SA
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y97NEH6tnsRjsy0/nVgMjQ
# You can replace this text with custom content, and it will be preserved on regeneration

6
src/lib/Hydra/Schema/BuildInputs.pm
View file

@ -50,7 +50,7 @@ __PACKAGE__->add_columns(
},
"revision",
{
data_type => "integer",
data_type => "text",
default_value => undef,
is_nullable => 1,
size => undef,
@ -107,8 +107,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RTeBJCvspGfTdHUR/TxFsg
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:skn8HFEXTvEiL4/6Q+ulvw
use Hydra::Helper::Nix;

4
src/lib/Hydra/Schema/BuildProducts.pm
View file

@ -94,8 +94,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8FZKxfabZQfchVpHfvnVHA
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6wJ0KJAILATio3ELRefU5Q
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildResultInfo.pm
View file

@ -89,8 +89,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:496WClgIZB6hXV0zCsBjbQ
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:K+kKzTgAlKSXevJWtLIwGA
__PACKAGE__->belongs_to(
"failedDep",

4
src/lib/Hydra/Schema/BuildSchedulingInfo.pm
View file

@ -46,8 +46,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:giLh18QewdxF8Zi/5juWqA
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eiaLNgxvTjSZGC4pMDtNWA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildSteps.pm
View file

@ -94,7 +94,7 @@ __PACKAGE__->set_primary_key("build", "stepnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5tx+fkjjKYUNYBYQS+kSOw
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5iUdHsNyEoM3D7i+tSteBQ
1;

4
src/lib/Hydra/Schema/Builds.pm
View file

@ -186,8 +186,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-26 14:22:27
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qbNPA5NkENlZXILuE3OGkA
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Un0iCqVS8PTpSdJiTjRXeA
use Hydra::Helper::Nix;

4
src/lib/Hydra/Schema/CachedPathInputs.pm
View file

@ -50,8 +50,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("srcpath", "sha256hash");
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vMJmHfs180MNWH9+pjS94g
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZEYeoP+fkE3y/cohKLoCLg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/CachedSubversionInputs.pm
View file

@ -43,8 +43,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("uri", "revision");
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3hG0+UX3MJMLjSTN29Le4Q
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:f3ZEnsrXJUjcpGkLNl24Rw
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Jobs.pm
View file

@ -78,8 +78,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:bY5DYFb1G7JRyNFpGbWXwA
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:I5DYg9q22g99Mstln/26fw
# You can replace this text with custom content, and it will be preserved on regeneration

6
src/lib/Hydra/Schema/JobsetInputAlts.pm
View file

@ -51,7 +51,7 @@ __PACKAGE__->add_columns(
},
"revision",
{
data_type => "integer",
data_type => "text",
default_value => undef,
is_nullable => 1,
size => undef,
@ -72,8 +72,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:XPfPlym2UDd6gUr1aKrXhg
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:B2BMIuiQ3IAoqEJ18pHCeQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/JobsetInputs.pm
View file

@ -68,8 +68,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:961DCIj2fAQFYAR6/SnJ8A
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:srhHJGx+LAdeo++jv1RmMg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Jobsets.pm
View file

@ -114,8 +114,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 14:04:55
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xWsqXneZw90uEw/vcEXc4w
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:05:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fVXvhb343Zw1625daVz40g
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Projects.pm
View file

@ -94,8 +94,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 14:04:55
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dWe2DEsuZuOjVj4IA8TwQg
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:05:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+HDJ8tIPcvj5+IwgHqTnaw
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/ReleaseMembers.pm
View file

@ -53,8 +53,8 @@ __PACKAGE__->belongs_to(
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DvQQfFaVP0ci1LMKfbl3tg
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fu21YcmM1U76pcgFY1qZ5A
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Releases.pm
View file

@ -53,8 +53,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zG8H+WLuEnvPl9UEJ3yyCQ
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cbt+hnRoIN5T6u8qC9uQBg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/SystemTypes.pm
View file

@ -24,8 +24,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("system");
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7HmnBjVA60uMDrV2Bc9TmA
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QtiooD8Th7dBKknK5IRhMQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/UserRoles.pm
View file

@ -31,8 +31,8 @@ __PACKAGE__->set_primary_key("username", "role");
__PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" });
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:l4FwcyoQw1cpvc+QKt6W7Q
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9oO99uYxlWqY3GxfUWjNzg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Users.pm
View file

@ -53,8 +53,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:FSfQfOaAf2oeQeFtBI7HoQ
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oBiKpCxHGZcGujhY/ZbbxA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/ViewJobs.pm
View file

@ -69,8 +69,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:FqMPRQo5hyHDy6zZIqdR5w
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RX9tEuV8mEg13dxEe9SJrQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Views.pm
View file

@ -45,8 +45,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DVmiIWbS8/PBOMvjjmns6g
# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BnM3PKkdJXAaT4rPR8gJsQ
# You can replace this text with custom content, and it will be preserved on regeneration

1
src/root/common.tt
View file

@ -5,6 +5,7 @@
[% inputTypes =
{ "svn" = "Subversion checkout"
, "cvs" = "CVS checkout"
, "git" = "Git checkout"
, "tarball" = "Download of a tarball"
, "string" = "String value"
, "boolean" = "Boolean"

24
src/sql/hydra.sql
View file

@ -72,7 +72,7 @@ create table JobsetInputAlts (
-- urgh
value text, -- for most types, a URI; for 'path', an absolute path; for 'string', an arbitrary value
revision integer, -- for type == 'svn'
revision text, -- for type == 'svn'
tag text, -- for type == 'cvs'
primary key (project, jobset, input, altnr),
@ -239,7 +239,7 @@ create table BuildInputs (
name text not null,
type text not null,
uri text,
revision integer,
revision text,
tag text,
value text,
dependency integer, -- build ID of the input, for type == 'build'
@ -292,6 +292,26 @@ create table CachedSubversionInputs (
primary key (uri, revision)
);
create table CachedGitInputs (
uri text not null,
timestamp integer not null, -- when we first saw this hash
lastSeen integer not null, -- when we last saw this hash
sha256hash text not null,
storePath text not null,
primary key (uri, sha256hash)
);
create table CachedCVSInputs (
uri text not null,
module text not null,
revision integer not null,
timestamp integer not null, -- when we first saw this hash
lastSeen integer not null, -- when we last saw this hash
sha256hash text not null,
storePath text not null,
primary key (uri, module, sha256hash)
);
create table SystemTypes (
system text primary key not null,