add support for git as jobinput

release.nix

@@ -88,6 +88,7 @@ let
           mv $out/libexec/hydra/script $out/bin
 
           cp ${"${nixpkgs}/pkgs/build-support/fetchsvn/nix-prefetch-svn"} $out/bin/nix-prefetch-svn
+          cp ${"${nixpkgs}/pkgs/build-support/fetchgit/nix-prefetch-git"} $out/bin/nix-prefetch-git
 
           make -C src/c NIX=${nix} ATERM=${aterm242fixes}
           cp src/c/hydra_eval_jobs $out/bin

src/lib/Hydra/Controller/Jobset.pm

@@ -120,7 +120,7 @@ sub checkInput {
     error($c, "Invalid input type: $inputType") unless
         $inputType eq "svn" || $inputType eq "cvs" || $inputType eq "tarball" ||
         $inputType eq "string" || $inputType eq "path" || $inputType eq "boolean" ||
-        $inputType eq "build";
+        $inputType eq "git" || $inputType eq "build";
 
     return ($inputName, $inputType);
 }

src/lib/Hydra/Helper/AddBuilds.pm

@@ -63,11 +63,9 @@ sub attrsToSQL {
     return $query;
 }
 
-
+sub fetchInputPath {
-sub fetchInput {
     my ($db, $project, $jobset, $name, $type, $value) = @_;
 
-    if ($type eq "path") {
         my $uri = $value;
 
         my $timestamp = time;
@@ -128,7 +126,9 @@ sub fetchInput {
             };
 }
 
-    elsif ($type eq "svn") {
+sub fetchInputSVN {
+    my ($db, $project, $jobset, $name, $type, $value) = @_;
+
         my $uri = $value;
 
         my $sha256;
@@ -180,7 +180,9 @@ sub fetchInput {
             };
 }
 
-    elsif ($type eq "build") {
+sub fetchInputBuild {
+    my ($db, $project, $jobset, $name, $type, $value) = @_;
+
         my ($projectName, $jobsetName, $jobName, $attrs) = parseJobName($value);
         $projectName ||= $project->name;
         $jobsetName ||= $jobset->name;
@@ -213,6 +215,86 @@ sub fetchInput {
             };
 }
 
+sub fetchInputGit {
+    my ($db, $project, $jobset, $name, $type, $value) = @_;
+
+    my $uri = $value;
+    my $timestamp = time;
+    my $sha256;
+    my $storePath;
+
+    # Some simple caching: don't check a path more than once every N seconds.
+    (my $cachedInput) = $db->resultset('CachedGitInputs')->search(
+	{uri => $uri, lastseen => {">", $timestamp - 3600}},
+	{rows => 1, order_by => "lastseen DESC"});
+
+    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+	$storePath = $cachedInput->storepath;
+	$sha256 = $cachedInput->sha256hash;
+	$timestamp = $cachedInput->timestamp;
+    } else {
+
+	# Then download this revision into the store.
+	print STDERR "checking out Git input from $uri";
+	$ENV{"NIX_HASH_ALGO"} = "sha256";
+	$ENV{"PRINT_PATH"} = "1";
+        my $stdout; my $stderr;
+	(my $res, $stdout, $stderr) = captureStdoutStderr(
+	    "nix-prefetch-git", $uri);
+	die "Cannot check out Git repository `$uri':\n$stderr" unless $res;
+
+	($sha256, $storePath) = split ' ', $stdout;
+	($cachedInput) = $db->resultset('CachedGitInputs')->search(
+	    {uri => $uri, sha256hash => $sha256});
+
+	if (!defined $cachedInput) {
+	    txn_do($db, sub {
+		$db->resultset('CachedGitInputs')->create(
+		    { uri => $uri
+                    , timestamp => $timestamp
+                    , lastseen => $timestamp
+                    , sha256hash => $sha256
+                    , storepath => $storePath
+                    });
+		   });
+	} else {
+	    $timestamp = $cachedInput->timestamp;
+	    txn_do($db, sub {
+		$cachedInput->update({lastseen => time});
+	    });
+	}
+    }
+
+    return
+       { type => $type
+       , uri => $uri
+       , storePath => $storePath
+       , sha256hash => $sha256
+       , revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
+       };
+
+}
+
+sub fetchInputCVS {
+    my ($db, $project, $jobset, $name, $type, $value) = @_;
+}
+
+sub fetchInput {
+    my ($db, $project, $jobset, $name, $type, $value) = @_;
+
+    if ($type eq "path") {
+	return fetchInputPath($db, $project, $jobset, $name, $type, $value);
+    }
+    elsif ($type eq "svn") {
+	return fetchInputSVN($db, $project, $jobset, $name, $type, $value);
+    }
+    elsif ($type eq "build") {
+	return fetchInputBuild($db, $project, $jobset, $name, $type, $value);
+    }
+    elsif ($type eq "git") {
+	return fetchInputGit($db, $project, $jobset, $name, $type, $value);
+    }
+    
     elsif ($type eq "string") {
         die unless defined $value;
         return {type => $type, value => $value};
@@ -242,7 +324,7 @@ sub inputsToArgs {
                 when ("boolean") {
                     push @res, "--arg", $input, $alt->{value};
                 }
-                when (["svn", "path", "build"]) {
+                when (["svn", "path", "build", "git", "cvs"]) {
                     push @res, "--arg", $input, (
                         "{ outPath = builtins.storePath " . $alt->{storePath} . "" .
                         (defined $alt->{revision} ? "; rev = \"" . $alt->{revision} . "\"" : "") .

src/lib/Hydra/Schema.pm

@@ -11,8 +11,8 @@ use base 'DBIx::Class::Schema';
 __PACKAGE__->load_classes;
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:s5XNrT7kEVqbLED5g/J/SA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y97NEH6tnsRjsy0/nVgMjQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/BuildInputs.pm

@@ -50,7 +50,7 @@ __PACKAGE__->add_columns(
   },
   "revision",
   {
-    data_type => "integer",
+    data_type => "text",
     default_value => undef,
     is_nullable => 1,
     size => undef,
@@ -107,8 +107,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RTeBJCvspGfTdHUR/TxFsg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:skn8HFEXTvEiL4/6Q+ulvw
 
 use Hydra::Helper::Nix;
 

src/lib/Hydra/Schema/BuildProducts.pm

@@ -94,8 +94,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
 __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8FZKxfabZQfchVpHfvnVHA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6wJ0KJAILATio3ELRefU5Q
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/BuildResultInfo.pm

@@ -89,8 +89,8 @@ __PACKAGE__->set_primary_key("id");
 __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:496WClgIZB6hXV0zCsBjbQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:K+kKzTgAlKSXevJWtLIwGA
 
 __PACKAGE__->belongs_to(
   "failedDep",

src/lib/Hydra/Schema/BuildSchedulingInfo.pm

@@ -46,8 +46,8 @@ __PACKAGE__->set_primary_key("id");
 __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:giLh18QewdxF8Zi/5juWqA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eiaLNgxvTjSZGC4pMDtNWA
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/BuildSteps.pm

@@ -94,7 +94,7 @@ __PACKAGE__->set_primary_key("build", "stepnr");
 __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5tx+fkjjKYUNYBYQS+kSOw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5iUdHsNyEoM3D7i+tSteBQ
 
 1;

src/lib/Hydra/Schema/Builds.pm

@@ -186,8 +186,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-26 14:22:27
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qbNPA5NkENlZXILuE3OGkA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Un0iCqVS8PTpSdJiTjRXeA
 
 use Hydra::Helper::Nix;
 

src/lib/Hydra/Schema/CachedPathInputs.pm

@@ -50,8 +50,8 @@ __PACKAGE__->add_columns(
 __PACKAGE__->set_primary_key("srcpath", "sha256hash");
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vMJmHfs180MNWH9+pjS94g
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZEYeoP+fkE3y/cohKLoCLg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/CachedSubversionInputs.pm

@@ -43,8 +43,8 @@ __PACKAGE__->add_columns(
 __PACKAGE__->set_primary_key("uri", "revision");
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3hG0+UX3MJMLjSTN29Le4Q
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:f3ZEnsrXJUjcpGkLNl24Rw
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/Jobs.pm

@@ -78,8 +78,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:bY5DYFb1G7JRyNFpGbWXwA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:I5DYg9q22g99Mstln/26fw
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/JobsetInputAlts.pm

@@ -51,7 +51,7 @@ __PACKAGE__->add_columns(
   },
   "revision",
   {
-    data_type => "integer",
+    data_type => "text",
     default_value => undef,
     is_nullable => 1,
     size => undef,
@@ -72,8 +72,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:XPfPlym2UDd6gUr1aKrXhg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:B2BMIuiQ3IAoqEJ18pHCeQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/JobsetInputs.pm

@@ -68,8 +68,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:961DCIj2fAQFYAR6/SnJ8A
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:srhHJGx+LAdeo++jv1RmMg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/Jobsets.pm

@@ -114,8 +114,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 14:04:55
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:05:10
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xWsqXneZw90uEw/vcEXc4w
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fVXvhb343Zw1625daVz40g
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/Projects.pm

@@ -94,8 +94,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 14:04:55
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:05:10
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dWe2DEsuZuOjVj4IA8TwQg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+HDJ8tIPcvj5+IwgHqTnaw
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/ReleaseMembers.pm

@@ -53,8 +53,8 @@ __PACKAGE__->belongs_to(
 __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DvQQfFaVP0ci1LMKfbl3tg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fu21YcmM1U76pcgFY1qZ5A
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/Releases.pm

@@ -53,8 +53,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zG8H+WLuEnvPl9UEJ3yyCQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cbt+hnRoIN5T6u8qC9uQBg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/SystemTypes.pm

@@ -24,8 +24,8 @@ __PACKAGE__->add_columns(
 __PACKAGE__->set_primary_key("system");
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7HmnBjVA60uMDrV2Bc9TmA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QtiooD8Th7dBKknK5IRhMQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/UserRoles.pm

@@ -31,8 +31,8 @@ __PACKAGE__->set_primary_key("username", "role");
 __PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:l4FwcyoQw1cpvc+QKt6W7Q
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9oO99uYxlWqY3GxfUWjNzg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/Users.pm

@@ -53,8 +53,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:FSfQfOaAf2oeQeFtBI7HoQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oBiKpCxHGZcGujhY/ZbbxA
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/ViewJobs.pm

@@ -69,8 +69,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:FqMPRQo5hyHDy6zZIqdR5w
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RX9tEuV8mEg13dxEe9SJrQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/lib/Hydra/Schema/Views.pm

@@ -45,8 +45,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-10-23 16:56:03
+# Created by DBIx::Class::Schema::Loader v0.04999_09 @ 2009-11-17 16:04:13
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DVmiIWbS8/PBOMvjjmns6g
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BnM3PKkdJXAaT4rPR8gJsQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/root/common.tt

@@ -5,6 +5,7 @@
 [% inputTypes =
     { "svn" = "Subversion checkout"
     , "cvs" = "CVS checkout"
+    , "git" = "Git checkout"
     , "tarball" = "Download of a tarball"
     , "string" = "String value"
     , "boolean" = "Boolean"

src/sql/hydra.sql

@@ -72,7 +72,7 @@ create table JobsetInputAlts (
 
     -- urgh
     value         text, -- for most types, a URI; for 'path', an absolute path; for 'string', an arbitrary value
-    revision      integer, -- for type == 'svn'
+    revision      text, -- for type == 'svn'
     tag           text, -- for type == 'cvs'
     
     primary key   (project, jobset, input, altnr),
@@ -239,7 +239,7 @@ create table BuildInputs (
     name          text not null,
     type          text not null,
     uri           text,
-    revision      integer,
+    revision      text,
     tag           text,
     value         text,
     dependency    integer, -- build ID of the input, for type == 'build'
@@ -292,6 +292,26 @@ create table CachedSubversionInputs (
     primary key   (uri, revision)
 );
 
+create table CachedGitInputs (
+    uri           text not null,
+    timestamp     integer not null, -- when we first saw this hash
+    lastSeen      integer not null, -- when we last saw this hash
+    sha256hash    text not null,
+    storePath     text not null,
+    primary key   (uri, sha256hash)
+);
+
+create table CachedCVSInputs (
+    uri           text not null,
+    module        text not null,
+    revision      integer not null,
+    timestamp     integer not null, -- when we first saw this hash
+    lastSeen      integer not null, -- when we last saw this hash
+    sha256hash    text not null,
+    storePath     text not null,
+    primary key   (uri, module, sha256hash)
+);
+
 
 create table SystemTypes (
     system        text primary key not null,