Merge pull request #81 from mornfall/master

DARCS support

release.nix

@@ -113,14 +113,14 @@ in rec {
 
       buildInputs =
         [ makeWrapper libtool unzip nukeReferences pkgconfig boehmgc sqlite
-          gitAndTools.topGit mercurial subversion bazaar openssl bzip2
+          gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2
           guile # optional, for Guile + Guix support
           perlDeps perl
         ];
 
       hydraPath = lib.makeSearchPath "bin" (
         [ libxslt sqlite subversion openssh nix coreutils findutils
-          gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial gnused graphviz bazaar
+          gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused graphviz bazaar
         ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
 
       preCheck = ''

src/lib/Hydra/Controller/Build.pm

@@ -158,7 +158,7 @@ sub showLog {
             . " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -"
             . " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2";
         $c->stash->{template} = 'log.tt';
-        $c->stash->{logtext} = `$pipeline`;
+        $c->stash->{logtext} = `ulimit -t 5 ; $pipeline`;
     }
 
     elsif ($mode eq "raw") {
@@ -298,7 +298,7 @@ sub contents : Chained('buildChain') PathPart Args(1) {
     notFound($c, "Product $path has disappeared.") unless -e $path;
 
     # Sanitize $path to prevent shell injection attacks.
-    $path =~ /^\/[\/[A-Za-z0-9_\-\.=]+$/ or die "Filename contains illegal characters.\n";
+    $path =~ /^\/[\/[A-Za-z0-9_\-\.=+:]+$/ or die "Filename contains illegal characters.\n";
 
     # FIXME: don't use shell invocations below.
 

src/lib/Hydra/Helper/AddBuilds.pm

@@ -149,7 +149,6 @@ sub fetchInputSystemBuild {
     return @inputs;
 }
 
-
 sub fetchInput {
     my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_;
     my @inputs;

src/lib/Hydra/Helper/CatalystUtils.pm

@@ -210,7 +210,7 @@ sub paramToList {
 
 
 # Security checking of filenames.
-Readonly our $pathCompRE    => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$]*)";
+Readonly our $pathCompRE    => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$:]*)";
 Readonly our $relPathRE     => "(?:$pathCompRE(?:/$pathCompRE)*)";
 Readonly our $relNameRE     => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)";
 Readonly our $attrNameRE    => "(?:[A-Za-z_][A-Za-z0-9-_]*)";

src/lib/Hydra/Helper/Nix.pm

@@ -253,13 +253,11 @@ sub getLatestSuccessfulViewResult {
 sub getDrvLogPath {
     my ($drvPath) = @_;
     my $base = basename $drvPath;
-    my $fn =
+    my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2);
-        ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/"
+    my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
-        . substr($base, 0, 2) . "/"
+    for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
-        . substr($base, 2);
+        return $_ if (-f $_);
-    return $fn if -f $fn;
+    }
-    $fn .= ".bz2";
-    return $fn if -f $fn;
     return undef;
 }
 

src/lib/Hydra/Plugin/DarcsInput.pm

@@ -0,0 +1,104 @@
+package Hydra::Plugin::DarcsInput;
+
+use strict;
+use parent 'Hydra::Plugin';
+use Digest::SHA qw(sha256_hex);
+use File::Path;
+use Hydra::Helper::Nix;
+use Nix::Store;
+
+sub supportedInputTypes {
+    my ($self, $inputTypes) = @_;
+    $inputTypes->{'darcs'} = 'Darcs checkout';
+}
+
+sub fetchInput {
+    my ($self, $type, $name, $uri) = @_;
+
+    return undef if $type ne "darcs";
+
+    my $timestamp = time;
+    my $sha256;
+    my $storePath;
+    my $revCount;
+
+    my $cacheDir = getSCMCacheDir . "/git";
+    mkpath($cacheDir);
+    my $clonePath = $cacheDir . "/" . sha256_hex($uri);
+    $uri =~ s|^file://||; # darcs wants paths, not file:// uris
+
+    my $stdout = ""; my $stderr = ""; my $res;
+    if (! -d $clonePath) {
+        # Clone the repository.
+        $res = run(timeout => 600,
+                   cmd => ["darcs", "get", "--lazy", $uri, $clonePath],
+                   dir => $ENV{"TMPDIR"});
+        die "Error getting darcs repo at `$uri':\n$stderr" if $res->{status};
+    }
+
+    # Update the repository to match $uri.
+    ($res, $stdout, $stderr) = captureStdoutStderr(600,
+        ("darcs", "pull", "-a", "--repodir", $clonePath, "$uri"));
+    die "Error fetching latest change from darcs repo at `$uri':\n$stderr" if $res;
+
+    ($res, $stdout, $stderr) = captureStdoutStderr(600,
+        ("darcs", "changes", "--last", "1", "--xml", "--repodir", $clonePath));
+    die "Error getting revision ID of darcs repo at `$uri':\n$stderr" if $res;
+
+    $stdout =~ /^<patch.*hash='([0-9a-fA-F-]+)'/sm; # sigh.
+    my $revision = $1;
+    die "Error obtaining revision from output: $stdout\nstderr = $stderr)" unless $revision =~ /^[0-9a-fA-F-]+$/;
+    die "Error getting a revision identifier at `$uri':\n$stderr" if $res;
+
+    # Some simple caching: don't check a uri/revision more than once.
+    my $cachedInput ;
+    ($cachedInput) = $self->{db}->resultset('CachedDarcsInputs')->search(
+        {uri => $uri, revision => $revision},
+        {rows => 1});
+
+    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+        $storePath = $cachedInput->storepath;
+        $sha256 = $cachedInput->sha256hash;
+        $revision = $cachedInput->revision;
+        $revCount = $cachedInput->revcount;
+    } else {
+        # Then download this revision into the store.
+        print STDERR "checking out darcs repo $uri\n";
+
+        my $tmpDir = File::Temp->newdir("hydra-darcs-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
+        (system "darcs", "get", "--lazy", $clonePath, "$tmpDir/export", "--quiet",
+                "--to-match", "hash $revision") == 0
+            or die "darcs export failed";
+        $revCount = `darcs changes --count --repodir $tmpDir/export`; chomp $revCount;
+        die "darcs changes --count failed" if $? != 0;
+
+        system "rm", "-rf", "$tmpDir/export/_darcs";
+        $storePath = addToStore("$tmpDir/export", 1, "sha256");
+        $sha256 = queryPathHash($storePath);
+        $sha256 =~ s/sha256://;
+
+        txn_do($self->{db}, sub {
+            $self->{db}->resultset('CachedDarcsInputs')->update_or_create(
+                { uri => $uri
+                , revision => $revision
+                , revcount => $revCount
+                , sha256hash => $sha256
+                , storepath => $storePath
+                });
+            });
+    }
+
+    $revision =~ /^([0-9]+)/;
+    my $shortRev = $1;
+
+    return
+        { uri => $uri
+        , storePath => $storePath
+        , sha256hash => $sha256
+        , revision => $revision
+        , revCount => int($revCount)
+        , shortRev => $shortRev
+        };
+}
+
+1;

src/lib/Hydra/Plugin/PathInput.pm

@@ -34,8 +34,13 @@ sub fetchInput {
     } else {
 
         print STDERR "copying input ", $name, " from $uri\n";
-        $storePath = `nix-store --add "$uri"`
+        if ( $uri =~ /^\// ) {
-            or die "cannot copy path $uri to the Nix store.\n";
+            $storePath = `nix-store --add "$uri"`
+                or die "cannot copy path $uri to the Nix store.\n";
+        } else {
+            $storePath = `PRINT_PATH=1 nix-prefetch-url "$uri" | tail -n 1`
+                or die "cannot fetch $uri to the Nix store.\n";
+        }
         chomp $storePath;
 
         $sha256 = (queryPathInfo($storePath, 0))[1] or die;

src/lib/Hydra/Schema/CachedDarcsInputs.pm

@@ -0,0 +1,86 @@
+use utf8;
+package Hydra::Schema::CachedDarcsInputs;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+=head1 NAME
+
+Hydra::Schema::CachedDarcsInputs
+
+=cut
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+=head1 TABLE: C<CachedDarcsInputs>
+
+=cut
+
+__PACKAGE__->table("CachedDarcsInputs");
+
+=head1 ACCESSORS
+
+=head2 uri
+
+  data_type: 'text'
+  is_nullable: 0
+
+=head2 branch
+
+  data_type: 'text'
+  is_nullable: 0
+
+=head2 revision
+
+  data_type: 'text'
+  is_nullable: 0
+
+=head2 sha256hash
+
+  data_type: 'text'
+  is_nullable: 0
+
+=head2 storepath
+
+  data_type: 'text'
+  is_nullable: 0
+
+=cut
+
+__PACKAGE__->add_columns(
+  "uri",
+  { data_type => "text", is_nullable => 0 },
+  "revision",
+  { data_type => "text", is_nullable => 0 },
+  "revcount",
+  { data_type => "integer", is_nullable => 0 },
+  "sha256hash",
+  { data_type => "text", is_nullable => 0 },
+  "storepath",
+  { data_type => "text", is_nullable => 0 },
+);
+
+=head1 PRIMARY KEY
+
+=over 4
+
+=item * L</uri>
+
+=item * L</branch>
+
+=item * L</revision>
+
+=back
+
+=cut
+
+__PACKAGE__->set_primary_key("uri", "revision");
+
+
+# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fx3yosWMmJ+MnvL/dSWtFA
+
+1;

src/sql/hydra.sql

@@ -322,6 +322,15 @@ create table CachedGitInputs (
     primary key   (uri, branch, revision)
 );
 
+create table CachedDarcsInputs (
+    uri           text not null,
+    revision      text not null,
+    sha256hash    text not null,
+    storePath     text not null,
+    revCount      integer not null,
+    primary key   (uri, revision)
+);
+
 create table CachedHgInputs (
     uri           text not null,
     branch        text not null,

src/sql/upgrade-darcs.sql

@@ -0,0 +1,8 @@
+create table CachedDarcsInputs (
+    uri           text not null,
+    revision      text not null,
+    sha256hash    text not null,
+    storePath     text not null,
+    revCount      integer not null,
+    primary key   (uri, revision)
+);

tests/evaluation-tests.pl

@@ -7,7 +7,7 @@ use Setup;
 
 my $db = Hydra::Model::DB->new;
 
-use Test::Simple tests => 68;
+use Test::Simple tests => 72;
 
 hydra_setup($db);
 
@@ -102,6 +102,13 @@ my @scminputs = (
         type => "hg",
         uri => "$jobsBaseUri/hg-repo",
         update => getcwd . "/jobs/hg-update.sh"
+    },
+    {
+        name => "darcs",
+        nixexpr => "darcs-input.nix",
+        type => "darcs",
+        uri => "$jobsBaseUri/darcs-repo",
+        update => getcwd . "/jobs/darcs-update.sh"
     }
 );
 

tests/jobs/darcs-input.nix

@@ -0,0 +1,10 @@
+with import ./config.nix;
+{ src }:
+{
+  copy =
+    mkDerivation {
+      name = "git-input";
+      builder = ./scm-builder.sh;
+      inherit src;
+    };
+}

tests/jobs/darcs-update.sh

@@ -0,0 +1,24 @@
+#! /bin/sh
+set -e
+
+repo="$1"
+STATE_FILE=$(pwd)/.hg-state
+if test -e $STATE_FILE; then
+    state=$(cat $STATE_FILE)
+    test $state -gt 1 && state=0
+else
+    state=0;
+fi
+
+case $state in
+    (0) echo "::Create repo. -- continue -- updated::"
+    mkdir darcs-repo
+    darcs init --repodir darcs-repo
+    touch darcs-repo/file
+    darcs add --repodir darcs-repo file
+    darcs record --repodir darcs-repo -a -l -m "add a file" file -A foobar@bar.bar
+    ;;
+    (*) echo "::End. -- stop -- nothing::" ;;
+esac
+
+echo $(($state + 1)) > $STATE_FILE

tests/query-all-tables.pl

@@ -7,7 +7,7 @@ my $db = Hydra::Model::DB->new;
 my @sources = $db->schema->sources;
 my $nrtables = scalar(@sources);
 
-use Test::Simple tests => 41;
+use Test::Simple tests => 42;
 
 foreach my $source (@sources) {
     my $title = "Basic select query for $source";