Merge pull request #81 from mornfall/master

DARCS support
This commit is contained in:
Rob Vermaas 2013-09-14 06:18:48 -07:00
commit 597fd827b1
14 changed files with 267 additions and 17 deletions

View file

@ -113,14 +113,14 @@ in rec {
buildInputs = buildInputs =
[ makeWrapper libtool unzip nukeReferences pkgconfig boehmgc sqlite [ makeWrapper libtool unzip nukeReferences pkgconfig boehmgc sqlite
gitAndTools.topGit mercurial subversion bazaar openssl bzip2 gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2
guile # optional, for Guile + Guix support guile # optional, for Guile + Guix support
perlDeps perl perlDeps perl
]; ];
hydraPath = lib.makeSearchPath "bin" ( hydraPath = lib.makeSearchPath "bin" (
[ libxslt sqlite subversion openssh nix coreutils findutils [ libxslt sqlite subversion openssh nix coreutils findutils
gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial gnused graphviz bazaar gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused graphviz bazaar
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
preCheck = '' preCheck = ''

View file

@ -158,7 +158,7 @@ sub showLog {
. " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -" . " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -"
. " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2"; . " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2";
$c->stash->{template} = 'log.tt'; $c->stash->{template} = 'log.tt';
$c->stash->{logtext} = `$pipeline`; $c->stash->{logtext} = `ulimit -t 5 ; $pipeline`;
} }
elsif ($mode eq "raw") { elsif ($mode eq "raw") {
@ -298,7 +298,7 @@ sub contents : Chained('buildChain') PathPart Args(1) {
notFound($c, "Product $path has disappeared.") unless -e $path; notFound($c, "Product $path has disappeared.") unless -e $path;
# Sanitize $path to prevent shell injection attacks. # Sanitize $path to prevent shell injection attacks.
$path =~ /^\/[\/[A-Za-z0-9_\-\.=]+$/ or die "Filename contains illegal characters.\n"; $path =~ /^\/[\/[A-Za-z0-9_\-\.=+:]+$/ or die "Filename contains illegal characters.\n";
# FIXME: don't use shell invocations below. # FIXME: don't use shell invocations below.

View file

@ -149,7 +149,6 @@ sub fetchInputSystemBuild {
return @inputs; return @inputs;
} }
sub fetchInput { sub fetchInput {
my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_; my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_;
my @inputs; my @inputs;

View file

@ -210,7 +210,7 @@ sub paramToList {
# Security checking of filenames. # Security checking of filenames.
Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$]*)"; Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$:]*)";
Readonly our $relPathRE => "(?:$pathCompRE(?:/$pathCompRE)*)"; Readonly our $relPathRE => "(?:$pathCompRE(?:/$pathCompRE)*)";
Readonly our $relNameRE => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)"; Readonly our $relNameRE => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)";
Readonly our $attrNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)"; Readonly our $attrNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)";

View file

@ -253,13 +253,11 @@ sub getLatestSuccessfulViewResult {
sub getDrvLogPath { sub getDrvLogPath {
my ($drvPath) = @_; my ($drvPath) = @_;
my $base = basename $drvPath; my $base = basename $drvPath;
my $fn = my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2);
($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/" my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
. substr($base, 0, 2) . "/" for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
. substr($base, 2); return $_ if (-f $_);
return $fn if -f $fn; }
$fn .= ".bz2";
return $fn if -f $fn;
return undef; return undef;
} }

View file

@ -0,0 +1,104 @@
package Hydra::Plugin::DarcsInput;
use strict;
use parent 'Hydra::Plugin';
use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
$inputTypes->{'darcs'} = 'Darcs checkout';
}
sub fetchInput {
my ($self, $type, $name, $uri) = @_;
return undef if $type ne "darcs";
my $timestamp = time;
my $sha256;
my $storePath;
my $revCount;
my $cacheDir = getSCMCacheDir . "/git";
mkpath($cacheDir);
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
$uri =~ s|^file://||; # darcs wants paths, not file:// uris
my $stdout = ""; my $stderr = ""; my $res;
if (! -d $clonePath) {
# Clone the repository.
$res = run(timeout => 600,
cmd => ["darcs", "get", "--lazy", $uri, $clonePath],
dir => $ENV{"TMPDIR"});
die "Error getting darcs repo at `$uri':\n$stderr" if $res->{status};
}
# Update the repository to match $uri.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("darcs", "pull", "-a", "--repodir", $clonePath, "$uri"));
die "Error fetching latest change from darcs repo at `$uri':\n$stderr" if $res;
($res, $stdout, $stderr) = captureStdoutStderr(600,
("darcs", "changes", "--last", "1", "--xml", "--repodir", $clonePath));
die "Error getting revision ID of darcs repo at `$uri':\n$stderr" if $res;
$stdout =~ /^<patch.*hash='([0-9a-fA-F-]+)'/sm; # sigh.
my $revision = $1;
die "Error obtaining revision from output: $stdout\nstderr = $stderr)" unless $revision =~ /^[0-9a-fA-F-]+$/;
die "Error getting a revision identifier at `$uri':\n$stderr" if $res;
# Some simple caching: don't check a uri/revision more than once.
my $cachedInput ;
($cachedInput) = $self->{db}->resultset('CachedDarcsInputs')->search(
{uri => $uri, revision => $revision},
{rows => 1});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
$revCount = $cachedInput->revcount;
} else {
# Then download this revision into the store.
print STDERR "checking out darcs repo $uri\n";
my $tmpDir = File::Temp->newdir("hydra-darcs-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "darcs", "get", "--lazy", $clonePath, "$tmpDir/export", "--quiet",
"--to-match", "hash $revision") == 0
or die "darcs export failed";
$revCount = `darcs changes --count --repodir $tmpDir/export`; chomp $revCount;
die "darcs changes --count failed" if $? != 0;
system "rm", "-rf", "$tmpDir/export/_darcs";
$storePath = addToStore("$tmpDir/export", 1, "sha256");
$sha256 = queryPathHash($storePath);
$sha256 =~ s/sha256://;
txn_do($self->{db}, sub {
$self->{db}->resultset('CachedDarcsInputs')->update_or_create(
{ uri => $uri
, revision => $revision
, revcount => $revCount
, sha256hash => $sha256
, storepath => $storePath
});
});
}
$revision =~ /^([0-9]+)/;
my $shortRev = $1;
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
, revCount => int($revCount)
, shortRev => $shortRev
};
}
1;

View file

@ -34,8 +34,13 @@ sub fetchInput {
} else { } else {
print STDERR "copying input ", $name, " from $uri\n"; print STDERR "copying input ", $name, " from $uri\n";
if ( $uri =~ /^\// ) {
$storePath = `nix-store --add "$uri"` $storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store.\n"; or die "cannot copy path $uri to the Nix store.\n";
} else {
$storePath = `PRINT_PATH=1 nix-prefetch-url "$uri" | tail -n 1`
or die "cannot fetch $uri to the Nix store.\n";
}
chomp $storePath; chomp $storePath;
$sha256 = (queryPathInfo($storePath, 0))[1] or die; $sha256 = (queryPathInfo($storePath, 0))[1] or die;

View file

@ -0,0 +1,86 @@
use utf8;
package Hydra::Schema::CachedDarcsInputs;
# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE
=head1 NAME
Hydra::Schema::CachedDarcsInputs
=cut
use strict;
use warnings;
use base 'DBIx::Class::Core';
=head1 TABLE: C<CachedDarcsInputs>
=cut
__PACKAGE__->table("CachedDarcsInputs");
=head1 ACCESSORS
=head2 uri
data_type: 'text'
is_nullable: 0
=head2 branch
data_type: 'text'
is_nullable: 0
=head2 revision
data_type: 'text'
is_nullable: 0
=head2 sha256hash
data_type: 'text'
is_nullable: 0
=head2 storepath
data_type: 'text'
is_nullable: 0
=cut
__PACKAGE__->add_columns(
"uri",
{ data_type => "text", is_nullable => 0 },
"revision",
{ data_type => "text", is_nullable => 0 },
"revcount",
{ data_type => "integer", is_nullable => 0 },
"sha256hash",
{ data_type => "text", is_nullable => 0 },
"storepath",
{ data_type => "text", is_nullable => 0 },
);
=head1 PRIMARY KEY
=over 4
=item * L</uri>
=item * L</branch>
=item * L</revision>
=back
=cut
__PACKAGE__->set_primary_key("uri", "revision");
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fx3yosWMmJ+MnvL/dSWtFA
1;

View file

@ -322,6 +322,15 @@ create table CachedGitInputs (
primary key (uri, branch, revision) primary key (uri, branch, revision)
); );
create table CachedDarcsInputs (
uri text not null,
revision text not null,
sha256hash text not null,
storePath text not null,
revCount integer not null,
primary key (uri, revision)
);
create table CachedHgInputs ( create table CachedHgInputs (
uri text not null, uri text not null,
branch text not null, branch text not null,

View file

@ -0,0 +1,8 @@
create table CachedDarcsInputs (
uri text not null,
revision text not null,
sha256hash text not null,
storePath text not null,
revCount integer not null,
primary key (uri, revision)
);

View file

@ -7,7 +7,7 @@ use Setup;
my $db = Hydra::Model::DB->new; my $db = Hydra::Model::DB->new;
use Test::Simple tests => 68; use Test::Simple tests => 72;
hydra_setup($db); hydra_setup($db);
@ -102,6 +102,13 @@ my @scminputs = (
type => "hg", type => "hg",
uri => "$jobsBaseUri/hg-repo", uri => "$jobsBaseUri/hg-repo",
update => getcwd . "/jobs/hg-update.sh" update => getcwd . "/jobs/hg-update.sh"
},
{
name => "darcs",
nixexpr => "darcs-input.nix",
type => "darcs",
uri => "$jobsBaseUri/darcs-repo",
update => getcwd . "/jobs/darcs-update.sh"
} }
); );

View file

@ -0,0 +1,10 @@
with import ./config.nix;
{ src }:
{
copy =
mkDerivation {
name = "git-input";
builder = ./scm-builder.sh;
inherit src;
};
}

24
tests/jobs/darcs-update.sh Executable file
View file

@ -0,0 +1,24 @@
#! /bin/sh
set -e
repo="$1"
STATE_FILE=$(pwd)/.hg-state
if test -e $STATE_FILE; then
state=$(cat $STATE_FILE)
test $state -gt 1 && state=0
else
state=0;
fi
case $state in
(0) echo "::Create repo. -- continue -- updated::"
mkdir darcs-repo
darcs init --repodir darcs-repo
touch darcs-repo/file
darcs add --repodir darcs-repo file
darcs record --repodir darcs-repo -a -l -m "add a file" file -A foobar@bar.bar
;;
(*) echo "::End. -- stop -- nothing::" ;;
esac
echo $(($state + 1)) > $STATE_FILE

View file

@ -7,7 +7,7 @@ my $db = Hydra::Model::DB->new;
my @sources = $db->schema->sources; my @sources = $db->schema->sources;
my $nrtables = scalar(@sources); my $nrtables = scalar(@sources);
use Test::Simple tests => 41; use Test::Simple tests => 42;
foreach my $source (@sources) { foreach my $source (@sources) {
my $title = "Basic select query for $source"; my $title = "Basic select query for $source";