forked from lix-project/hydra
commit
597fd827b1
|
@ -113,14 +113,14 @@ in rec {
|
||||||
|
|
||||||
buildInputs =
|
buildInputs =
|
||||||
[ makeWrapper libtool unzip nukeReferences pkgconfig boehmgc sqlite
|
[ makeWrapper libtool unzip nukeReferences pkgconfig boehmgc sqlite
|
||||||
gitAndTools.topGit mercurial subversion bazaar openssl bzip2
|
gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2
|
||||||
guile # optional, for Guile + Guix support
|
guile # optional, for Guile + Guix support
|
||||||
perlDeps perl
|
perlDeps perl
|
||||||
];
|
];
|
||||||
|
|
||||||
hydraPath = lib.makeSearchPath "bin" (
|
hydraPath = lib.makeSearchPath "bin" (
|
||||||
[ libxslt sqlite subversion openssh nix coreutils findutils
|
[ libxslt sqlite subversion openssh nix coreutils findutils
|
||||||
gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial gnused graphviz bazaar
|
gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused graphviz bazaar
|
||||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
|
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
|
||||||
|
|
||||||
preCheck = ''
|
preCheck = ''
|
||||||
|
|
|
@ -158,7 +158,7 @@ sub showLog {
|
||||||
. " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -"
|
. " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -"
|
||||||
. " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2";
|
. " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2";
|
||||||
$c->stash->{template} = 'log.tt';
|
$c->stash->{template} = 'log.tt';
|
||||||
$c->stash->{logtext} = `$pipeline`;
|
$c->stash->{logtext} = `ulimit -t 5 ; $pipeline`;
|
||||||
}
|
}
|
||||||
|
|
||||||
elsif ($mode eq "raw") {
|
elsif ($mode eq "raw") {
|
||||||
|
@ -298,7 +298,7 @@ sub contents : Chained('buildChain') PathPart Args(1) {
|
||||||
notFound($c, "Product $path has disappeared.") unless -e $path;
|
notFound($c, "Product $path has disappeared.") unless -e $path;
|
||||||
|
|
||||||
# Sanitize $path to prevent shell injection attacks.
|
# Sanitize $path to prevent shell injection attacks.
|
||||||
$path =~ /^\/[\/[A-Za-z0-9_\-\.=]+$/ or die "Filename contains illegal characters.\n";
|
$path =~ /^\/[\/[A-Za-z0-9_\-\.=+:]+$/ or die "Filename contains illegal characters.\n";
|
||||||
|
|
||||||
# FIXME: don't use shell invocations below.
|
# FIXME: don't use shell invocations below.
|
||||||
|
|
||||||
|
|
|
@ -149,7 +149,6 @@ sub fetchInputSystemBuild {
|
||||||
return @inputs;
|
return @inputs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
sub fetchInput {
|
sub fetchInput {
|
||||||
my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_;
|
my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_;
|
||||||
my @inputs;
|
my @inputs;
|
||||||
|
|
|
@ -210,7 +210,7 @@ sub paramToList {
|
||||||
|
|
||||||
|
|
||||||
# Security checking of filenames.
|
# Security checking of filenames.
|
||||||
Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$]*)";
|
Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$:]*)";
|
||||||
Readonly our $relPathRE => "(?:$pathCompRE(?:/$pathCompRE)*)";
|
Readonly our $relPathRE => "(?:$pathCompRE(?:/$pathCompRE)*)";
|
||||||
Readonly our $relNameRE => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)";
|
Readonly our $relNameRE => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)";
|
||||||
Readonly our $attrNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)";
|
Readonly our $attrNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)";
|
||||||
|
|
|
@ -253,13 +253,11 @@ sub getLatestSuccessfulViewResult {
|
||||||
sub getDrvLogPath {
|
sub getDrvLogPath {
|
||||||
my ($drvPath) = @_;
|
my ($drvPath) = @_;
|
||||||
my $base = basename $drvPath;
|
my $base = basename $drvPath;
|
||||||
my $fn =
|
my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2);
|
||||||
($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/"
|
my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
|
||||||
. substr($base, 0, 2) . "/"
|
for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
|
||||||
. substr($base, 2);
|
return $_ if (-f $_);
|
||||||
return $fn if -f $fn;
|
}
|
||||||
$fn .= ".bz2";
|
|
||||||
return $fn if -f $fn;
|
|
||||||
return undef;
|
return undef;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
104
src/lib/Hydra/Plugin/DarcsInput.pm
Normal file
104
src/lib/Hydra/Plugin/DarcsInput.pm
Normal file
|
@ -0,0 +1,104 @@
|
||||||
|
package Hydra::Plugin::DarcsInput;
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use parent 'Hydra::Plugin';
|
||||||
|
use Digest::SHA qw(sha256_hex);
|
||||||
|
use File::Path;
|
||||||
|
use Hydra::Helper::Nix;
|
||||||
|
use Nix::Store;
|
||||||
|
|
||||||
|
sub supportedInputTypes {
|
||||||
|
my ($self, $inputTypes) = @_;
|
||||||
|
$inputTypes->{'darcs'} = 'Darcs checkout';
|
||||||
|
}
|
||||||
|
|
||||||
|
sub fetchInput {
|
||||||
|
my ($self, $type, $name, $uri) = @_;
|
||||||
|
|
||||||
|
return undef if $type ne "darcs";
|
||||||
|
|
||||||
|
my $timestamp = time;
|
||||||
|
my $sha256;
|
||||||
|
my $storePath;
|
||||||
|
my $revCount;
|
||||||
|
|
||||||
|
my $cacheDir = getSCMCacheDir . "/git";
|
||||||
|
mkpath($cacheDir);
|
||||||
|
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||||
|
$uri =~ s|^file://||; # darcs wants paths, not file:// uris
|
||||||
|
|
||||||
|
my $stdout = ""; my $stderr = ""; my $res;
|
||||||
|
if (! -d $clonePath) {
|
||||||
|
# Clone the repository.
|
||||||
|
$res = run(timeout => 600,
|
||||||
|
cmd => ["darcs", "get", "--lazy", $uri, $clonePath],
|
||||||
|
dir => $ENV{"TMPDIR"});
|
||||||
|
die "Error getting darcs repo at `$uri':\n$stderr" if $res->{status};
|
||||||
|
}
|
||||||
|
|
||||||
|
# Update the repository to match $uri.
|
||||||
|
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||||
|
("darcs", "pull", "-a", "--repodir", $clonePath, "$uri"));
|
||||||
|
die "Error fetching latest change from darcs repo at `$uri':\n$stderr" if $res;
|
||||||
|
|
||||||
|
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||||
|
("darcs", "changes", "--last", "1", "--xml", "--repodir", $clonePath));
|
||||||
|
die "Error getting revision ID of darcs repo at `$uri':\n$stderr" if $res;
|
||||||
|
|
||||||
|
$stdout =~ /^<patch.*hash='([0-9a-fA-F-]+)'/sm; # sigh.
|
||||||
|
my $revision = $1;
|
||||||
|
die "Error obtaining revision from output: $stdout\nstderr = $stderr)" unless $revision =~ /^[0-9a-fA-F-]+$/;
|
||||||
|
die "Error getting a revision identifier at `$uri':\n$stderr" if $res;
|
||||||
|
|
||||||
|
# Some simple caching: don't check a uri/revision more than once.
|
||||||
|
my $cachedInput ;
|
||||||
|
($cachedInput) = $self->{db}->resultset('CachedDarcsInputs')->search(
|
||||||
|
{uri => $uri, revision => $revision},
|
||||||
|
{rows => 1});
|
||||||
|
|
||||||
|
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||||
|
$storePath = $cachedInput->storepath;
|
||||||
|
$sha256 = $cachedInput->sha256hash;
|
||||||
|
$revision = $cachedInput->revision;
|
||||||
|
$revCount = $cachedInput->revcount;
|
||||||
|
} else {
|
||||||
|
# Then download this revision into the store.
|
||||||
|
print STDERR "checking out darcs repo $uri\n";
|
||||||
|
|
||||||
|
my $tmpDir = File::Temp->newdir("hydra-darcs-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
|
||||||
|
(system "darcs", "get", "--lazy", $clonePath, "$tmpDir/export", "--quiet",
|
||||||
|
"--to-match", "hash $revision") == 0
|
||||||
|
or die "darcs export failed";
|
||||||
|
$revCount = `darcs changes --count --repodir $tmpDir/export`; chomp $revCount;
|
||||||
|
die "darcs changes --count failed" if $? != 0;
|
||||||
|
|
||||||
|
system "rm", "-rf", "$tmpDir/export/_darcs";
|
||||||
|
$storePath = addToStore("$tmpDir/export", 1, "sha256");
|
||||||
|
$sha256 = queryPathHash($storePath);
|
||||||
|
$sha256 =~ s/sha256://;
|
||||||
|
|
||||||
|
txn_do($self->{db}, sub {
|
||||||
|
$self->{db}->resultset('CachedDarcsInputs')->update_or_create(
|
||||||
|
{ uri => $uri
|
||||||
|
, revision => $revision
|
||||||
|
, revcount => $revCount
|
||||||
|
, sha256hash => $sha256
|
||||||
|
, storepath => $storePath
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
$revision =~ /^([0-9]+)/;
|
||||||
|
my $shortRev = $1;
|
||||||
|
|
||||||
|
return
|
||||||
|
{ uri => $uri
|
||||||
|
, storePath => $storePath
|
||||||
|
, sha256hash => $sha256
|
||||||
|
, revision => $revision
|
||||||
|
, revCount => int($revCount)
|
||||||
|
, shortRev => $shortRev
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
1;
|
|
@ -34,8 +34,13 @@ sub fetchInput {
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
print STDERR "copying input ", $name, " from $uri\n";
|
print STDERR "copying input ", $name, " from $uri\n";
|
||||||
|
if ( $uri =~ /^\// ) {
|
||||||
$storePath = `nix-store --add "$uri"`
|
$storePath = `nix-store --add "$uri"`
|
||||||
or die "cannot copy path $uri to the Nix store.\n";
|
or die "cannot copy path $uri to the Nix store.\n";
|
||||||
|
} else {
|
||||||
|
$storePath = `PRINT_PATH=1 nix-prefetch-url "$uri" | tail -n 1`
|
||||||
|
or die "cannot fetch $uri to the Nix store.\n";
|
||||||
|
}
|
||||||
chomp $storePath;
|
chomp $storePath;
|
||||||
|
|
||||||
$sha256 = (queryPathInfo($storePath, 0))[1] or die;
|
$sha256 = (queryPathInfo($storePath, 0))[1] or die;
|
||||||
|
|
86
src/lib/Hydra/Schema/CachedDarcsInputs.pm
Normal file
86
src/lib/Hydra/Schema/CachedDarcsInputs.pm
Normal file
|
@ -0,0 +1,86 @@
|
||||||
|
use utf8;
|
||||||
|
package Hydra::Schema::CachedDarcsInputs;
|
||||||
|
|
||||||
|
# Created by DBIx::Class::Schema::Loader
|
||||||
|
# DO NOT MODIFY THE FIRST PART OF THIS FILE
|
||||||
|
|
||||||
|
=head1 NAME
|
||||||
|
|
||||||
|
Hydra::Schema::CachedDarcsInputs
|
||||||
|
|
||||||
|
=cut
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
|
||||||
|
use base 'DBIx::Class::Core';
|
||||||
|
|
||||||
|
=head1 TABLE: C<CachedDarcsInputs>
|
||||||
|
|
||||||
|
=cut
|
||||||
|
|
||||||
|
__PACKAGE__->table("CachedDarcsInputs");
|
||||||
|
|
||||||
|
=head1 ACCESSORS
|
||||||
|
|
||||||
|
=head2 uri
|
||||||
|
|
||||||
|
data_type: 'text'
|
||||||
|
is_nullable: 0
|
||||||
|
|
||||||
|
=head2 branch
|
||||||
|
|
||||||
|
data_type: 'text'
|
||||||
|
is_nullable: 0
|
||||||
|
|
||||||
|
=head2 revision
|
||||||
|
|
||||||
|
data_type: 'text'
|
||||||
|
is_nullable: 0
|
||||||
|
|
||||||
|
=head2 sha256hash
|
||||||
|
|
||||||
|
data_type: 'text'
|
||||||
|
is_nullable: 0
|
||||||
|
|
||||||
|
=head2 storepath
|
||||||
|
|
||||||
|
data_type: 'text'
|
||||||
|
is_nullable: 0
|
||||||
|
|
||||||
|
=cut
|
||||||
|
|
||||||
|
__PACKAGE__->add_columns(
|
||||||
|
"uri",
|
||||||
|
{ data_type => "text", is_nullable => 0 },
|
||||||
|
"revision",
|
||||||
|
{ data_type => "text", is_nullable => 0 },
|
||||||
|
"revcount",
|
||||||
|
{ data_type => "integer", is_nullable => 0 },
|
||||||
|
"sha256hash",
|
||||||
|
{ data_type => "text", is_nullable => 0 },
|
||||||
|
"storepath",
|
||||||
|
{ data_type => "text", is_nullable => 0 },
|
||||||
|
);
|
||||||
|
|
||||||
|
=head1 PRIMARY KEY
|
||||||
|
|
||||||
|
=over 4
|
||||||
|
|
||||||
|
=item * L</uri>
|
||||||
|
|
||||||
|
=item * L</branch>
|
||||||
|
|
||||||
|
=item * L</revision>
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
=cut
|
||||||
|
|
||||||
|
__PACKAGE__->set_primary_key("uri", "revision");
|
||||||
|
|
||||||
|
|
||||||
|
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
|
||||||
|
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fx3yosWMmJ+MnvL/dSWtFA
|
||||||
|
|
||||||
|
1;
|
|
@ -322,6 +322,15 @@ create table CachedGitInputs (
|
||||||
primary key (uri, branch, revision)
|
primary key (uri, branch, revision)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
create table CachedDarcsInputs (
|
||||||
|
uri text not null,
|
||||||
|
revision text not null,
|
||||||
|
sha256hash text not null,
|
||||||
|
storePath text not null,
|
||||||
|
revCount integer not null,
|
||||||
|
primary key (uri, revision)
|
||||||
|
);
|
||||||
|
|
||||||
create table CachedHgInputs (
|
create table CachedHgInputs (
|
||||||
uri text not null,
|
uri text not null,
|
||||||
branch text not null,
|
branch text not null,
|
||||||
|
|
8
src/sql/upgrade-darcs.sql
Normal file
8
src/sql/upgrade-darcs.sql
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
create table CachedDarcsInputs (
|
||||||
|
uri text not null,
|
||||||
|
revision text not null,
|
||||||
|
sha256hash text not null,
|
||||||
|
storePath text not null,
|
||||||
|
revCount integer not null,
|
||||||
|
primary key (uri, revision)
|
||||||
|
);
|
|
@ -7,7 +7,7 @@ use Setup;
|
||||||
|
|
||||||
my $db = Hydra::Model::DB->new;
|
my $db = Hydra::Model::DB->new;
|
||||||
|
|
||||||
use Test::Simple tests => 68;
|
use Test::Simple tests => 72;
|
||||||
|
|
||||||
hydra_setup($db);
|
hydra_setup($db);
|
||||||
|
|
||||||
|
@ -102,6 +102,13 @@ my @scminputs = (
|
||||||
type => "hg",
|
type => "hg",
|
||||||
uri => "$jobsBaseUri/hg-repo",
|
uri => "$jobsBaseUri/hg-repo",
|
||||||
update => getcwd . "/jobs/hg-update.sh"
|
update => getcwd . "/jobs/hg-update.sh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name => "darcs",
|
||||||
|
nixexpr => "darcs-input.nix",
|
||||||
|
type => "darcs",
|
||||||
|
uri => "$jobsBaseUri/darcs-repo",
|
||||||
|
update => getcwd . "/jobs/darcs-update.sh"
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
10
tests/jobs/darcs-input.nix
Normal file
10
tests/jobs/darcs-input.nix
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
with import ./config.nix;
|
||||||
|
{ src }:
|
||||||
|
{
|
||||||
|
copy =
|
||||||
|
mkDerivation {
|
||||||
|
name = "git-input";
|
||||||
|
builder = ./scm-builder.sh;
|
||||||
|
inherit src;
|
||||||
|
};
|
||||||
|
}
|
24
tests/jobs/darcs-update.sh
Executable file
24
tests/jobs/darcs-update.sh
Executable file
|
@ -0,0 +1,24 @@
|
||||||
|
#! /bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
repo="$1"
|
||||||
|
STATE_FILE=$(pwd)/.hg-state
|
||||||
|
if test -e $STATE_FILE; then
|
||||||
|
state=$(cat $STATE_FILE)
|
||||||
|
test $state -gt 1 && state=0
|
||||||
|
else
|
||||||
|
state=0;
|
||||||
|
fi
|
||||||
|
|
||||||
|
case $state in
|
||||||
|
(0) echo "::Create repo. -- continue -- updated::"
|
||||||
|
mkdir darcs-repo
|
||||||
|
darcs init --repodir darcs-repo
|
||||||
|
touch darcs-repo/file
|
||||||
|
darcs add --repodir darcs-repo file
|
||||||
|
darcs record --repodir darcs-repo -a -l -m "add a file" file -A foobar@bar.bar
|
||||||
|
;;
|
||||||
|
(*) echo "::End. -- stop -- nothing::" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo $(($state + 1)) > $STATE_FILE
|
|
@ -7,7 +7,7 @@ my $db = Hydra::Model::DB->new;
|
||||||
my @sources = $db->schema->sources;
|
my @sources = $db->schema->sources;
|
||||||
my $nrtables = scalar(@sources);
|
my $nrtables = scalar(@sources);
|
||||||
|
|
||||||
use Test::Simple tests => 41;
|
use Test::Simple tests => 42;
|
||||||
|
|
||||||
foreach my $source (@sources) {
|
foreach my $source (@sources) {
|
||||||
my $title = "Basic select query for $source";
|
my $title = "Basic select query for $source";
|
||||||
|
|
Loading…
Reference in a new issue