From 5a35912956b40f73e0c7235581fbdea998df2966 Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Tue, 5 Feb 2013 19:50:58 +0100 Subject: [PATCH 1/8] Add support for darcs repositories. --- src/lib/Hydra/Helper/AddBuilds.pm | 1 - src/lib/Hydra/Plugin/DarcsInput.pm | 105 ++++++++++++++++++++++ src/lib/Hydra/Schema/CachedDarcsInputs.pm | 86 ++++++++++++++++++ src/sql/hydra.sql | 9 ++ src/sql/upgrade-darcs.sql | 8 ++ tests/query-all-tables.pl | 2 +- 6 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 src/lib/Hydra/Plugin/DarcsInput.pm create mode 100644 src/lib/Hydra/Schema/CachedDarcsInputs.pm create mode 100644 src/sql/upgrade-darcs.sql diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index ec62811d..cf6d7243 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -149,7 +149,6 @@ sub fetchInputSystemBuild { return @inputs; } - sub fetchInput { my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_; my @inputs; diff --git a/src/lib/Hydra/Plugin/DarcsInput.pm b/src/lib/Hydra/Plugin/DarcsInput.pm new file mode 100644 index 00000000..4c593842 --- /dev/null +++ b/src/lib/Hydra/Plugin/DarcsInput.pm @@ -0,0 +1,105 @@ +package Hydra::Plugin::DarcsInput; + +use strict; +use parent 'Hydra::Plugin'; +use Digest::SHA qw(sha256_hex); +use File::Path; +use Hydra::Helper::Nix; +use Nix::Store; + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'darcs'} = 'Darcs checkout'; +} + +sub fetchInput { + my ($self, $type, $name, $uri) = @_; + + return undef if $type ne "darcs"; + + my $timestamp = time; + my $sha256; + my $storePath; + my $revCount; + + my $cacheDir = getSCMCacheDir . "/git"; + mkpath($cacheDir); + my $clonePath = $cacheDir . "/" . sha256_hex($uri); + $uri =~ s|^file://||; # darcs wants paths, not file:// uris + + chdir $ENV{"TMPDIR"}; # sigh. darcs needs a writeable working directory + + my $stdout = ""; my $stderr = ""; my $res; + if (! -d $clonePath) { + # Clone the repository. + ($res, $stdout, $stderr) = captureStdoutStderr(600, + ("darcs", "get", "--lazy", $uri, $clonePath)); + die "Error getting darcs repo at `$uri':\n$stderr" if $res; + } + + # Update the repository to match $uri. + ($res, $stdout, $stderr) = captureStdoutStderr(600, + ("darcs", "pull", "-a", "--repodir", $clonePath, "$uri")); + die "Error fetching latest change from darcs repo at `$uri':\n$stderr" if $res; + + ($res, $stdout, $stderr) = captureStdoutStderr(600, + ("darcs", "changes", "--last", "1", "--xml", "--repodir", $clonePath)); + die "Error getting revision ID of darcs repo at `$uri':\n$stderr" if $res; + + $stdout =~ /^{db}->resultset('CachedDarcsInputs')->search( + {uri => $uri, revision => $revision}, + {rows => 1}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + $revision = $cachedInput->revision; + $revCount = $cachedInput->revcount; + } else { + # Then download this revision into the store. + print STDERR "checking out darcs repo $uri\n"; + + my $tmpDir = File::Temp->newdir("hydra-darcs-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die; + (system "darcs", "get", "--lazy", $clonePath, "$tmpDir/export", "--quiet", + "--to-match", "hash $revision") == 0 + or die "darcs export failed"; + $revCount = `darcs changes --count --repodir $tmpDir/export`; chomp $revCount; + die "darcs changes --count failed" if $? != 0; + + system "rm", "-rf", "$tmpDir/export/_darcs"; + $storePath = addToStore("$tmpDir/export", 1, "sha256"); + $sha256 = queryPathHash($storePath); + $sha256 =~ s/sha256://; + + txn_do($self->{db}, sub { + $self->{db}->resultset('CachedDarcsInputs')->update_or_create( + { uri => $uri + , revision => $revision + , revcount => $revCount + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } + + $revision =~ /^([0-9]+)/; + my $shortRev = $1; + + return + { uri => $uri + , storePath => $storePath + , sha256hash => $sha256 + , revision => $revision + , revCount => int($revCount) + , shortRev => $shortRev + }; +} + +1; diff --git a/src/lib/Hydra/Schema/CachedDarcsInputs.pm b/src/lib/Hydra/Schema/CachedDarcsInputs.pm new file mode 100644 index 00000000..426f6f2f --- /dev/null +++ b/src/lib/Hydra/Schema/CachedDarcsInputs.pm @@ -0,0 +1,86 @@ +use utf8; +package Hydra::Schema::CachedDarcsInputs; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::CachedDarcsInputs + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("CachedDarcsInputs"); + +=head1 ACCESSORS + +=head2 uri + + data_type: 'text' + is_nullable: 0 + +=head2 branch + + data_type: 'text' + is_nullable: 0 + +=head2 revision + + data_type: 'text' + is_nullable: 0 + +=head2 sha256hash + + data_type: 'text' + is_nullable: 0 + +=head2 storepath + + data_type: 'text' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "uri", + { data_type => "text", is_nullable => 0 }, + "revision", + { data_type => "text", is_nullable => 0 }, + "revcount", + { data_type => "integer", is_nullable => 0 }, + "sha256hash", + { data_type => "text", is_nullable => 0 }, + "storepath", + { data_type => "text", is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("uri", "revision"); + + +# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fx3yosWMmJ+MnvL/dSWtFA + +1; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 7bbd26b3..12e56ff2 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -322,6 +322,15 @@ create table CachedGitInputs ( primary key (uri, branch, revision) ); +create table CachedDarcsInputs ( + uri text not null, + revision text not null, + sha256hash text not null, + storePath text not null, + revCount integer not null, + primary key (uri, revision) +); + create table CachedHgInputs ( uri text not null, branch text not null, diff --git a/src/sql/upgrade-darcs.sql b/src/sql/upgrade-darcs.sql new file mode 100644 index 00000000..17df74c0 --- /dev/null +++ b/src/sql/upgrade-darcs.sql @@ -0,0 +1,8 @@ +create table CachedDarcsInputs ( + uri text not null, + revision text not null, + sha256hash text not null, + storePath text not null, + revCount integer not null, + primary key (uri, revision) +); diff --git a/tests/query-all-tables.pl b/tests/query-all-tables.pl index c484c4a8..d786505c 100755 --- a/tests/query-all-tables.pl +++ b/tests/query-all-tables.pl @@ -7,7 +7,7 @@ my $db = Hydra::Model::DB->new; my @sources = $db->schema->sources; my $nrtables = scalar(@sources); -use Test::Simple tests => 41; +use Test::Simple tests => 45; foreach my $source (@sources) { my $title = "Basic select query for $source"; From bb8059e50ca5e8cdecd2360382668f68cafd139d Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Tue, 5 Feb 2013 19:52:46 +0100 Subject: [PATCH 2/8] Add support for fetching URLs as inputs. --- src/lib/Hydra/Plugin/PathInput.pm | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib/Hydra/Plugin/PathInput.pm b/src/lib/Hydra/Plugin/PathInput.pm index d913782d..551fc94a 100644 --- a/src/lib/Hydra/Plugin/PathInput.pm +++ b/src/lib/Hydra/Plugin/PathInput.pm @@ -34,8 +34,13 @@ sub fetchInput { } else { print STDERR "copying input ", $name, " from $uri\n"; - $storePath = `nix-store --add "$uri"` - or die "cannot copy path $uri to the Nix store.\n"; + if ( $uri =~ /^\// ) { + $storePath = `nix-store --add "$uri"` + or die "cannot copy path $uri to the Nix store.\n"; + } else { + $storePath = `PRINT_PATH=1 nix-prefetch-url "$uri" | tail -n 1` + or die "cannot fetch $uri to the Nix store.\n"; + } chomp $storePath; $sha256 = (queryPathInfo($storePath, 0))[1] or die; From dcf386cfedc392cd68f194e9f70f289d22441d8b Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Wed, 13 Feb 2013 13:09:07 +0100 Subject: [PATCH 3/8] Make getDrvLogPath work with both bucketed and non-bucketed nix logs. --- src/lib/Hydra/Helper/Nix.pm | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 93dc3bf7..024194a6 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -253,13 +253,11 @@ sub getLatestSuccessfulViewResult { sub getDrvLogPath { my ($drvPath) = @_; my $base = basename $drvPath; - my $fn = - ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/" - . substr($base, 0, 2) . "/" - . substr($base, 2); - return $fn if -f $fn; - $fn .= ".bz2"; - return $fn if -f $fn; + my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2); + my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/"; + for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) { + return $_ if (-f $_); + } return undef; } From bb38bdfe359a035dd46dfeecd6129d26b96c67c0 Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Mon, 27 May 2013 12:42:52 +0200 Subject: [PATCH 4/8] Allow : and + in build product filenames. --- src/lib/Hydra/Controller/Build.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 167bab12..a347dd24 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -298,7 +298,7 @@ sub contents : Chained('buildChain') PathPart Args(1) { notFound($c, "Product $path has disappeared.") unless -e $path; # Sanitize $path to prevent shell injection attacks. - $path =~ /^\/[\/[A-Za-z0-9_\-\.=]+$/ or die "Filename contains illegal characters.\n"; + $path =~ /^\/[\/[A-Za-z0-9_\-\.=+:]+$/ or die "Filename contains illegal characters.\n"; # FIXME: don't use shell invocations below. From 1e1a1f083872ea27d6239fbbc19e1d058b62ca94 Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Mon, 27 May 2013 13:01:23 +0200 Subject: [PATCH 5/8] Also allow : (colon) in pathCompRE. --- src/lib/Hydra/Helper/CatalystUtils.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index a13f2570..67bfe88e 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -210,7 +210,7 @@ sub paramToList { # Security checking of filenames. -Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$]*)"; +Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$:]*)"; Readonly our $relPathRE => "(?:$pathCompRE(?:/$pathCompRE)*)"; Readonly our $relNameRE => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)"; Readonly our $attrNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)"; From d31e4469bb6dd11f8555f1797ebdc6186cfa9b42 Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Sun, 2 Jun 2013 14:34:19 +0200 Subject: [PATCH 6/8] Put a 5-second CPU time limit on the log processing pipeline. --- src/lib/Hydra/Controller/Build.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index a347dd24..e17056bf 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -158,7 +158,7 @@ sub showLog { . " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -" . " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2"; $c->stash->{template} = 'log.tt'; - $c->stash->{logtext} = `$pipeline`; + $c->stash->{logtext} = `ulimit -t 5 ; $pipeline`; } elsif ($mode eq "raw") { From 66f3e60e2a85081ebdc4862ced32429a3e525934 Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Wed, 10 Jul 2013 11:50:24 +0200 Subject: [PATCH 7/8] Add a test for darcs inputs. --- release.nix | 4 ++-- tests/evaluation-tests.pl | 9 ++++++++- tests/jobs/darcs-input.nix | 10 ++++++++++ tests/jobs/darcs-update.sh | 24 ++++++++++++++++++++++++ tests/query-all-tables.pl | 2 +- 5 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 tests/jobs/darcs-input.nix create mode 100755 tests/jobs/darcs-update.sh diff --git a/release.nix b/release.nix index 868ba424..05366f74 100644 --- a/release.nix +++ b/release.nix @@ -112,14 +112,14 @@ in rec { buildInputs = [ makeWrapper libtool unzip nukeReferences pkgconfig boehmgc sqlite - gitAndTools.topGit mercurial subversion bazaar openssl bzip2 + gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 guile # optional, for Guile + Guix support perlDeps perl ]; hydraPath = lib.makeSearchPath "bin" ( [ libxslt sqlite subversion openssh nix coreutils findutils - gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial gnused graphviz bazaar + gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused graphviz bazaar ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); preCheck = '' diff --git a/tests/evaluation-tests.pl b/tests/evaluation-tests.pl index 2044c1be..90ae41df 100755 --- a/tests/evaluation-tests.pl +++ b/tests/evaluation-tests.pl @@ -7,7 +7,7 @@ use Setup; my $db = Hydra::Model::DB->new; -use Test::Simple tests => 68; +use Test::Simple tests => 72; hydra_setup($db); @@ -102,6 +102,13 @@ my @scminputs = ( type => "hg", uri => "$jobsBaseUri/hg-repo", update => getcwd . "/jobs/hg-update.sh" + }, + { + name => "darcs", + nixexpr => "darcs-input.nix", + type => "darcs", + uri => "$jobsBaseUri/darcs-repo", + update => getcwd . "/jobs/darcs-update.sh" } ); diff --git a/tests/jobs/darcs-input.nix b/tests/jobs/darcs-input.nix new file mode 100644 index 00000000..9374b1f6 --- /dev/null +++ b/tests/jobs/darcs-input.nix @@ -0,0 +1,10 @@ +with import ./config.nix; +{ src }: +{ + copy = + mkDerivation { + name = "git-input"; + builder = ./scm-builder.sh; + inherit src; + }; +} diff --git a/tests/jobs/darcs-update.sh b/tests/jobs/darcs-update.sh new file mode 100755 index 00000000..164a9e9d --- /dev/null +++ b/tests/jobs/darcs-update.sh @@ -0,0 +1,24 @@ +#! /bin/sh +set -e + +repo="$1" +STATE_FILE=$(pwd)/.hg-state +if test -e $STATE_FILE; then + state=$(cat $STATE_FILE) + test $state -gt 1 && state=0 +else + state=0; +fi + +case $state in + (0) echo "::Create repo. -- continue -- updated::" + mkdir darcs-repo + darcs init --repodir darcs-repo + touch darcs-repo/file + darcs add --repodir darcs-repo file + darcs record --repodir darcs-repo -a -l -m "add a file" file -A foobar@bar.bar + ;; + (*) echo "::End. -- stop -- nothing::" ;; +esac + +echo $(($state + 1)) > $STATE_FILE diff --git a/tests/query-all-tables.pl b/tests/query-all-tables.pl index d786505c..3b25057d 100755 --- a/tests/query-all-tables.pl +++ b/tests/query-all-tables.pl @@ -7,7 +7,7 @@ my $db = Hydra::Model::DB->new; my @sources = $db->schema->sources; my $nrtables = scalar(@sources); -use Test::Simple tests => 45; +use Test::Simple tests => 42; foreach my $source (@sources) { my $title = "Basic select query for $source"; From 0041c336d1311bff0478e7cdb41316ccd1dc82eb Mon Sep 17 00:00:00 2001 From: Petr Rockai Date: Sat, 7 Sep 2013 14:47:49 +0200 Subject: [PATCH 8/8] DarcsInput: Avoid a chdir. --- src/lib/Hydra/Plugin/DarcsInput.pm | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/Hydra/Plugin/DarcsInput.pm b/src/lib/Hydra/Plugin/DarcsInput.pm index 4c593842..c6227123 100644 --- a/src/lib/Hydra/Plugin/DarcsInput.pm +++ b/src/lib/Hydra/Plugin/DarcsInput.pm @@ -27,14 +27,13 @@ sub fetchInput { my $clonePath = $cacheDir . "/" . sha256_hex($uri); $uri =~ s|^file://||; # darcs wants paths, not file:// uris - chdir $ENV{"TMPDIR"}; # sigh. darcs needs a writeable working directory - my $stdout = ""; my $stderr = ""; my $res; if (! -d $clonePath) { # Clone the repository. - ($res, $stdout, $stderr) = captureStdoutStderr(600, - ("darcs", "get", "--lazy", $uri, $clonePath)); - die "Error getting darcs repo at `$uri':\n$stderr" if $res; + $res = run(timeout => 600, + cmd => ["darcs", "get", "--lazy", $uri, $clonePath], + dir => $ENV{"TMPDIR"}); + die "Error getting darcs repo at `$uri':\n$stderr" if $res->{status}; } # Update the repository to match $uri.