Support revision control systems via plugins
This commit is contained in:
Eelco Dolstra
parent
5e0542d3af
commit
1f1615e80b
|
|
@ -4,6 +4,7 @@ use strict;
|
|||
use warnings;
|
||||
use parent 'Catalyst';
|
||||
use Moose;
|
||||
use Hydra::Plugin;
|
||||
use Hydra::Model::DB;
|
||||
use Catalyst::Runtime '5.70';
|
||||
use Catalyst qw/ConfigLoader
|
||||
|
|
@ -82,6 +83,18 @@ __PACKAGE__->config(
|
|||
|
||||
__PACKAGE__->apply_request_class_roles(qw/Catalyst::TraitFor::Request::ProxyBase/);
|
||||
|
||||
my $plugins;
|
||||
|
||||
has 'hydra_plugins' => (
|
||||
is => 'ro',
|
||||
default => sub { return $plugins; }
|
||||
);
|
||||
|
||||
after setup_finalize => sub {
|
||||
my $class = shift;
|
||||
$plugins = [Hydra::Plugin->plugins(db => $class->model('DB'), config => $class->config)];
|
||||
};
|
||||
|
||||
__PACKAGE__->setup();
|
||||
|
||||
1;
|
||||
|
|
|
|||
|
|
@ -219,15 +219,17 @@ sub scmdiff : Chained('api') PathPart('scmdiff') Args(0) {
|
|||
|
||||
die("invalid revisions: [$rev1] [$rev2]") if $rev1 !~ m/^[a-zA-Z0-9_.]+$/ || $rev2 !~ m/^[a-zA-Z0-9_.]+$/;
|
||||
|
||||
# FIXME: injection danger.
|
||||
|
||||
my $diff = "";
|
||||
if ($type eq "hg") {
|
||||
my $clonePath = scmPath . "/" . sha256_hex($uri);
|
||||
my $clonePath = getSCMCacheDir . "/hg/" . sha256_hex($uri);
|
||||
die if ! -d $clonePath;
|
||||
$branch = `(cd $clonePath; hg log --template '{branch}' -r $rev2)`;
|
||||
$diff .= `(cd $clonePath; hg log -r $rev1 -r $rev2 -b $branch)`;
|
||||
$diff .= `(cd $clonePath; hg diff -r $rev1:$rev2)`;
|
||||
} elsif ($type eq "git") {
|
||||
my $clonePath = scmPath . "/" . sha256_hex($uri);
|
||||
my $clonePath = getSCMCacheDir . "/git/" . sha256_hex($uri);
|
||||
die if ! -d $clonePath;
|
||||
$diff .= `(cd $clonePath; git log $rev1..$rev2)`;
|
||||
$diff .= `(cd $clonePath; git diff $rev1..$rev2)`;
|
||||
|
|
|
|||
|
|
@ -533,7 +533,7 @@ sub clone_submit : Chained('build') PathPart('clone/submit') Args(0) {
|
|||
# should be done asynchronously. But then error reporting
|
||||
# becomes harder.
|
||||
my $info = fetchInput(
|
||||
$c->model('DB'), $build->project, $build->jobset,
|
||||
$c->hydra_plugins, $c->model('DB'), $build->project, $build->jobset,
|
||||
$inputName, $inputType, $inputValue);
|
||||
push @{$$inputInfo{$inputName}}, $info if defined $info;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -28,6 +28,15 @@ sub begin :Private {
|
|||
$c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count();
|
||||
$c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count();
|
||||
}
|
||||
|
||||
# Gather the supported input types.
|
||||
$c->stash->{inputTypes} = {
|
||||
'string' => 'String value',
|
||||
'boolean' => 'Boolean',
|
||||
'build' => 'Build output',
|
||||
'sysbuild' => 'Build output (same system)'
|
||||
};
|
||||
$_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package Hydra::Helper::AddBuilds;
|
|||
use strict;
|
||||
use feature 'switch';
|
||||
use XML::Simple;
|
||||
use POSIX qw(strftime);
|
||||
use IPC::Run;
|
||||
use Nix::Store;
|
||||
use Nix::Config;
|
||||
|
|
@ -19,31 +18,12 @@ use File::Slurp;
|
|||
|
||||
our @ISA = qw(Exporter);
|
||||
our @EXPORT = qw(
|
||||
fetchInput evalJobs checkBuild inputsToArgs captureStdoutStderr
|
||||
getReleaseName addBuildProducts restartBuild scmPath
|
||||
fetchInput evalJobs checkBuild inputsToArgs
|
||||
getReleaseName addBuildProducts restartBuild
|
||||
getPrevJobsetEval
|
||||
);
|
||||
|
||||
|
||||
sub scmPath {
|
||||
return Hydra::Model::DB::getHydraPath . "/scm" ;
|
||||
}
|
||||
|
||||
|
||||
sub getStorePathHash {
|
||||
my ($storePath) = @_;
|
||||
my $hash = `nix-store --query --hash $storePath`
|
||||
or die "cannot get hash of $storePath";
|
||||
chomp $hash;
|
||||
die unless $hash =~ /^sha256:(.*)$/;
|
||||
$hash = $1;
|
||||
$hash = `nix-hash --to-base16 --type sha256 $hash`
|
||||
or die "cannot convert hash";
|
||||
chomp $hash;
|
||||
return $hash;
|
||||
}
|
||||
|
||||
|
||||
sub getReleaseName {
|
||||
my ($outPath) = @_;
|
||||
return undef unless -f "$outPath/nix-support/hydra-release-name";
|
||||
|
|
@ -92,140 +72,6 @@ sub attrsToSQL {
|
|||
}
|
||||
|
||||
|
||||
sub fetchInputPath {
|
||||
my ($db, $project, $jobset, $name, $value) = @_;
|
||||
|
||||
my $uri = $value;
|
||||
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
# Some simple caching: don't check a path more than once every N seconds.
|
||||
(my $cachedInput) = $db->resultset('CachedPathInputs')->search(
|
||||
{srcpath => $uri, lastseen => {">", $timestamp - 30}},
|
||||
{rows => 1, order_by => "lastseen DESC"});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
} else {
|
||||
|
||||
print STDERR "copying input ", $name, " from $uri\n";
|
||||
$storePath = `nix-store --add "$uri"`
|
||||
or die "cannot copy path $uri to the Nix store.\n";
|
||||
chomp $storePath;
|
||||
|
||||
$sha256 = getStorePathHash $storePath;
|
||||
|
||||
($cachedInput) = $db->resultset('CachedPathInputs')->search(
|
||||
{srcpath => $uri, sha256hash => $sha256});
|
||||
|
||||
# Path inputs don't have a natural notion of a "revision", so
|
||||
# we simulate it by using the timestamp that we first saw this
|
||||
# path have this SHA-256 hash. So if the contents of the path
|
||||
# changes, we get a new "revision", but if it doesn't change
|
||||
# (or changes back), we don't get a new "revision".
|
||||
if (!defined $cachedInput) {
|
||||
txn_do($db, sub {
|
||||
$db->resultset('CachedPathInputs')->update_or_create(
|
||||
{ srcpath => $uri
|
||||
, timestamp => $timestamp
|
||||
, lastseen => $timestamp
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
} else {
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
txn_do($db, sub {
|
||||
$cachedInput->update({lastseen => time});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
sub fetchInputSVN {
|
||||
my ($db, $project, $jobset, $name, $value, $checkout) = @_;
|
||||
|
||||
# Allow users to specify a revision number next to the URI.
|
||||
my ($uri, $revision) = split ' ', $value;
|
||||
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
my $stdout; my $stderr;
|
||||
|
||||
unless (defined $revision) {
|
||||
# First figure out the last-modified revision of the URI.
|
||||
my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],
|
||||
"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
|
||||
IPC::Run::run(@cmd, \$stdout, \$stderr);
|
||||
die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?;
|
||||
$revision = $stdout; $revision =~ s/\s*([0-9]+)\s*/$1/sm;
|
||||
}
|
||||
|
||||
die unless $revision =~ /^\d+$/;
|
||||
|
||||
# Do we already have this revision in the store?
|
||||
# !!! This needs to take $checkout into account! Otherwise "svn"
|
||||
# and "svn-checkout" inputs can get mixed up.
|
||||
(my $cachedInput) = $db->resultset('CachedSubversionInputs')->search(
|
||||
{uri => $uri, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
|
||||
# No, do a checkout. The working copy is reused between
|
||||
# invocations to speed things up.
|
||||
my $wcPath = scmPath . "/svn/" . sha256_hex($uri) . "/svn-checkout";
|
||||
|
||||
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath);
|
||||
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
if ($checkout) {
|
||||
$storePath = addToStore($wcPath, 1, "sha256");
|
||||
} else {
|
||||
# Hm, if the Nix Perl bindings supported filters in
|
||||
# addToStore(), then we wouldn't need to make a copy here.
|
||||
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
|
||||
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
|
||||
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
|
||||
}
|
||||
|
||||
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
|
||||
|
||||
txn_do($db, sub {
|
||||
$db->resultset('CachedSubversionInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
sub fetchInputBuild {
|
||||
my ($db, $project, $jobset, $name, $value) = @_;
|
||||
|
||||
|
|
@ -278,7 +124,7 @@ sub fetchInputSystemBuild {
|
|||
|
||||
if (scalar(@validBuilds) == 0) {
|
||||
print STDERR "input `", $name, "': no previous build available\n";
|
||||
return undef;
|
||||
return ();
|
||||
}
|
||||
|
||||
my @inputs = ();
|
||||
|
|
@ -303,323 +149,37 @@ sub fetchInputSystemBuild {
|
|||
}
|
||||
|
||||
|
||||
sub fetchInputGit {
|
||||
my ($db, $project, $jobset, $name, $value) = @_;
|
||||
|
||||
(my $uri, my $branch, my $deepClone) = split ' ', $value;
|
||||
$branch = defined $branch ? $branch : "master";
|
||||
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
mkpath(scmPath);
|
||||
my $clonePath = scmPath . "/" . sha256_hex($uri);
|
||||
|
||||
my $stdout = ""; my $stderr = ""; my $res;
|
||||
if (! -d $clonePath) {
|
||||
# Clone everything and fetch the branch.
|
||||
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath);
|
||||
die "error cloning git repo at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir
|
||||
|
||||
# This command force the update of the local branch to be in the same as
|
||||
# the remote branch for whatever the repository state is. This command mirror
|
||||
# only one branch of the remote repository.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"git", "fetch", "-fu", "origin", "+$branch:$branch");
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"git", "fetch", "-fu", "origin") if $res;
|
||||
die "error fetching latest change from git repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
("git", "rev-parse", "$branch"));
|
||||
die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
my ($revision) = split /\n/, $stdout;
|
||||
die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout"
|
||||
unless $revision =~ /^[0-9a-fA-F]+$/;
|
||||
|
||||
my $ref = "refs/heads/$branch";
|
||||
|
||||
# If deepClone is defined, then we look at the content of the repository
|
||||
# to determine if this is a top-git branch.
|
||||
if (defined $deepClone) {
|
||||
|
||||
# Checkout the branch to look at its content.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch");
|
||||
die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
if (-f ".topdeps") {
|
||||
# This is a TopGit branch. Fetch all the topic branches so
|
||||
# that builders can run "tg patch" and similar.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"tg", "remote", "--populate", "origin");
|
||||
print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res;
|
||||
}
|
||||
}
|
||||
|
||||
# Some simple caching: don't check a uri/branch/revision more than once.
|
||||
# TODO: Fix case where the branch is reset to a previous commit.
|
||||
my $cachedInput ;
|
||||
($cachedInput) = $db->resultset('CachedGitInputs')->search(
|
||||
{uri => $uri, branch => $branch, revision => $revision},
|
||||
{rows => 1});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$revision = $cachedInput->revision;
|
||||
} else {
|
||||
# Then download this revision into the store.
|
||||
print STDERR "checking out Git branch $branch from $uri\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
|
||||
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
|
||||
|
||||
if (defined $deepClone) {
|
||||
# Checked out code often wants to be able to run `git
|
||||
# describe', e.g., code that uses Gnulib's `git-version-gen'
|
||||
# script. Thus, we leave `.git' in there. Same for
|
||||
# Subversion (e.g., libgcrypt's build system uses that.)
|
||||
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
|
||||
|
||||
# Ask for a "deep clone" to allow "git describe" and similar
|
||||
# tools to work. See
|
||||
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
|
||||
# for a discussion.
|
||||
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
|
||||
}
|
||||
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "nix-prefetch-git", $clonePath, $revision);
|
||||
die "cannot check out Git repository branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($db, sub {
|
||||
$db->resultset('CachedGitInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
# For convenience in producing readable version names, pass the
|
||||
# number of commits in the history of this revision (‘revCount’)
|
||||
# the output of git-describe (‘gitTag’), and the abbreviated
|
||||
# revision (‘shortRev’).
|
||||
my $revCount = `git rev-list $revision | wc -l`; chomp $revCount;
|
||||
die "git rev-list failed" if $? != 0;
|
||||
my $gitTag = `git describe --always $revision`; chomp $gitTag;
|
||||
die "git describe failed" if $? != 0;
|
||||
my $shortRev = `git rev-parse --short $revision`; chomp $shortRev;
|
||||
die "git rev-parse failed" if $? != 0;
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
, revCount => int($revCount)
|
||||
, gitTag => $gitTag
|
||||
, shortRev => $shortRev
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
sub fetchInputBazaar {
|
||||
my ($db, $project, $jobset, $name, $value, $checkout) = @_;
|
||||
|
||||
my $uri = $value;
|
||||
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
my $stdout; my $stderr;
|
||||
|
||||
mkpath(scmPath);
|
||||
my $clonePath = scmPath . "/" . sha256_hex($uri);
|
||||
|
||||
if (! -d $clonePath) {
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath);
|
||||
die "error cloning bazaar branch at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
chdir $clonePath or die $!;
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull");
|
||||
die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res;
|
||||
|
||||
# First figure out the last-modified revision of the URI.
|
||||
my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
|
||||
|
||||
IPC::Run::run(@cmd, \$stdout, \$stderr);
|
||||
die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?;
|
||||
my $revision = $stdout; chomp $revision;
|
||||
die unless $revision =~ /^\d+$/;
|
||||
|
||||
(my $cachedInput) = $db->resultset('CachedBazaarInputs')->search(
|
||||
{uri => $uri, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
|
||||
# Then download this revision into the store.
|
||||
print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
$ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = "$checkout";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"nix-prefetch-bzr", $clonePath, $revision);
|
||||
die "cannot check out Bazaar branch `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($db, sub {
|
||||
$db->resultset('CachedBazaarInputs')->create(
|
||||
{ uri => $uri
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
sub fetchInputHg {
|
||||
my ($db, $project, $jobset, $name, $value) = @_;
|
||||
|
||||
(my $uri, my $id) = split ' ', $value;
|
||||
$id = defined $id ? $id : "default";
|
||||
|
||||
# init local hg clone
|
||||
|
||||
my $stdout = ""; my $stderr = "";
|
||||
|
||||
mkpath(scmPath);
|
||||
my $clonePath = scmPath . "/" . sha256_hex($uri);
|
||||
|
||||
if (! -d $clonePath) {
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"hg", "clone", $uri, $clonePath);
|
||||
die "error cloning mercurial repo at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
# hg pull + check rev
|
||||
chdir $clonePath or die $!;
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull");
|
||||
die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}");
|
||||
die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1;
|
||||
|
||||
my ($revision, $revCount, $branch) = split ' ', $stdout;
|
||||
|
||||
my $storePath;
|
||||
my $sha256;
|
||||
(my $cachedInput) = $db->resultset('CachedHgInputs')->search(
|
||||
{uri => $uri, branch => $branch, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
print STDERR "checking out Mercurial input from $uri $branch revision $revision\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"nix-prefetch-hg", $clonePath, $revision);
|
||||
die "cannot check out Mercurial repository `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($db, sub {
|
||||
$db->resultset('CachedHgInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
, revCount => int($revCount)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
sub fetchInput {
|
||||
my ($db, $project, $jobset, $name, $type, $value) = @_;
|
||||
my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_;
|
||||
my @inputs;
|
||||
|
||||
if ($type eq "path") {
|
||||
push @inputs, fetchInputPath($db, $project, $jobset, $name, $value);
|
||||
}
|
||||
elsif ($type eq "svn") {
|
||||
push @inputs, fetchInputSVN($db, $project, $jobset, $name, $value, 0);
|
||||
}
|
||||
elsif ($type eq "svn-checkout") {
|
||||
push @inputs, fetchInputSVN($db, $project, $jobset, $name, $value, 1);
|
||||
}
|
||||
elsif ($type eq "build") {
|
||||
push @inputs, fetchInputBuild($db, $project, $jobset, $name, $value);
|
||||
if ($type eq "build") {
|
||||
@inputs = fetchInputBuild($db, $project, $jobset, $name, $value);
|
||||
}
|
||||
elsif ($type eq "sysbuild") {
|
||||
push @inputs, fetchInputSystemBuild($db, $project, $jobset, $name, $value);
|
||||
}
|
||||
elsif ($type eq "git") {
|
||||
push @inputs, fetchInputGit($db, $project, $jobset, $name, $value);
|
||||
}
|
||||
elsif ($type eq "hg") {
|
||||
push @inputs, fetchInputHg($db, $project, $jobset, $name, $value);
|
||||
}
|
||||
elsif ($type eq "bzr") {
|
||||
push @inputs, fetchInputBazaar($db, $project, $jobset, $name, $value, 0);
|
||||
}
|
||||
elsif ($type eq "bzr-checkout") {
|
||||
push @inputs, fetchInputBazaar($db, $project, $jobset, $name, $value, 1);
|
||||
@inputs = fetchInputSystemBuild($db, $project, $jobset, $name, $value);
|
||||
}
|
||||
elsif ($type eq "string") {
|
||||
die unless defined $value;
|
||||
push @inputs, { value => $value };
|
||||
@inputs = { value => $value };
|
||||
}
|
||||
elsif ($type eq "boolean") {
|
||||
die unless defined $value && ($value eq "true" || $value eq "false");
|
||||
push @inputs, { value => $value };
|
||||
@inputs = { value => $value };
|
||||
}
|
||||
else {
|
||||
die "input `" . $name . "' has unknown type `$type'.";
|
||||
my $found = 0;
|
||||
foreach my $plugin (@{$plugins}) {
|
||||
@inputs = $plugin->fetchInput($type, $name, $value);
|
||||
if (defined $inputs[0]) {
|
||||
$found = 1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
die "input `$name' has unknown type `$type'." unless $found;
|
||||
}
|
||||
|
||||
foreach my $input (@inputs) {
|
||||
$input->{type} = $type if defined $input;
|
||||
}
|
||||
$_->{type} = $type foreach @inputs;
|
||||
|
||||
return @inputs;
|
||||
}
|
||||
|
|
@ -641,6 +201,7 @@ sub booleanToString {
|
|||
return $result;
|
||||
}
|
||||
|
||||
|
||||
sub buildInputToString {
|
||||
my ($exprType, $input) = @_;
|
||||
my $result;
|
||||
|
|
@ -654,6 +215,7 @@ sub buildInputToString {
|
|||
")";
|
||||
} else {
|
||||
$result = "{ outPath = builtins.storePath " . $input->{storePath} . "" .
|
||||
(defined $input->{revNumber} ? "; rev = " . $input->{revNumber} . "" : "") .
|
||||
(defined $input->{revision} ? "; rev = \"" . $input->{revision} . "\"" : "") .
|
||||
(defined $input->{revCount} ? "; revCount = " . $input->{revCount} . "" : "") .
|
||||
(defined $input->{gitTag} ? "; gitTag = \"" . $input->{gitTag} . "\"" : "") .
|
||||
|
|
@ -681,16 +243,9 @@ sub inputsToArgs {
|
|||
when ("boolean") {
|
||||
push @res, "--arg", $input, booleanToString($exprType, $alt->{value});
|
||||
}
|
||||
when (["path", "build", "git", "hg", "sysbuild"]) {
|
||||
default {
|
||||
push @res, "--arg", $input, buildInputToString($exprType, $alt);
|
||||
}
|
||||
when (["svn", "svn-checkout", "bzr", "bzr-checkout"]) {
|
||||
push @res, "--arg", $input, (
|
||||
"{ outPath = builtins.storePath " . $alt->{storePath} . "" .
|
||||
(defined $alt->{revision} ? "; rev = " . $alt->{revision} . "" : "") .
|
||||
";}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -699,28 +254,6 @@ sub inputsToArgs {
|
|||
}
|
||||
|
||||
|
||||
sub captureStdoutStderr {
|
||||
my ($timeout, @cmd) = @_;
|
||||
my $stdin = "";
|
||||
my $stdout;
|
||||
my $stderr;
|
||||
|
||||
eval {
|
||||
local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required
|
||||
alarm $timeout;
|
||||
IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr);
|
||||
alarm 0;
|
||||
};
|
||||
|
||||
if ($@) {
|
||||
die unless $@ eq "timeout\n"; # propagate unexpected errors
|
||||
return (-1, "", "timeout\n");
|
||||
} else {
|
||||
return ($?, $stdout, $stderr);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub evalJobs {
|
||||
my ($inputInfo, $exprType, $nixExprInputName, $nixExprPath) = @_;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use Hydra::Model::DB;
|
|||
our @ISA = qw(Exporter);
|
||||
our @EXPORT = qw(
|
||||
getHydraHome getHydraConfig txn_do
|
||||
getSCMCacheDir
|
||||
registerRoot getGCRootsDir gcRootFor
|
||||
getPrimaryBuildsForView
|
||||
getPrimaryBuildTotal
|
||||
|
|
@ -18,7 +19,8 @@ our @EXPORT = qw(
|
|||
jobsetOverview removeAsciiEscapes getDrvLogPath logContents
|
||||
getMainOutput
|
||||
getEvals getMachines
|
||||
pathIsInsidePrefix);
|
||||
pathIsInsidePrefix
|
||||
captureStdoutStderr);
|
||||
|
||||
|
||||
sub getHydraHome {
|
||||
|
|
@ -50,6 +52,11 @@ sub txn_do {
|
|||
}
|
||||
|
||||
|
||||
sub getSCMCacheDir {
|
||||
return Hydra::Model::DB::getHydraPath . "/scm" ;
|
||||
}
|
||||
|
||||
|
||||
sub getGCRootsDir {
|
||||
die unless defined $ENV{LOGNAME};
|
||||
my $dir = ($ENV{NIX_STATE_DIR} || "/nix/var/nix" ) . "/gcroots/per-user/$ENV{LOGNAME}/hydra-roots";
|
||||
|
|
@ -443,4 +450,26 @@ sub pathIsInsidePrefix {
|
|||
}
|
||||
|
||||
|
||||
sub captureStdoutStderr {
|
||||
my ($timeout, @cmd) = @_;
|
||||
my $stdin = "";
|
||||
my $stdout;
|
||||
my $stderr;
|
||||
|
||||
eval {
|
||||
local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required
|
||||
alarm $timeout;
|
||||
IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr);
|
||||
alarm 0;
|
||||
};
|
||||
|
||||
if ($@) {
|
||||
die unless $@ eq "timeout\n"; # propagate unexpected errors
|
||||
return (-1, "", "timeout\n");
|
||||
} else {
|
||||
return ($?, $stdout, $stderr);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
1;
|
||||
|
|
|
|||
|
|
@ -20,4 +20,18 @@ sub buildFinished {
|
|||
my ($self, $build, $dependents) = @_;
|
||||
}
|
||||
|
||||
# Called to determine the set of supported input types. The plugin
|
||||
# should add these to the $inputTypes hashref, e.g. $inputTypes{'svn'}
|
||||
# = 'Subversion checkout'.
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
}
|
||||
|
||||
# Called to fetch an input of type ‘$type’. ‘$value’ is the input
|
||||
# location, typically the repository URL.
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
return undef;
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
|||
87
src/lib/Hydra/Plugin/BazaarInput.pm
Normal file
87
src/lib/Hydra/Plugin/BazaarInput.pm
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
package Hydra::Plugin::BazaarInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Path;
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'bzr'} = 'Bazaar export';
|
||||
$inputTypes->{'bzr-checkout'} = 'Bazaar checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "bzr" && $type ne "bzr-checkout";
|
||||
|
||||
my $uri = $value;
|
||||
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
my $stdout; my $stderr;
|
||||
|
||||
my $cacheDir = getSCMCacheDir . "/bzr";
|
||||
mkpath($cacheDir);
|
||||
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||
|
||||
if (! -d $clonePath) {
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath);
|
||||
die "error cloning bazaar branch at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
chdir $clonePath or die $!;
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull");
|
||||
die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res;
|
||||
|
||||
# First figure out the last-modified revision of the URI.
|
||||
my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
|
||||
|
||||
IPC::Run::run(@cmd, \$stdout, \$stderr);
|
||||
die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?;
|
||||
my $revision = $stdout; chomp $revision;
|
||||
die unless $revision =~ /^\d+$/;
|
||||
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
|
||||
{uri => $uri, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
|
||||
# Then download this revision into the store.
|
||||
print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
$ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = $type eq "bzr-checkout" ? "1" : "0";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"nix-prefetch-bzr", $clonePath, $revision);
|
||||
die "cannot check out Bazaar branch `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedBazaarInputs')->create(
|
||||
{ uri => $uri
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
148
src/lib/Hydra/Plugin/GitInput.pm
Normal file
148
src/lib/Hydra/Plugin/GitInput.pm
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
package Hydra::Plugin::GitInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Path;
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'git'} = 'Git checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "git";
|
||||
|
||||
(my $uri, my $branch, my $deepClone) = split ' ', $value;
|
||||
$branch = defined $branch ? $branch : "master";
|
||||
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
my $cacheDir = getSCMCacheDir . "/git";
|
||||
mkpath($cacheDir);
|
||||
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||
|
||||
my $stdout = ""; my $stderr = ""; my $res;
|
||||
if (! -d $clonePath) {
|
||||
# Clone everything and fetch the branch.
|
||||
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath);
|
||||
die "error cloning git repo at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir
|
||||
|
||||
# This command force the update of the local branch to be in the same as
|
||||
# the remote branch for whatever the repository state is. This command mirror
|
||||
# only one branch of the remote repository.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"git", "fetch", "-fu", "origin", "+$branch:$branch");
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"git", "fetch", "-fu", "origin") if $res;
|
||||
die "error fetching latest change from git repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
("git", "rev-parse", "$branch"));
|
||||
die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
my ($revision) = split /\n/, $stdout;
|
||||
die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout"
|
||||
unless $revision =~ /^[0-9a-fA-F]+$/;
|
||||
|
||||
my $ref = "refs/heads/$branch";
|
||||
|
||||
# If deepClone is defined, then we look at the content of the repository
|
||||
# to determine if this is a top-git branch.
|
||||
if (defined $deepClone) {
|
||||
|
||||
# Checkout the branch to look at its content.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch");
|
||||
die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
if (-f ".topdeps") {
|
||||
# This is a TopGit branch. Fetch all the topic branches so
|
||||
# that builders can run "tg patch" and similar.
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"tg", "remote", "--populate", "origin");
|
||||
print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res;
|
||||
}
|
||||
}
|
||||
|
||||
# Some simple caching: don't check a uri/branch/revision more than once.
|
||||
# TODO: Fix case where the branch is reset to a previous commit.
|
||||
my $cachedInput ;
|
||||
($cachedInput) = $self->{db}->resultset('CachedGitInputs')->search(
|
||||
{uri => $uri, branch => $branch, revision => $revision},
|
||||
{rows => 1});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$revision = $cachedInput->revision;
|
||||
} else {
|
||||
# Then download this revision into the store.
|
||||
print STDERR "checking out Git branch $branch from $uri\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
|
||||
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
|
||||
|
||||
if (defined $deepClone) {
|
||||
# Checked out code often wants to be able to run `git
|
||||
# describe', e.g., code that uses Gnulib's `git-version-gen'
|
||||
# script. Thus, we leave `.git' in there. Same for
|
||||
# Subversion (e.g., libgcrypt's build system uses that.)
|
||||
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
|
||||
|
||||
# Ask for a "deep clone" to allow "git describe" and similar
|
||||
# tools to work. See
|
||||
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
|
||||
# for a discussion.
|
||||
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
|
||||
}
|
||||
|
||||
($res, $stdout, $stderr) = captureStdoutStderr(600, "nix-prefetch-git", $clonePath, $revision);
|
||||
die "cannot check out Git repository branch '$branch' at `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedGitInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
# For convenience in producing readable version names, pass the
|
||||
# number of commits in the history of this revision (‘revCount’)
|
||||
# the output of git-describe (‘gitTag’), and the abbreviated
|
||||
# revision (‘shortRev’).
|
||||
my $revCount = `git rev-list $revision | wc -l`; chomp $revCount;
|
||||
die "git rev-list failed" if $? != 0;
|
||||
my $gitTag = `git describe --always $revision`; chomp $gitTag;
|
||||
die "git describe failed" if $? != 0;
|
||||
my $shortRev = `git rev-parse --short $revision`; chomp $shortRev;
|
||||
die "git rev-parse failed" if $? != 0;
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
, revCount => int($revCount)
|
||||
, gitTag => $gitTag
|
||||
, shortRev => $shortRev
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
88
src/lib/Hydra/Plugin/MercurialInput.pm
Normal file
88
src/lib/Hydra/Plugin/MercurialInput.pm
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
package Hydra::Plugin::MercurialInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use File::Path;
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'hg'} = 'Mercurial checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "hg";
|
||||
|
||||
(my $uri, my $id) = split ' ', $value;
|
||||
$id = defined $id ? $id : "default";
|
||||
|
||||
# init local hg clone
|
||||
|
||||
my $stdout = ""; my $stderr = "";
|
||||
|
||||
my $cacheDir = getSCMCacheDir . "/hg";
|
||||
mkpath($cacheDir);
|
||||
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
|
||||
|
||||
if (! -d $clonePath) {
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"hg", "clone", $uri, $clonePath);
|
||||
die "error cloning mercurial repo at `$uri':\n$stderr" if $res;
|
||||
}
|
||||
|
||||
# hg pull + check rev
|
||||
chdir $clonePath or die $!;
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull");
|
||||
die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}");
|
||||
die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1;
|
||||
|
||||
my ($revision, $revCount, $branch) = split ' ', $stdout;
|
||||
|
||||
my $storePath;
|
||||
my $sha256;
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
|
||||
{uri => $uri, branch => $branch, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
print STDERR "checking out Mercurial input from $uri $branch revision $revision\n";
|
||||
$ENV{"NIX_HASH_ALGO"} = "sha256";
|
||||
$ENV{"PRINT_PATH"} = "1";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
|
||||
"nix-prefetch-hg", $clonePath, $revision);
|
||||
die "cannot check out Mercurial repository `$uri':\n$stderr" if $res;
|
||||
|
||||
($sha256, $storePath) = split ' ', $stdout;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedHgInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, branch => $branch
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => $revision
|
||||
, revCount => int($revCount)
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
77
src/lib/Hydra/Plugin/PathInput.pm
Normal file
77
src/lib/Hydra/Plugin/PathInput.pm
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
package Hydra::Plugin::PathInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use POSIX qw(strftime);
|
||||
use Hydra::Helper::Nix;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'path'} = 'Local path';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "path";
|
||||
|
||||
my $uri = $value;
|
||||
|
||||
my $timestamp = time;
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
|
||||
# Some simple caching: don't check a path more than once every N seconds.
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
|
||||
{srcpath => $uri, lastseen => {">", $timestamp - 30}},
|
||||
{rows => 1, order_by => "lastseen DESC"});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
} else {
|
||||
|
||||
print STDERR "copying input ", $name, " from $uri\n";
|
||||
$storePath = `nix-store --add "$uri"`
|
||||
or die "cannot copy path $uri to the Nix store.\n";
|
||||
chomp $storePath;
|
||||
|
||||
$sha256 = (queryPathInfo($storePath, 0))[1] or die;
|
||||
|
||||
($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
|
||||
{srcpath => $uri, sha256hash => $sha256});
|
||||
|
||||
# Path inputs don't have a natural notion of a "revision", so
|
||||
# we simulate it by using the timestamp that we first saw this
|
||||
# path have this SHA-256 hash. So if the contents of the path
|
||||
# changes, we get a new "revision", but if it doesn't change
|
||||
# (or changes back), we don't get a new "revision".
|
||||
if (!defined $cachedInput) {
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedPathInputs')->update_or_create(
|
||||
{ srcpath => $uri
|
||||
, timestamp => $timestamp
|
||||
, lastseen => $timestamp
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
} else {
|
||||
$timestamp = $cachedInput->timestamp;
|
||||
txn_do($self->{db}, sub {
|
||||
$cachedInput->update({lastseen => time});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
90
src/lib/Hydra/Plugin/SubversionInput.pm
Normal file
90
src/lib/Hydra/Plugin/SubversionInput.pm
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
package Hydra::Plugin::SubversionInput;
|
||||
|
||||
use strict;
|
||||
use parent 'Hydra::Plugin';
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use Hydra::Helper::Nix;
|
||||
use IPC::Run;
|
||||
use Nix::Store;
|
||||
|
||||
sub supportedInputTypes {
|
||||
my ($self, $inputTypes) = @_;
|
||||
$inputTypes->{'svn'} = 'Subversion export';
|
||||
$inputTypes->{'svn-checkout'} = 'Subversion checkout';
|
||||
}
|
||||
|
||||
sub fetchInput {
|
||||
my ($self, $type, $name, $value) = @_;
|
||||
|
||||
return undef if $type ne "svn" && $type ne "svn-checkout";
|
||||
|
||||
# Allow users to specify a revision number next to the URI.
|
||||
my ($uri, $revision) = split ' ', $value;
|
||||
|
||||
my $sha256;
|
||||
my $storePath;
|
||||
my $stdout; my $stderr;
|
||||
|
||||
unless (defined $revision) {
|
||||
# First figure out the last-modified revision of the URI.
|
||||
my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],
|
||||
"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
|
||||
IPC::Run::run(@cmd, \$stdout, \$stderr);
|
||||
die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?;
|
||||
$revision = int($stdout); $revision =~ s/\s*([0-9]+)\s*/$1/sm;
|
||||
}
|
||||
|
||||
die unless $revision =~ /^\d+$/;
|
||||
$revision = int($revision);
|
||||
|
||||
# Do we already have this revision in the store?
|
||||
# !!! This needs to take $checkout into account! Otherwise "svn"
|
||||
# and "svn-checkout" inputs can get mixed up.
|
||||
(my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
|
||||
{uri => $uri, revision => $revision});
|
||||
|
||||
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
|
||||
$storePath = $cachedInput->storepath;
|
||||
$sha256 = $cachedInput->sha256hash;
|
||||
} else {
|
||||
|
||||
# No, do a checkout. The working copy is reused between
|
||||
# invocations to speed things up.
|
||||
my $wcPath = getSCMCacheDir . "/svn/" . sha256_hex($uri) . "/svn-checkout";
|
||||
|
||||
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
|
||||
|
||||
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath);
|
||||
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
|
||||
|
||||
if ($type eq "svn-checkout") {
|
||||
$storePath = addToStore($wcPath, 1, "sha256");
|
||||
} else {
|
||||
# Hm, if the Nix Perl bindings supported filters in
|
||||
# addToStore(), then we wouldn't need to make a copy here.
|
||||
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
|
||||
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
|
||||
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
|
||||
}
|
||||
|
||||
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
|
||||
|
||||
txn_do($self->{db}, sub {
|
||||
$self->{db}->resultset('CachedSubversionInputs')->update_or_create(
|
||||
{ uri => $uri
|
||||
, revision => $revision
|
||||
, sha256hash => $sha256
|
||||
, storepath => $storePath
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return
|
||||
{ uri => $uri
|
||||
, storePath => $storePath
|
||||
, sha256hash => $sha256
|
||||
, revNumber => $revision
|
||||
};
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
@ -4,20 +4,6 @@ USE String;
|
|||
USE HTML;
|
||||
|
||||
|
||||
inputTypes =
|
||||
{ "svn" = "Subversion export"
|
||||
, "svn-checkout" = "Subversion checkout"
|
||||
, "bzr" = "Bazaar export"
|
||||
, "bzr-checkout" = "Bazaar checkout"
|
||||
, "git" = "Git checkout"
|
||||
, "hg" = "Mercurial checkout"
|
||||
, "string" = "String value"
|
||||
, "boolean" = "Boolean"
|
||||
, "path" = "Local path"
|
||||
, "build" = "Build output"
|
||||
, "sysbuild" = "Build output (same system)"
|
||||
};
|
||||
|
||||
BLOCK renderDateTime;
|
||||
date.format(timestamp, '%Y-%m-%d %H:%M:%S');
|
||||
END;
|
||||
|
|
|
|||
|
|
@ -167,5 +167,3 @@
|
|||
</form>
|
||||
|
||||
[% END %]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
use strict;
|
||||
use feature 'switch';
|
||||
use Hydra::Schema;
|
||||
use Hydra::Plugin;
|
||||
use Hydra::Helper::Nix;
|
||||
use Hydra::Helper::AddBuilds;
|
||||
use Hydra::Model::DB;
|
||||
|
|
@ -20,6 +21,8 @@ STDOUT->autoflush();
|
|||
my $db = Hydra::Model::DB->new();
|
||||
my $config = getHydraConfig();
|
||||
|
||||
my $plugins = [Hydra::Plugin->plugins(db => $db, config => $config)];
|
||||
|
||||
# Don't check a jobset more than once every five minutes.
|
||||
my $minCheckInterval = 5 * 60;
|
||||
|
||||
|
|
@ -29,10 +32,8 @@ sub fetchInputs {
|
|||
my ($project, $jobset, $inputInfo) = @_;
|
||||
foreach my $input ($jobset->jobsetinputs->all) {
|
||||
foreach my $alt ($input->jobsetinputalts->all) {
|
||||
my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
|
||||
foreach my $info_el (@info) {
|
||||
push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
|
||||
}
|
||||
push @{$$inputInfo{$input->name}}, $_
|
||||
foreach fetchInput($plugins, $db, $project, $jobset, $input->name, $input->type, $alt->value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue