diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index a9539387..9d857874 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -273,7 +273,7 @@ sub restart : Chained('build') PathPart Args(0) { requireProjectOwner($c, $build->project); - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { error($c, "This build cannot be restarted.") unless $build->finished && ($build->resultInfo->buildstatus == 3 || @@ -304,7 +304,7 @@ sub cancel : Chained('build') PathPart Args(0) { requireProjectOwner($c, $build->project); - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { error($c, "This build cannot be cancelled.") if $build->finished || $build->schedulingInfo->busy; @@ -340,7 +340,7 @@ sub keep : Chained('build') PathPart Args(1) { registerRoot $build->outpath if $newStatus == 1; - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { $build->resultInfo->update({keep => int $newStatus}); }); diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 93b2741a..78d255e1 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -64,7 +64,7 @@ sub submit : Chained('jobset') PathPart Args(0) { requireProjectOwner($c, $c->stash->{project}); requirePost($c); - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { updateJobset($c, $c->stash->{jobset}); }); @@ -79,7 +79,7 @@ sub delete : Chained('jobset') PathPart Args(0) { requireProjectOwner($c, $c->stash->{project}); requirePost($c); - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { $c->stash->{jobset}->delete; }); diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm index 22084153..e815280f 100644 --- a/src/lib/Hydra/Controller/Project.pm +++ b/src/lib/Hydra/Controller/Project.pm @@ -44,7 +44,7 @@ sub submit : Chained('project') PathPart Args(0) { requireProjectOwner($c, $c->stash->{project}); requirePost($c); - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { updateProject($c, $c->stash->{project}); }); @@ -58,7 +58,7 @@ sub delete : Chained('project') PathPart Args(0) { requireProjectOwner($c, $c->stash->{project}); requirePost($c); - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { $c->stash->{project}->delete; }); @@ -94,7 +94,7 @@ sub create_submit : Path('/create-project/submit') { my $projectName = trim $c->request->params->{name}; - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { # Note: $projectName is validated in updateProject, # which will abort the transaction if the name isn't # valid. Idem for the owner. @@ -127,7 +127,7 @@ sub create_jobset_submit : Chained('project') PathPart('create-jobset/submit') A my $jobsetName = trim $c->request->params->{name}; - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { # Note: $jobsetName is validated in updateProject, which will # abort the transaction if the name isn't valid. my $jobset = $c->stash->{project}->jobsets->create( diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index d211eafb..5d42a25f 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -146,14 +146,14 @@ sub releases :Local { } elsif ($subcommand eq "submit") { - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { updateReleaseSet($c, $releaseSet); }); return $c->res->redirect($c->uri_for("/releases", $projectName, $releaseSet->name)); } elsif ($subcommand eq "delete") { - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { $releaseSet->delete; }); return $c->res->redirect($c->uri_for($c->controller('Project')->action_for('view'), [$project->name])); @@ -181,7 +181,7 @@ sub create_releaseset :Local { if (defined $subcommand && $subcommand eq "submit") { my $releaseSetName = $c->request->params->{name}; - $c->model('DB')->schema->txn_do(sub { + txn_do($c->model('DB')->schema, sub { # Note: $releaseSetName is validated in updateProject, # which will abort the transaction if the name isn't # valid. diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index d65e96a3..562b2915 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -8,7 +8,7 @@ use File::Basename; our @ISA = qw(Exporter); our @EXPORT = qw( isValidPath queryPathInfo - getHydraPath getHydraDBPath openHydraDB + getHydraPath getHydraDBPath openHydraDB txn_do registerRoot getGCRootsDir gcRootFor getPrimaryBuildsForReleaseSet getRelease getLatestSuccessfulRelease ); @@ -78,6 +78,21 @@ sub openHydraDB { } +# Awful hack to handle timeouts in SQLite: just retry the transaction. +# DBD::SQLite *has* a 30 second retry window, but apparently it +# doesn't work. +sub txn_do { + my ($db, $coderef) = @_; + while (1) { + eval { + $db->txn_do($coderef); + }; + last if !$@; + die $@ unless $@ =~ "database is locked"; + } +} + + sub getGCRootsDir { die unless defined $ENV{LOGNAME}; my $dir = "/nix/var/nix/gcroots/per-user/$ENV{LOGNAME}/hydra-roots"; diff --git a/src/script/hydra_build.pl b/src/script/hydra_build.pl index fa5ef015..8cb4f4d1 100755 --- a/src/script/hydra_build.pl +++ b/src/script/hydra_build.pl @@ -65,7 +65,7 @@ sub doBuild { if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { my $drvPathStep = $1; - $db->txn_do(sub { + txn_do($db, sub { $build->buildsteps->create( { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++) , type => 0 # = build @@ -80,7 +80,7 @@ sub doBuild { elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) { my $drvPathStep = $1; - $db->txn_do(sub { + txn_do($db, sub { my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die; $step->update({busy => 0, status => 0, stoptime => time}); }); @@ -92,7 +92,7 @@ sub doBuild { $thisBuildFailed = 1 if $drvPath eq $drvPathStep; my $errorMsg = $4; $errorMsg = "build failed previously (cached)" if $3 eq "cached"; - $db->txn_do(sub { + txn_do($db, sub { if ($buildSteps{$drvPathStep}) { my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die; $step->update({busy => 0, status => 1, errormsg => $errorMsg, stoptime => time}); @@ -119,7 +119,7 @@ sub doBuild { elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) { my $outPath = $1; - $db->txn_do(sub { + txn_do($db, sub { $build->buildsteps->create( { stepnr => ($buildSteps{$outPath} = $buildStepNr++) , type => 1 # = substitution @@ -132,7 +132,7 @@ sub doBuild { elsif (/^@\s+substituter-succeeded\s+(\S+)$/) { my $outPath = $1; - $db->txn_do(sub { + txn_do($db, sub { my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die; $step->update({busy => 0, status => 0, stoptime => time}); }); @@ -140,7 +140,7 @@ sub doBuild { elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) { my $outPath = $1; - $db->txn_do(sub { + txn_do($db, sub { my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die; $step->update({busy => 0, status => 1, errormsg => $3, stoptime => time}); }); @@ -169,7 +169,7 @@ sub doBuild { done: - $db->txn_do(sub { + txn_do($db, sub { $build->update({finished => 1, timestamp => time}); my $releaseName; @@ -265,7 +265,7 @@ print STDERR "performing build $buildId\n"; # children (i.e. the build.pl instances) can continue to run and won't # have the lock taken away. my $build; -$db->txn_do(sub { +txn_do($db, sub { $build = $db->resultset('Builds')->find($buildId); die "build $buildId doesn't exist" unless defined $build; die "build $buildId already done" if defined $build->resultInfo; @@ -287,7 +287,7 @@ eval { }; if ($@) { warn $@; - $db->txn_do(sub { + txn_do($db, sub { $build->schedulingInfo->update({busy => 0, locker => $$}); }); } diff --git a/src/script/hydra_queue_runner.pl b/src/script/hydra_queue_runner.pl index 8d7c5edd..030aeee8 100755 --- a/src/script/hydra_queue_runner.pl +++ b/src/script/hydra_queue_runner.pl @@ -19,7 +19,7 @@ die "The HYDRA_HOME environment variable is not set!\n" unless defined $hydraHom sub unlockDeadBuilds { # Unlock builds whose building process has died. - $db->txn_do(sub { + txn_do($db, sub { my @builds = $db->resultset('Builds')->search( {finished => 0, busy => 1}, {join => 'schedulingInfo'}); foreach my $build (@builds) { @@ -54,7 +54,7 @@ sub checkBuilds { my @buildsStarted; - $db->txn_do(sub { + txn_do($db, sub { # Get the system types for the runnable builds. my @systemTypes = $db->resultset('Builds')->search( @@ -123,7 +123,7 @@ sub checkBuilds { }; if ($@) { warn $@; - $db->txn_do(sub { + txn_do($db, sub { $build->schedulingInfo->busy(0); $build->schedulingInfo->locker($$); $build->schedulingInfo->update; diff --git a/src/script/hydra_scheduler.pl b/src/script/hydra_scheduler.pl index 7f82cc98..d2afc316 100755 --- a/src/script/hydra_scheduler.pl +++ b/src/script/hydra_scheduler.pl @@ -111,7 +111,7 @@ sub fetchInputAlt { # but if it doesn't change (or changes back), we don't get # a new "revision". if (!defined $cachedInput) { - $db->txn_do(sub { + txn_do($db, sub { $db->resultset('CachedPathInputs')->create( { srcpath => $uri , timestamp => $timestamp @@ -122,7 +122,7 @@ sub fetchInputAlt { }); } else { $timestamp = $cachedInput->timestamp; - $db->txn_do(sub { + txn_do($db, sub { $cachedInput->update({lastseen => time}); }); } @@ -170,7 +170,7 @@ sub fetchInputAlt { ($sha256, $storePath) = split ' ', $stdout; - $db->txn_do(sub { + txn_do($db, sub { $db->resultset('CachedSubversionInputs')->create( { uri => $uri , revision => $revision @@ -260,7 +260,7 @@ sub checkJob { $priority = int($job->{schedulingPriority}) if $job->{schedulingPriority} =~ /^\d+$/; - $db->txn_do(sub { + txn_do($db, sub { # Mark this job as active in the database. my $jobInDB = $jobset->jobs->update_or_create( { name => $jobName @@ -326,7 +326,7 @@ sub checkJob { sub setJobsetError { my ($jobset, $errorMsg) = @_; eval { - $db->txn_do(sub { + txn_do($db, sub { $jobset->update({errormsg => $errorMsg, errortime => time}); }); }; @@ -370,9 +370,10 @@ sub checkJobset { fetchInputs($project, $jobset, $inputInfo); # Evaluate the job expression. - die "not supported" if scalar @{$inputInfo->{$jobset->nixexprinput}} != 1; my $nixExprInput = $inputInfo->{$jobset->nixexprinput}->[0] or die "cannot find the input containing the job expression"; + die "multiple alternatives for the input containing the Nix expression are not supported" + if scalar @{$inputInfo->{$jobset->nixexprinput}} != 1; my $nixExprPath = $nixExprInput->{storePath} . "/" . $jobset->nixexprpath; (my $res, my $jobsXml, my $stderr) = captureStdoutStderr( @@ -402,7 +403,7 @@ sub checkJobset { my %failedJobNames; push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}}; - $db->txn_do(sub { + txn_do($db, sub { $jobset->update({lastcheckedtime => time}); foreach my $jobInDB ($jobset->jobs->all) { @@ -448,7 +449,7 @@ sub checkJobsetWrapped { if ($@) { my $msg = $@; print "error evaluating jobset ", $jobset->name, ": $msg"; - $db->txn_do(sub { + txn_do($db, sub { $jobset->update({lastcheckedtime => time}); setJobsetError($jobset, $msg); });