From 9ddc6e355f69b0cdb1424e3e48ab5bae7e1d7be7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 22 Feb 2021 13:29:52 -0500
Subject: [PATCH 01/13] flake: add TestPostgreSQL for per-test DBs

---
 flake.nix | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/flake.nix b/flake.nix
index baa6c1e8..e7bfb65b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -37,6 +37,66 @@
 
         # Add LDAP dependencies that aren't currently found within nixpkgs.
         perlPackages = prev.perlPackages // {
+          TestPostgreSQL = final.perlPackages.buildPerlModule {
+            pname = "Test-PostgreSQL";
+            version = "1.27";
+            src = final.fetchurl {
+              url = "mirror://cpan/authors/id/T/TJ/TJC/Test-PostgreSQL-1.27.tar.gz";
+              sha256 = "b1bd231693100cc40905fb0ba3173173201621de9c8301f21c5b593b0a46f907";
+            };
+            buildInputs = with final.perlPackages; [ ModuleBuildTiny TestSharedFork pkgs.postgresql ];
+            propagatedBuildInputs = with final.perlPackages; [ DBDPg DBI FileWhich FunctionParameters Moo TieHashMethod TryTiny TypeTiny ];
+
+            makeMakerFlags = "POSTGRES_HOME=${final.postgresql}";
+
+            meta = {
+              homepage = https://github.com/TJC/Test-postgresql;
+              description = "PostgreSQL runner for tests";
+              license = with final.lib.licenses; [ artistic2 ];
+            };
+          };
+
+          FunctionParameters = final.buildPerlPackage {
+            pname = "Function-Parameters";
+            version = "2.001003";
+            src = final.fetchurl {
+              url = "mirror://cpan/authors/id/M/MA/MAUKE/Function-Parameters-2.001003.tar.gz";
+              sha256 = "eaa22c6b43c02499ec7db0758c2dd218a3b2ab47a714b2bdf8010b5ee113c242";
+            };
+            buildInputs = with final.perlPackages; [ DirSelf TestFatal ];
+            meta = {
+              description = "Define functions and methods with parameter lists (\"subroutine signatures\")";
+              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
+            };
+          };
+
+          DirSelf = final.buildPerlPackage {
+            pname = "Dir-Self";
+            version = "0.11";
+            src = final.fetchurl {
+              url = "mirror://cpan/authors/id/M/MA/MAUKE/Dir-Self-0.11.tar.gz";
+              sha256 = "e251a51abc7d9ba3e708f73c2aa208e09d47a0c528d6254710fa78cc8d6885b5";
+            };
+            meta = {
+              homepage = "https://github.com/mauke/Dir-Self";
+              description = "A __DIR__ constant for the directory your source file is in";
+              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
+            };
+          };
+
+          TieHashMethod = final.buildPerlPackage {
+              pname = "Tie-Hash-Method";
+              version = "0.02";
+              src = final.fetchurl {
+                url = "mirror://cpan/authors/id/Y/YV/YVES/Tie-Hash-Method-0.02.tar.gz";
+                sha256 = "d513fbb51413f7ca1e64a1bdce6194df7ec6076dea55066d67b950191eec32a9";
+              };
+              meta = {
+                description = "Tied hash with specific methods overriden by callbacks";
+                license = with final.lib.licenses; [ artistic1 ];
+              };
+            };
+
           Test2Harness = final.buildPerlPackage {
             pname = "Test2-Harness";
             version = "1.000042";
@@ -222,6 +282,7 @@
                 SysHostnameLong
                 TermSizeAny
                 TestMore
+                TestPostgreSQL
                 TextDiff
                 Test2Harness
                 TextTable

From b15d8edab17478290bb92fba0da988eb0fd23788 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 22 Feb 2021 13:33:25 -0500
Subject: [PATCH 02/13] tests: delete set-up.pl / tear-down.pl

We'll set these up on a per-test basis.
---
 tests/Makefile.am  |  4 +---
 tests/set-up.pl    |  5 -----
 tests/tear-down.pl | 12 ------------
 3 files changed, 1 insertion(+), 20 deletions(-)
 delete mode 100644 tests/set-up.pl
 delete mode 100644 tests/tear-down.pl

diff --git a/tests/Makefile.am b/tests/Makefile.am
index f5a92239..4f84154c 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -25,9 +25,7 @@ EXTRA_DIST = \
   $(TESTS)
 
 TESTS = \
-  set-up.pl \
-  test.pl \
-  tear-down.pl
+  test.pl
 
 check_SCRIPTS = repos
 
diff --git a/tests/set-up.pl b/tests/set-up.pl
deleted file mode 100644
index 4fb99a49..00000000
--- a/tests/set-up.pl
+++ /dev/null
@@ -1,5 +0,0 @@
-use strict;
-system("initdb -D postgres --locale C.UTF-8 ") == 0 or die;
-system("pg_ctl -D postgres -o \"-F -p 6433  -h '' -k /tmp \" -w start") == 0 or die;
-system("createdb -l C.UTF-8 -p 6433 hydra-test-suite") == 0 or die;
-system("hydra-init") == 0 or die;
diff --git a/tests/tear-down.pl b/tests/tear-down.pl
deleted file mode 100644
index f30bb278..00000000
--- a/tests/tear-down.pl
+++ /dev/null
@@ -1,12 +0,0 @@
-use strict;
-
-my $fail = 0;
-
-system("dropdb -p 6433 hydra-test-suite") == 0 or $fail = 1;
-system("pg_ctl -D postgres -w stop") == 0 or $fail = 1;
-
-system("chmod -R a+w nix") == 0 or $fail = 1;
-system("rm -rf postgres data nix git-repo hg-repo svn-repo svn-checkout svn-checkout-repo bzr-repo bzr-checkout-repo darcs-repo") == 0 or $fail = 1;
-system("rm -f .*-state") == 0 or $fail = 1;
-
-exit $fail;

From fe1f2f0806f8ff3134d09141ee28f17093ad85ce Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 22 Feb 2021 13:47:41 -0500
Subject: [PATCH 03/13] Create an ephemeral PostgreSQL database per test

---
 src/libhydra/db.hh | 14 ++++++++++----
 tests/evaluation.t |  9 ++++++---
 tests/lib/Setup.pm | 21 ++++++++++++++++++---
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh
index 35d78edf..ec9a024b 100644
--- a/src/libhydra/db.hh
+++ b/src/libhydra/db.hh
@@ -13,10 +13,16 @@ struct Connection : pqxx::connection
     {
         using namespace nix;
         auto s = getEnv("HYDRA_DBI").value_or("dbi:Pg:dbname=hydra;");
-        std::string prefix = "dbi:Pg:";
-        if (std::string(s, 0, prefix.size()) != prefix)
-            throw Error("$HYDRA_DBI does not denote a PostgreSQL database");
-        return concatStringsSep(" ", tokenizeString<Strings>(string(s, prefix.size()), ";"));
+
+        std::string lower_prefix = "dbi:Pg:";
+        std::string upper_prefix = "DBI:Pg:";
+
+        if ((std::string(s, 0, lower_prefix.size()) == lower_prefix) ||
+            (std::string(s, 0, upper_prefix.size()) == upper_prefix)) {
+            return concatStringsSep(" ", tokenizeString<Strings>(string(s, lower_prefix.size()), ";"));
+        }
+
+        throw Error("$HYDRA_DBI does not denote a PostgreSQL database");
     }
 };
 
diff --git a/tests/evaluation.t b/tests/evaluation.t
index 1d4c37a5..04dc144e 100644
--- a/tests/evaluation.t
+++ b/tests/evaluation.t
@@ -1,13 +1,16 @@
 use strict;
-use Hydra::Schema;
-use Hydra::Model::DB;
 use Cwd;
 use Setup;
 
-my $db = Hydra::Model::DB->new;
+my $pgsql = dbinit();
+my $dsn = $pgsql->dsn;
+
+require Hydra::Schema;
+require Hydra::Model::DB;
 
 use Test::Simple tests => 76;
 
+my $db = Hydra::Model::DB->new;
 hydra_setup($db);
 
 my $res;
diff --git a/tests/lib/Setup.pm b/tests/lib/Setup.pm
index 6ddd0162..2989120b 100644
--- a/tests/lib/Setup.pm
+++ b/tests/lib/Setup.pm
@@ -2,12 +2,27 @@ package Setup;
 
 use strict;
 use Exporter;
-use Hydra::Helper::Nix;
-use Hydra::Model::DB;
+use Test::PostgreSQL;
 use Cwd;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
+our @EXPORT = qw(dbinit hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
+
+sub dbinit() {
+    my $pgsql = Test::PostgreSQL->new();
+    $ENV{'HYDRA_DBI'} = $pgsql->dsn;
+    system("hydra-init") == 0 or die;
+    return $pgsql;
+}
+
+sub captureStdoutStderr {
+    # "Lazy"-load Hydra::Helper::Nix to avoid the compile-time
+    # import of Hydra::Model::DB. Early loading of the DB class
+    # causes fixation of the DSN, and we need to fixate it after
+    # the temporary DB is setup.
+    require Hydra::Helper::Nix;
+    return Hydra::Helper::Nix::captureStdoutStderr(@_)
+}
 
 sub hydra_setup {
     my ($db) = @_;

From 62b2880dfc893381d594b32e977fc7344ca80849 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 22 Feb 2021 16:52:38 -0500
Subject: [PATCH 04/13] Give each test its own Nix directories

Otherwise we risk tripping over confusing statuses where a build is
"done" and "Cached", but we were expecting to run it.
---
 tests/evaluate-basic.t | 32 ++++++++++++++++++++++++++++++++
 tests/evaluation.t     | 17 ++---------------
 tests/lib/Setup.pm     | 29 +++++++++++++++++++++++++----
 3 files changed, 59 insertions(+), 19 deletions(-)
 create mode 100644 tests/evaluate-basic.t

diff --git a/tests/evaluate-basic.t b/tests/evaluate-basic.t
new file mode 100644
index 00000000..0d58d08f
--- /dev/null
+++ b/tests/evaluate-basic.t
@@ -0,0 +1,32 @@
+use feature 'unicode_strings';
+use strict;
+use Cwd;
+use Setup;
+
+(my $datadir, my $pgsql) = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+# Most basic test case, no parameters
+my $jobset = createBaseJobset("basic", "basic.nix");
+
+ok(evalSucceeds($jobset),               "Evaluating jobs/basic.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 3, "Evaluating jobs/basic.nix should result in 3 builds");
+
+for my $build (queuedBuildsForJobset($jobset)) {
+    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/basic.nix should be finished.");
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
+    is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/basic.nix should have buildstatus $expected.");
+}
+
+done_testing;
\ No newline at end of file
diff --git a/tests/evaluation.t b/tests/evaluation.t
index 04dc144e..56a7c0a0 100644
--- a/tests/evaluation.t
+++ b/tests/evaluation.t
@@ -2,13 +2,12 @@ use strict;
 use Cwd;
 use Setup;
 
-my $pgsql = dbinit();
-my $dsn = $pgsql->dsn;
+(my $datadir, my $pgsql) = test_init();
 
 require Hydra::Schema;
 require Hydra::Model::DB;
 
-use Test::Simple tests => 76;
+use Test::Simple tests => 68;
 
 my $db = Hydra::Model::DB->new;
 hydra_setup($db);
@@ -21,18 +20,6 @@ my $jobsBaseUri = "file://".getcwd;
 my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
 my $jobset;
 
-# Most basic test case, no parameters
-$jobset = createBaseJobset("basic", "basic.nix");
-
-ok(evalSucceeds($jobset),                  "Evaluating jobs/basic.nix should exit with return code 0");
-ok(nrQueuedBuildsForJobset($jobset) == 3 , "Evaluating jobs/basic.nix should result in 3 builds");
-
-for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
-    my $newbuild = $db->resultset('Builds')->find($build->id);
-    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
-    ok($newbuild->finished == 1 && $newbuild->buildstatus == $expected, "Build '".$build->job."' from jobs/basic.nix should have buildstatus $expected");
-}
 
 # Test jobset with 2 jobs, one has parameter of succeeded build of the other
 $jobset = createJobsetWithOneInput("build-output-as-input", "build-output-as-input.nix", "build1", "build", "build1");
diff --git a/tests/lib/Setup.pm b/tests/lib/Setup.pm
index 2989120b..03c7046a 100644
--- a/tests/lib/Setup.pm
+++ b/tests/lib/Setup.pm
@@ -3,16 +3,37 @@ package Setup;
 use strict;
 use Exporter;
 use Test::PostgreSQL;
+use File::Temp;
+use File::Path qw(make_path);
 use Cwd;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(dbinit hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
+our @EXPORT = qw(test_init hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
 
-sub dbinit() {
-    my $pgsql = Test::PostgreSQL->new();
+sub test_init() {
+    my $dir = File::Temp->newdir();
+
+    $ENV{'HYDRA_DATA'} = "$dir/hydra-data";
+    mkdir $ENV{'HYDRA_DATA'};
+    $ENV{'NIX_CONF_DIR'} = "$dir/nix/etc/nix";
+    make_path($ENV{'NIX_CONF_DIR'});
+    my $nixconf = "$ENV{'NIX_CONF_DIR'}/nix.conf";
+    open(my $fh, '>', $nixconf) or die "Could not open file '$nixconf' $!";
+    print $fh "sandbox = false\n";
+    close $fh;
+
+    $ENV{'NIX_STATE_DIR'} = "$dir/nix/var/nix";
+
+    $ENV{'NIX_MANIFESTS_DIR'} = "$dir/nix/var/nix/manifests";
+    $ENV{'NIX_STORE_DIR'} = "$dir/nix/store";
+    $ENV{'NIX_LOG_DIR'} = "$dir/nix/var/log/nix";
+
+    my $pgsql = Test::PostgreSQL->new(
+        extra_initdb_args => "--locale C.UTF-8"
+    );
     $ENV{'HYDRA_DBI'} = $pgsql->dsn;
     system("hydra-init") == 0 or die;
-    return $pgsql;
+    return ($dir, $pgsql);
 }
 
 sub captureStdoutStderr {

From e4cda87b5a31e572cb52d7aa644234d2f4e07689 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 24 Feb 2021 07:00:26 -0500
Subject: [PATCH 05/13] db.hh: use hasPrefix for prefix comparisons

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libhydra/db.hh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh
index ec9a024b..7d5bdc58 100644
--- a/src/libhydra/db.hh
+++ b/src/libhydra/db.hh
@@ -17,8 +17,7 @@ struct Connection : pqxx::connection
         std::string lower_prefix = "dbi:Pg:";
         std::string upper_prefix = "DBI:Pg:";
 
-        if ((std::string(s, 0, lower_prefix.size()) == lower_prefix) ||
-            (std::string(s, 0, upper_prefix.size()) == upper_prefix)) {
+        if (hasPrefix(s, lower_prefix) || hasPrefix(s, upper_prefix)) {
             return concatStringsSep(" ", tokenizeString<Strings>(string(s, lower_prefix.size()), ";"));
         }
 

From 9590bababc6c0ba92c36c4ba235d21dc90467f6f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 12:55:52 -0500
Subject: [PATCH 06/13] Split out dependent tests in to its own .t

---
 tests/evaluate-dependent-jobsets.t | 34 ++++++++++++++++++++++++++++++
 tests/evaluation.t                 | 22 +------------------
 2 files changed, 35 insertions(+), 21 deletions(-)
 create mode 100644 tests/evaluate-dependent-jobsets.t

diff --git a/tests/evaluate-dependent-jobsets.t b/tests/evaluate-dependent-jobsets.t
new file mode 100644
index 00000000..7797b486
--- /dev/null
+++ b/tests/evaluate-dependent-jobsets.t
@@ -0,0 +1,34 @@
+use strict;
+use Cwd;
+use Setup;
+
+(my $datadir, my $pgsql) = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+# Test jobset with 2 jobs, one has parameter of succeeded build of the other
+my $jobset = createJobsetWithOneInput("build-output-as-input", "build-output-as-input.nix", "build1", "build", "build1");
+
+ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix should exit with return code 0");
+ok(nrQueuedBuildsForJobset($jobset) == 1 , "Evaluating jobs/build-output-as-input.nix for first time should result in 1 build in queue");
+for my $build (queuedBuildsForJobset($jobset)) {
+    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/basic.nix should have buildstatus 0");
+}
+
+ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix for second time should exit with return code 0");
+ok(nrQueuedBuildsForJobset($jobset) == 1 , "Evaluating jobs/build-output-as-input.nix for second time after building build1 should result in 1 build in queue");
+for my $build (queuedBuildsForJobset($jobset)) {
+    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/basic.nix should have buildstatus 0");
+}
+
+done_testing;
diff --git a/tests/evaluation.t b/tests/evaluation.t
index 56a7c0a0..9a47d44b 100644
--- a/tests/evaluation.t
+++ b/tests/evaluation.t
@@ -7,7 +7,7 @@ use Setup;
 require Hydra::Schema;
 require Hydra::Model::DB;
 
-use Test::Simple tests => 68;
+use Test::Simple tests => 60;
 
 my $db = Hydra::Model::DB->new;
 hydra_setup($db);
@@ -21,26 +21,6 @@ my $project = $db->resultset('Projects')->create({name => "tests", displayname =
 my $jobset;
 
 
-# Test jobset with 2 jobs, one has parameter of succeeded build of the other
-$jobset = createJobsetWithOneInput("build-output-as-input", "build-output-as-input.nix", "build1", "build", "build1");
-
-ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix should exit with return code 0");
-ok(nrQueuedBuildsForJobset($jobset) == 1 , "Evaluating jobs/build-output-as-input.nix for first time should result in 1 build in queue");
-for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
-    my $newbuild = $db->resultset('Builds')->find($build->id);
-    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/basic.nix should have buildstatus 0");
-}
-
-ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix for second time should exit with return code 0");
-ok(nrQueuedBuildsForJobset($jobset) == 1 , "Evaluating jobs/build-output-as-input.nix for second time after building build1 should result in 1 build in queue");
-for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
-    my $newbuild = $db->resultset('Builds')->find($build->id);
-    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/basic.nix should have buildstatus 0");
-}
-
-
 # Test scm inputs
 my @scminputs = (
     {

From 0b693ad8e85aad2390890b7025eb5f0c1acb6cf2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 13:23:18 -0500
Subject: [PATCH 07/13] Use is in evaluate-dependent-jobsets

---
 tests/evaluate-dependent-jobsets.t | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/evaluate-dependent-jobsets.t b/tests/evaluate-dependent-jobsets.t
index 7797b486..7180fbc8 100644
--- a/tests/evaluate-dependent-jobsets.t
+++ b/tests/evaluate-dependent-jobsets.t
@@ -16,19 +16,21 @@ hydra_setup($db);
 my $jobset = createJobsetWithOneInput("build-output-as-input", "build-output-as-input.nix", "build1", "build", "build1");
 
 ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix should exit with return code 0");
-ok(nrQueuedBuildsForJobset($jobset) == 1 , "Evaluating jobs/build-output-as-input.nix for first time should result in 1 build in queue");
+is(nrQueuedBuildsForJobset($jobset), 1 , "Evaluating jobs/build-output-as-input.nix for first time should result in 1 build in queue");
 for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
+    ok(runBuild($build), "Build '".$build->job."' from jobs/build-output-as-input.nix should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
-    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/basic.nix should have buildstatus 0");
+    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/build-output-as-input.nix should be finished.");
+    is($newbuild->buildstatus, 0, "Build '".$build->job."' from jobs/build-output-as-input.nix should have buildstatus 0.");
 }
 
 ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix for second time should exit with return code 0");
-ok(nrQueuedBuildsForJobset($jobset) == 1 , "Evaluating jobs/build-output-as-input.nix for second time after building build1 should result in 1 build in queue");
+is(nrQueuedBuildsForJobset($jobset), 1 , "Evaluating jobs/build-output-as-input.nix for second time after building build1 should result in 1 build in queue");
 for my $build (queuedBuildsForJobset($jobset)) {
     ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
-    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/basic.nix should have buildstatus 0");
+    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/build-output-as-input.nix should be finished.");
+    is($newbuild->buildstatus, 0, "Build '".$build->job."' from jobs/build-output-as-input.nix should have buildstatus 0.");
 }
 
 done_testing;

From c8df544046b4d730ccbb333e73f0b6827ef29e94 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 14:16:03 -0500
Subject: [PATCH 08/13] evaluate-dependent-jobsets: clean up test to be more
 clear

---
 tests/evaluate-dependent-jobsets.t | 37 ++++++++++++++++++------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/tests/evaluate-dependent-jobsets.t b/tests/evaluate-dependent-jobsets.t
index 7180fbc8..279e1f33 100644
--- a/tests/evaluate-dependent-jobsets.t
+++ b/tests/evaluate-dependent-jobsets.t
@@ -15,22 +15,29 @@ hydra_setup($db);
 # Test jobset with 2 jobs, one has parameter of succeeded build of the other
 my $jobset = createJobsetWithOneInput("build-output-as-input", "build-output-as-input.nix", "build1", "build", "build1");
 
-ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 1 , "Evaluating jobs/build-output-as-input.nix for first time should result in 1 build in queue");
-for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/build-output-as-input.nix should exit with code 0");
-    my $newbuild = $db->resultset('Builds')->find($build->id);
-    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/build-output-as-input.nix should be finished.");
-    is($newbuild->buildstatus, 0, "Build '".$build->job."' from jobs/build-output-as-input.nix should have buildstatus 0.");
-}
+ok(evalSucceeds($jobset), "Evaluating jobs/build-output-as-input.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 1 , "Evaluation should result in 1 build in queue");
 
-ok(evalSucceeds($jobset),                  "Evaluating jobs/build-output-as-input.nix for second time should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 1 , "Evaluating jobs/build-output-as-input.nix for second time after building build1 should result in 1 build in queue");
-for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/basic.nix should exit with code 0");
+subtest "For the 'build1' job" => sub {
+    my ($build) = queuedBuildsForJobset($jobset);
+    is($build->job, "build1", "Verify the only job we got is for 'build1'");
+
+    ok(runBuild($build), "Build should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
-    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/build-output-as-input.nix should be finished.");
-    is($newbuild->buildstatus, 0, "Build '".$build->job."' from jobs/build-output-as-input.nix should have buildstatus 0.");
-}
+    is($newbuild->finished, 1, "Build should be finished.");
+    is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+};
+
+ok(evalSucceeds($jobset), "Evaluating jobs/build-output-as-input.nix for second time should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 1 , "The second evaluation should result in 1 new build in queue: build2");
+subtest "For the 'build2' job" => sub {
+    my ($build) = queuedBuildsForJobset($jobset);
+    is($build->job, "build2", "Verify the only job we got is for 'build2'");
+
+    ok(runBuild($build), "Build should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    is($newbuild->finished, 1, "Build should be finished.");
+    is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+};
 
 done_testing;

From 2776ae6c785276724254f53462733736a277ce7a Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 14:25:40 -0500
Subject: [PATCH 09/13] Move tests for SCM inputs in to its own .t

---
 tests/evaluation.t  | 104 +--------------------------------------
 tests/input-types.t | 116 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+), 102 deletions(-)
 create mode 100644 tests/input-types.t

diff --git a/tests/evaluation.t b/tests/evaluation.t
index 9a47d44b..1368b0f9 100644
--- a/tests/evaluation.t
+++ b/tests/evaluation.t
@@ -7,7 +7,7 @@ use Setup;
 require Hydra::Schema;
 require Hydra::Model::DB;
 
-use Test::Simple tests => 60;
+use Test::Simple tests => 8;
 
 my $db = Hydra::Model::DB->new;
 hydra_setup($db);
@@ -20,109 +20,9 @@ my $jobsBaseUri = "file://".getcwd;
 my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
 my $jobset;
 
-
-# Test scm inputs
-my @scminputs = (
-    {
-        name => "svn",
-        nixexpr => "svn-input.nix",
-        type => "svn",
-        uri => "$jobsBaseUri/svn-repo",
-        update => getcwd . "/jobs/svn-update.sh"
-    },
-    {
-        name => "svn-checkout",
-        nixexpr => "svn-checkout-input.nix",
-        type => "svn-checkout",
-        uri => "$jobsBaseUri/svn-checkout-repo",
-        update => getcwd . "/jobs/svn-checkout-update.sh"
-    },
-    {
-        name => "git",
-        nixexpr => "git-input.nix",
-        type => "git",
-        uri => "$jobsBaseUri/git-repo",
-        update => getcwd . "/jobs/git-update.sh"
-    },
-    {
-        name => "git-rev",
-        nixexpr => "git-rev-input.nix",
-        type => "git",
-        uri => "$jobsBaseUri/git-repo 7f60df502b96fd54bbfa64dd94b56d936a407701",
-        update => getcwd . "/jobs/git-rev-update.sh"
-    },
-    {
-        name => "deepgit",
-        nixexpr => "deepgit-input.nix",
-        type => "git",
-        uri => "$jobsBaseUri/git-repo master 1",
-        update => getcwd . "/jobs/git-update.sh"
-    },
-    {
-        name => "bzr",
-        nixexpr => "bzr-input.nix",
-        type => "bzr",
-        uri => "$jobsBaseUri/bzr-repo",
-        update => getcwd . "/jobs/bzr-update.sh"
-    },
-    {
-        name => "bzr-checkout",
-        nixexpr => "bzr-checkout-input.nix",
-        type => "bzr-checkout",
-        uri => "$jobsBaseUri/bzr-checkout-repo",
-        update => getcwd . "/jobs/bzr-checkout-update.sh"
-    },
-    {
-        name => "hg",
-        nixexpr => "hg-input.nix",
-        type => "hg",
-        uri => "$jobsBaseUri/hg-repo",
-        update => getcwd . "/jobs/hg-update.sh"
-    },
-    {
-        name => "darcs",
-        nixexpr => "darcs-input.nix",
-        type => "darcs",
-        uri => "$jobsBaseUri/darcs-repo",
-        update => getcwd . "/jobs/darcs-update.sh"
-    }
-);
-
-foreach my $scm ( @scminputs ) {
-    my $scmName = $scm->{"name"};
-    my $nixexpr = $scm->{"nixexpr"};
-    my $type = $scm->{"type"};
-    my $uri = $scm->{"uri"};
-    my $update = $scm->{"update"};
-    $jobset = createJobsetWithOneInput($scmName, $nixexpr, "src", $type, $uri);
-
-    my $state = 0;
-    my $q = 0;
-    my ($loop, $updated) = updateRepository($scmName, $update);
-    while($loop) {
-        my $c = 0;
-
-        # Verify that it can be fetched and possibly queued.
-        ok(evalSucceeds($jobset),                  "$scmName:$state.$c: Evaluating nix-expression."); $c++;
-
-        # Verify that the evaluation has queued a new job and evaluate again to ...
-        if ($updated) {
-            $q++;
-            ok(nrQueuedBuildsForJobset($jobset) == $q, "$scmName:$state.$c: Expect $q jobs in the queue."); $c++;
-            ok(evalSucceeds($jobset),                  "$scmName:$state.$c: Evaluating nix-expression again."); $c++;
-        }
-
-        # ... check that it is deterministic and not queued again.
-        ok(nrQueuedBuildsForJobset($jobset) == $q, "$scmName:$state.$c: Expect $q jobs in the queue."); $c++;
-
-        $state++;
-        ($loop, $updated) = updateRepository($scmName, $update, getcwd . "/$scmName-repo/");
-    }
-}
-
 # Test build products
 
-$jobset = createBaseJobset("build-products", "build-products.nix");
+my $jobset = createBaseJobset("build-products", "build-products.nix");
 
 ok(evalSucceeds($jobset),                  "Evaluating jobs/build-products.nix should exit with return code 0");
 ok(nrQueuedBuildsForJobset($jobset) == 2 , "Evaluating jobs/build-products.nix should result in 2 builds");
diff --git a/tests/input-types.t b/tests/input-types.t
new file mode 100644
index 00000000..c41b8e55
--- /dev/null
+++ b/tests/input-types.t
@@ -0,0 +1,116 @@
+use strict;
+use Cwd;
+use Setup;
+
+(my $datadir, my $pgsql) = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $jobsBaseUri = "file://".getcwd;
+
+# Test scm inputs
+my @scminputs = (
+    {
+        name => "svn",
+        nixexpr => "svn-input.nix",
+        type => "svn",
+        uri => "$jobsBaseUri/svn-repo",
+        update => getcwd . "/jobs/svn-update.sh"
+    },
+    {
+        name => "svn-checkout",
+        nixexpr => "svn-checkout-input.nix",
+        type => "svn-checkout",
+        uri => "$jobsBaseUri/svn-checkout-repo",
+        update => getcwd . "/jobs/svn-checkout-update.sh"
+    },
+    {
+        name => "git",
+        nixexpr => "git-input.nix",
+        type => "git",
+        uri => "$jobsBaseUri/git-repo",
+        update => getcwd . "/jobs/git-update.sh"
+    },
+    {
+        name => "git-rev",
+        nixexpr => "git-rev-input.nix",
+        type => "git",
+        uri => "$jobsBaseUri/git-repo 7f60df502b96fd54bbfa64dd94b56d936a407701",
+        update => getcwd . "/jobs/git-rev-update.sh"
+    },
+    {
+        name => "deepgit",
+        nixexpr => "deepgit-input.nix",
+        type => "git",
+        uri => "$jobsBaseUri/git-repo master 1",
+        update => getcwd . "/jobs/git-update.sh"
+    },
+    {
+        name => "bzr",
+        nixexpr => "bzr-input.nix",
+        type => "bzr",
+        uri => "$jobsBaseUri/bzr-repo",
+        update => getcwd . "/jobs/bzr-update.sh"
+    },
+    {
+        name => "bzr-checkout",
+        nixexpr => "bzr-checkout-input.nix",
+        type => "bzr-checkout",
+        uri => "$jobsBaseUri/bzr-checkout-repo",
+        update => getcwd . "/jobs/bzr-checkout-update.sh"
+    },
+    {
+        name => "hg",
+        nixexpr => "hg-input.nix",
+        type => "hg",
+        uri => "$jobsBaseUri/hg-repo",
+        update => getcwd . "/jobs/hg-update.sh"
+    },
+    {
+        name => "darcs",
+        nixexpr => "darcs-input.nix",
+        type => "darcs",
+        uri => "$jobsBaseUri/darcs-repo",
+        update => getcwd . "/jobs/darcs-update.sh"
+    }
+);
+
+foreach my $scm ( @scminputs ) {
+    my $scmName = $scm->{"name"};
+    my $nixexpr = $scm->{"nixexpr"};
+    my $type = $scm->{"type"};
+    my $uri = $scm->{"uri"};
+    my $update = $scm->{"update"};
+    my $jobset = createJobsetWithOneInput($scmName, $nixexpr, "src", $type, $uri);
+
+    my $state = 0;
+    my $q = 0;
+    my ($loop, $updated) = updateRepository($scmName, $update);
+    while($loop) {
+        my $c = 0;
+
+        # Verify that it can be fetched and possibly queued.
+        ok(evalSucceeds($jobset),                  "$scmName:$state.$c: Evaluating nix-expression."); $c++;
+
+        # Verify that the evaluation has queued a new job and evaluate again to ...
+        if ($updated) {
+            $q++;
+            ok(nrQueuedBuildsForJobset($jobset) == $q, "$scmName:$state.$c: Expect $q jobs in the queue."); $c++;
+            ok(evalSucceeds($jobset),                  "$scmName:$state.$c: Evaluating nix-expression again."); $c++;
+        }
+
+        # ... check that it is deterministic and not queued again.
+        ok(nrQueuedBuildsForJobset($jobset) == $q, "$scmName:$state.$c: Expect $q jobs in the queue."); $c++;
+
+        $state++;
+        ($loop, $updated) = updateRepository($scmName, $update, getcwd . "/$scmName-repo/");
+    }
+}
+
+done_testing;

From 0df9c68422dc1a696190e14ca843f58413314c76 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 14:28:57 -0500
Subject: [PATCH 10/13] Relocate the final evalutation tests to a
 build-products specific test.

---
 tests/{evaluation.t => build-products.t} | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)
 rename tests/{evaluation.t => build-products.t} (86%)

diff --git a/tests/evaluation.t b/tests/build-products.t
similarity index 86%
rename from tests/evaluation.t
rename to tests/build-products.t
index 1368b0f9..bda1d90d 100644
--- a/tests/evaluation.t
+++ b/tests/build-products.t
@@ -7,18 +7,11 @@ use Setup;
 require Hydra::Schema;
 require Hydra::Model::DB;
 
-use Test::Simple tests => 8;
+use Test2::V0;
 
 my $db = Hydra::Model::DB->new;
 hydra_setup($db);
 
-my $res;
-my $stdout;
-my $stderr;
-
-my $jobsBaseUri = "file://".getcwd;
-my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
-my $jobset;
 
 # Test build products
 
@@ -41,3 +34,5 @@ for my $build (queuedBuildsForJobset($jobset)) {
         ok($buildproduct->name eq "some text.txt", "We should have: \"some text.txt\", but found: ".$buildproduct->name."\n");
     }
 }
+
+done_testing;

From 371826f93113a79fffff757c8bfaab28bc63f6c7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 14:44:23 -0500
Subject: [PATCH 11/13] Tests: build-products: use `is` for good errors on
 failures

---
 tests/build-products.t | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tests/build-products.t b/tests/build-products.t
index bda1d90d..3431324c 100644
--- a/tests/build-products.t
+++ b/tests/build-products.t
@@ -17,22 +17,27 @@ hydra_setup($db);
 
 my $jobset = createBaseJobset("build-products", "build-products.nix");
 
-ok(evalSucceeds($jobset),                  "Evaluating jobs/build-products.nix should exit with return code 0");
-ok(nrQueuedBuildsForJobset($jobset) == 2 , "Evaluating jobs/build-products.nix should result in 2 builds");
+ok(evalSucceeds($jobset),               "Evaluating jobs/build-products.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 2, "Evaluating jobs/build-products.nix should result in 2 builds");
 
 for my $build (queuedBuildsForJobset($jobset)) {
-    ok(runBuild($build), "Build '".$build->job."' from jobs/build-products.nix should exit with code 0");
-    my $newbuild = $db->resultset('Builds')->find($build->id);
-    ok($newbuild->finished == 1 && $newbuild->buildstatus == 0, "Build '".$build->job."' from jobs/build-products.nix should have buildstatus 0");
+    subtest "For the build job '" . $build->job . "'" => sub {
+        ok(runBuild($build), "Build should exit with code 0");
+        my $newbuild = $db->resultset('Builds')->find($build->id);
 
-    my $buildproducts = $db->resultset('BuildProducts')->search({ build => $build->id });
-    my $buildproduct = $buildproducts->next;
+        is($newbuild->finished, 1, "Build should have finished");
+        is($newbuild->buildstatus, 0, "Build should have buildstatus 0");
+
+        my $buildproducts = $db->resultset('BuildProducts')->search({ build => $build->id });
+        my $buildproduct = $buildproducts->next;
+
+        if($build->job eq "simple") {
+            is($buildproduct->name, "text.txt", "We should have \"text.txt\"");
+        } elsif ($build->job eq "with_spaces") {
+            is($buildproduct->name, "some text.txt", "We should have: \"some text.txt\"");
+        }
+    };
 
-    if($build->job eq "simple") {
-        ok($buildproduct->name eq "text.txt", "We should have text.txt, but found: ".$buildproduct->name."\n");
-    } elsif ($build->job eq "with_spaces") {
-        ok($buildproduct->name eq "some text.txt", "We should have: \"some text.txt\", but found: ".$buildproduct->name."\n");
-    }
 }
 
 done_testing;

From 611d7b71f230eef1e92efdc09e9a28706b9ff8d3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 14:52:37 -0500
Subject: [PATCH 12/13] input-types: use is() for test comparisons

---
 tests/input-types.t | 51 ++++++++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/tests/input-types.t b/tests/input-types.t
index c41b8e55..68035519 100644
--- a/tests/input-types.t
+++ b/tests/input-types.t
@@ -83,34 +83,37 @@ my @scminputs = (
 
 foreach my $scm ( @scminputs ) {
     my $scmName = $scm->{"name"};
-    my $nixexpr = $scm->{"nixexpr"};
-    my $type = $scm->{"type"};
-    my $uri = $scm->{"uri"};
-    my $update = $scm->{"update"};
-    my $jobset = createJobsetWithOneInput($scmName, $nixexpr, "src", $type, $uri);
 
-    my $state = 0;
-    my $q = 0;
-    my ($loop, $updated) = updateRepository($scmName, $update);
-    while($loop) {
-        my $c = 0;
+    subtest "With the SCM input named $scmName" => sub {
+        my $nixexpr = $scm->{"nixexpr"};
+        my $type = $scm->{"type"};
+        my $uri = $scm->{"uri"};
+        my $update = $scm->{"update"};
+        my $jobset = createJobsetWithOneInput($scmName, $nixexpr, "src", $type, $uri);
 
-        # Verify that it can be fetched and possibly queued.
-        ok(evalSucceeds($jobset),                  "$scmName:$state.$c: Evaluating nix-expression."); $c++;
+        my $state = 0;
+        my $q = 0;
+        my ($loop, $updated) = updateRepository($scmName, $update);
+        while($loop) {
+            subtest "Mutation number $state" => sub {
+                # Verify that it can be fetched and possibly queued.
+                ok(evalSucceeds($jobset),                  "Evaluating nix-expression.");
 
-        # Verify that the evaluation has queued a new job and evaluate again to ...
-        if ($updated) {
-            $q++;
-            ok(nrQueuedBuildsForJobset($jobset) == $q, "$scmName:$state.$c: Expect $q jobs in the queue."); $c++;
-            ok(evalSucceeds($jobset),                  "$scmName:$state.$c: Evaluating nix-expression again."); $c++;
+                # Verify that the evaluation has queued a new job and evaluate again to ...
+                if ($updated) {
+                    $q++;
+                    is(nrQueuedBuildsForJobset($jobset), $q, "Expect $q jobs in the queue.");
+                    ok(evalSucceeds($jobset),                "Evaluating nix-expression again.");
+                }
+
+                # ... check that it is deterministic and not queued again.
+                is(nrQueuedBuildsForJobset($jobset), $q, "Expect deterministic evaluation.");
+
+                $state++;
+                ($loop, $updated) = updateRepository($scmName, $update, getcwd . "/$scmName-repo/");
+            };
         }
-
-        # ... check that it is deterministic and not queued again.
-        ok(nrQueuedBuildsForJobset($jobset) == $q, "$scmName:$state.$c: Expect $q jobs in the queue."); $c++;
-
-        $state++;
-        ($loop, $updated) = updateRepository($scmName, $update, getcwd . "/$scmName-repo/");
-    }
+    };
 }
 
 done_testing;

From cccdc701627e0300920c6ace1dd65c89b6a08cad Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 23 Feb 2021 21:42:59 -0500
Subject: [PATCH 13/13] input-types.t: don't litter ./tests/

---
 tests/input-types.t | 27 +++++++++++++++------------
 tests/lib/Setup.pm  |  5 ++++-
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/tests/input-types.t b/tests/input-types.t
index 68035519..bda15b96 100644
--- a/tests/input-types.t
+++ b/tests/input-types.t
@@ -12,7 +12,10 @@ use Test2::V0;
 my $db = Hydra::Model::DB->new;
 hydra_setup($db);
 
-my $jobsBaseUri = "file://".getcwd;
+my $testdir = getcwd;
+my $scratchdir = "$datadir/scratch";
+mkdir $scratchdir;
+my $jobsBaseUri = "file://".$scratchdir;
 
 # Test scm inputs
 my @scminputs = (
@@ -21,63 +24,63 @@ my @scminputs = (
         nixexpr => "svn-input.nix",
         type => "svn",
         uri => "$jobsBaseUri/svn-repo",
-        update => getcwd . "/jobs/svn-update.sh"
+        update => $testdir . "/jobs/svn-update.sh"
     },
     {
         name => "svn-checkout",
         nixexpr => "svn-checkout-input.nix",
         type => "svn-checkout",
         uri => "$jobsBaseUri/svn-checkout-repo",
-        update => getcwd . "/jobs/svn-checkout-update.sh"
+        update => $testdir . "/jobs/svn-checkout-update.sh"
     },
     {
         name => "git",
         nixexpr => "git-input.nix",
         type => "git",
         uri => "$jobsBaseUri/git-repo",
-        update => getcwd . "/jobs/git-update.sh"
+        update => $testdir . "/jobs/git-update.sh"
     },
     {
         name => "git-rev",
         nixexpr => "git-rev-input.nix",
         type => "git",
         uri => "$jobsBaseUri/git-repo 7f60df502b96fd54bbfa64dd94b56d936a407701",
-        update => getcwd . "/jobs/git-rev-update.sh"
+        update => $testdir . "/jobs/git-rev-update.sh"
     },
     {
         name => "deepgit",
         nixexpr => "deepgit-input.nix",
         type => "git",
         uri => "$jobsBaseUri/git-repo master 1",
-        update => getcwd . "/jobs/git-update.sh"
+        update => $testdir . "/jobs/git-update.sh"
     },
     {
         name => "bzr",
         nixexpr => "bzr-input.nix",
         type => "bzr",
         uri => "$jobsBaseUri/bzr-repo",
-        update => getcwd . "/jobs/bzr-update.sh"
+        update => $testdir . "/jobs/bzr-update.sh"
     },
     {
         name => "bzr-checkout",
         nixexpr => "bzr-checkout-input.nix",
         type => "bzr-checkout",
         uri => "$jobsBaseUri/bzr-checkout-repo",
-        update => getcwd . "/jobs/bzr-checkout-update.sh"
+        update => $testdir . "/jobs/bzr-checkout-update.sh"
     },
     {
         name => "hg",
         nixexpr => "hg-input.nix",
         type => "hg",
         uri => "$jobsBaseUri/hg-repo",
-        update => getcwd . "/jobs/hg-update.sh"
+        update => $testdir . "/jobs/hg-update.sh"
     },
     {
         name => "darcs",
         nixexpr => "darcs-input.nix",
         type => "darcs",
         uri => "$jobsBaseUri/darcs-repo",
-        update => getcwd . "/jobs/darcs-update.sh"
+        update => $testdir . "/jobs/darcs-update.sh"
     }
 );
 
@@ -93,7 +96,7 @@ foreach my $scm ( @scminputs ) {
 
         my $state = 0;
         my $q = 0;
-        my ($loop, $updated) = updateRepository($scmName, $update);
+        my ($loop, $updated) = updateRepository($scmName, $update, $scratchdir);
         while($loop) {
             subtest "Mutation number $state" => sub {
                 # Verify that it can be fetched and possibly queued.
@@ -110,7 +113,7 @@ foreach my $scm ( @scminputs ) {
                 is(nrQueuedBuildsForJobset($jobset), $q, "Expect deterministic evaluation.");
 
                 $state++;
-                ($loop, $updated) = updateRepository($scmName, $update, getcwd . "/$scmName-repo/");
+                ($loop, $updated) = updateRepository($scmName, $update, $scratchdir);
             };
         }
     };
diff --git a/tests/lib/Setup.pm b/tests/lib/Setup.pm
index 03c7046a..eba61aef 100644
--- a/tests/lib/Setup.pm
+++ b/tests/lib/Setup.pm
@@ -115,8 +115,11 @@ sub runBuild {
 }
 
 sub updateRepository {
-    my ($scm, $update) = @_;
+    my ($scm, $update, $scratchdir) = @_;
+    my $curdir = getcwd;
+    chdir "$scratchdir";
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ($update, $scm));
+    chdir "$curdir";
     die "unexpected update error with $scm: $stderr\n" if $res;
     my ($message, $loop, $status) = $stdout =~ m/::(.*) -- (.*) -- (.*)::/;
     print STDOUT "Update $scm repository: $message\n";