Update "make check" for the new queue runner

Also, if the machines file contains an entry for localhost, then run
"nix-store --serve" directly, without going through SSH.

src/hydra-queue-runner/Makefile.am

@@ -1,6 +1,7 @@
 bin_PROGRAMS = hydra-queue-runner
 
-hydra_queue_runner_SOURCES = hydra-queue-runner.cc build-result.cc build-remote.cc
+hydra_queue_runner_SOURCES = hydra-queue-runner.cc build-result.cc build-remote.cc \
+ build-remote.hh build-result.hh counter.hh pool.hh sync.hh token-server.hh
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall

src/hydra-queue-runner/build-remote.cc

@@ -21,6 +21,12 @@ struct Child
 };
 
 
+static void append(Strings & dst, const Strings & src)
+{
+    dst.insert(dst.end(), src.begin(), src.end());
+}
+
+
 static void openConnection(const string & sshName, const string & sshKey,
     int stderrFD, Child & child)
 {
@@ -39,13 +45,18 @@ static void openConnection(const string & sshName, const string & sshKey,
         if (dup2(stderrFD, STDERR_FILENO) == -1)
             throw SysError("cannot dup stderr");
 
-        // FIXME: connection timeouts
+        Strings argv;
-        Strings argv(
+        if (sshName == "localhost")
-            { "ssh", sshName, "-i", sshKey, "-x", "-a"
+            argv = {"nix-store", "--serve", "--write"};
-            , "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+        else {
+            argv = {"ssh", sshName};
+            if (sshKey != "" && sshKey != "-") append(argv, {"-i", sshKey});
+            append(argv,
+                { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
                 , "--", "nix-store", "--serve", "--write" });
+        }
 
-        execvp("ssh", (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
+        execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
 
         throw SysError("cannot start ssh");
     });

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -232,7 +232,7 @@ private:
 
     /* The build machines. */
     typedef std::map<string, Machine::ptr> Machines;
-    Sync<Machines> machines;
+    Sync<Machines> machines; // FIXME: use atomic_shared_ptr
 
     Path machinesFile;
     struct stat machinesFileStat;
@@ -269,6 +269,9 @@ private:
     Sync<std::queue<NotificationItem>> notificationSenderQueue;
     std::condition_variable_any notificationSenderWakeup;
 
+    /* Specific build to do for --build-one (testing only). */
+    BuildID buildOne;
+
 public:
     State();
 
@@ -342,7 +345,7 @@ public:
 
     void unlock();
 
-    void run();
+    void run(BuildID buildOne = 0);
 };
 
 
@@ -562,6 +565,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
         for (auto const & row : res) {
             auto builds_(builds.lock());
             BuildID id = row["id"].as<BuildID>();
+            if (buildOne && id != buildOne) continue;
             if (id > lastBuildId) lastBuildId = id;
             if (has(*builds_, id)) continue;
 
@@ -1122,6 +1126,8 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
             % step->drvPath % machine->sshName % build->id % (dependents.size() - 1));
     }
 
+    bool quit = build->id == buildOne;
+
     auto conn(dbPool.get());
 
     RemoteResult result;
@@ -1188,6 +1194,7 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
             finishBuildStep(txn, result.startTime, result.stopTime, build->id,
                 stepNr, machine->sshName, bssAborted, result.errorMsg);
             txn.commit();
+            if (quit) exit(1);
             return true;
         }
     }
@@ -1379,6 +1386,7 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
                 b->finishedInDB = true;
                 builds_->erase(b->id);
                 dependentIDs.push_back(b->id);
+                if (buildOne == b->id) quit = true;
             }
         }
 
@@ -1399,6 +1407,8 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
     machine->state->totalStepTime += stepStopTime - stepStartTime;
     machine->state->totalStepBuildTime += result.stopTime - result.startTime;
 
+    if (quit) exit(0); // testing hack
+
     return false;
 }
 
@@ -1694,9 +1704,10 @@ void State::unlock()
 }
 
 
-void State::run()
+void State::run(BuildID buildOne)
 {
     startedAt = time(0);
+    this->buildOne = buildOne;
 
     auto lock = acquireGlobalLock();
     if (!lock)
@@ -1752,13 +1763,17 @@ int main(int argc, char * * argv)
 
         bool unlock = false;
         bool status = false;
+        BuildID buildOne = 0;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--unlock")
                 unlock = true;
             else if (*arg == "--status")
                 status = true;
-            else
+            else if (*arg == "--build-one") {
+                if (!string2Int<BuildID>(getArg(*arg, arg, end), buildOne))
+                    throw Error("‘--build-one’ requires a build ID");
+            } else
                 return false;
             return true;
         });
@@ -1773,6 +1788,6 @@ int main(int argc, char * * argv)
         else if (unlock)
             state.unlock();
         else
-            state.run();
+            state.run(buildOne);
     });
 }

tests/Makefile.am

@@ -1,6 +1,6 @@
 TESTS_ENVIRONMENT = \
   BZR_HOME="$(abs_builddir)/data"			\
-  HYDRA_DBI="dbi:SQLite:db.sqlite"			\
+  HYDRA_DBI="dbi:Pg:dbname=hydra-test-suite;"		\
   HYDRA_DATA="$(abs_builddir)/data"			\
   HYDRA_HOME="$(top_srcdir)/src"			\
   HYDRA_CONFIG=						\
@@ -22,15 +22,11 @@ EXTRA_DIST = \
   $(TESTS)
 
 TESTS = \
-  query-all-tables.pl \
+  set-up.pl \
-  evaluation-tests.pl
+  evaluation-tests.pl \
+  tear-down.pl
 
-clean:
+check_SCRIPTS = repos
-	chmod -R a+w nix || true
-	rm -rf db.sqlite data nix git-repo hg-repo svn-repo svn-checkout svn-checkout-repo bzr-repo bzr-checkout-repo darcs-repo
-	rm -f .*-state
-
-check_SCRIPTS = db.sqlite repos
 
 db.sqlite: $(top_srcdir)/src/sql/hydra-sqlite.sql
 	$(TESTS_ENVIRONMENT) $(top_srcdir)/src/script/hydra-init

tests/Setup.pm

@@ -71,8 +71,7 @@ sub evalSucceeds {
 
 sub runBuild {
     my ($build) = @_;
-    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-build", $build->id));
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));
-    print "STDERR: $stderr" if $stderr ne "";
     return !$res;
 }
 

tests/evaluation-tests.pl

@@ -28,7 +28,7 @@ ok(nrQueuedBuildsForJobset($jobset) == 3 , "Evaluating jobs/basic.nix should res
 for my $build (queuedBuildsForJobset($jobset)) {
     ok(runBuild($build), "Build '".$build->job->name."' from jobs/basic.nix should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
-    my $expected = $build->job->name eq "fails" ? 1 : 0;
+    my $expected = $build->job->name eq "fails" ? 1 : $build->job->name =~ /with_failed/ ? 6 : 0;
     ok($newbuild->finished == 1 && $newbuild->buildstatus == $expected, "Build '".$build->job->name."' from jobs/basic.nix should have buildstatus $expected");
 }
 

tests/query-all-tables.pl

@@ -1,21 +0,0 @@
-use strict;
-use Hydra::Schema;
-use Hydra::Model::DB;
-
-my $db = Hydra::Model::DB->new;
-
-my @sources = $db->schema->sources;
-my $nrtables = scalar(@sources);
-
-use Test::Simple tests => 38;
-
-foreach my $source (@sources) {
-    my $title = "Basic select query for $source";
-    if ($source eq "SchemaVersion" || $source eq "NrBuilds") {
-        ok(scalar($db->resultset($source)->all) == 1, $title);
-    } elsif( $source !~ m/^LatestSucceeded/) {
-        ok(scalar($db->resultset($source)->all) == 0, $title);
-    } else {
-        ok(scalar($db->resultset($source)->search({},{ bind => ["", "", ""] })) == 0, $title);
-    }
-}

tests/set-up.pl

@@ -0,0 +1,3 @@
+use strict;
+system("createdb hydra-test-suite") == 0 or die;
+system("hydra-init") == 0 or die;

tests/tear-down.pl

@@ -0,0 +1,5 @@
+use strict;
+system("chmod -R a+w nix") == 0 or die;
+system("rm -rf data nix git-repo hg-repo svn-repo svn-checkout svn-checkout-repo bzr-repo bzr-checkout-repo darcs-repo") == 0 or die;
+system("rm -f .*-state") == 0 or die;
+system("dropdb hydra-test-suite") == 0 or die;