From 18a3c3ff1c1cb6b601a1db36b28ca052c765a85c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Jun 2015 15:29:22 +0200 Subject: [PATCH] Update "make check" for the new queue runner Also, if the machines file contains an entry for localhost, then run "nix-store --serve" directly, without going through SSH. --- src/hydra-queue-runner/Makefile.am | 3 ++- src/hydra-queue-runner/build-remote.cc | 23 +++++++++++++----- src/hydra-queue-runner/hydra-queue-runner.cc | 25 ++++++++++++++++---- tests/Makefile.am | 14 ++++------- tests/Setup.pm | 3 +-- tests/evaluation-tests.pl | 2 +- tests/query-all-tables.pl | 21 ---------------- tests/s3-backup-test.pl | 0 tests/set-up.pl | 3 +++ tests/tear-down.pl | 5 ++++ 10 files changed, 54 insertions(+), 45 deletions(-) mode change 100755 => 100644 tests/evaluation-tests.pl delete mode 100755 tests/query-all-tables.pl mode change 100755 => 100644 tests/s3-backup-test.pl create mode 100644 tests/set-up.pl create mode 100644 tests/tear-down.pl diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 00aa254d..699a22a5 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -1,6 +1,7 @@ bin_PROGRAMS = hydra-queue-runner -hydra_queue_runner_SOURCES = hydra-queue-runner.cc build-result.cc build-remote.cc +hydra_queue_runner_SOURCES = hydra-queue-runner.cc build-result.cc build-remote.cc \ + build-remote.hh build-result.hh counter.hh pool.hh sync.hh token-server.hh hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx AM_CXXFLAGS = $(NIX_CFLAGS) -Wall diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 73cf294c..4fa9da0c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -21,6 +21,12 @@ struct Child }; +static void append(Strings & dst, const Strings & src) +{ + dst.insert(dst.end(), src.begin(), src.end()); +} + + static void openConnection(const string & sshName, const string & sshKey, int stderrFD, Child & child) { @@ -39,13 +45,18 @@ static void openConnection(const string & sshName, const string & sshKey, if (dup2(stderrFD, STDERR_FILENO) == -1) throw SysError("cannot dup stderr"); - // FIXME: connection timeouts - Strings argv( - { "ssh", sshName, "-i", sshKey, "-x", "-a" - , "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" - , "--", "nix-store", "--serve", "--write" }); + Strings argv; + if (sshName == "localhost") + argv = {"nix-store", "--serve", "--write"}; + else { + argv = {"ssh", sshName}; + if (sshKey != "" && sshKey != "-") append(argv, {"-i", sshKey}); + append(argv, + { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" + , "--", "nix-store", "--serve", "--write" }); + } - execvp("ssh", (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast + execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast throw SysError("cannot start ssh"); }); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index ccb3e829..9037ee10 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -232,7 +232,7 @@ private: /* The build machines. */ typedef std::map Machines; - Sync machines; + Sync machines; // FIXME: use atomic_shared_ptr Path machinesFile; struct stat machinesFileStat; @@ -269,6 +269,9 @@ private: Sync> notificationSenderQueue; std::condition_variable_any notificationSenderWakeup; + /* Specific build to do for --build-one (testing only). */ + BuildID buildOne; + public: State(); @@ -342,7 +345,7 @@ public: void unlock(); - void run(); + void run(BuildID buildOne = 0); }; @@ -562,6 +565,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, for (auto const & row : res) { auto builds_(builds.lock()); BuildID id = row["id"].as(); + if (buildOne && id != buildOne) continue; if (id > lastBuildId) lastBuildId = id; if (has(*builds_, id)) continue; @@ -1122,6 +1126,8 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, % step->drvPath % machine->sshName % build->id % (dependents.size() - 1)); } + bool quit = build->id == buildOne; + auto conn(dbPool.get()); RemoteResult result; @@ -1188,6 +1194,7 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, machine->sshName, bssAborted, result.errorMsg); txn.commit(); + if (quit) exit(1); return true; } } @@ -1379,6 +1386,7 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, b->finishedInDB = true; builds_->erase(b->id); dependentIDs.push_back(b->id); + if (buildOne == b->id) quit = true; } } @@ -1399,6 +1407,8 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, machine->state->totalStepTime += stepStopTime - stepStartTime; machine->state->totalStepBuildTime += result.stopTime - result.startTime; + if (quit) exit(0); // testing hack + return false; } @@ -1694,9 +1704,10 @@ void State::unlock() } -void State::run() +void State::run(BuildID buildOne) { startedAt = time(0); + this->buildOne = buildOne; auto lock = acquireGlobalLock(); if (!lock) @@ -1752,13 +1763,17 @@ int main(int argc, char * * argv) bool unlock = false; bool status = false; + BuildID buildOne = 0; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--unlock") unlock = true; else if (*arg == "--status") status = true; - else + else if (*arg == "--build-one") { + if (!string2Int(getArg(*arg, arg, end), buildOne)) + throw Error("‘--build-one’ requires a build ID"); + } else return false; return true; }); @@ -1773,6 +1788,6 @@ int main(int argc, char * * argv) else if (unlock) state.unlock(); else - state.run(); + state.run(buildOne); }); } diff --git a/tests/Makefile.am b/tests/Makefile.am index 165903d3..8b8880f7 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTS_ENVIRONMENT = \ BZR_HOME="$(abs_builddir)/data" \ - HYDRA_DBI="dbi:SQLite:db.sqlite" \ + HYDRA_DBI="dbi:Pg:dbname=hydra-test-suite;" \ HYDRA_DATA="$(abs_builddir)/data" \ HYDRA_HOME="$(top_srcdir)/src" \ HYDRA_CONFIG= \ @@ -22,15 +22,11 @@ EXTRA_DIST = \ $(TESTS) TESTS = \ - query-all-tables.pl \ - evaluation-tests.pl + set-up.pl \ + evaluation-tests.pl \ + tear-down.pl -clean: - chmod -R a+w nix || true - rm -rf db.sqlite data nix git-repo hg-repo svn-repo svn-checkout svn-checkout-repo bzr-repo bzr-checkout-repo darcs-repo - rm -f .*-state - -check_SCRIPTS = db.sqlite repos +check_SCRIPTS = repos db.sqlite: $(top_srcdir)/src/sql/hydra-sqlite.sql $(TESTS_ENVIRONMENT) $(top_srcdir)/src/script/hydra-init diff --git a/tests/Setup.pm b/tests/Setup.pm index 96aecde6..209d950d 100644 --- a/tests/Setup.pm +++ b/tests/Setup.pm @@ -71,8 +71,7 @@ sub evalSucceeds { sub runBuild { my ($build) = @_; - my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-build", $build->id)); - print "STDERR: $stderr" if $stderr ne ""; + my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id)); return !$res; } diff --git a/tests/evaluation-tests.pl b/tests/evaluation-tests.pl old mode 100755 new mode 100644 index 90ae41df..71673f79 --- a/tests/evaluation-tests.pl +++ b/tests/evaluation-tests.pl @@ -28,7 +28,7 @@ ok(nrQueuedBuildsForJobset($jobset) == 3 , "Evaluating jobs/basic.nix should res for my $build (queuedBuildsForJobset($jobset)) { ok(runBuild($build), "Build '".$build->job->name."' from jobs/basic.nix should exit with code 0"); my $newbuild = $db->resultset('Builds')->find($build->id); - my $expected = $build->job->name eq "fails" ? 1 : 0; + my $expected = $build->job->name eq "fails" ? 1 : $build->job->name =~ /with_failed/ ? 6 : 0; ok($newbuild->finished == 1 && $newbuild->buildstatus == $expected, "Build '".$build->job->name."' from jobs/basic.nix should have buildstatus $expected"); } diff --git a/tests/query-all-tables.pl b/tests/query-all-tables.pl deleted file mode 100755 index eb67a7f2..00000000 --- a/tests/query-all-tables.pl +++ /dev/null @@ -1,21 +0,0 @@ -use strict; -use Hydra::Schema; -use Hydra::Model::DB; - -my $db = Hydra::Model::DB->new; - -my @sources = $db->schema->sources; -my $nrtables = scalar(@sources); - -use Test::Simple tests => 38; - -foreach my $source (@sources) { - my $title = "Basic select query for $source"; - if ($source eq "SchemaVersion" || $source eq "NrBuilds") { - ok(scalar($db->resultset($source)->all) == 1, $title); - } elsif( $source !~ m/^LatestSucceeded/) { - ok(scalar($db->resultset($source)->all) == 0, $title); - } else { - ok(scalar($db->resultset($source)->search({},{ bind => ["", "", ""] })) == 0, $title); - } -} diff --git a/tests/s3-backup-test.pl b/tests/s3-backup-test.pl old mode 100755 new mode 100644 diff --git a/tests/set-up.pl b/tests/set-up.pl new file mode 100644 index 00000000..ff72483f --- /dev/null +++ b/tests/set-up.pl @@ -0,0 +1,3 @@ +use strict; +system("createdb hydra-test-suite") == 0 or die; +system("hydra-init") == 0 or die; diff --git a/tests/tear-down.pl b/tests/tear-down.pl new file mode 100644 index 00000000..32e51a87 --- /dev/null +++ b/tests/tear-down.pl @@ -0,0 +1,5 @@ +use strict; +system("chmod -R a+w nix") == 0 or die; +system("rm -rf data nix git-repo hg-repo svn-repo svn-checkout svn-checkout-repo bzr-repo bzr-checkout-repo darcs-repo") == 0 or die; +system("rm -f .*-state") == 0 or die; +system("dropdb hydra-test-suite") == 0 or die;