From 4d26546d3c137cf4afb2403c60c28a1053599894 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2015 00:57:30 +0200 Subject: [PATCH] Add support for tracking custom metrics Builds can now emit metrics that Hydra will store in its database and render as time series via flot charts. Typical applications are to keep track of performance indicators, coverage percentages, artifact sizes, and so on. For example, a coverage build can emit the coverage percentage as follows: echo "lineCoverage $pct %" > $out/nix-support/hydra-metrics Graphs of all metrics for a job can be seen at http://.../job///#tabs-charts Specific metrics are also visible at http://.../job////metric/ The latter URL also allows getting the data in JSON format (e.g. via "curl -H 'Accept: application/json'"). --- src/hydra-queue-runner/build-result.cc | 47 +++-- src/hydra-queue-runner/build-result.hh | 8 + src/hydra-queue-runner/hydra-queue-runner.cc | 13 ++ src/hydra-queue-runner/queue-monitor.cc | 9 +- src/hydra-queue-runner/state.hh | 9 +- src/lib/Hydra/Controller/Job.pm | 17 ++ src/lib/Hydra/Schema/BuildMetrics.pm | 187 +++++++++++++++++++ src/lib/Hydra/Schema/Builds.pm | 20 +- src/lib/Hydra/Schema/Jobs.pm | 23 ++- src/lib/Hydra/Schema/Jobsets.pm | 22 ++- src/lib/Hydra/Schema/Projects.pm | 19 +- src/root/build.tt | 21 ++- src/root/common.tt | 8 +- src/root/job.tt | 8 + src/root/metric.tt | 7 + src/root/static/css/hydra.css | 4 + src/sql/hydra.sql | 23 +++ src/sql/upgrade-39.sql | 22 +++ 18 files changed, 437 insertions(+), 30 deletions(-) create mode 100644 src/lib/Hydra/Schema/BuildMetrics.pm create mode 100644 src/root/metric.tt create mode 100644 src/sql/upgrade-39.sql diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index c2c2c8af..5f516d23 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -7,6 +7,21 @@ using namespace nix; +static std::tuple secureRead(Path fileName) +{ + auto fail = std::make_tuple(false, ""); + + if (!pathExists(fileName)) return fail; + + try { + /* For security, resolve symlinks. */ + fileName = canonPath(fileName, true); + if (!isInStore(fileName)) return fail; + return std::make_tuple(true, readFile(fileName)); + } catch (Error & e) { return fail; } +} + + BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & drv) { BuildOutput res; @@ -40,22 +55,12 @@ BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & d Path failedFile = output + "/nix-support/failed"; if (pathExists(failedFile)) res.failed = true; - Path productsFile = output + "/nix-support/hydra-build-products"; - if (!pathExists(productsFile)) continue; + auto file = secureRead(output + "/nix-support/hydra-build-products"); + if (!std::get<0>(file)) continue; + explicitProducts = true; - /* For security, resolve symlinks. */ - try { - productsFile = canonPath(productsFile, true); - } catch (Error & e) { continue; } - if (!isInStore(productsFile)) continue; - - string contents; - try { - contents = readFile(productsFile); - } catch (Error & e) { continue; } - - for (auto & line : tokenizeString(contents, "\n")) { + for (auto & line : tokenizeString(std::get<1>(file), "\n")) { BuildProduct product; Regex::Subs subs; @@ -122,5 +127,19 @@ BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & d // FIXME: validate release name } + /* Get metrics. */ + for (auto & output : outputs) { + auto file = secureRead(output + "/nix-support/hydra-metrics"); + for (auto & line : tokenizeString(std::get<1>(file), "\n")) { + auto fields = tokenizeString>(line); + if (fields.size() < 2) continue; + BuildMetric metric; + metric.name = fields[0]; // FIXME: validate + metric.value = atof(fields[1].c_str()); // FIXME + metric.unit = fields.size() >= 3 ? fields[2] : ""; + res.metrics[metric.name] = metric; + } + } + return res; } diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh index 65fbf948..e40e7dcd 100644 --- a/src/hydra-queue-runner/build-result.hh +++ b/src/hydra-queue-runner/build-result.hh @@ -15,6 +15,12 @@ struct BuildProduct BuildProduct() { } }; +struct BuildMetric +{ + std::string name, unit; + double value; +}; + struct BuildOutput { /* Whether this build has failed with output, i.e., the build @@ -27,6 +33,8 @@ struct BuildOutput unsigned long long closureSize = 0, size = 0; std::list products; + + std::map metrics; }; BuildOutput getBuildOutput(std::shared_ptr store, const nix::Derivation & drv); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 2860b854..10fd1cd1 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -238,6 +238,19 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, (product.defaultPath).exec(); } + for (auto & metric : res.metrics) { + txn.parameterized + ("insert into BuildMetrics (build, name, unit, value, project, jobset, job, timestamp) values ($1, $2, $3, $4, $5, $6, $7, $8)") + (build->id) + (metric.second.name) + (metric.second.unit, metric.second.unit != "") + (metric.second.value) + (build->projectName) + (build->jobsetName) + (build->jobName) + (build->timestamp).exec(); + } + nrBuildsDone++; } diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index bd9b2aef..8448cb68 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -64,7 +64,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, { pqxx::work txn(conn); - auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); + auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); for (auto const & row : res) { auto builds_(builds.lock()); @@ -76,9 +76,12 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, auto build = std::make_shared(); build->id = id; build->drvPath = row["drvPath"].as(); - build->fullJobName = row["project"].as() + ":" + row["jobset"].as() + ":" + row["job"].as(); + build->projectName = row["project"].as(); + build->jobsetName = row["jobset"].as(); + build->jobName = row["job"].as(); build->maxSilentTime = row["maxsilent"].as(); build->buildTimeout = row["timeout"].as(); + build->timestamp = row["timestamp"].as(); newBuilds.emplace(std::make_pair(build->drvPath, build)); } @@ -89,7 +92,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, std::function createBuild; createBuild = [&](Build::ptr build) { - printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName); + printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); nrAdded++; if (!store->isValidPath(build->drvPath)) { diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 761a100b..2bc3a56c 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -38,6 +38,7 @@ typedef enum { bssFailed = 1, bssAborted = 4, bssTimedOut = 7, + bssCachedFailure = 8, bssUnsupported = 9, bssBusy = 100, // not stored } BuildStepStatus; @@ -67,12 +68,18 @@ struct Build BuildID id; nix::Path drvPath; std::map outputs; - std::string fullJobName; + std::string projectName, jobsetName, jobName; + time_t timestamp; unsigned int maxSilentTime, buildTimeout; std::shared_ptr toplevel; std::atomic_bool finishedInDB{false}; + + std::string fullJobName() + { + return projectName + ":" + jobsetName + ":" + jobName; + } }; diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 274d9604..475eca37 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -77,6 +77,9 @@ sub overview : Chained('job') PathPart('') Args(0) { , jobset => $c->stash->{jobset}->name , job => $c->stash->{job}->name })->count == 1 if $c->user_exists; + + $c->stash->{metrics} = [ $job->buildmetrics->search( + { }, { select => ["name"], distinct => 1, order_by => "timestamp desc", }) ]; } @@ -110,6 +113,20 @@ sub output_sizes : Chained('job') PathPart('output-sizes') Args(0) { } +sub metric : Chained('job') PathPart('metric') Args(1) { + my ($self, $c, $metricName) = @_; + + $c->stash->{template} = 'metric.tt'; + $c->stash->{metricName} = $metricName; + + my @res = $c->stash->{job}->buildmetrics->search( + { name => $metricName }, + { order_by => "timestamp", columns => [ "build", "name", "timestamp", "value", "unit" ] }); + + $self->status_ok($c, entity => [ map { { id => $_->get_column("build"), timestamp => $_ ->timestamp, value => $_->value, unit => $_->unit } } @res ]); +} + + # Hydra::Base::Controller::ListBuilds needs this. sub get_builds : Chained('job') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; diff --git a/src/lib/Hydra/Schema/BuildMetrics.pm b/src/lib/Hydra/Schema/BuildMetrics.pm new file mode 100644 index 00000000..58bbed94 --- /dev/null +++ b/src/lib/Hydra/Schema/BuildMetrics.pm @@ -0,0 +1,187 @@ +use utf8; +package Hydra::Schema::BuildMetrics; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::BuildMetrics + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("BuildMetrics"); + +=head1 ACCESSORS + +=head2 build + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 name + + data_type: 'text' + is_nullable: 0 + +=head2 unit + + data_type: 'text' + is_nullable: 1 + +=head2 value + + data_type: 'double precision' + is_nullable: 0 + +=head2 project + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 jobset + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 job + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 timestamp + + data_type: 'integer' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "build", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "name", + { data_type => "text", is_nullable => 0 }, + "unit", + { data_type => "text", is_nullable => 1 }, + "value", + { data_type => "double precision", is_nullable => 0 }, + "project", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "jobset", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "job", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "timestamp", + { data_type => "integer", is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("build", "name"); + +=head1 RELATIONS + +=head2 build + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "build", + "Hydra::Schema::Builds", + { id => "build" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + +=head2 job + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "job", + "Hydra::Schema::Jobs", + { jobset => "jobset", name => "job", project => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + +=head2 jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset", + "Hydra::Schema::Jobsets", + { name => "jobset", project => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + +=head2 project + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "project", + "Hydra::Schema::Projects", + { name => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qoPm5/le+sVHigW4Dmum2Q + +sub json_hint { + return { columns => ['value', 'unit'] }; +} + +1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index a5abce08..5e502449 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -341,6 +341,21 @@ __PACKAGE__->has_many( undef, ); +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { "foreign.build" => "self.id" }, + undef, +); + =head2 buildoutputs Type: has_many @@ -535,8 +550,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:03:55 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EwxiaQpqbdzI9RvU0uUtLQ +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y2lDtgY8EBLOuCHAI8fWRQ __PACKAGE__->has_many( "dependents", @@ -630,6 +645,7 @@ my %hint = ( buildoutputs => 'name', buildinputs_builds => 'name', buildproducts => 'productnr', + buildmetrics => 'name', } ); diff --git a/src/lib/Hydra/Schema/Jobs.pm b/src/lib/Hydra/Schema/Jobs.pm index 883c1bee..cd89ed3d 100644 --- a/src/lib/Hydra/Schema/Jobs.pm +++ b/src/lib/Hydra/Schema/Jobs.pm @@ -81,6 +81,25 @@ __PACKAGE__->set_primary_key("project", "jobset", "name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { + "foreign.job" => "self.name", + "foreign.jobset" => "self.jobset", + "foreign.project" => "self.project", + }, + undef, +); + =head2 builds Type: has_many @@ -150,7 +169,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-09-29 19:41:42 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lnZSd0gDXgLk8WQeAFqByA +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vDAo9bzLca+QWfhOb9OLMg 1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index f304f770..682b3ab0 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -184,6 +184,24 @@ __PACKAGE__->set_primary_key("project", "name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { + "foreign.jobset" => "self.name", + "foreign.project" => "self.project", + }, + undef, +); + =head2 builds Type: has_many @@ -320,8 +338,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:51 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CO0aE+jrjB+UrwGRzWZLlw +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Coci9FdBAvUO9T3st2NEqA my %hint = ( columns => [ diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index 9bf9dde3..e04b1f8e 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -106,6 +106,21 @@ __PACKAGE__->set_primary_key("name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { "foreign.project" => "self.name" }, + undef, +); + =head2 builds Type: has_many @@ -267,8 +282,8 @@ Composing rels: L -> username __PACKAGE__->many_to_many("usernames", "projectmembers", "username"); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:08 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fkd9ruEoVSBGIktmAj4u4g +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:67kWIE0IGmEJTvOIATAKaw my %hint = ( columns => [ diff --git a/src/root/build.tt b/src/root/build.tt index 9ecd5cd6..e3eb43f9 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -106,7 +106,7 @@ [% END %] - +
  • Summary
    • [% IF isAggregate %]
  • Constituents
    • [% END %]
  • Details
    • @@ -385,6 +385,25 @@ [% END %] + + [% IF build.finished && build.buildmetrics %] +

    Metrics

    + + + + + + + [% FOREACH metric IN build.buildmetrics %] + + + + + [% END %] + +
    NameValue
    [%HTML.escape(metric.name)%][%metric.value%][%metric.unit%]
    + [% END %] +
    diff --git a/src/root/common.tt b/src/root/common.tt index 19e2462e..dafae8aa 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -566,12 +566,14 @@ BLOCK createChart %] success: function(data) { var ids = []; var d = []; - var max = 0; + var maxTime = 0; + var minTime = Number.MAX_SAFE_INTEGER; data.forEach(function(x) { var t = x.timestamp * 1000; ids[t] = x.id; d.push([t, x.value [% IF yaxis == "mib" %] / (1024.0 * 1024.0)[% END %]]); - max = Math.max(t, max); + maxTime = Math.max(t, maxTime); + minTime = Math.min(t, minTime); }); var options = { @@ -634,7 +636,7 @@ BLOCK createChart %] }); // Zoom in to the last two months by default. - plot.setSelection({ xaxis: { from: max - 60 * 24 * 60 * 60 * 1000, to: max } }); + plot.setSelection({ xaxis: { from: Math.max(minTime, maxTime - 60 * 24 * 60 * 60 * 1000), to: maxTime } }); } }); }); diff --git a/src/root/job.tt b/src/root/job.tt index 998bc083..a9e1ca77 100644 --- a/src/root/job.tt +++ b/src/root/job.tt @@ -98,6 +98,14 @@ removed or had an evaluation error.
    [% INCLUDE createChart id="output-size" yaxis="mib" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'output-sizes') %] + [% FOREACH metric IN metrics %] + +

    Metric: [%HTML.escape(metric.name)%]

    + + [% INCLUDE createChart id="metric-${metric.name}" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'metric' metric.name) %] + + [% END %] +