diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index c2c2c8af..5f516d23 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -7,6 +7,21 @@ using namespace nix; +static std::tuple secureRead(Path fileName) +{ + auto fail = std::make_tuple(false, ""); + + if (!pathExists(fileName)) return fail; + + try { + /* For security, resolve symlinks. */ + fileName = canonPath(fileName, true); + if (!isInStore(fileName)) return fail; + return std::make_tuple(true, readFile(fileName)); + } catch (Error & e) { return fail; } +} + + BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & drv) { BuildOutput res; @@ -40,22 +55,12 @@ BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & d Path failedFile = output + "/nix-support/failed"; if (pathExists(failedFile)) res.failed = true; - Path productsFile = output + "/nix-support/hydra-build-products"; - if (!pathExists(productsFile)) continue; + auto file = secureRead(output + "/nix-support/hydra-build-products"); + if (!std::get<0>(file)) continue; + explicitProducts = true; - /* For security, resolve symlinks. */ - try { - productsFile = canonPath(productsFile, true); - } catch (Error & e) { continue; } - if (!isInStore(productsFile)) continue; - - string contents; - try { - contents = readFile(productsFile); - } catch (Error & e) { continue; } - - for (auto & line : tokenizeString(contents, "\n")) { + for (auto & line : tokenizeString(std::get<1>(file), "\n")) { BuildProduct product; Regex::Subs subs; @@ -122,5 +127,19 @@ BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & d // FIXME: validate release name } + /* Get metrics. */ + for (auto & output : outputs) { + auto file = secureRead(output + "/nix-support/hydra-metrics"); + for (auto & line : tokenizeString(std::get<1>(file), "\n")) { + auto fields = tokenizeString>(line); + if (fields.size() < 2) continue; + BuildMetric metric; + metric.name = fields[0]; // FIXME: validate + metric.value = atof(fields[1].c_str()); // FIXME + metric.unit = fields.size() >= 3 ? fields[2] : ""; + res.metrics[metric.name] = metric; + } + } + return res; } diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh index 65fbf948..e40e7dcd 100644 --- a/src/hydra-queue-runner/build-result.hh +++ b/src/hydra-queue-runner/build-result.hh @@ -15,6 +15,12 @@ struct BuildProduct BuildProduct() { } }; +struct BuildMetric +{ + std::string name, unit; + double value; +}; + struct BuildOutput { /* Whether this build has failed with output, i.e., the build @@ -27,6 +33,8 @@ struct BuildOutput unsigned long long closureSize = 0, size = 0; std::list products; + + std::map metrics; }; BuildOutput getBuildOutput(std::shared_ptr store, const nix::Derivation & drv); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 2860b854..10fd1cd1 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -238,6 +238,19 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, (product.defaultPath).exec(); } + for (auto & metric : res.metrics) { + txn.parameterized + ("insert into BuildMetrics (build, name, unit, value, project, jobset, job, timestamp) values ($1, $2, $3, $4, $5, $6, $7, $8)") + (build->id) + (metric.second.name) + (metric.second.unit, metric.second.unit != "") + (metric.second.value) + (build->projectName) + (build->jobsetName) + (build->jobName) + (build->timestamp).exec(); + } + nrBuildsDone++; } diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index bd9b2aef..8448cb68 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -64,7 +64,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, { pqxx::work txn(conn); - auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); + auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); for (auto const & row : res) { auto builds_(builds.lock()); @@ -76,9 +76,12 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, auto build = std::make_shared(); build->id = id; build->drvPath = row["drvPath"].as(); - build->fullJobName = row["project"].as() + ":" + row["jobset"].as() + ":" + row["job"].as(); + build->projectName = row["project"].as(); + build->jobsetName = row["jobset"].as(); + build->jobName = row["job"].as(); build->maxSilentTime = row["maxsilent"].as(); build->buildTimeout = row["timeout"].as(); + build->timestamp = row["timestamp"].as(); newBuilds.emplace(std::make_pair(build->drvPath, build)); } @@ -89,7 +92,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, std::function createBuild; createBuild = [&](Build::ptr build) { - printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName); + printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); nrAdded++; if (!store->isValidPath(build->drvPath)) { diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 761a100b..2bc3a56c 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -38,6 +38,7 @@ typedef enum { bssFailed = 1, bssAborted = 4, bssTimedOut = 7, + bssCachedFailure = 8, bssUnsupported = 9, bssBusy = 100, // not stored } BuildStepStatus; @@ -67,12 +68,18 @@ struct Build BuildID id; nix::Path drvPath; std::map outputs; - std::string fullJobName; + std::string projectName, jobsetName, jobName; + time_t timestamp; unsigned int maxSilentTime, buildTimeout; std::shared_ptr toplevel; std::atomic_bool finishedInDB{false}; + + std::string fullJobName() + { + return projectName + ":" + jobsetName + ":" + jobName; + } }; diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 274d9604..475eca37 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -77,6 +77,9 @@ sub overview : Chained('job') PathPart('') Args(0) { , jobset => $c->stash->{jobset}->name , job => $c->stash->{job}->name })->count == 1 if $c->user_exists; + + $c->stash->{metrics} = [ $job->buildmetrics->search( + { }, { select => ["name"], distinct => 1, order_by => "timestamp desc", }) ]; } @@ -110,6 +113,20 @@ sub output_sizes : Chained('job') PathPart('output-sizes') Args(0) { } +sub metric : Chained('job') PathPart('metric') Args(1) { + my ($self, $c, $metricName) = @_; + + $c->stash->{template} = 'metric.tt'; + $c->stash->{metricName} = $metricName; + + my @res = $c->stash->{job}->buildmetrics->search( + { name => $metricName }, + { order_by => "timestamp", columns => [ "build", "name", "timestamp", "value", "unit" ] }); + + $self->status_ok($c, entity => [ map { { id => $_->get_column("build"), timestamp => $_ ->timestamp, value => $_->value, unit => $_->unit } } @res ]); +} + + # Hydra::Base::Controller::ListBuilds needs this. sub get_builds : Chained('job') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; diff --git a/src/lib/Hydra/Schema/BuildMetrics.pm b/src/lib/Hydra/Schema/BuildMetrics.pm new file mode 100644 index 00000000..58bbed94 --- /dev/null +++ b/src/lib/Hydra/Schema/BuildMetrics.pm @@ -0,0 +1,187 @@ +use utf8; +package Hydra::Schema::BuildMetrics; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::BuildMetrics + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("BuildMetrics"); + +=head1 ACCESSORS + +=head2 build + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 name + + data_type: 'text' + is_nullable: 0 + +=head2 unit + + data_type: 'text' + is_nullable: 1 + +=head2 value + + data_type: 'double precision' + is_nullable: 0 + +=head2 project + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 jobset + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 job + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 timestamp + + data_type: 'integer' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "build", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "name", + { data_type => "text", is_nullable => 0 }, + "unit", + { data_type => "text", is_nullable => 1 }, + "value", + { data_type => "double precision", is_nullable => 0 }, + "project", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "jobset", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "job", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "timestamp", + { data_type => "integer", is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("build", "name"); + +=head1 RELATIONS + +=head2 build + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "build", + "Hydra::Schema::Builds", + { id => "build" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + +=head2 job + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "job", + "Hydra::Schema::Jobs", + { jobset => "jobset", name => "job", project => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + +=head2 jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset", + "Hydra::Schema::Jobsets", + { name => "jobset", project => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + +=head2 project + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "project", + "Hydra::Schema::Projects", + { name => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qoPm5/le+sVHigW4Dmum2Q + +sub json_hint { + return { columns => ['value', 'unit'] }; +} + +1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index a5abce08..5e502449 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -341,6 +341,21 @@ __PACKAGE__->has_many( undef, ); +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { "foreign.build" => "self.id" }, + undef, +); + =head2 buildoutputs Type: has_many @@ -535,8 +550,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:03:55 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EwxiaQpqbdzI9RvU0uUtLQ +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y2lDtgY8EBLOuCHAI8fWRQ __PACKAGE__->has_many( "dependents", @@ -630,6 +645,7 @@ my %hint = ( buildoutputs => 'name', buildinputs_builds => 'name', buildproducts => 'productnr', + buildmetrics => 'name', } ); diff --git a/src/lib/Hydra/Schema/Jobs.pm b/src/lib/Hydra/Schema/Jobs.pm index 883c1bee..cd89ed3d 100644 --- a/src/lib/Hydra/Schema/Jobs.pm +++ b/src/lib/Hydra/Schema/Jobs.pm @@ -81,6 +81,25 @@ __PACKAGE__->set_primary_key("project", "jobset", "name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { + "foreign.job" => "self.name", + "foreign.jobset" => "self.jobset", + "foreign.project" => "self.project", + }, + undef, +); + =head2 builds Type: has_many @@ -150,7 +169,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-09-29 19:41:42 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lnZSd0gDXgLk8WQeAFqByA +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vDAo9bzLca+QWfhOb9OLMg 1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index f304f770..682b3ab0 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -184,6 +184,24 @@ __PACKAGE__->set_primary_key("project", "name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { + "foreign.jobset" => "self.name", + "foreign.project" => "self.project", + }, + undef, +); + =head2 builds Type: has_many @@ -320,8 +338,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:51 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CO0aE+jrjB+UrwGRzWZLlw +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Coci9FdBAvUO9T3st2NEqA my %hint = ( columns => [ diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index 9bf9dde3..e04b1f8e 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -106,6 +106,21 @@ __PACKAGE__->set_primary_key("name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { "foreign.project" => "self.name" }, + undef, +); + =head2 builds Type: has_many @@ -267,8 +282,8 @@ Composing rels: L -> username __PACKAGE__->many_to_many("usernames", "projectmembers", "username"); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:08 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fkd9ruEoVSBGIktmAj4u4g +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:67kWIE0IGmEJTvOIATAKaw my %hint = ( columns => [ diff --git a/src/root/build.tt b/src/root/build.tt index 9ecd5cd6..e3eb43f9 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -106,7 +106,7 @@ [% END %] - +
  • Summary
    • [% IF isAggregate %]
  • Constituents
    • [% END %]
  • Details
    • @@ -385,6 +385,25 @@ [% END %] + + [% IF build.finished && build.buildmetrics %] +

    Metrics

    + + + + + + + [% FOREACH metric IN build.buildmetrics %] + + + + + [% END %] + +
    NameValue
    [%HTML.escape(metric.name)%][%metric.value%][%metric.unit%]
    + [% END %] +
    diff --git a/src/root/common.tt b/src/root/common.tt index 19e2462e..dafae8aa 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -566,12 +566,14 @@ BLOCK createChart %] success: function(data) { var ids = []; var d = []; - var max = 0; + var maxTime = 0; + var minTime = Number.MAX_SAFE_INTEGER; data.forEach(function(x) { var t = x.timestamp * 1000; ids[t] = x.id; d.push([t, x.value [% IF yaxis == "mib" %] / (1024.0 * 1024.0)[% END %]]); - max = Math.max(t, max); + maxTime = Math.max(t, maxTime); + minTime = Math.min(t, minTime); }); var options = { @@ -634,7 +636,7 @@ BLOCK createChart %] }); // Zoom in to the last two months by default. - plot.setSelection({ xaxis: { from: max - 60 * 24 * 60 * 60 * 1000, to: max } }); + plot.setSelection({ xaxis: { from: Math.max(minTime, maxTime - 60 * 24 * 60 * 60 * 1000), to: maxTime } }); } }); }); diff --git a/src/root/job.tt b/src/root/job.tt index 998bc083..a9e1ca77 100644 --- a/src/root/job.tt +++ b/src/root/job.tt @@ -98,6 +98,14 @@ removed or had an evaluation error.
    [% INCLUDE createChart id="output-size" yaxis="mib" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'output-sizes') %] + [% FOREACH metric IN metrics %] + +

    Metric: [%HTML.escape(metric.name)%]

    + + [% INCLUDE createChart id="metric-${metric.name}" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'metric' metric.name) %] + + [% END %] +