Merge branch 'master' into job_headers

release.nix

@@ -37,8 +37,9 @@ in
 rec {
 
   build = genAttrs' (system:
+    let pkgs = import nixpkgs { inherit system; }; in
 
-    with import nixpkgs { inherit system; };
+    with pkgs;
 
     let
 

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -29,7 +29,7 @@ static void findJobs(EvalState & state, JSONObject & top,
     Bindings & autoArgs, Value & v, const string & attrPath);
 
 
-static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name)
+static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute)
 {
     Strings res;
     std::function<void(Value & v)> rec;
@@ -42,7 +42,7 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string &
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 rec(*v.listElems()[n]);
         else if (v.type == tAttrs) {
-            auto a = v.attrs->find(state.symbols.create("shortName"));
+            auto a = v.attrs->find(state.symbols.create(subAttribute));
             if (a != v.attrs->end())
                 res.push_back(state.forceString(*a->value));
         }
@@ -117,9 +117,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top,
             res.attr("system", drv->querySystem());
             res.attr("drvPath", drvPath = drv->queryDrvPath());
             res.attr("description", drv->queryMetaString("description"));
-            res.attr("license", queryMetaStrings(state, *drv, "license"));
+            res.attr("license", queryMetaStrings(state, *drv, "license", "shortName"));
             res.attr("homepage", drv->queryMetaString("homepage"));
-            res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers"));
+            res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email"));
             res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100));
             res.attr("timeout", drv->queryMetaInt("timeout", 36000));
             res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200));

src/hydra-queue-runner/build-remote.cc

@@ -29,6 +29,7 @@ static void append(Strings & dst, const Strings & src)
 
 static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
 {
+    string pgmName;
     Pipe to, from;
     to.create();
     from.create();
@@ -47,9 +48,12 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
             throw SysError("cannot dup stderr");
 
         Strings argv;
-        if (machine->sshName == "localhost")
+        if (machine->sshName == "localhost") {
+            pgmName = "nix-store";
             argv = {"nix-store", "--serve", "--write"};
+        }
         else {
+            pgmName = "ssh";
             argv = {"ssh", machine->sshName};
             if (machine->sshKey != "") append(argv, {"-i", machine->sshKey});
             if (machine->sshPublicHostKey != "") {
@@ -66,7 +70,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
 
         execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
 
-        throw SysError("cannot start ssh");
+        throw SysError("cannot start %s", pgmName);
     });
 
     to.readSide = -1;

src/hydra-queue-runner/dispatcher.cc

@@ -152,7 +152,7 @@ system_time State::doDispatch()
              establish priority between builds in the same jobset, but
              here it's used between steps in different jobsets if they
              happen to have the same lowest used scheduling share. But
-             that's not every likely.
+             that's not very likely.
 
            - The lowest ID of the builds depending on the step;
              i.e. older builds take priority over new ones.

src/lib/Hydra/Controller/Root.pm

@@ -237,7 +237,13 @@ sub end : ActionClass('RenderView') {
 
     elsif (scalar @{$c->error}) {
         $c->stash->{resource} = { error => join "\n", @{$c->error} };
-        $c->stash->{template} = 'error.tt';
+        if ($c->stash->{lazy}) {
+            $c->response->headers->header('X-Hydra-Lazy', 'Yes');
+            $c->stash->{template} = 'lazy_error.tt';
+        }
+        else {
+            $c->stash->{template} = 'error.tt';
+        }
         $c->stash->{errors} = $c->error;
         $c->response->status(500) if $c->response->status == 200;
         if ($c->response->status >= 300) {

src/lib/Hydra/Controller/User.pm

@@ -349,9 +349,10 @@ sub dashboard :Chained('dashboard_base') :PathPart('') :Args(0) {
 
 sub my_jobs_tab :Chained('dashboard_base') :PathPart('my-jobs-tab') :Args(0) {
     my ($self, $c) = @_;
+    $c->stash->{lazy} = 1;
     $c->stash->{template} = 'dashboard-my-jobs-tab.tt';
 
-    die unless $c->stash->{user}->emailaddress;
+    error($c, "No email address is set for this user.") unless $c->stash->{user}->emailaddress;
 
     # Get all current builds of which this user is a maintainer.
     $c->stash->{builds} = [$c->model('DB::Builds')->search(

src/lib/Hydra/Plugin/GitlabPulls.pm

@@ -0,0 +1,95 @@
+# This plugin allows to build Gitlab merge requests.
+#
+# The declarative project spec.json file must contains an input such as
+#   "pulls": {
+#      "type": "gitlabpulls",
+#      "value": "https://gitlab.com 42",
+#      "emailresponsible": false
+#   }
+# where 42 is the project id of a repository.
+#
+# The values `target_repo_url` and `iid` can then be used to
+# build the git input value, e.g.:
+# "${target_repo_url} merge-requests/${iid}/head".
+
+package Hydra::Plugin::GitlabPulls;
+
+use strict;
+use parent 'Hydra::Plugin';
+use HTTP::Request;
+use LWP::UserAgent;
+use JSON;
+use Hydra::Helper::CatalystUtils;
+use File::Temp;
+use POSIX qw(strftime);
+
+sub supportedInputTypes {
+    my ($self, $inputTypes) = @_;
+    $inputTypes->{'gitlabpulls'} = 'Open Gitlab Merge Requests';
+}
+
+sub _query {
+    my ($url, $ua) = @_;
+    my $req = HTTP::Request->new('GET', $url);
+    my $res = $ua->request($req);
+    my $content = $res->decoded_content;
+    die "Error pulling from the gitlab pulls API: $content\n"
+        unless $res->is_success;
+    return (decode_json $content, $res);
+}
+
+sub _iterate {
+    my ($url, $baseUrl, $pulls, $ua, $target_repo_url) = @_;
+    my ($pulls_list, $res) = _query($url, $ua);
+
+    foreach my $pull (@$pulls_list) {
+        $pull->{target_repo_url} = $target_repo_url;
+        $pulls->{$pull->{iid}} = $pull;
+    }
+    # TODO Make Link header parsing more robust!!!
+    my @links = split ',', $res->header("Link");
+    my $next = "";
+    foreach my $link (@links) {
+        my ($url, $rel) = split ";", $link;
+        if (trim($rel) eq 'rel="next"') {
+            $next = substr trim($url), 1, -1;
+            last;
+        }
+    }
+    _iterate($next, $baseUrl, $pulls, $ua, $target_repo_url) unless $next eq "";
+}
+
+sub fetchInput {
+    my ($self, $type, $name, $value, $project, $jobset) = @_;
+    return undef if $type ne "gitlabpulls";
+
+    (my $baseUrl, my $projectId) = split ' ', $value;
+    my $url = "$baseUrl/api/v4/projects/$projectId/merge_requests?per_page=100&state=opened";
+
+    my $accessToken = $self->{config}->{gitlab_authorization}->{$projectId};
+
+    my %pulls;
+    my $ua = LWP::UserAgent->new();
+    $ua->default_header('Private-Token' => $accessToken) if defined $accessToken;
+
+    # Get the target project URL, as it is the one we need to build the pull
+    # urls from later
+    (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua);
+    my $target_repo_url = $repo->{http_url_to_repo};
+
+    _iterate($url, $baseUrl, \%pulls, $ua, $target_repo_url);
+
+    my $tempdir = File::Temp->newdir("gitlab-pulls" . "XXXXX", TMPDIR => 1);
+    my $filename = "$tempdir/gitlab-pulls.json";
+    open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";
+    print $fh encode_json \%pulls;
+    close $fh;
+    system("jq -S . < $filename > $tempdir/gitlab-pulls-sorted.json");
+    my $storePath = trim(`nix-store --add "$tempdir/gitlab-pulls-sorted.json"`
+        or die "cannot copy path $filename to the Nix store.\n");
+    chomp $storePath;
+    my $timestamp = time;
+    return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) };
+}
+
+1;

src/lib/Hydra/Script/DevServer.pm

@@ -0,0 +1,7 @@
+package Hydra::Script::DevServer;
+use Moose;
+use namespace::autoclean;
+
+extends 'Catalyst::Script::Server';
+
+1;

src/root/common.tt

@@ -7,13 +7,27 @@ USE Math;
 
 USE mibs=format("%.2f");
 
-
+# Formatted date time, in hydra-local timezone.
-BLOCK renderDateTime;
+# Use only where an HTML element cannot be used.
+BLOCK dateTimeText;
 date.format(timestamp, '%Y-%m-%d %H:%M:%S');
 END;
 
+# HTML-rendered date. Formatted in hydra-local timezone.
+# It is enhanced with JavaScript to show user-local and UTC time zones.
+BLOCK renderDateTime %]
+<time [% HTML.attributes(
+    "data-timestamp" => timestamp,
+    title => date.format(timestamp, '%Y-%m-%d %H:%M:%S (%Z)'),
+    datetime => date.format(timestamp, "%Y-%m-%dT%H:%M:%SZ", gmt => 1),
+) %] class="date is-absolute">
+[% INCLUDE dateTimeText %]
+</time>
+[% END;
 
-BLOCK renderRelativeDate;
+# Relative date, as text.
+# Use only where an HTML element cannot be used.
+BLOCK relativeDateText;
   ago = date.now - timestamp;
   IF ago >= 0 && ago < 60; THEN;
     ago _ 's ago';
@@ -28,6 +42,17 @@ BLOCK renderRelativeDate;
   END;
 END;
 
+# HTML-rendered relative date.
+# It is enhanced with JavaScript to show user-local and UTC time zones.
+BLOCK renderRelativeDate %]
+<time [% HTML.attributes(
+    "data-timestamp" => timestamp,
+    title => date.format(timestamp, '%Y-%m-%d %H:%M:%S (%Z)'),
+    datetime => date.format(timestamp, "%Y-%m-%dT%H:%M:%SZ", gmt => 1),
+) %] class="date is-relative">
+[% INCLUDE relativeDateText %]
+</time>
+[% END;
 
 BLOCK renderProjectName %]
 <a [% IF inRow %]class="row-link"[% END %] href="[% c.uri_for('/project' project) %]"><tt>[% project %]</tt></a>

src/root/layout.tt

@@ -13,6 +13,7 @@
 
     <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-1.12.3.min.js") %]"></script>
     <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
+    <script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
@@ -57,7 +58,7 @@
             [% IF logo == "" %]
               Hydra
             [% ELSE %]
-              <img src="[% logo %]" alt="Hydra Logo" class="logo" />
+              <img src="[% c.uri_for(logo) %]" alt="Hydra Logo" class="logo" />
             [% END %]
           </a>
           <div class="nav-collapse collapse">

src/root/lazy_error.tt

@@ -0,0 +1,5 @@
+[% PROCESS common.tt %]
+
+[% FOREACH error IN errors %]
+   <div class="alert alert-error">[% HTML.escape(error).replace('\n', '<br/>') %]</div>
+[% END %]

src/root/static/css/hydra.css

@@ -147,3 +147,8 @@ td.step-status span.warn {
 .table thead th {
     border-top: 0;
 }
+
+.date {
+	cursor: help;
+	border-bottom: 1px dotted #999;
+}

src/root/static/js/common.js

@@ -97,6 +97,35 @@ $(document).ready(function() {
             }
         });
     });
+
+    /* Makes dates more user friendly. */
+    // Friendly date format
+    var DATE_FORMAT = "YYYY-MM-DD HH:mm:ss";
+    // Local timezone offset to display.
+    var tz = moment().format("Z");
+    $("time.date").each(function(i, el) {
+        var el = $(el);
+        var localTime = moment(el.data("timestamp"), "%X");
+        var hydraTime = el.attr("title");
+        if (el.hasClass("is-absolute")) {
+            el.attr( "title", [
+                "Adjusted to local time (" + tz + ")",
+                "Other timezones:",
+                "  UTC: " + localTime.clone().utc().format(DATE_FORMAT),
+                "  As Hydra reported: " + hydraTime,
+            ].join("\n"));
+            el.text(localTime.format(DATE_FORMAT));
+            el.addClass("is-local");
+        }
+        else if (el.hasClass("is-relative")) {
+            el.attr( "title", [
+                "Local (" + tz + "): " + localTime.format(DATE_FORMAT),
+                "UTC: " + localTime.clone().utc().format(DATE_FORMAT),
+                "As Hydra reported: " + hydraTime,
+            ].join("\n"));
+            el.addClass("is-local");
+        }
+    });
 });
 
 var tabsLoaded = {};
@@ -108,9 +137,13 @@ function makeLazyTab(tabName, uri) {
         if (id == '#' + tabName && !tabsLoaded[id]) {
           tabsLoaded[id] = 1;
           $('#' + tabName).load(uri, function(response, status, xhr) {
-            if (status == "error") {
+            var lazy = xhr.getResponseHeader("X-Hydra-Lazy") === "Yes";
+            if (status == "error" && !lazy) {
               $('#' + tabName).html("<div class='alert alert-error'>Error loading tab: " + xhr.status + " " + xhr.statusText + "</div>");
             }
+            else {
+              $('#' + tabName).html(response);
+            }
           });
         }
     });

src/script/hydra-dev-server

@@ -0,0 +1,59 @@
+#! /usr/bin/env perl
+
+BEGIN {
+    $ENV{CATALYST_SCRIPT_GEN} = 40;
+}
+
+use Catalyst::ScriptRunner;
+Catalyst::ScriptRunner->run('Hydra', 'DevServer');
+
+1;
+
+=head1 NAME
+
+hydra_server.pl - Catalyst Test Server
+
+=head1 SYNOPSIS
+
+hydra_server.pl [options]
+
+   -d --debug           force debug mode
+   -f --fork            handle each request in a new process
+                        (defaults to false)
+   -? --help            display this help and exits
+   -h --host            host (defaults to all)
+   -p --port            port (defaults to 3000)
+   -k --keepalive       enable keep-alive connections
+   -r --restart         restart when files get modified
+                        (defaults to false)
+   -rd --restart_delay  delay between file checks
+                        (ignored if you have Linux::Inotify2 installed)
+   -rr --restart_regex  regex match files that trigger
+                        a restart when modified
+                        (defaults to '\.yml$|\.yaml$|\.conf|\.pm$')
+   --restart_directory  the directory to search for
+                        modified files, can be set multiple times
+                        (defaults to '[SCRIPT_DIR]/..')
+   --follow_symlinks    follow symlinks in search directories
+                        (defaults to false. this is a no-op on Win32)
+   --background         run the process in the background
+   --pidfile            specify filename for pid file
+
+ See also:
+   perldoc Catalyst::Manual
+   perldoc Catalyst::Manual::Intro
+
+=head1 DESCRIPTION
+
+Run a Catalyst Testserver for this application.
+
+=head1 AUTHORS
+
+Catalyst Contributors, see Catalyst.pm
+
+=head1 COPYRIGHT
+
+This library is free software. You can redistribute it and/or modify
+it under the same terms as Perl itself.
+
+=cut

tests/Makefile.am

@@ -40,5 +40,6 @@ dirs:
 	mkdir -p nix
 	mkdir -p nix/etc/nix
 	touch nix/etc/nix/nix.conf
+	echo "sandbox = false" >> nix/etc/nix/nix.conf
 	mkdir -p nix/store
 	mkdir -p nix/var