From 68aad22d1930020a6e4417604950c6522e6a4245 Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Mon, 3 Sep 2018 23:41:30 +0200 Subject: [PATCH 01/13] Add GitlabPulls input plugin --- src/lib/Hydra/Plugin/GitlabPulls.pm | 94 +++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/lib/Hydra/Plugin/GitlabPulls.pm diff --git a/src/lib/Hydra/Plugin/GitlabPulls.pm b/src/lib/Hydra/Plugin/GitlabPulls.pm new file mode 100644 index 00000000..de371f9a --- /dev/null +++ b/src/lib/Hydra/Plugin/GitlabPulls.pm @@ -0,0 +1,94 @@ +# This plugin allows to build Gitlab merge requests. +# +# The declarative project spec.json file must contains an input such as +# "pulls": { +# "type": "gitlabpulls", +# "value": "https://gitlab.com 42", +# "emailresponsible": false +# } +# where 42 is the project id of a repository. +# +# The values source_repo_url and source_branch can then be used to +# build the git input value. + +package Hydra::Plugin::GitlabPulls; + +use strict; +use parent 'Hydra::Plugin'; +use HTTP::Request; +use LWP::UserAgent; +use JSON; +use Hydra::Helper::CatalystUtils; +use File::Temp; +use POSIX qw(strftime); + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'gitlabpulls'} = 'Open Gitlab Merge Requests'; +} + +sub _query { + my ($url, $ua) = @_; + my $req = HTTP::Request->new('GET', $url); + my $res = $ua->request($req); + my $content = $res->decoded_content; + die "Error pulling from the gitlab pulls API: $content\n" + unless $res->is_success; + return (decode_json $content, $res); +} + +# We need to query the Gitlab API for each merge request to get the +# source repository URL. +sub _enhanceGitlabPull { + my ($pull, $baseUrl, $ua) = @_; + my $projectId = $pull->{source_project_id}; + (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua); + $pull->{source_repo_url} = $repo->{http_url_to_repo}; +} + +sub _iterate { + my ($url, $baseUrl, $pulls, $ua) = @_; + my ($pulls_list, $res) = _query($url, $ua); + + foreach my $pull (@$pulls_list) { + _enhanceGitlabPull($pull, $baseUrl, $ua); + $pulls->{$pull->{iid}} = $pull; + } + # TODO Make Link header parsing more robust!!! + my @links = split ',', $res->header("Link"); + my $next = ""; + foreach my $link (@links) { + my ($url, $rel) = split ";", $link; + if (trim($rel) eq 'rel="next"') { + $next = substr trim($url), 1, -1; + last; + } + } + _iterate($next, $baseUrl, $pulls, $ua) unless $next eq ""; +} + +sub fetchInput { + my ($self, $type, $name, $value, $project, $jobset) = @_; + return undef if $type ne "gitlabpulls"; + + (my $baseUrl, my $projectId) = split ' ', $value; + my $url = "$baseUrl/api/v4/projects/$projectId/merge_requests?per_page=100&state=opened"; + + my %pulls; + my $ua = LWP::UserAgent->new(); + _iterate($url, $baseUrl, \%pulls, $ua); + + my $tempdir = File::Temp->newdir("gitlab-pulls" . "XXXXX", TMPDIR => 1); + my $filename = "$tempdir/gitlab-pulls.json"; + open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!"; + print $fh encode_json \%pulls; + close $fh; + system("jq -S . < $filename > $tempdir/gitlab-pulls-sorted.json"); + my $storePath = trim(`nix-store --add "$tempdir/gitlab-pulls-sorted.json"` + or die "cannot copy path $filename to the Nix store.\n"); + chomp $storePath; + my $timestamp = time; + return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) }; +} + +1; From d9253543e4fe29f57d9d47eaf85fea1f709237d8 Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Tue, 20 Nov 2018 16:27:40 +0100 Subject: [PATCH 02/13] plugin/GitLabPulls: support for using a personal access token (PAT) In order to access protected or private repositories. Using the target repository URL along with the merge-request ref instead of the source repository url and branch is necessary to avoid running into issues if the source repository is not actually accessible to the user Hydra is authenticating as. Thanks Alexei Robyn for this patch. --- src/lib/Hydra/Plugin/GitlabPulls.pm | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/lib/Hydra/Plugin/GitlabPulls.pm b/src/lib/Hydra/Plugin/GitlabPulls.pm index de371f9a..b44a1a98 100644 --- a/src/lib/Hydra/Plugin/GitlabPulls.pm +++ b/src/lib/Hydra/Plugin/GitlabPulls.pm @@ -8,8 +8,9 @@ # } # where 42 is the project id of a repository. # -# The values source_repo_url and source_branch can then be used to -# build the git input value. +# The values `target_repo_url` and `iid` can then be used to +# build the git input value, e.g.: +# "${target_repo_url} merge-requests/${iid}/head". package Hydra::Plugin::GitlabPulls; @@ -37,21 +38,12 @@ sub _query { return (decode_json $content, $res); } -# We need to query the Gitlab API for each merge request to get the -# source repository URL. -sub _enhanceGitlabPull { - my ($pull, $baseUrl, $ua) = @_; - my $projectId = $pull->{source_project_id}; - (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua); - $pull->{source_repo_url} = $repo->{http_url_to_repo}; -} - sub _iterate { - my ($url, $baseUrl, $pulls, $ua) = @_; + my ($url, $baseUrl, $pulls, $ua, $target_repo_url) = @_; my ($pulls_list, $res) = _query($url, $ua); foreach my $pull (@$pulls_list) { - _enhanceGitlabPull($pull, $baseUrl, $ua); + $pull->{target_repo_url} = $target_repo_url; $pulls->{$pull->{iid}} = $pull; } # TODO Make Link header parsing more robust!!! @@ -64,7 +56,7 @@ sub _iterate { last; } } - _iterate($next, $baseUrl, $pulls, $ua) unless $next eq ""; + _iterate($next, $baseUrl, $pulls, $ua, $target_repo_url) unless $next eq ""; } sub fetchInput { @@ -74,10 +66,19 @@ sub fetchInput { (my $baseUrl, my $projectId) = split ' ', $value; my $url = "$baseUrl/api/v4/projects/$projectId/merge_requests?per_page=100&state=opened"; + my $accessToken = $self->{config}->{gitlab_authorization}->{$projectId}; + my %pulls; my $ua = LWP::UserAgent->new(); - _iterate($url, $baseUrl, \%pulls, $ua); - + $ua->default_header('Private-Token' => $accessToken) if defined $accessToken; + + # Get the target project URL, as it is the one we need to build the pull + # urls from later + (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua); + my $target_repo_url = $repo->{http_url_to_repo}; + + _iterate($url, $baseUrl, \%pulls, $ua, $target_repo_url); + my $tempdir = File::Temp->newdir("gitlab-pulls" . "XXXXX", TMPDIR => 1); my $filename = "$tempdir/gitlab-pulls.json"; open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!"; From 9986053e73c1ecf74353991c9cb0ef77fd46dcc8 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Sat, 1 Dec 2018 13:39:10 -0500 Subject: [PATCH 03/13] Controllers: allows lazy tabs to return custom errors. --- src/lib/Hydra/Controller/Root.pm | 8 +++++++- src/root/lazy_error.tt | 5 +++++ src/root/static/js/common.js | 6 +++++- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 src/root/lazy_error.tt diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 146c37a4..a486091f 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -237,7 +237,13 @@ sub end : ActionClass('RenderView') { elsif (scalar @{$c->error}) { $c->stash->{resource} = { error => join "\n", @{$c->error} }; - $c->stash->{template} = 'error.tt'; + if ($c->stash->{lazy}) { + $c->response->headers->header('X-Hydra-Lazy', 'Yes'); + $c->stash->{template} = 'lazy_error.tt'; + } + else { + $c->stash->{template} = 'error.tt'; + } $c->stash->{errors} = $c->error; $c->response->status(500) if $c->response->status == 200; if ($c->response->status >= 300) { diff --git a/src/root/lazy_error.tt b/src/root/lazy_error.tt new file mode 100644 index 00000000..38bcfcb6 --- /dev/null +++ b/src/root/lazy_error.tt @@ -0,0 +1,5 @@ +[% PROCESS common.tt %] + +[% FOREACH error IN errors %] +
[% HTML.escape(error).replace('\n', '
') %]
+[% END %] diff --git a/src/root/static/js/common.js b/src/root/static/js/common.js index 1fe2f778..baa74ae6 100644 --- a/src/root/static/js/common.js +++ b/src/root/static/js/common.js @@ -108,9 +108,13 @@ function makeLazyTab(tabName, uri) { if (id == '#' + tabName && !tabsLoaded[id]) { tabsLoaded[id] = 1; $('#' + tabName).load(uri, function(response, status, xhr) { - if (status == "error") { + var lazy = xhr.getResponseHeader("X-Hydra-Lazy") === "Yes"; + if (status == "error" && !lazy) { $('#' + tabName).html("
Error loading tab: " + xhr.status + " " + xhr.statusText + "
"); } + else { + $('#' + tabName).html(response); + } }); } }); From ed85daf2ac9e4f06ae5746bda0f5f1cf8a15fd43 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Sat, 1 Dec 2018 13:39:28 -0500 Subject: [PATCH 04/13] User: jobs tab returns its error as a lazy error. --- src/lib/Hydra/Controller/User.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Hydra/Controller/User.pm b/src/lib/Hydra/Controller/User.pm index e9953a83..cad19d78 100644 --- a/src/lib/Hydra/Controller/User.pm +++ b/src/lib/Hydra/Controller/User.pm @@ -349,9 +349,10 @@ sub dashboard :Chained('dashboard_base') :PathPart('') :Args(0) { sub my_jobs_tab :Chained('dashboard_base') :PathPart('my-jobs-tab') :Args(0) { my ($self, $c) = @_; + $c->stash->{lazy} = 1; $c->stash->{template} = 'dashboard-my-jobs-tab.tt'; - die unless $c->stash->{user}->emailaddress; + error($c, "No email address is set for this user.") unless $c->stash->{user}->emailaddress; # Get all current builds of which this user is a maintainer. $c->stash->{builds} = [$c->model('DB::Builds')->search( From 423c0440eaee8b66706ca1f00f90e2ece41b36b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Dec 2018 12:07:02 +0100 Subject: [PATCH 05/13] Typo --- src/hydra-queue-runner/dispatcher.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 531c6b46..068d5c57 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -152,7 +152,7 @@ system_time State::doDispatch() establish priority between builds in the same jobset, but here it's used between steps in different jobsets if they happen to have the same lowest used scheduling share. But - that's not every likely. + that's not very likely. - The lowest ID of the builds depending on the step; i.e. older builds take priority over new ones. From 3a7068898561f6a7bd4108c8872925fd837decea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 27 Dec 2018 17:08:20 +0100 Subject: [PATCH 06/13] Support $X-Request-Base for the Logo --- src/root/layout.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/layout.tt b/src/root/layout.tt index 554e7b45..7d1788d4 100644 --- a/src/root/layout.tt +++ b/src/root/layout.tt @@ -57,7 +57,7 @@ [% IF logo == "" %] Hydra [% ELSE %] - + [% END %]
From aa87e7a52e3d37acfc345c22a06420f467d2c80a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 29 Dec 2018 14:47:03 +0100 Subject: [PATCH 07/13] hydra-eval-jobs: fix maintainer resolution Some time ago the data structure for maintainer descriptions in `nixpkgs` changed from a simple attr set with maintainer emails as values to an attribute set where the maintainer' nick is associated to an attribute set with email, GitHub handle and full name. Hydra can either parse a Nix list or fetches `shortName` from the associated attribute set (which is used for `meta.licenses` as each value in it contains a `shortName`). This behavior needs to be replicated for maintainers to retrieve the emails for `hydra-notify`. This change is backwards-compatible since `queryMetaStrings` is still able to understand lists, so old versions of `nixpkgs` or packages using the old maintainer data structure remain usable. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2be3dd80..ae737527 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -29,7 +29,7 @@ static void findJobs(EvalState & state, JSONObject & top, Bindings & autoArgs, Value & v, const string & attrPath); -static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name) +static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) { Strings res; std::function rec; @@ -42,7 +42,7 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & for (unsigned int n = 0; n < v.listSize(); ++n) rec(*v.listElems()[n]); else if (v.type == tAttrs) { - auto a = v.attrs->find(state.symbols.create("shortName")); + auto a = v.attrs->find(state.symbols.create(subAttribute)); if (a != v.attrs->end()) res.push_back(state.forceString(*a->value)); } @@ -117,9 +117,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top, res.attr("system", drv->querySystem()); res.attr("drvPath", drvPath = drv->queryDrvPath()); res.attr("description", drv->queryMetaString("description")); - res.attr("license", queryMetaStrings(state, *drv, "license")); + res.attr("license", queryMetaStrings(state, *drv, "license", "shortName")); res.attr("homepage", drv->queryMetaString("homepage")); - res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers")); + res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email")); res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100)); res.attr("timeout", drv->queryMetaInt("timeout", 36000)); res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200)); From 2430b12ec9e080fb1650ce2a0ab60921c264901e Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Mon, 21 Jan 2019 19:57:10 -0500 Subject: [PATCH 08/13] Adds moment.js for client-side date manipulations --- src/root/layout.tt | 1 + src/root/static/js/moment/moment-2.24.0.min.js | 1 + 2 files changed, 2 insertions(+) create mode 100644 src/root/static/js/moment/moment-2.24.0.min.js diff --git a/src/root/layout.tt b/src/root/layout.tt index 554e7b45..b3f758ad 100644 --- a/src/root/layout.tt +++ b/src/root/layout.tt @@ -13,6 +13,7 @@ + diff --git a/src/root/static/js/moment/moment-2.24.0.min.js b/src/root/static/js/moment/moment-2.24.0.min.js new file mode 100644 index 00000000..5787a408 --- /dev/null +++ b/src/root/static/js/moment/moment-2.24.0.min.js @@ -0,0 +1 @@ +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):e.moment=t()}(this,function(){"use strict";var e,i;function c(){return e.apply(null,arguments)}function o(e){return e instanceof Array||"[object Array]"===Object.prototype.toString.call(e)}function u(e){return null!=e&&"[object Object]"===Object.prototype.toString.call(e)}function l(e){return void 0===e}function h(e){return"number"==typeof e||"[object Number]"===Object.prototype.toString.call(e)}function d(e){return e instanceof Date||"[object Date]"===Object.prototype.toString.call(e)}function f(e,t){var n,s=[];for(n=0;n>>0,s=0;sSe(e)?(r=e+1,o-Se(e)):(r=e,o),{year:r,dayOfYear:a}}function Ie(e,t,n){var s,i,r=Ve(e.year(),t,n),a=Math.floor((e.dayOfYear()-r-1)/7)+1;return a<1?s=a+Ae(i=e.year()-1,t,n):a>Ae(e.year(),t,n)?(s=a-Ae(e.year(),t,n),i=e.year()+1):(i=e.year(),s=a),{week:s,year:i}}function Ae(e,t,n){var s=Ve(e,t,n),i=Ve(e+1,t,n);return(Se(e)-s+i)/7}I("w",["ww",2],"wo","week"),I("W",["WW",2],"Wo","isoWeek"),C("week","w"),C("isoWeek","W"),F("week",5),F("isoWeek",5),ue("w",B),ue("ww",B,z),ue("W",B),ue("WW",B,z),fe(["w","ww","W","WW"],function(e,t,n,s){t[s.substr(0,1)]=D(e)});function je(e,t){return e.slice(t,7).concat(e.slice(0,t))}I("d",0,"do","day"),I("dd",0,0,function(e){return this.localeData().weekdaysMin(this,e)}),I("ddd",0,0,function(e){return this.localeData().weekdaysShort(this,e)}),I("dddd",0,0,function(e){return this.localeData().weekdays(this,e)}),I("e",0,0,"weekday"),I("E",0,0,"isoWeekday"),C("day","d"),C("weekday","e"),C("isoWeekday","E"),F("day",11),F("weekday",11),F("isoWeekday",11),ue("d",B),ue("e",B),ue("E",B),ue("dd",function(e,t){return t.weekdaysMinRegex(e)}),ue("ddd",function(e,t){return t.weekdaysShortRegex(e)}),ue("dddd",function(e,t){return t.weekdaysRegex(e)}),fe(["dd","ddd","dddd"],function(e,t,n,s){var i=n._locale.weekdaysParse(e,s,n._strict);null!=i?t.d=i:g(n).invalidWeekday=e}),fe(["d","e","E"],function(e,t,n,s){t[s]=D(e)});var Ze="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_");var ze="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_");var $e="Su_Mo_Tu_We_Th_Fr_Sa".split("_");var qe=ae;var Je=ae;var Be=ae;function Qe(){function e(e,t){return t.length-e.length}var t,n,s,i,r,a=[],o=[],u=[],l=[];for(t=0;t<7;t++)n=y([2e3,1]).day(t),s=this.weekdaysMin(n,""),i=this.weekdaysShort(n,""),r=this.weekdays(n,""),a.push(s),o.push(i),u.push(r),l.push(s),l.push(i),l.push(r);for(a.sort(e),o.sort(e),u.sort(e),l.sort(e),t=0;t<7;t++)o[t]=he(o[t]),u[t]=he(u[t]),l[t]=he(l[t]);this._weekdaysRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+u.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+o.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+a.join("|")+")","i")}function Xe(){return this.hours()%12||12}function Ke(e,t){I(e,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),t)})}function et(e,t){return t._meridiemParse}I("H",["HH",2],0,"hour"),I("h",["hh",2],0,Xe),I("k",["kk",2],0,function(){return this.hours()||24}),I("hmm",0,0,function(){return""+Xe.apply(this)+L(this.minutes(),2)}),I("hmmss",0,0,function(){return""+Xe.apply(this)+L(this.minutes(),2)+L(this.seconds(),2)}),I("Hmm",0,0,function(){return""+this.hours()+L(this.minutes(),2)}),I("Hmmss",0,0,function(){return""+this.hours()+L(this.minutes(),2)+L(this.seconds(),2)}),Ke("a",!0),Ke("A",!1),C("hour","h"),F("hour",13),ue("a",et),ue("A",et),ue("H",B),ue("h",B),ue("k",B),ue("HH",B,z),ue("hh",B,z),ue("kk",B,z),ue("hmm",Q),ue("hmmss",X),ue("Hmm",Q),ue("Hmmss",X),ce(["H","HH"],ge),ce(["k","kk"],function(e,t,n){var s=D(e);t[ge]=24===s?0:s}),ce(["a","A"],function(e,t,n){n._isPm=n._locale.isPM(e),n._meridiem=e}),ce(["h","hh"],function(e,t,n){t[ge]=D(e),g(n).bigHour=!0}),ce("hmm",function(e,t,n){var s=e.length-2;t[ge]=D(e.substr(0,s)),t[ve]=D(e.substr(s)),g(n).bigHour=!0}),ce("hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[ge]=D(e.substr(0,s)),t[ve]=D(e.substr(s,2)),t[pe]=D(e.substr(i)),g(n).bigHour=!0}),ce("Hmm",function(e,t,n){var s=e.length-2;t[ge]=D(e.substr(0,s)),t[ve]=D(e.substr(s))}),ce("Hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[ge]=D(e.substr(0,s)),t[ve]=D(e.substr(s,2)),t[pe]=D(e.substr(i))});var tt,nt=Te("Hours",!0),st={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:Ce,monthsShort:He,week:{dow:0,doy:6},weekdays:Ze,weekdaysMin:$e,weekdaysShort:ze,meridiemParse:/[ap]\.?m?\.?/i},it={},rt={};function at(e){return e?e.toLowerCase().replace("_","-"):e}function ot(e){var t=null;if(!it[e]&&"undefined"!=typeof module&&module&&module.exports)try{t=tt._abbr,require("./locale/"+e),ut(t)}catch(e){}return it[e]}function ut(e,t){var n;return e&&((n=l(t)?ht(e):lt(e,t))?tt=n:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+e+" not found. Did you forget to load it?")),tt._abbr}function lt(e,t){if(null===t)return delete it[e],null;var n,s=st;if(t.abbr=e,null!=it[e])T("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),s=it[e]._config;else if(null!=t.parentLocale)if(null!=it[t.parentLocale])s=it[t.parentLocale]._config;else{if(null==(n=ot(t.parentLocale)))return rt[t.parentLocale]||(rt[t.parentLocale]=[]),rt[t.parentLocale].push({name:e,config:t}),null;s=n._config}return it[e]=new P(x(s,t)),rt[e]&&rt[e].forEach(function(e){lt(e.name,e.config)}),ut(e),it[e]}function ht(e){var t;if(e&&e._locale&&e._locale._abbr&&(e=e._locale._abbr),!e)return tt;if(!o(e)){if(t=ot(e))return t;e=[e]}return function(e){for(var t,n,s,i,r=0;r=t&&a(i,n,!0)>=t-1)break;t--}r++}return tt}(e)}function dt(e){var t,n=e._a;return n&&-2===g(e).overflow&&(t=n[_e]<0||11Pe(n[me],n[_e])?ye:n[ge]<0||24Ae(n,r,a)?g(e)._overflowWeeks=!0:null!=u?g(e)._overflowWeekday=!0:(o=Ee(n,s,i,r,a),e._a[me]=o.year,e._dayOfYear=o.dayOfYear)}(e),null!=e._dayOfYear&&(r=ct(e._a[me],s[me]),(e._dayOfYear>Se(r)||0===e._dayOfYear)&&(g(e)._overflowDayOfYear=!0),n=Ge(r,0,e._dayOfYear),e._a[_e]=n.getUTCMonth(),e._a[ye]=n.getUTCDate()),t=0;t<3&&null==e._a[t];++t)e._a[t]=a[t]=s[t];for(;t<7;t++)e._a[t]=a[t]=null==e._a[t]?2===t?1:0:e._a[t];24===e._a[ge]&&0===e._a[ve]&&0===e._a[pe]&&0===e._a[we]&&(e._nextDay=!0,e._a[ge]=0),e._d=(e._useUTC?Ge:function(e,t,n,s,i,r,a){var o;return e<100&&0<=e?(o=new Date(e+400,t,n,s,i,r,a),isFinite(o.getFullYear())&&o.setFullYear(e)):o=new Date(e,t,n,s,i,r,a),o}).apply(null,a),i=e._useUTC?e._d.getUTCDay():e._d.getDay(),null!=e._tzm&&e._d.setUTCMinutes(e._d.getUTCMinutes()-e._tzm),e._nextDay&&(e._a[ge]=24),e._w&&void 0!==e._w.d&&e._w.d!==i&&(g(e).weekdayMismatch=!0)}}var mt=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,_t=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,yt=/Z|[+-]\d\d(?::?\d\d)?/,gt=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],vt=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],pt=/^\/?Date\((\-?\d+)/i;function wt(e){var t,n,s,i,r,a,o=e._i,u=mt.exec(o)||_t.exec(o);if(u){for(g(e).iso=!0,t=0,n=gt.length;tn.valueOf():n.valueOf()this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},mn.isLocal=function(){return!!this.isValid()&&!this._isUTC},mn.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},mn.isUtc=Et,mn.isUTC=Et,mn.zoneAbbr=function(){return this._isUTC?"UTC":""},mn.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},mn.dates=n("dates accessor is deprecated. Use date instead.",un),mn.months=n("months accessor is deprecated. Use month instead",Ue),mn.years=n("years accessor is deprecated. Use year instead",Oe),mn.zone=n("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",function(e,t){return null!=e?("string"!=typeof e&&(e=-e),this.utcOffset(e,t),this):-this.utcOffset()}),mn.isDSTShifted=n("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",function(){if(!l(this._isDSTShifted))return this._isDSTShifted;var e={};if(w(e,this),(e=Ot(e))._a){var t=e._isUTC?y(e._a):bt(e._a);this._isDSTShifted=this.isValid()&&0 Date: Mon, 21 Jan 2019 19:58:53 -0500 Subject: [PATCH 09/13] Makes user-facing dates local when possible Also ensures it makes available the hydra timezone in the title of the dates. --- src/root/common.tt | 31 ++++++++++++++++++++++++++++--- src/root/static/css/hydra.css | 5 +++++ src/root/static/js/common.js | 29 +++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/root/common.tt b/src/root/common.tt index 47b26e86..5ea82d04 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -7,13 +7,27 @@ USE Math; USE mibs=format("%.2f"); - -BLOCK renderDateTime; +# Formatted date time, in hydra-local timezone. +# Use only where an HTML element cannot be used. +BLOCK dateTimeText; date.format(timestamp, '%Y-%m-%d %H:%M:%S'); END; +# HTML-rendered date. Formatted in hydra-local timezone. +# It is enhanced with JavaScript to show user-local and UTC time zones. +BLOCK renderDateTime %] + +[% END; -BLOCK renderRelativeDate; +# Relative date, as text. +# Use only where an HTML element cannot be used. +BLOCK relativeDateText; ago = date.now - timestamp; IF ago >= 0 && ago < 60; THEN; ago _ 's ago'; @@ -28,6 +42,17 @@ BLOCK renderRelativeDate; END; END; +# HTML-rendered relative date. +# It is enhanced with JavaScript to show user-local and UTC time zones. +BLOCK renderRelativeDate %] + +[% END; BLOCK renderProjectName %] [% project %] diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css index a871485e..c3bd9946 100644 --- a/src/root/static/css/hydra.css +++ b/src/root/static/css/hydra.css @@ -142,3 +142,8 @@ td.step-status span.warn { color: #aaaa00; font-weight: bold; } + +.date { + cursor: help; + border-bottom: 1px dotted #999; +} diff --git a/src/root/static/js/common.js b/src/root/static/js/common.js index baa74ae6..e9c5e576 100644 --- a/src/root/static/js/common.js +++ b/src/root/static/js/common.js @@ -97,6 +97,35 @@ $(document).ready(function() { } }); }); + + /* Makes dates more user friendly. */ + // Friendly date format + var DATE_FORMAT = "YYYY-MM-DD HH:mm:ss"; + // Local timezone offset to display. + var tz = moment().format("Z"); + $("time.date").each(function(i, el) { + var el = $(el); + var localTime = moment(el.data("timestamp"), "%X"); + var hydraTime = el.attr("title"); + if (el.hasClass("is-absolute")) { + el.attr( "title", [ + "Adjusted to local time (" + tz + ")", + "Other timezones:", + " UTC: " + localTime.clone().utc().format(DATE_FORMAT), + " As Hydra reported: " + hydraTime, + ].join("\n")); + el.text(localTime.format(DATE_FORMAT)); + el.addClass("is-local"); + } + else if (el.hasClass("is-relative")) { + el.attr( "title", [ + "Local (" + tz + "): " + localTime.format(DATE_FORMAT), + "UTC: " + localTime.clone().utc().format(DATE_FORMAT), + "As Hydra reported: " + hydraTime, + ].join("\n")); + el.addClass("is-local"); + } + }); }); var tabsLoaded = {}; From 04ff9e217baa13c231835a05163759038bad2f8e Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Tue, 22 Jan 2019 20:34:21 -0500 Subject: [PATCH 10/13] Add hydra-dev-server which uses the classic Catalyst server This, in turns allows - Using --restart for reloading the perl code - Printing traces on error --- src/lib/Hydra/Script/DevServer.pm | 7 ++++ src/script/hydra-dev-server | 59 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 src/lib/Hydra/Script/DevServer.pm create mode 100755 src/script/hydra-dev-server diff --git a/src/lib/Hydra/Script/DevServer.pm b/src/lib/Hydra/Script/DevServer.pm new file mode 100644 index 00000000..72934081 --- /dev/null +++ b/src/lib/Hydra/Script/DevServer.pm @@ -0,0 +1,7 @@ +package Hydra::Script::DevServer; +use Moose; +use namespace::autoclean; + +extends 'Catalyst::Script::Server'; + +1; diff --git a/src/script/hydra-dev-server b/src/script/hydra-dev-server new file mode 100755 index 00000000..8d30624f --- /dev/null +++ b/src/script/hydra-dev-server @@ -0,0 +1,59 @@ +#! /usr/bin/env perl + +BEGIN { + $ENV{CATALYST_SCRIPT_GEN} = 40; +} + +use Catalyst::ScriptRunner; +Catalyst::ScriptRunner->run('Hydra', 'DevServer'); + +1; + +=head1 NAME + +hydra_server.pl - Catalyst Test Server + +=head1 SYNOPSIS + +hydra_server.pl [options] + + -d --debug force debug mode + -f --fork handle each request in a new process + (defaults to false) + -? --help display this help and exits + -h --host host (defaults to all) + -p --port port (defaults to 3000) + -k --keepalive enable keep-alive connections + -r --restart restart when files get modified + (defaults to false) + -rd --restart_delay delay between file checks + (ignored if you have Linux::Inotify2 installed) + -rr --restart_regex regex match files that trigger + a restart when modified + (defaults to '\.yml$|\.yaml$|\.conf|\.pm$') + --restart_directory the directory to search for + modified files, can be set multiple times + (defaults to '[SCRIPT_DIR]/..') + --follow_symlinks follow symlinks in search directories + (defaults to false. this is a no-op on Win32) + --background run the process in the background + --pidfile specify filename for pid file + + See also: + perldoc Catalyst::Manual + perldoc Catalyst::Manual::Intro + +=head1 DESCRIPTION + +Run a Catalyst Testserver for this application. + +=head1 AUTHORS + +Catalyst Contributors, see Catalyst.pm + +=head1 COPYRIGHT + +This library is free software. You can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut From 9a73ec6455a3c422c0281951fa94791d4d31db2f Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Tue, 22 Jan 2019 17:27:36 +0100 Subject: [PATCH 11/13] hydra-queue-runner: better error message if nix-store can not be started MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hydra-queue-runner opens a connection to the builder. If the builder is 'localhost' it starts `nix-store`, otherwise it starts 'ssh'. Currently, if the hydra-queue-runner can not start `nix-store` (not in the PATH for instance), the error message is: cannot connect to ‘localhost’: error: cannot start ssh: No such file or directory This is not useful since ssh is actually not started:/ With this patch the error message is now: cannot connect to ‘localhost’: error: cannot start nix-store: No such file or directory --- src/hydra-queue-runner/build-remote.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index dccc4469..69c430eb 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -29,6 +29,7 @@ static void append(Strings & dst, const Strings & src) static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child) { + string pgmName; Pipe to, from; to.create(); from.create(); @@ -47,9 +48,12 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil throw SysError("cannot dup stderr"); Strings argv; - if (machine->sshName == "localhost") + if (machine->sshName == "localhost") { + pgmName = "nix-store"; argv = {"nix-store", "--serve", "--write"}; + } else { + pgmName = "ssh"; argv = {"ssh", machine->sshName}; if (machine->sshKey != "") append(argv, {"-i", machine->sshKey}); if (machine->sshPublicHostKey != "") { @@ -66,7 +70,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast - throw SysError("cannot start ssh"); + throw SysError("cannot start %s", pgmName); }); to.readSide = -1; From 4160132278ac629f6a74ca287ed6f467487ff861 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 1 Feb 2019 04:29:07 +0100 Subject: [PATCH 12/13] tests: disable sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior, tests would all fail to build, causing, roughly, the following error (roughly, because I added some debug log messages :)): ok 68 - Evaluating jobs/build-products.nix should result in 2 builds Queue runner stderr: using 4185024512 bytes for the NAR buffer locking path '/build/source/tests/data/queue-runner/lock' lock acquired on '/build/source/tests/data/queue-runner/lock.lock' warning: unknown setting 'max-connection-age' warning: unknown setting 'max-connections' dispatcher woken up dispatcher woken up dispatcher sleeping for 7674380800s adding new machine ‘localhost’ dispatcher woken up checking the queue for builds > 0... dispatcher sleeping for 7674380800s sending notification about build 1 loading build 18 (tests:build-products:simple) considering derivation ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ sending notification about build 2 creating build step ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ added build 18 (top-level step /build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv, 1 new steps) got 1 new runnable steps from 1 new builds step ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ is now runnable dispatcher woken up dispatcher sleeping for 7674380800s performing step ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ 1 times on ‘localhost’ (needed by build 18 and 0 others) sending closure of ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ to ‘localhost’ building ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ on ‘localhost’ killing process 10462 marking build 18 as failed finishing build step ‘/build/source/tests/nix/store/24h0i450d4k00a4jhhk6r7qpqdvzskw6-build-product-simple.drv’ ok 69 - Build 'simple' from jobs/build-products.nix should exit with code 0 ok 70 - newbuild->finished was '1' instead of 1 not ok 71 - newbuild->buildstatus was '1' instead of 0 not ok 72 - Build 'simple' from jobs/build-products.nix should have buildstatus 0 Can't call method "name" on an undefined value at ./evaluation-tests.pl line 173. FAIL: evaluation-tests.pl --- tests/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Makefile.am b/tests/Makefile.am index fde2d01f..cff12f09 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -40,5 +40,6 @@ dirs: mkdir -p nix mkdir -p nix/etc/nix touch nix/etc/nix/nix.conf + echo "sandbox = false" >> nix/etc/nix/nix.conf mkdir -p nix/store mkdir -p nix/var From e9d032bfe9c4256c8c03929b499e9cedc534736c Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Fri, 15 Mar 2019 11:22:10 +0100 Subject: [PATCH 13/13] Fix Linux build on Darwin Without this patch running the following on MacOS: nix-build release.nix -A build.x86_64-linux results in the following error during the configuration phase (note that Nix should be configured with a x86_64-linux build machine): building '/nix/store/jb6ca1gmplyb69ayd43z7fb0y9npxd53-hydra-0.1.2581.8b5948f4cf12424c04df67a6eb136c9846fb2cfd.drv' on 'ssh://my-linux-build-machine'... ... checking whether /nix/store/s6bhdppx66bkgf741vk4d29hgsj1h1zp-hydra-perl-deps/bin/nix-store is recent enough... ./configure: line 16254: /nix/store/s6bhdppx66bkgf741vk4d29hgsj1h1zp-hydra-perl-deps/bin/nix-store: cannot execute binary file: Exec format error no configure: error: `/nix/store/s6bhdppx66bkgf741vk4d29hgsj1h1zp-hydra-perl-deps/bin/nix-store' doesn't support `--timeout'; please use a newer version. build time elapsed: 0m1.624s 0m1.774s 0m9.366s 0m6.110s builder for '/nix/store/jb6ca1gmplyb69ayd43z7fb0y9npxd53-hydra-0.1.2581.8b5948f4cf12424c04df67a6eb136c9846fb2cfd.drv' failed with exit code 1 This problem is that the `nix` dependency of hydra is selected from a nixpkgs set configured with a default `system` parameter, i.e. `builtin.currentSystem`. This means that the hydra derivation which is build for and on Linux depends on the nix derivation build for Darwin. The fix is to select nix from the nixpkgs set configured with a system specified by the user. --- release.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 372e31f8..b20435f8 100644 --- a/release.nix +++ b/release.nix @@ -37,8 +37,9 @@ in rec { build = genAttrs' (system: + let pkgs = import nixpkgs { inherit system; }; in - with import nixpkgs { inherit system; }; + with pkgs; let