forked from lix-project/hydra
Merge pull request #877 from immae/mdbook
Convert documentation to markdown / mdbook
This commit is contained in:
Graham Christensen
GitHub
commit
bfcc50f978
|
|
@ -1,33 +1,6 @@
|
||||||
DOCBOOK_FILES = installation.xml introduction.xml manual.xml projects.xml hacking.xml
|
MD_FILES = src/*.md
|
||||||
|
|
||||||
EXTRA_DIST = $(DOCBOOK_FILES)
|
EXTRA_DIST = $(MD_FILES)
|
||||||
|
|
||||||
xsltproc_opts = \
|
install: $(MD_FILES)
|
||||||
--param callout.graphics.extension \'.gif\' \
|
mdbook build . -d $(docdir)
|
||||||
--param section.autolabel 1 \
|
|
||||||
--param section.label.includes.component.label 1
|
|
||||||
|
|
||||||
|
|
||||||
# Include the manual in the tarball.
|
|
||||||
dist_html_DATA = manual.html
|
|
||||||
|
|
||||||
# Embed Docbook's callout images in the distribution.
|
|
||||||
EXTRA_DIST += images
|
|
||||||
|
|
||||||
manual.html: $(DOCBOOK_FILES)
|
|
||||||
$(XSLTPROC) $(xsltproc_opts) --nonet --xinclude \
|
|
||||||
--output manual.html \
|
|
||||||
$(docbookxsl)/xhtml/docbook.xsl manual.xml
|
|
||||||
|
|
||||||
images:
|
|
||||||
$(MKDIR_P) images/callouts
|
|
||||||
cp $(docbookxsl)/images/callouts/*.gif images/callouts
|
|
||||||
chmod +wx images images/callouts
|
|
||||||
|
|
||||||
install-data-hook: images
|
|
||||||
$(INSTALL) -d $(DESTDIR)$(htmldir)/images/callouts
|
|
||||||
$(INSTALL_DATA) images/callouts/* $(DESTDIR)$(htmldir)/images/callouts
|
|
||||||
ln -sfn manual.html $(DESTDIR)$(htmldir)/index.html
|
|
||||||
|
|
||||||
distclean-hook:
|
|
||||||
-rm -rf images
|
|
||||||
|
|
|
||||||
|
|
@ -1,334 +0,0 @@
|
||||||
<chapter xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
|
||||||
xml:id="chap-api">
|
|
||||||
|
|
||||||
<title>Using the external API</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To be able to create integrations with other services, Hydra exposes
|
|
||||||
an external API that you can manage projects with.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The API is accessed over HTTP(s) where all data is sent and received
|
|
||||||
as JSON.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Creating resources requires the caller to be authenticated, while
|
|
||||||
retrieving resources does not.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The API does not have a separate URL structure for it's endpoints.
|
|
||||||
Instead you request the pages of the web interface as
|
|
||||||
<literal>application/json</literal> to use the API.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>List projects</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To list all the <literal>projects</literal> of the Hydra install:
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET /
|
|
||||||
Accept: application/json
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
This will give you a list of <literal>projects</literal>, where each
|
|
||||||
<literal>project</literal> contains general information and a list
|
|
||||||
of its <literal>job sets</literal>.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Example</emphasis>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
curl -i -H 'Accept: application/json' \
|
|
||||||
https://hydra.nixos.org
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Note:</emphasis> this response is truncated
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET https://hydra.nixos.org/
|
|
||||||
HTTP/1.1 200 OK
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"displayname": "Acoda",
|
|
||||||
"name": "acoda",
|
|
||||||
"description": "Acoda is a tool set for automatic data migration along an evolving data model",
|
|
||||||
"enabled": 0,
|
|
||||||
"owner": "sander",
|
|
||||||
"hidden": 1,
|
|
||||||
"jobsets": [
|
|
||||||
"trunk"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"displayname": "cabal2nix",
|
|
||||||
"name": "cabal2nix",
|
|
||||||
"description": "Convert Cabal files into Nix build instructions",
|
|
||||||
"enabled": 0,
|
|
||||||
"owner": "simons@cryp.to",
|
|
||||||
"hidden": 1,
|
|
||||||
"jobsets": [
|
|
||||||
"master"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
</programlisting>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Get a single project</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To get a single <literal>project</literal> by identifier:
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET /project/:project-identifier
|
|
||||||
Accept: application/json
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Example</emphasis>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
curl -i -H 'Accept: application/json' \
|
|
||||||
https://hydra.nixos.org/project/hydra
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET https://hydra.nixos.org/project/hydra
|
|
||||||
HTTP/1.1 200 OK
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"description": "Hydra, the Nix-based continuous build system",
|
|
||||||
"hidden": 0,
|
|
||||||
"displayname": "Hydra",
|
|
||||||
"jobsets": [
|
|
||||||
"hydra-master",
|
|
||||||
"hydra-ant-logger-trunk",
|
|
||||||
"master",
|
|
||||||
"build-ng"
|
|
||||||
],
|
|
||||||
"name": "hydra",
|
|
||||||
"enabled": 1,
|
|
||||||
"owner": "eelco"
|
|
||||||
}
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Get a single job set</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To get a single <literal>job set</literal> by identifier:
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET /jobset/:project-identifier/:jobset-identifier
|
|
||||||
Content-Type: application/json
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Example</emphasis>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
curl -i -H 'Accept: application/json' \
|
|
||||||
https://hydra.nixos.org/jobset/hydra/build-ng
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET https://hydra.nixos.org/jobset/hydra/build-ng
|
|
||||||
HTTP/1.1 200 OK
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"errormsg": "evaluation failed due to signal 9 (Killed)",
|
|
||||||
"fetcherrormsg": null,
|
|
||||||
"nixexprpath": "release.nix",
|
|
||||||
"nixexprinput": "hydraSrc",
|
|
||||||
"emailoverride": "rob.vermaas@gmail.com, eelco.dolstra@logicblox.com",
|
|
||||||
"jobsetinputs": {
|
|
||||||
"officialRelease": {
|
|
||||||
"jobsetinputalts": [
|
|
||||||
"false"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"hydraSrc": {
|
|
||||||
"jobsetinputalts": [
|
|
||||||
"https://github.com/NixOS/hydra.git build-ng"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"nixpkgs": {
|
|
||||||
"jobsetinputalts": [
|
|
||||||
"https://github.com/NixOS/nixpkgs.git release-14.12"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"enabled": 0
|
|
||||||
}
|
|
||||||
</programlisting>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>List evaluations</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To list the <literal>evaluations</literal> of a
|
|
||||||
<literal>job set</literal> by identifier:
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET /jobset/:project-identifier/:jobset-identifier/evals
|
|
||||||
Content-Type: application/json
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Example</emphasis>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
curl -i -H 'Accept: application/json' \
|
|
||||||
https://hydra.nixos.org/jobset/hydra/build-ng/evals
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Note:</emphasis> this response is truncated
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET https://hydra.nixos.org/jobset/hydra/build-ng/evals
|
|
||||||
HTTP/1.1 200 OK
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"evals": [
|
|
||||||
{
|
|
||||||
"jobsetevalinputs": {
|
|
||||||
"nixpkgs": {
|
|
||||||
"dependency": null,
|
|
||||||
"type": "git",
|
|
||||||
"value": null,
|
|
||||||
"uri": "https://github.com/NixOS/nixpkgs.git",
|
|
||||||
"revision": "f60e48ce81b6f428d072d3c148f6f2e59f1dfd7a"
|
|
||||||
},
|
|
||||||
"hydraSrc": {
|
|
||||||
"dependency": null,
|
|
||||||
"type": "git",
|
|
||||||
"value": null,
|
|
||||||
"uri": "https://github.com/NixOS/hydra.git",
|
|
||||||
"revision": "48d6f0de2ab94f728d287b9c9670c4d237e7c0f6"
|
|
||||||
},
|
|
||||||
"officialRelease": {
|
|
||||||
"dependency": null,
|
|
||||||
"value": "false",
|
|
||||||
"type": "boolean",
|
|
||||||
"uri": null,
|
|
||||||
"revision": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"hasnewbuilds": 1,
|
|
||||||
"builds": [
|
|
||||||
24670686,
|
|
||||||
24670684,
|
|
||||||
24670685,
|
|
||||||
24670687
|
|
||||||
],
|
|
||||||
"id": 1213758
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"first": "?page=1",
|
|
||||||
"last": "?page=1"
|
|
||||||
}
|
|
||||||
</programlisting>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Get a single build</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To get a single <literal>build</literal> by its id:
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET /build/:build-id
|
|
||||||
Content-Type: application/json
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis role="strong">Example</emphasis>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
curl -i -H 'Accept: application/json' \
|
|
||||||
https://hydra.nixos.org/build/24670686
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
GET /build/24670686
|
|
||||||
HTTP/1.1 200 OK
|
|
||||||
Content-Type: application/json
|
|
||||||
|
|
||||||
{
|
|
||||||
"job": "tests.api.x86_64-linux",
|
|
||||||
"jobsetevals": [
|
|
||||||
1213758
|
|
||||||
],
|
|
||||||
"buildstatus": 0,
|
|
||||||
"buildmetrics": null,
|
|
||||||
"project": "hydra",
|
|
||||||
"system": "x86_64-linux",
|
|
||||||
"priority": 100,
|
|
||||||
"releasename": null,
|
|
||||||
"starttime": 1439402853,
|
|
||||||
"nixname": "vm-test-run-unnamed",
|
|
||||||
"timestamp": 1439388618,
|
|
||||||
"id": 24670686,
|
|
||||||
"stoptime": 1439403403,
|
|
||||||
"jobset": "build-ng",
|
|
||||||
"buildoutputs": {
|
|
||||||
"out": {
|
|
||||||
"path": "/nix/store/lzrxkjc35mhp8w7r8h82g0ljyizfchma-vm-test-run-unnamed"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"buildproducts": {
|
|
||||||
"1": {
|
|
||||||
"path": "/nix/store/lzrxkjc35mhp8w7r8h82g0ljyizfchma-vm-test-run-unnamed",
|
|
||||||
"defaultpath": "log.html",
|
|
||||||
"type": "report",
|
|
||||||
"sha256hash": null,
|
|
||||||
"filesize": null,
|
|
||||||
"name": "",
|
|
||||||
"subtype": "testlog"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"finished": 1
|
|
||||||
}
|
|
||||||
</programlisting>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
</chapter>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
Local Variables:
|
|
||||||
indent-tabs-mode: nil
|
|
||||||
ispell-local-dictionary: "american"
|
|
||||||
End:
|
|
||||||
-->
|
|
||||||
|
|
@ -1,196 +0,0 @@
|
||||||
<section xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xml:id="sec-declarative-projects">
|
|
||||||
|
|
||||||
<title>Declarative projects</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Hydra supports declaratively configuring a project's jobsets. This
|
|
||||||
configuration can be done statically, or generated by a build job.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<note><para>
|
|
||||||
Hydra will treat the project's declarative input as a static definition
|
|
||||||
if and only if the spec file contains a dictionary of dictionaries.
|
|
||||||
If the value of any key in the spec is not a dictionary, it will
|
|
||||||
treat the spec as a generated declarative spec.
|
|
||||||
</para></note>
|
|
||||||
|
|
||||||
<section xml:id="sec-static-declarative-projects">
|
|
||||||
|
|
||||||
<title>Static, Declarative Projects</title>
|
|
||||||
<para>
|
|
||||||
Hydra supports declarative projects, where jobsets are configured
|
|
||||||
from a static JSON document in a repository.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To configure a static declarative project, take the following steps:
|
|
||||||
</para>
|
|
||||||
<orderedlist numeration="arabic" spacing="compact">
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
Create a Hydra-fetchable source like a Git repository or local path.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
In that source, create a file called <filename>spec.json</filename>,
|
|
||||||
and add the specification for all of the jobsets. Each key is jobset
|
|
||||||
and each value is a jobset's specification. For example:
|
|
||||||
|
|
||||||
<programlisting language="json">
|
|
||||||
{
|
|
||||||
"nixpkgs": {
|
|
||||||
"enabled": 1,
|
|
||||||
"hidden": false,
|
|
||||||
"description": "Nixpkgs",
|
|
||||||
"nixexprinput": "nixpkgs",
|
|
||||||
"nixexprpath": "pkgs/top-level/release.nix",
|
|
||||||
"checkinterval": 300,
|
|
||||||
"schedulingshares": 100,
|
|
||||||
"enableemail": false,
|
|
||||||
"emailoverride": "",
|
|
||||||
"keepnr": 3,
|
|
||||||
"inputs": {
|
|
||||||
"nixpkgs": {
|
|
||||||
"type": "git",
|
|
||||||
"value": "git://github.com/NixOS/nixpkgs.git master",
|
|
||||||
"emailresponsible": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nixos": {
|
|
||||||
"enabled": 1,
|
|
||||||
"hidden": false,
|
|
||||||
"description": "NixOS: Small Evaluation",
|
|
||||||
"nixexprinput": "nixpkgs",
|
|
||||||
"nixexprpath": "nixos/release-small.nix",
|
|
||||||
"checkinterval": 300,
|
|
||||||
"schedulingshares": 100,
|
|
||||||
"enableemail": false,
|
|
||||||
"emailoverride": "",
|
|
||||||
"keepnr": 3,
|
|
||||||
"inputs": {
|
|
||||||
"nixpkgs": {
|
|
||||||
"type": "git",
|
|
||||||
"value": "git://github.com/NixOS/nixpkgs.git master",
|
|
||||||
"emailresponsible": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</programlisting>
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
Create a new project, and set the project's declarative input type,
|
|
||||||
declarative input value, and declarative spec file to point to the
|
|
||||||
source and JSON file you created in step 2.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
</orderedlist>
|
|
||||||
<para>
|
|
||||||
Hydra will create a special jobset named <literal>.jobsets</literal>.
|
|
||||||
When the <literal>.jobsets</literal> jobset is evaluated, this static
|
|
||||||
specification will be used for configuring the rest of the project's
|
|
||||||
jobsets.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section xml:id="sec-generated-declarative-projects">
|
|
||||||
|
|
||||||
<title>Generated, Declarative Projects</title>
|
|
||||||
<para>
|
|
||||||
Hydra also supports generated declarative projects, where jobsets are
|
|
||||||
configured automatically from specification files instead of being
|
|
||||||
managed through the UI. A jobset specification is a JSON object
|
|
||||||
containing the configuration of the jobset, for example:
|
|
||||||
</para>
|
|
||||||
<programlisting language="json">
|
|
||||||
{
|
|
||||||
"enabled": 1,
|
|
||||||
"hidden": false,
|
|
||||||
"description": "js",
|
|
||||||
"nixexprinput": "src",
|
|
||||||
"nixexprpath": "release.nix",
|
|
||||||
"checkinterval": 300,
|
|
||||||
"schedulingshares": 100,
|
|
||||||
"enableemail": false,
|
|
||||||
"emailoverride": "",
|
|
||||||
"keepnr": 3,
|
|
||||||
"inputs": {
|
|
||||||
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
|
|
||||||
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</programlisting>
|
|
||||||
<para>
|
|
||||||
To configure a declarative project, take the following steps:
|
|
||||||
</para>
|
|
||||||
<orderedlist numeration="arabic" spacing="compact">
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
Create a jobset repository in the normal way (e.g. a git repo with
|
|
||||||
a <literal>release.nix</literal> file, any other needed helper
|
|
||||||
files, and taking any kind of hydra input), but without adding it
|
|
||||||
to the UI. The nix expression of this repository should contain a
|
|
||||||
single job, named <literal>jobsets</literal>. The output of the
|
|
||||||
<literal>jobsets</literal> job should be a JSON file containing an
|
|
||||||
object of jobset specifications. Each member of the object will
|
|
||||||
become a jobset of the project, configured by the corresponding
|
|
||||||
jobset specification.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
In some hydra-fetchable source (potentially, but not necessarily,
|
|
||||||
the same repo you created in step 1), create a JSON file
|
|
||||||
containing a jobset specification that points to the jobset
|
|
||||||
repository you created in the first step, specifying any needed
|
|
||||||
inputs (e.g. nixpkgs) as necessary.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
In the project creation/edit page, set declarative input type,
|
|
||||||
declarative input value, and declarative spec file to point to the
|
|
||||||
source and JSON file you created in step 2.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
</orderedlist>
|
|
||||||
<para>
|
|
||||||
Hydra will create a special jobset named <literal>.jobsets</literal>,
|
|
||||||
which whenever evaluated will go through the steps above in reverse
|
|
||||||
order:
|
|
||||||
</para>
|
|
||||||
<orderedlist numeration="arabic" spacing="compact">
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
Hydra will fetch the input specified by the declarative input type
|
|
||||||
and value.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
Hydra will use the configuration given in the declarative spec
|
|
||||||
file as the jobset configuration for this evaluation. In addition
|
|
||||||
to any inputs specified in the spec file, hydra will also pass the
|
|
||||||
<literal>declInput</literal> argument corresponding to the input
|
|
||||||
fetched in step 1.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
<para>
|
|
||||||
As normal, hydra will build the jobs specified in the jobset
|
|
||||||
repository, which in this case is the single
|
|
||||||
<literal>jobsets</literal> job. When that job completes, hydra
|
|
||||||
will read the created jobset specifications and create
|
|
||||||
corresponding jobsets in the project, disabling any jobsets that
|
|
||||||
used to exist but are not present in the current spec.
|
|
||||||
</para>
|
|
||||||
</listitem>
|
|
||||||
</orderedlist>
|
|
||||||
</section>
|
|
||||||
</section>
|
|
||||||
|
|
@ -1,39 +0,0 @@
|
||||||
<appendix xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xml:id="chap-hacking">
|
|
||||||
|
|
||||||
<title>Hacking</title>
|
|
||||||
|
|
||||||
<para>This section provides some notes on how to hack on Hydra. To
|
|
||||||
get the latest version of Hydra from GitHub:
|
|
||||||
<screen>
|
|
||||||
$ git clone git://github.com/NixOS/hydra.git
|
|
||||||
$ cd hydra
|
|
||||||
</screen>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>To build it and its dependencies:
|
|
||||||
<screen>
|
|
||||||
$ nix-build release.nix -A build.x86_64-linux
|
|
||||||
</screen>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>To build all dependencies and start a shell in which all
|
|
||||||
environment variables (such as <envar>PERL5LIB</envar>) are set up so
|
|
||||||
that those dependencies can be found:
|
|
||||||
<screen>
|
|
||||||
$ nix-shell
|
|
||||||
</screen>
|
|
||||||
To build Hydra, you should then do:
|
|
||||||
<screen>
|
|
||||||
[nix-shell]$ ./bootstrap
|
|
||||||
[nix-shell]$ configurePhase
|
|
||||||
[nix-shell]$ make
|
|
||||||
</screen>
|
|
||||||
You can run the Hydra web server in your source tree as follows:
|
|
||||||
<screen>
|
|
||||||
$ ./src/script/hydra-server
|
|
||||||
</screen>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</appendix>
|
|
||||||
|
|
@ -1,338 +0,0 @@
|
||||||
<chapter xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xml:id="chap-installation">
|
|
||||||
|
|
||||||
<title>Installation</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
This chapter explains how to install Hydra on your own build farm server.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Prerequisites</title>
|
|
||||||
<para>
|
|
||||||
To install and use Hydra you need to have installed the following dependencies:
|
|
||||||
|
|
||||||
<itemizedlist>
|
|
||||||
<listitem><para>Nix</para></listitem>
|
|
||||||
<listitem><para>PostgreSQL</para></listitem>
|
|
||||||
<listitem><para>many Perl packages, notably Catalyst, EmailSender,
|
|
||||||
and NixPerl (see the <link
|
|
||||||
xlink:href="https://github.com/NixOS/hydra/blob/master/release.nix">Hydra
|
|
||||||
expression in Nixpkgs</link> for the complete
|
|
||||||
list)</para></listitem>
|
|
||||||
</itemizedlist>
|
|
||||||
|
|
||||||
At the moment, Hydra runs only on GNU/Linux
|
|
||||||
(<emphasis>i686-linux</emphasis> and
|
|
||||||
<emphasis>x86_64_linux</emphasis>).
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
For small projects, Hydra can be run on any reasonably modern
|
|
||||||
machine. For individual projects you can even run Hydra on a
|
|
||||||
laptop. However, the charm of a buildfarm server is usually that
|
|
||||||
it operates without disturbing the developer's working
|
|
||||||
environment and can serve releases over the internet. In
|
|
||||||
conjunction you should typically have your source code
|
|
||||||
administered in a version management system, such as
|
|
||||||
subversion. Therefore, you will probably want to install a
|
|
||||||
server that is connected to the internet. To scale up to large
|
|
||||||
and/or many projects, you will need at least a considerable
|
|
||||||
amount of diskspace to store builds. Since Hydra can schedule
|
|
||||||
multiple simultaneous build jobs, it can be useful to have a
|
|
||||||
multi-core machine, and/or attach multiple build machines in a
|
|
||||||
network to the central Hydra server.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Of course we think it is a good idea to use the <link
|
|
||||||
xlink:href="http://nixos.org/nixos">NixOS</link> GNU/Linux
|
|
||||||
distribution for your buildfarm server. But this is not a
|
|
||||||
requirement. The Nix software deployment system can be
|
|
||||||
installed on any GNU/Linux distribution in parallel to the
|
|
||||||
regular package management system. Thus, you can use Hydra on a
|
|
||||||
Debian, Fedora, SuSE, or Ubuntu system.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Getting Nix</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
If your server runs NixOS you are all set to continue with
|
|
||||||
installation of Hydra. Otherwise you first need to install Nix.
|
|
||||||
The latest stable version can be found one <link
|
|
||||||
xlink:href="http://nixos.org/nix/download.html">the Nix web
|
|
||||||
site</link>, along with a manual, which includes installation
|
|
||||||
instructions.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Installation</title>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
<para>
|
|
||||||
Hydra can be installed using Nixpkgs:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
nix-env -f /path/to/nixpkgs -iA hydra</screen>
|
|
||||||
|
|
||||||
This makes the tools available in your Nix user environment,
|
|
||||||
<literal>$HOME/.nix-profile</literal> by default.
|
|
||||||
</para>
|
|
||||||
-->
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The latest development snapshot of Hydra can be installed
|
|
||||||
by visiting the URL <link
|
|
||||||
xlink:href="http://hydra.nixos.org/view/hydra/unstable"><literal>http://hydra.nixos.org/view/hydra/unstable</literal></link>
|
|
||||||
and using the one-click install available at one of the build
|
|
||||||
pages. You can also install Hydra through the channel by
|
|
||||||
performing the following commands:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
nix-channel --add http://hydra.nixos.org/jobset/hydra/master/channel/latest
|
|
||||||
nix-channel --update
|
|
||||||
nix-env -i hydra</screen>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Command completion should reveal a number of command-line tools
|
|
||||||
from Hydra, such as <command>hydra-queue-runner</command>.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Creating the database</title>
|
|
||||||
<para>
|
|
||||||
Hydra stores its results in a PostgreSQL database.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To setup a PostgreSQL database with <emphasis>hydra</emphasis>
|
|
||||||
as database name and user name, issue the following commands on
|
|
||||||
the PostgreSQL server:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
createuser -S -D -R -P hydra
|
|
||||||
createdb -O hydra hydra</screen>
|
|
||||||
|
|
||||||
Note that <emphasis>$prefix</emphasis> is the location of Hydra
|
|
||||||
in the nix store.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Hydra uses an environment variable to know which database should
|
|
||||||
be used, and a variable which point to a location that holds
|
|
||||||
some state. To set these variables for a PostgreSQL database,
|
|
||||||
add the following to the file <filename>~/.profile</filename> of
|
|
||||||
the user running the Hydra services.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
export HYDRA_DBI="dbi:Pg:dbname=hydra;host=dbserver.example.org;user=hydra;"
|
|
||||||
export HYDRA_DATA=/var/lib/hydra</screen>
|
|
||||||
|
|
||||||
You can provide the username and password in the file
|
|
||||||
<filename>~/.pgpass</filename>, e.g.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
dbserver.example.org:*:hydra:hydra:password</screen>
|
|
||||||
|
|
||||||
Make sure that the <emphasis>HYDRA_DATA</emphasis> directory
|
|
||||||
exists and is writable for the user which will run the Hydra
|
|
||||||
services.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Having set these environment variables, you can now initialise
|
|
||||||
the database by doing:
|
|
||||||
<screen>
|
|
||||||
hydra-init</screen>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To create projects, you need to create a user with
|
|
||||||
<emphasis>admin</emphasis> privileges. This can be done using
|
|
||||||
the command <command>hydra-create-user</command>:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ hydra-create-user alice --full-name 'Alice Q. User' \
|
|
||||||
--email-address 'alice@example.org' --password foobar --role admin
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
Additional users can be created through the web interface.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Upgrading</title>
|
|
||||||
|
|
||||||
<para>If you're upgrading Hydra from a previous version, you
|
|
||||||
should do the following to perform any necessary database schema migrations:
|
|
||||||
<screen>
|
|
||||||
hydra-init</screen>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Getting Started</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To start the Hydra web server, execute:
|
|
||||||
<screen>
|
|
||||||
hydra-server</screen>
|
|
||||||
|
|
||||||
When the server is started, you can browse to
|
|
||||||
<ulink>http://localhost:3000/</ulink> to start configuring
|
|
||||||
your Hydra instance.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The <command>hydra-server</command> command launches the web
|
|
||||||
server. There are two other processes that come into play:
|
|
||||||
|
|
||||||
<itemizedlist>
|
|
||||||
<listitem>
|
|
||||||
The <emphasis>evaluator</emphasis> is responsible for
|
|
||||||
periodically evaluating job sets, checking out their
|
|
||||||
dependencies off their version control systems (VCS), and
|
|
||||||
queueing new builds if the result of the evaluation changed.
|
|
||||||
It is launched by the <command>hydra-evaluator</command>
|
|
||||||
command.
|
|
||||||
</listitem>
|
|
||||||
<listitem>
|
|
||||||
The <emphasis>queue runner</emphasis> launches builds (using
|
|
||||||
Nix) as they are queued by the evaluator, scheduling them
|
|
||||||
onto the configured Nix hosts. It is launched using the
|
|
||||||
<command>hydra-queue-runner</command> command.
|
|
||||||
</listitem>
|
|
||||||
</itemizedlist>
|
|
||||||
|
|
||||||
All three processes must be running for Hydra to be fully
|
|
||||||
functional, though it's possible to temporarily stop any one of
|
|
||||||
them for maintenance purposes, for instance.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title> Serving behind reverse proxy </title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To serve hydra web server behind reverse proxy like
|
|
||||||
<emphasis>nginx</emphasis> or <emphasis>httpd</emphasis> some
|
|
||||||
additional configuration must be made.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Edit your <literal>hydra.conf</literal> file in a similar way to
|
|
||||||
this example:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
using_frontend_proxy 1
|
|
||||||
base_uri example.com</screen>
|
|
||||||
|
|
||||||
<literal>base_uri</literal> should be your hydra servers proxied URL.
|
|
||||||
|
|
||||||
If you are using Hydra nixos module then setting <literal>hydraURL</literal>
|
|
||||||
option should be enough.
|
|
||||||
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
|
|
||||||
If you want to serve Hydra with a prefix path, for example
|
|
||||||
<ulink>http://example.com/hydra</ulink> then you need to configure your
|
|
||||||
reverse proxy to pass <literal>X-Request-Base</literal> to hydra, with
|
|
||||||
prefix path as value.
|
|
||||||
|
|
||||||
For example if you are using nginx, then use configuration similar to following:
|
|
||||||
<screen>
|
|
||||||
server {
|
|
||||||
listen 433 ssl;
|
|
||||||
server_name example.com;
|
|
||||||
.. other configuration ..
|
|
||||||
location /hydra/ {
|
|
||||||
|
|
||||||
proxy_pass http://127.0.0.1:3000;
|
|
||||||
proxy_redirect http://127.0.0.1:3000 https://example.com/hydra;
|
|
||||||
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_set_header X-Request-Base /hydra;
|
|
||||||
}
|
|
||||||
}</screen>
|
|
||||||
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
<section>
|
|
||||||
<title>Using LDAP as authentication backend (optional)</title>
|
|
||||||
<para>
|
|
||||||
Instead of using Hydra's built-in user management you can optionally use LDAP to manage roles and users.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The <command>hydra-server</command> accepts the environment
|
|
||||||
variable <emphasis>HYDRA_LDAP_CONFIG</emphasis>. The value of
|
|
||||||
the variable should point to a valid YAML file containing the
|
|
||||||
Catalyst LDAP configuration. The format of the configuration
|
|
||||||
file is describe in the
|
|
||||||
<link xlink:href="https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS">
|
|
||||||
<emphasis>Catalyst::Authentication::Store::LDAP</emphasis> documentation</link>.
|
|
||||||
An example is given below.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Roles can be assigned to users based on their LDAP group membership
|
|
||||||
(<emphasis>use_roles: 1</emphasis> in the below example).
|
|
||||||
For a user to have the role <emphasis>admin</emphasis> assigned to them
|
|
||||||
they should be in the group <emphasis>hydra_admin</emphasis>. In general
|
|
||||||
any LDAP group of the form <emphasis>hydra_some_role</emphasis>
|
|
||||||
(notice the <emphasis>hydra_</emphasis> prefix) will work.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
credential:
|
|
||||||
class: Password
|
|
||||||
password_field: password
|
|
||||||
password_type: self_check
|
|
||||||
store:
|
|
||||||
class: LDAP
|
|
||||||
ldap_server: localhost
|
|
||||||
ldap_server_options.timeout: 30
|
|
||||||
binddn: "cn=root,dc=example"
|
|
||||||
bindpw: notapassword
|
|
||||||
start_tls: 0
|
|
||||||
start_tls_options
|
|
||||||
verify: none
|
|
||||||
user_basedn: "ou=users,dc=example"
|
|
||||||
user_filter: "(&(objectClass=inetOrgPerson)(cn=%s))"
|
|
||||||
user_scope: one
|
|
||||||
user_field: cn
|
|
||||||
user_search_options:
|
|
||||||
deref: always
|
|
||||||
use_roles: 1
|
|
||||||
role_basedn: "ou=groups,dc=example"
|
|
||||||
role_filter: "(&(objectClass=groupOfNames)(member=%s))"
|
|
||||||
role_scope: one
|
|
||||||
role_field: cn
|
|
||||||
role_value: dn
|
|
||||||
role_search_options:
|
|
||||||
deref: always
|
|
||||||
</screen>
|
|
||||||
</section>
|
|
||||||
</chapter>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
Local Variables:
|
|
||||||
indent-tabs-mode: nil
|
|
||||||
ispell-local-dictionary: "american"
|
|
||||||
End:
|
|
||||||
-->
|
|
||||||
|
|
@ -1,267 +0,0 @@
|
||||||
<chapter xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xml:id="chap-introduction">
|
|
||||||
|
|
||||||
<title>Introduction</title>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>About Hydra</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Hydra is a tool for continuous integration testing and software
|
|
||||||
release that uses a purely functional language to describe build jobs
|
|
||||||
and their dependencies. Continuous integration is a simple technique
|
|
||||||
to improve the quality of the software development process. An
|
|
||||||
automated system continuously or periodically checks out the source
|
|
||||||
code of a project, builds it, runs tests, and produces reports for the
|
|
||||||
developers. Thus, various errors that might accidentally be committed
|
|
||||||
into the code base are automatically caught. Such a system allows
|
|
||||||
more in-depth testing than what developers could feasibly do manually:
|
|
||||||
|
|
||||||
<itemizedlist>
|
|
||||||
<listitem> <emphasis>Portability testing</emphasis>: The
|
|
||||||
software may need to be built and tested on many different
|
|
||||||
platforms. It is infeasible for each developer to do this
|
|
||||||
before every commit.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> Likewise, many projects have very large test sets
|
|
||||||
(e.g., regression tests in a compiler, or stress tests in a
|
|
||||||
DBMS) that can take hours or days to run to completion.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> Many kinds of static and dynamic analyses can be
|
|
||||||
performed as part of the tests, such as code coverage runs and
|
|
||||||
static analyses.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> It may also be necessary to build many different
|
|
||||||
<emphasis>variants</emphasis> of the software. For instance,
|
|
||||||
it may be necessary to verify that the component builds with
|
|
||||||
various versions of a compiler.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> Developers typically use incremental building to
|
|
||||||
test their changes (since a full build may take too long), but
|
|
||||||
this is unreliable with many build management tools (such as
|
|
||||||
Make), i.e., the result of the incremental build might differ
|
|
||||||
from a full build.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> It ensures that the software can be built from the
|
|
||||||
sources under revision control. Users of version management
|
|
||||||
systems such as CVS and Subversion often forget to place
|
|
||||||
source files under revision control.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> The machines on which the continuous integration
|
|
||||||
system runs ideally provides a clean, well-defined build
|
|
||||||
environment. If this environment is administered through
|
|
||||||
proper SCM techniques, then builds produced by the system can
|
|
||||||
be reproduced. In contrast, developer work environments are
|
|
||||||
typically not under any kind of SCM control.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> In large projects, developers often work on a
|
|
||||||
particular component of the project, and do not build and test
|
|
||||||
the composition of those components (again since this is
|
|
||||||
likely to take too long). To prevent the phenomenon of ``big
|
|
||||||
bang integration'', where components are only tested together
|
|
||||||
near the end of the development process, it is important to
|
|
||||||
test components together as soon as possible (hence
|
|
||||||
<emphasis>continuous integration</emphasis>).
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> It allows software to be
|
|
||||||
<emphasis>released</emphasis> by automatically creating
|
|
||||||
packages that users can download and install. To do this
|
|
||||||
manually represents an often prohibitive amount of work, as
|
|
||||||
one may want to produce releases for many different platforms:
|
|
||||||
e.g., installers for Windows and Mac OS X, RPM or Debian
|
|
||||||
packages for certain Linux distributions, and so on.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
</itemizedlist>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
In its simplest form, a continuous integration tool sits in a
|
|
||||||
loop building and releasing software components from a version
|
|
||||||
management system. For each component, it performs the
|
|
||||||
following tasks:
|
|
||||||
|
|
||||||
<itemizedlist>
|
|
||||||
|
|
||||||
<listitem>It obtains the latest version of the component's
|
|
||||||
source code from the version management system.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> It runs the component's build process (which
|
|
||||||
presumably includes the execution of the component's test
|
|
||||||
set).
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> It presents the results of the build (such as error
|
|
||||||
logs and releases) to the developers, e.g., by producing a web
|
|
||||||
page.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
</itemizedlist>
|
|
||||||
|
|
||||||
Examples of continuous integration tools include Jenkins,
|
|
||||||
CruiseControl Tinderbox, Sisyphus, Anthill and BuildBot. These
|
|
||||||
tools have various limitations.
|
|
||||||
|
|
||||||
<itemizedlist>
|
|
||||||
|
|
||||||
<listitem> They do not manage the <emphasis>build
|
|
||||||
environment</emphasis>. The build environment consists of the
|
|
||||||
dependencies necessary to perform a build action, e.g.,
|
|
||||||
compilers, libraries, etc. Setting up the environment is
|
|
||||||
typically done manually, and without proper SCM control (so it
|
|
||||||
may be hard to reproduce a build at a later time). Manual
|
|
||||||
management of the environment scales poorly in the number of
|
|
||||||
configurations that must be supported. For instance, suppose
|
|
||||||
that we want to build a component that requires a certain
|
|
||||||
compiler X. We then have to go to each machine and install X.
|
|
||||||
If we later need a newer version of X, the process must be
|
|
||||||
repeated all over again. An ever worse problem occurs if
|
|
||||||
there are conflicting, mutually exclusive versions of the
|
|
||||||
dependencies. Thus, simply installing the latest version is
|
|
||||||
not an option. Of course, we can install these components in
|
|
||||||
different directories and manually pass the appropriate paths
|
|
||||||
to the build processes of the various components. But this is
|
|
||||||
a rather tiresome and error-prone process.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
<listitem> They do not easily support <emphasis>variability in software
|
|
||||||
systems</emphasis>. A system may have a great deal of build-time
|
|
||||||
variability: optional functionality, whether to build a debug or
|
|
||||||
production version, different versions of dependencies, and so on.
|
|
||||||
(For instance, the Linux kernel now has over 2,600 build-time
|
|
||||||
configuration switches.) It is therefore important that a continuous
|
|
||||||
integration tool can easily select and test different instances from
|
|
||||||
the configuration space of the system to reveal problems, such as
|
|
||||||
erroneous interactions between features. In a continuous integration
|
|
||||||
setting, it is also useful to test different combinations of versions
|
|
||||||
of subsystems, e.g., the head revision of a component against stable
|
|
||||||
releases of its dependencies, and vice versa, as this can reveal
|
|
||||||
various integration problems.
|
|
||||||
</listitem>
|
|
||||||
|
|
||||||
</itemizedlist>
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<emphasis>Hydra</emphasis>, is a continuous integration tool
|
|
||||||
that solves these problems. It is built on top of the <link
|
|
||||||
xlink:href="http://nixos.org/nix/">Nix package manager</link>,
|
|
||||||
which has a purely functional language for describing package
|
|
||||||
build actions and their dependencies. This allows the build
|
|
||||||
environment for projects to be produced automatically and
|
|
||||||
deterministically, and variability in components to be expressed
|
|
||||||
naturally using functions; and as such is an ideal fit for a
|
|
||||||
continuous build system.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>About Us</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Hydra is the successor of the Nix Buildfarm, which was developed
|
|
||||||
in tandem with the Nix software deployment system. Nix was
|
|
||||||
originally developed at the Department of Information and
|
|
||||||
Computing Sciences, Utrecht University by the TraCE project
|
|
||||||
(2003-2008). The project was funded by the Software Engineering
|
|
||||||
Research Program Jacquard to improve the support for variability
|
|
||||||
in software systems. Funding for the development of Nix and
|
|
||||||
Hydra is now provided by the NIRICT LaQuSo Build Farm project.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>About this Manual</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
This manual tells you how to install the Hydra buildfarm
|
|
||||||
software on your own server and how to operate that server using
|
|
||||||
its web interface.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>License</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Hydra is free software: you can redistribute it and/or
|
|
||||||
modify it under the terms of the GNU General Public License as
|
|
||||||
published by the Free Software Foundation, either version 3 of
|
|
||||||
the License, or (at your option) any later version.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Hydra is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
<link xlink:href="http://www.gnu.org/licenses/">GNU General
|
|
||||||
Public License</link> for more details.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Hydra at <literal>nixos.org</literal></title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The <literal>nixos.org</literal> installation of Hydra runs at
|
|
||||||
<link
|
|
||||||
xlink:href="http://hydra.nixos.org/"><literal>http://hydra.nixos.org/</literal></link>.
|
|
||||||
|
|
||||||
That installation is used to build software components from the
|
|
||||||
<link xlink:href="http://nixos.org">Nix</link>,
|
|
||||||
<link xlink:href="http://nixos.org/nixos">NixOS</link>,
|
|
||||||
<link xlink:href="http://www.gnu.org/">GNU</link>,
|
|
||||||
<link xlink:href="http://strategoxt.org">Stratego/XT</link>,
|
|
||||||
and related projects.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
If you are one of the developers on those projects, it is likely
|
|
||||||
that you will be using the NixOS Hydra server in some way. If
|
|
||||||
you need to administer automatic builds for your project, you
|
|
||||||
should pull the right strings to get an account on the
|
|
||||||
server. This manual will tell you how to set up new projects and
|
|
||||||
build jobs within those projects and write a release.nix file to
|
|
||||||
describe the build process of your project to Hydra. You can
|
|
||||||
skip the next chapter.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
If your project does not yet have automatic builds within the
|
|
||||||
NixOS Hydra server, it may actually be eligible. We are in the
|
|
||||||
process of setting up a large buildfarm that should be able to
|
|
||||||
support open source and academic software projects. Get in
|
|
||||||
touch.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Hydra on your own buildfarm</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
If you need to run your own Hydra installation, <xref
|
|
||||||
linkend="chap-installation" /> explains how to download and
|
|
||||||
install the system on your own server.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
</chapter>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
Local Variables:
|
|
||||||
indent-tabs-mode: nil
|
|
||||||
ispell-local-dictionary: "american"
|
|
||||||
End:
|
|
||||||
-->
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<book xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xi="http://www.w3.org/2001/XInclude">
|
|
||||||
|
|
||||||
<info>
|
|
||||||
|
|
||||||
<title>Hydra User's Guide</title>
|
|
||||||
|
|
||||||
<subtitle>Draft</subtitle>
|
|
||||||
|
|
||||||
<authorgroup>
|
|
||||||
<author>
|
|
||||||
<personname>
|
|
||||||
<firstname>Eelco</firstname>
|
|
||||||
<surname>Dolstra</surname>
|
|
||||||
</personname>
|
|
||||||
<affiliation>
|
|
||||||
<orgname>Delft University of Technology</orgname>
|
|
||||||
<orgdiv>Department of Software Technology</orgdiv>
|
|
||||||
</affiliation>
|
|
||||||
<contrib>Author</contrib>
|
|
||||||
</author>
|
|
||||||
<author>
|
|
||||||
<personname>
|
|
||||||
<firstname>Rob</firstname>
|
|
||||||
<surname>Vermaas</surname>
|
|
||||||
</personname>
|
|
||||||
<affiliation>
|
|
||||||
<orgname>Delft University of Technology</orgname>
|
|
||||||
<orgdiv>Department of Software Technology</orgdiv>
|
|
||||||
</affiliation>
|
|
||||||
<contrib>Author</contrib>
|
|
||||||
</author>
|
|
||||||
<author>
|
|
||||||
<personname>
|
|
||||||
<firstname>Eelco</firstname>
|
|
||||||
<surname>Visser</surname>
|
|
||||||
</personname>
|
|
||||||
<affiliation>
|
|
||||||
<orgname>Delft University of Technology</orgname>
|
|
||||||
<orgdiv>Department of Software Technology</orgdiv>
|
|
||||||
</affiliation>
|
|
||||||
<contrib>Author</contrib>
|
|
||||||
</author>
|
|
||||||
<author>
|
|
||||||
<personname>
|
|
||||||
<firstname>Ludovic</firstname>
|
|
||||||
<surname>Courtès</surname>
|
|
||||||
</personname>
|
|
||||||
<contrib>Author</contrib>
|
|
||||||
</author>
|
|
||||||
</authorgroup>
|
|
||||||
|
|
||||||
|
|
||||||
<copyright>
|
|
||||||
<year>2009-2013</year>
|
|
||||||
<holder>Eelco Dolstra</holder>
|
|
||||||
</copyright>
|
|
||||||
|
|
||||||
<date>March 2010</date>
|
|
||||||
|
|
||||||
</info>
|
|
||||||
|
|
||||||
<xi:include href="introduction.xml" />
|
|
||||||
<xi:include href="installation.xml" />
|
|
||||||
<xi:include href="projects.xml" />
|
|
||||||
<xi:include href="api.xml" />
|
|
||||||
<xi:include href="hacking.xml" />
|
|
||||||
|
|
||||||
|
|
||||||
</book>
|
|
||||||
|
|
@ -1,496 +0,0 @@
|
||||||
<chapter xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
|
||||||
xml:id="chap-projects">
|
|
||||||
|
|
||||||
<title>Creating and Managing Projects</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Once Hydra is installed and running, the next step is to add
|
|
||||||
projects to the build farm. We follow the example of the <link
|
|
||||||
xlink:href="http://nixos.org/patchelf.html">Patchelf
|
|
||||||
project</link>, a software tool written in C and using the GNU
|
|
||||||
Build System (GNU Autoconf and GNU Automake).
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Log in to the web interface of your Hydra installation using the
|
|
||||||
user name and password you inserted in the database (by default,
|
|
||||||
Hydra's web server listens on <link
|
|
||||||
xlink:href="http://localhost:3000/"><literal>localhost:3000</literal></link>).
|
|
||||||
Then follow the "Create Project" link to create a new project.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Project Information</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
A project definition consists of some general information and a
|
|
||||||
set of job sets. The general information identifies a project,
|
|
||||||
its owner, and current state of activity.
|
|
||||||
|
|
||||||
Here's what we fill in for the patchelf project:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
Identifier: patchelf
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The <emphasis>identifier</emphasis> is the identity of the
|
|
||||||
project. It is used in URLs and in the names of build results.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The identifier should be a unique name (it is the primary
|
|
||||||
database key for the project table in the database). If you try
|
|
||||||
to create a project with an already existing identifier you'd
|
|
||||||
get an error message from the database.
|
|
||||||
|
|
||||||
So try to create the project after entering just the general
|
|
||||||
information to figure out if you have chosen a unique name.
|
|
||||||
Job sets can be added once the project has been created.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
Display name: Patchelf
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The <emphasis>display name</emphasis> is used in menus.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
Description: A tool for modifying ELF binaries
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The <emphasis>description</emphasis> is used as short
|
|
||||||
documentation of the nature of the project.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
Owner: eelco
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The <emphasis>owner</emphasis> of a project can create and edit
|
|
||||||
job sets.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
Enabled: Yes
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
Only if the project is <emphasis>enabled</emphasis> are builds
|
|
||||||
performed.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Once created there should be an entry for the project in the
|
|
||||||
sidebar. Go to the project page for the <link
|
|
||||||
xlink:href="http://localhost:3000/project/patchelf">Patchelf</link>
|
|
||||||
project.
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Job Sets</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
A project can consist of multiple <emphasis>job sets</emphasis>
|
|
||||||
(hereafter <emphasis>jobsets</emphasis>), separate tasks that
|
|
||||||
can be built separately, but may depend on each other (without
|
|
||||||
cyclic dependencies, of course). Go to the <link
|
|
||||||
xlink:href="http://localhost:3000/project/patchelf/edit">Edit</link>
|
|
||||||
page of the Patchelf project and "Add a new jobset" by providing
|
|
||||||
the following "Information":
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
Identifier: trunk
|
|
||||||
Description: Trunk
|
|
||||||
Nix expression: release.nix in input patchelfSrc
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
This states that in order to build the <literal>trunk</literal>
|
|
||||||
jobset, the Nix expression in the file
|
|
||||||
<filename>release.nix</filename>, which can be obtained from
|
|
||||||
input <literal>patchelfSrc</literal>, should be
|
|
||||||
evaluated. (We'll have a look at
|
|
||||||
<filename>release.nix</filename> later.)
|
|
||||||
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To realize a job we probably need a number of inputs, which can
|
|
||||||
be declared in the table below. As many inputs as required can
|
|
||||||
be added. For patchelf we declare the following inputs.
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
patchelfSrc
|
|
||||||
'Git checkout' https://github.com/NixOS/patchelf
|
|
||||||
|
|
||||||
nixpkgs 'Git checkout' https://github.com/NixOS/nixpkgs
|
|
||||||
|
|
||||||
officialRelease Boolean false
|
|
||||||
|
|
||||||
system String value "i686-linux"
|
|
||||||
</screen>
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Building Jobs</title>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Build Recipes</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Build jobs and <emphasis>build recipes</emphasis> for a jobset are
|
|
||||||
specified in a text file written in the <link
|
|
||||||
xlink:href="http://nixos.org/nix/">Nix language</link>. The
|
|
||||||
recipe is actually called a <emphasis>Nix expression</emphasis> in
|
|
||||||
Nix parlance. By convention this file is often called
|
|
||||||
<filename>release.nix</filename>.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The <filename>release.nix</filename> file is typically kept under
|
|
||||||
version control, and the repository that contains it one of the
|
|
||||||
build inputs of the corresponding–often called
|
|
||||||
<literal>hydraConfig</literal> by convention. The repository for
|
|
||||||
that file and the actual file name are specified on the web
|
|
||||||
interface of Hydra under the <literal>Setup</literal> tab of the
|
|
||||||
jobset's overview page, under the <literal>Nix
|
|
||||||
expression</literal> heading. See, for example, the <link
|
|
||||||
xlink:href="http://hydra.nixos.org/jobset/patchelf/trunk">jobset
|
|
||||||
overview page</link> of the PatchELF project, and <link
|
|
||||||
xlink:href="https://github.com/NixOS/patchelf/blob/master/release.nix">
|
|
||||||
the corresponding Nix file</link>.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Knowledge of the Nix language is recommended, but the example
|
|
||||||
below should already give a good idea of how it works:
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<example xml:id='ex-hello'>
|
|
||||||
<title><filename>release.nix</filename> file for GNU Hello</title>
|
|
||||||
<programlisting>
|
|
||||||
let
|
|
||||||
pkgs = import <nixpkgs> {}; <co xml:id='ex-hello-co-import-nixpkgs' />
|
|
||||||
|
|
||||||
jobs = rec { <co xml:id='ex-hello-co-jobs' />
|
|
||||||
|
|
||||||
tarball = <co xml:id='ex-hello-co-tarball' />
|
|
||||||
pkgs.releaseTools.sourceTarball { <co xml:id='ex-hello-co-source-tarball' />
|
|
||||||
name = "hello-tarball";
|
|
||||||
src = <hello>; <co xml:id='ex-hello-co-tarball-args' />
|
|
||||||
buildInputs = (with pkgs; [ gettext texLive texinfo ]);
|
|
||||||
};
|
|
||||||
|
|
||||||
build = <co xml:id='ex-hello-co-build' />
|
|
||||||
{ system ? builtins.currentSystem }: <co xml:id='ex-hello-co-build-args' />
|
|
||||||
|
|
||||||
let pkgs = import <nixpkgs> { inherit system; }; in
|
|
||||||
pkgs.releaseTools.nixBuild { <co xml:id='ex-hello-co-nix-build' />
|
|
||||||
name = "hello";
|
|
||||||
src = jobs.tarball;
|
|
||||||
configureFlags = [ "--disable-silent-rules" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
in
|
|
||||||
jobs <co xml:id='ex-hello-co-body' />
|
|
||||||
</programlisting>
|
|
||||||
</example>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<xref linkend='ex-hello' /> shows what a
|
|
||||||
<filename>release.nix</filename> file for <link
|
|
||||||
xlink:href="http://www.gnu.org/software/hello/">GNU Hello</link>
|
|
||||||
would look like. GNU Hello is representative of many GNU
|
|
||||||
and non-GNU free software projects:
|
|
||||||
|
|
||||||
<itemizedlist>
|
|
||||||
<listitem>it uses the GNU Build System, namely GNU Autoconf,
|
|
||||||
and GNU Automake; for users, it means it can be installed
|
|
||||||
using the <link
|
|
||||||
xlink:href="http://www.gnu.org/prep/standards/html_node/Managing-Releases.html">usual
|
|
||||||
<literal>./configure && make install</literal>
|
|
||||||
procedure</link>;
|
|
||||||
</listitem>
|
|
||||||
<listitem>it uses Gettext for internationalization;</listitem>
|
|
||||||
<listitem>it has a Texinfo manual, which can be rendered as PDF
|
|
||||||
with TeX.</listitem>
|
|
||||||
</itemizedlist>
|
|
||||||
|
|
||||||
The file defines a jobset consisting of two jobs:
|
|
||||||
<literal>tarball</literal>, and <literal>build</literal>. It
|
|
||||||
contains the following elements (referenced from the figure by
|
|
||||||
numbers):
|
|
||||||
|
|
||||||
<calloutlist>
|
|
||||||
|
|
||||||
<callout arearefs='ex-hello-co-import-nixpkgs'>
|
|
||||||
<para>
|
|
||||||
This defines a variable <varname>pkgs</varname> holding
|
|
||||||
the set of packages provided by <link
|
|
||||||
xlink:href="http://nixos.org/nixpkgs/">Nixpkgs</link>.
|
|
||||||
</para>
|
|
||||||
<para>
|
|
||||||
Since <varname>nixpkgs</varname> appears in angle brackets,
|
|
||||||
there must be a build input of that name in the Nix search
|
|
||||||
path. In this case, the web interface should show a
|
|
||||||
<varname>nixpkgs</varname> build input, which is a checkout
|
|
||||||
of the Nixpkgs source code repository; Hydra then adds this
|
|
||||||
and other build inputs to the Nix search path when
|
|
||||||
evaluating <filename>release.nix</filename>.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
|
|
||||||
<callout arearefs='ex-hello-co-jobs'>
|
|
||||||
<para>
|
|
||||||
This defines a variable holding the two Hydra
|
|
||||||
jobs–an <emphasis>attribute set</emphasis> in Nix.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
|
|
||||||
<callout arearefs='ex-hello-co-tarball'>
|
|
||||||
<para>
|
|
||||||
This is the definition of the first job, named
|
|
||||||
<varname>tarball</varname>. The purpose of this job is to
|
|
||||||
produce a usable source code tarball.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
<callout arearefs='ex-hello-co-source-tarball'>
|
|
||||||
<para>
|
|
||||||
The <varname>tarball</varname> job calls the
|
|
||||||
<varname>sourceTarball</varname> function, which (roughly)
|
|
||||||
runs <command>autoreconf && ./configure &&
|
|
||||||
make dist</command> on the checkout. The
|
|
||||||
<varname>buildInputs</varname> attribute specifies
|
|
||||||
additional software dependencies for the
|
|
||||||
job<footnote><para>The package names used in
|
|
||||||
<varname>buildInputs</varname>–e.g.,
|
|
||||||
<varname>texLive</varname>–are the names of the
|
|
||||||
<emphasis>attributes</emphasis> corresponding to these
|
|
||||||
packages in Nixpkgs, specifically in the <link
|
|
||||||
xlink:href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/all-packages.nix"><filename>all-packages.nix</filename></link>
|
|
||||||
file. See the section entitled “Package Naming” in the
|
|
||||||
Nixpkgs manual for more information.</para></footnote>.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
<callout arearefs='ex-hello-co-tarball-args'>
|
|
||||||
<para>
|
|
||||||
The <varname>tarball</varname> jobs expects a
|
|
||||||
<varname>hello</varname> build input to be available in the
|
|
||||||
Nix search path. Again, this input is passed by Hydra and
|
|
||||||
is meant to be a checkout of GNU Hello's source code
|
|
||||||
repository.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
|
|
||||||
<callout arearefs='ex-hello-co-build'>
|
|
||||||
<para>
|
|
||||||
This is the definition of the <varname>build</varname>
|
|
||||||
job, whose purpose is to build Hello from the tarball
|
|
||||||
produced above.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
<callout arearefs='ex-hello-co-build-args'>
|
|
||||||
<para>
|
|
||||||
The <varname>build</varname> function takes one
|
|
||||||
parameter, <varname>system</varname>, which should be a string
|
|
||||||
defining the Nix system type–e.g.,
|
|
||||||
<literal>"x86_64-linux"</literal>. Additionally, it refers
|
|
||||||
to <varname>jobs.tarball</varname>, seen above.
|
|
||||||
</para>
|
|
||||||
<para>
|
|
||||||
Hydra inspects the formal argument list of the function
|
|
||||||
(here, the <varname>system</varname> argument) and passes it
|
|
||||||
the corresponding parameter specified as a build input on
|
|
||||||
Hydra's web interface. Here, <varname>system</varname> is
|
|
||||||
passed by Hydra when it calls <varname>build</varname>.
|
|
||||||
Thus, it must be defined as a build input of type string in
|
|
||||||
Hydra, which could take one of several values.
|
|
||||||
</para>
|
|
||||||
<para>
|
|
||||||
The question mark after <literal>system</literal> defines
|
|
||||||
the default value for this argument, and is only useful when
|
|
||||||
debugging locally.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
<callout arearefs='ex-hello-co-nix-build'>
|
|
||||||
<para>
|
|
||||||
The <varname>build</varname> job calls the
|
|
||||||
<varname>nixBuild</varname> function, which unpacks the
|
|
||||||
tarball, then runs <command>./configure && make
|
|
||||||
&& make check && make install</command>.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
|
|
||||||
<callout arearefs='ex-hello-co-body'>
|
|
||||||
<para>
|
|
||||||
Finally, the set of jobs is returned to Hydra, as a Nix
|
|
||||||
attribute set.
|
|
||||||
</para>
|
|
||||||
</callout>
|
|
||||||
</calloutlist>
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Building from the Command Line</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
It is often useful to test a build recipe, for instance before
|
|
||||||
it is actually used by Hydra, when testing changes, or when
|
|
||||||
debugging a build issue. Since build recipes for Hydra jobsets
|
|
||||||
are just plain Nix expressions, they can be evaluated using the
|
|
||||||
standard Nix tools.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
To evaluate the <varname>tarball</varname> jobset of <xref
|
|
||||||
linkend='ex-hello' />, just run:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-build release.nix -A tarball
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
However, doing this with <xref linkend='ex-hello' /> as is will
|
|
||||||
probably yield an error like this:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
error: user-thrown exception: file `hello' was not found in the Nix search path (add it using $NIX_PATH or -I)
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The error is self-explanatory. Assuming
|
|
||||||
<filename>$HOME/src/hello</filename> points to a checkout of
|
|
||||||
Hello, this can be fixed this way:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-build -I ~/src release.nix -A tarball
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
Similarly, the <varname>build</varname> jobset can be evaluated:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-build -I ~/src release.nix -A build
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The <varname>build</varname> job reuses the result of the
|
|
||||||
<varname>tarball</varname> job, rebuilding it only if it needs to.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Adding More Jobs</title>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
<xref linkend='ex-hello' /> illustrates how to write the most
|
|
||||||
basic jobs, <varname>tarball</varname> and
|
|
||||||
<varname>build</varname>. In practice, much more can be done by
|
|
||||||
using features readily provided by Nixpkgs or by creating new jobs
|
|
||||||
as customizations of existing jobs.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
For instance, test coverage report for projects compiled with GCC
|
|
||||||
can be automatically generated using the
|
|
||||||
<varname>coverageAnalysis</varname> function provided by Nixpkgs
|
|
||||||
instead of <varname>nixBuild</varname>. Back to our GNU Hello
|
|
||||||
example, we can define a <varname>coverage</varname> job that
|
|
||||||
produces an HTML code coverage report directly readable from the
|
|
||||||
corresponding Hydra build page:
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
coverage =
|
|
||||||
{ system ? builtins.currentSystem }:
|
|
||||||
|
|
||||||
let pkgs = import nixpkgs { inherit system; }; in
|
|
||||||
pkgs.releaseTools.coverageAnalysis {
|
|
||||||
name = "hello";
|
|
||||||
src = jobs.tarball;
|
|
||||||
configureFlags = [ "--disable-silent-rules" ];
|
|
||||||
};
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
As can be seen, the only difference compared to
|
|
||||||
<varname>build</varname> is the use of
|
|
||||||
<varname>coverageAnalysis</varname>.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Nixpkgs provides many more build tools, including the ability to
|
|
||||||
run build in virtual machines, which can themselves run another
|
|
||||||
GNU/Linux distribution, which allows for the creation of packages
|
|
||||||
for these distributions. Please see <link
|
|
||||||
xlink:href="https://github.com/NixOS/nixpkgs/tree/master/pkgs/build-support/release">the
|
|
||||||
<filename>pkgs/build-support/release</filename> directory</link>
|
|
||||||
of Nixpkgs for more. The NixOS manual also contains information
|
|
||||||
about whole-system testing in virtual machine.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
Now, assume we want to build Hello with an old version of GCC, and
|
|
||||||
with different <command>configure</command> flags. A new
|
|
||||||
<varname>build_exotic</varname> job can be written that simply
|
|
||||||
<emphasis>overrides</emphasis> the relevant arguments passed to
|
|
||||||
<varname>nixBuild</varname>:
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
build_exotic =
|
|
||||||
{ system ? builtins.currentSystem }:
|
|
||||||
|
|
||||||
let
|
|
||||||
pkgs = import nixpkgs { inherit system; };
|
|
||||||
build = jobs.build { inherit system; };
|
|
||||||
in
|
|
||||||
pkgs.lib.overrideDerivation build (attrs: {
|
|
||||||
buildInputs = [ pkgs.gcc33 ];
|
|
||||||
preConfigure = "gcc --version";
|
|
||||||
configureFlags =
|
|
||||||
attrs.configureFlags ++ [ "--disable-nls" ];
|
|
||||||
});
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
The <varname>build_exotic</varname> job reuses
|
|
||||||
<varname>build</varname> and overrides some of its arguments: it
|
|
||||||
adds a dependency on GCC 3.3, a pre-configure phase that runs
|
|
||||||
<command>gcc --version</command>, and adds the
|
|
||||||
<literal>--disable-nls</literal> configure flags.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
This customization mechanism is very powerful. For instance, it
|
|
||||||
can be used to change the way Hello and <emphasis>all</emphasis>
|
|
||||||
its dependencies–including the C library and compiler used to
|
|
||||||
build it–are built. See the Nixpkgs manual for more.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<xi:include href="declarative-projects.xml" />
|
|
||||||
|
|
||||||
<section>
|
|
||||||
<title>Email Notifications</title>
|
|
||||||
<para>
|
|
||||||
Hydra can send email notifications when the status of a build changes. This provides
|
|
||||||
immediate feedback to maintainers or committers when a change causes build failures.
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>
|
|
||||||
The simplest approach to enable Email Notifications is to use the ssmtp package, which
|
|
||||||
simply hands off the emails to another SMTP server. For details on how to configure ssmtp,
|
|
||||||
see the documentation for the <varname>networking.defaultMailServer</varname> option.
|
|
||||||
To use ssmtp for the Hydra email notifications, add it to the path option of the Hydra services
|
|
||||||
in your <filename>/etc/nixos/configuration.nix</filename> file:
|
|
||||||
<programlisting>
|
|
||||||
systemd.services.hydra-queue-runner.path = [ pkgs.ssmtp ];
|
|
||||||
systemd.services.hydra-server.path = [ pkgs.ssmtp ];
|
|
||||||
</programlisting>
|
|
||||||
</para>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
</chapter>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
Local Variables:
|
|
||||||
indent-tabs-mode: nil
|
|
||||||
ispell-local-dictionary: "american"
|
|
||||||
End:
|
|
||||||
-->
|
|
||||||
9
doc/manual/src/SUMMARY.md
Normal file
9
doc/manual/src/SUMMARY.md
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
# Hydra User's Guide
|
||||||
|
|
||||||
|
- [Introduction](introduction.md)
|
||||||
|
- [Installation](installation.md)
|
||||||
|
- [Creating and Managing Projects](projects.md)
|
||||||
|
- [Using the external API](api.md)
|
||||||
|
-----------
|
||||||
|
[About](about.md)
|
||||||
|
[Hacking](hacking.md)
|
||||||
6
doc/manual/src/about.md
Normal file
6
doc/manual/src/about.md
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
# Authors
|
||||||
|
|
||||||
|
* Eelco Dolstra, Delft University of Technology, Department of Software Technology
|
||||||
|
* Rob Vermaas, Delft University of Technology, Department of Software Technology
|
||||||
|
* Eelco Visser, Delft University of Technology, Department of Software Technology
|
||||||
|
* Ludovic Courtès
|
||||||
249
doc/manual/src/api.md
Normal file
249
doc/manual/src/api.md
Normal file
|
|
@ -0,0 +1,249 @@
|
||||||
|
Using the external API
|
||||||
|
======================
|
||||||
|
|
||||||
|
To be able to create integrations with other services, Hydra exposes an
|
||||||
|
external API that you can manage projects with.
|
||||||
|
|
||||||
|
The API is accessed over HTTP(s) where all data is sent and received as
|
||||||
|
JSON.
|
||||||
|
|
||||||
|
Creating resources requires the caller to be authenticated, while
|
||||||
|
retrieving resources does not.
|
||||||
|
|
||||||
|
The API does not have a separate URL structure for it\'s endpoints.
|
||||||
|
Instead you request the pages of the web interface as `application/json`
|
||||||
|
to use the API.
|
||||||
|
|
||||||
|
List projects
|
||||||
|
-------------
|
||||||
|
|
||||||
|
To list all the `projects` of the Hydra install:
|
||||||
|
|
||||||
|
GET /
|
||||||
|
Accept: application/json
|
||||||
|
|
||||||
|
This will give you a list of `projects`, where each `project` contains
|
||||||
|
general information and a list of its `job sets`.
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
curl -i -H 'Accept: application/json' \
|
||||||
|
https://hydra.nixos.org
|
||||||
|
|
||||||
|
**Note:** this response is truncated
|
||||||
|
|
||||||
|
GET https://hydra.nixos.org/
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"displayname": "Acoda",
|
||||||
|
"name": "acoda",
|
||||||
|
"description": "Acoda is a tool set for automatic data migration along an evolving data model",
|
||||||
|
"enabled": 0,
|
||||||
|
"owner": "sander",
|
||||||
|
"hidden": 1,
|
||||||
|
"jobsets": [
|
||||||
|
"trunk"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"displayname": "cabal2nix",
|
||||||
|
"name": "cabal2nix",
|
||||||
|
"description": "Convert Cabal files into Nix build instructions",
|
||||||
|
"enabled": 0,
|
||||||
|
"owner": "simons@cryp.to",
|
||||||
|
"hidden": 1,
|
||||||
|
"jobsets": [
|
||||||
|
"master"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
Get a single project
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
To get a single `project` by identifier:
|
||||||
|
|
||||||
|
GET /project/:project-identifier
|
||||||
|
Accept: application/json
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
curl -i -H 'Accept: application/json' \
|
||||||
|
https://hydra.nixos.org/project/hydra
|
||||||
|
|
||||||
|
GET https://hydra.nixos.org/project/hydra
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"description": "Hydra, the Nix-based continuous build system",
|
||||||
|
"hidden": 0,
|
||||||
|
"displayname": "Hydra",
|
||||||
|
"jobsets": [
|
||||||
|
"hydra-master",
|
||||||
|
"hydra-ant-logger-trunk",
|
||||||
|
"master",
|
||||||
|
"build-ng"
|
||||||
|
],
|
||||||
|
"name": "hydra",
|
||||||
|
"enabled": 1,
|
||||||
|
"owner": "eelco"
|
||||||
|
}
|
||||||
|
|
||||||
|
Get a single job set
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
To get a single `job set` by identifier:
|
||||||
|
|
||||||
|
GET /jobset/:project-identifier/:jobset-identifier
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
curl -i -H 'Accept: application/json' \
|
||||||
|
https://hydra.nixos.org/jobset/hydra/build-ng
|
||||||
|
|
||||||
|
GET https://hydra.nixos.org/jobset/hydra/build-ng
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"errormsg": "evaluation failed due to signal 9 (Killed)",
|
||||||
|
"fetcherrormsg": null,
|
||||||
|
"nixexprpath": "release.nix",
|
||||||
|
"nixexprinput": "hydraSrc",
|
||||||
|
"emailoverride": "rob.vermaas@gmail.com, eelco.dolstra@logicblox.com",
|
||||||
|
"jobsetinputs": {
|
||||||
|
"officialRelease": {
|
||||||
|
"jobsetinputalts": [
|
||||||
|
"false"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hydraSrc": {
|
||||||
|
"jobsetinputalts": [
|
||||||
|
"https://github.com/NixOS/hydra.git build-ng"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"jobsetinputalts": [
|
||||||
|
"https://github.com/NixOS/nixpkgs.git release-14.12"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"enabled": 0
|
||||||
|
}
|
||||||
|
|
||||||
|
List evaluations
|
||||||
|
----------------
|
||||||
|
|
||||||
|
To list the `evaluations` of a `job set` by identifier:
|
||||||
|
|
||||||
|
GET /jobset/:project-identifier/:jobset-identifier/evals
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
curl -i -H 'Accept: application/json' \
|
||||||
|
https://hydra.nixos.org/jobset/hydra/build-ng/evals
|
||||||
|
|
||||||
|
**Note:** this response is truncated
|
||||||
|
|
||||||
|
GET https://hydra.nixos.org/jobset/hydra/build-ng/evals
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"evals": [
|
||||||
|
{
|
||||||
|
"jobsetevalinputs": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"dependency": null,
|
||||||
|
"type": "git",
|
||||||
|
"value": null,
|
||||||
|
"uri": "https://github.com/NixOS/nixpkgs.git",
|
||||||
|
"revision": "f60e48ce81b6f428d072d3c148f6f2e59f1dfd7a"
|
||||||
|
},
|
||||||
|
"hydraSrc": {
|
||||||
|
"dependency": null,
|
||||||
|
"type": "git",
|
||||||
|
"value": null,
|
||||||
|
"uri": "https://github.com/NixOS/hydra.git",
|
||||||
|
"revision": "48d6f0de2ab94f728d287b9c9670c4d237e7c0f6"
|
||||||
|
},
|
||||||
|
"officialRelease": {
|
||||||
|
"dependency": null,
|
||||||
|
"value": "false",
|
||||||
|
"type": "boolean",
|
||||||
|
"uri": null,
|
||||||
|
"revision": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"hasnewbuilds": 1,
|
||||||
|
"builds": [
|
||||||
|
24670686,
|
||||||
|
24670684,
|
||||||
|
24670685,
|
||||||
|
24670687
|
||||||
|
],
|
||||||
|
"id": 1213758
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"first": "?page=1",
|
||||||
|
"last": "?page=1"
|
||||||
|
}
|
||||||
|
|
||||||
|
Get a single build
|
||||||
|
------------------
|
||||||
|
|
||||||
|
To get a single `build` by its id:
|
||||||
|
|
||||||
|
GET /build/:build-id
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
curl -i -H 'Accept: application/json' \
|
||||||
|
https://hydra.nixos.org/build/24670686
|
||||||
|
|
||||||
|
GET /build/24670686
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"job": "tests.api.x86_64-linux",
|
||||||
|
"jobsetevals": [
|
||||||
|
1213758
|
||||||
|
],
|
||||||
|
"buildstatus": 0,
|
||||||
|
"buildmetrics": null,
|
||||||
|
"project": "hydra",
|
||||||
|
"system": "x86_64-linux",
|
||||||
|
"priority": 100,
|
||||||
|
"releasename": null,
|
||||||
|
"starttime": 1439402853,
|
||||||
|
"nixname": "vm-test-run-unnamed",
|
||||||
|
"timestamp": 1439388618,
|
||||||
|
"id": 24670686,
|
||||||
|
"stoptime": 1439403403,
|
||||||
|
"jobset": "build-ng",
|
||||||
|
"buildoutputs": {
|
||||||
|
"out": {
|
||||||
|
"path": "/nix/store/lzrxkjc35mhp8w7r8h82g0ljyizfchma-vm-test-run-unnamed"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"buildproducts": {
|
||||||
|
"1": {
|
||||||
|
"path": "/nix/store/lzrxkjc35mhp8w7r8h82g0ljyizfchma-vm-test-run-unnamed",
|
||||||
|
"defaultpath": "log.html",
|
||||||
|
"type": "report",
|
||||||
|
"sha256hash": null,
|
||||||
|
"filesize": null,
|
||||||
|
"name": "",
|
||||||
|
"subtype": "testlog"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"finished": 1
|
||||||
|
}
|
||||||
28
doc/manual/src/hacking.md
Normal file
28
doc/manual/src/hacking.md
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
Hacking
|
||||||
|
=======
|
||||||
|
|
||||||
|
This section provides some notes on how to hack on Hydra. To get the
|
||||||
|
latest version of Hydra from GitHub:
|
||||||
|
|
||||||
|
$ git clone git://github.com/NixOS/hydra.git
|
||||||
|
$ cd hydra
|
||||||
|
|
||||||
|
To build it and its dependencies:
|
||||||
|
|
||||||
|
$ nix-build release.nix -A build.x86_64-linux
|
||||||
|
|
||||||
|
To build all dependencies and start a shell in which all environment
|
||||||
|
variables (such as PERL5LIB) are set up so that those dependencies can
|
||||||
|
be found:
|
||||||
|
|
||||||
|
$ nix-shell
|
||||||
|
|
||||||
|
To build Hydra, you should then do:
|
||||||
|
|
||||||
|
[nix-shell]$ ./bootstrap
|
||||||
|
[nix-shell]$ configurePhase
|
||||||
|
[nix-shell]$ make
|
||||||
|
|
||||||
|
You can run the Hydra web server in your source tree as follows:
|
||||||
|
|
||||||
|
$ ./src/script/hydra-server
|
||||||
237
doc/manual/src/installation.md
Normal file
237
doc/manual/src/installation.md
Normal file
|
|
@ -0,0 +1,237 @@
|
||||||
|
Installation
|
||||||
|
============
|
||||||
|
|
||||||
|
This chapter explains how to install Hydra on your own build farm
|
||||||
|
server.
|
||||||
|
|
||||||
|
Prerequisites
|
||||||
|
-------------
|
||||||
|
|
||||||
|
To install and use Hydra you need to have installed the following
|
||||||
|
dependencies:
|
||||||
|
|
||||||
|
- Nix
|
||||||
|
|
||||||
|
- PostgreSQL
|
||||||
|
|
||||||
|
- many Perl packages, notably Catalyst, EmailSender, and NixPerl (see
|
||||||
|
the [Hydra expression in
|
||||||
|
Nixpkgs](https://github.com/NixOS/hydra/blob/master/release.nix) for
|
||||||
|
the complete list)
|
||||||
|
|
||||||
|
At the moment, Hydra runs only on GNU/Linux (*i686-linux* and
|
||||||
|
*x86\_64\_linux*).
|
||||||
|
|
||||||
|
For small projects, Hydra can be run on any reasonably modern machine.
|
||||||
|
For individual projects you can even run Hydra on a laptop. However, the
|
||||||
|
charm of a buildfarm server is usually that it operates without
|
||||||
|
disturbing the developer\'s working environment and can serve releases
|
||||||
|
over the internet. In conjunction you should typically have your source
|
||||||
|
code administered in a version management system, such as subversion.
|
||||||
|
Therefore, you will probably want to install a server that is connected
|
||||||
|
to the internet. To scale up to large and/or many projects, you will
|
||||||
|
need at least a considerable amount of diskspace to store builds. Since
|
||||||
|
Hydra can schedule multiple simultaneous build jobs, it can be useful to
|
||||||
|
have a multi-core machine, and/or attach multiple build machines in a
|
||||||
|
network to the central Hydra server.
|
||||||
|
|
||||||
|
Of course we think it is a good idea to use the
|
||||||
|
[NixOS](http://nixos.org/nixos) GNU/Linux distribution for your
|
||||||
|
buildfarm server. But this is not a requirement. The Nix software
|
||||||
|
deployment system can be installed on any GNU/Linux distribution in
|
||||||
|
parallel to the regular package management system. Thus, you can use
|
||||||
|
Hydra on a Debian, Fedora, SuSE, or Ubuntu system.
|
||||||
|
|
||||||
|
Getting Nix
|
||||||
|
-----------
|
||||||
|
|
||||||
|
If your server runs NixOS you are all set to continue with installation
|
||||||
|
of Hydra. Otherwise you first need to install Nix. The latest stable
|
||||||
|
version can be found one [the Nix web
|
||||||
|
site](http://nixos.org/nix/download.html), along with a manual, which
|
||||||
|
includes installation instructions.
|
||||||
|
|
||||||
|
Installation
|
||||||
|
------------
|
||||||
|
|
||||||
|
The latest development snapshot of Hydra can be installed by visiting
|
||||||
|
the URL
|
||||||
|
[`http://hydra.nixos.org/view/hydra/unstable`](http://hydra.nixos.org/view/hydra/unstable)
|
||||||
|
and using the one-click install available at one of the build pages. You
|
||||||
|
can also install Hydra through the channel by performing the following
|
||||||
|
commands:
|
||||||
|
|
||||||
|
nix-channel --add http://hydra.nixos.org/jobset/hydra/master/channel/latest
|
||||||
|
nix-channel --update
|
||||||
|
nix-env -i hydra
|
||||||
|
|
||||||
|
Command completion should reveal a number of command-line tools from
|
||||||
|
Hydra, such as `hydra-queue-runner`.
|
||||||
|
|
||||||
|
Creating the database
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
Hydra stores its results in a PostgreSQL database.
|
||||||
|
|
||||||
|
To setup a PostgreSQL database with *hydra* as database name and user
|
||||||
|
name, issue the following commands on the PostgreSQL server:
|
||||||
|
|
||||||
|
createuser -S -D -R -P hydra
|
||||||
|
createdb -O hydra hydra
|
||||||
|
|
||||||
|
Note that *\$prefix* is the location of Hydra in the nix store.
|
||||||
|
|
||||||
|
Hydra uses an environment variable to know which database should be
|
||||||
|
used, and a variable which point to a location that holds some state. To
|
||||||
|
set these variables for a PostgreSQL database, add the following to the
|
||||||
|
file `~/.profile` of the user running the Hydra services.
|
||||||
|
|
||||||
|
export HYDRA_DBI="dbi:Pg:dbname=hydra;host=dbserver.example.org;user=hydra;"
|
||||||
|
export HYDRA_DATA=/var/lib/hydra
|
||||||
|
|
||||||
|
You can provide the username and password in the file `~/.pgpass`, e.g.
|
||||||
|
|
||||||
|
dbserver.example.org:*:hydra:hydra:password
|
||||||
|
|
||||||
|
Make sure that the *HYDRA\_DATA* directory exists and is writable for
|
||||||
|
the user which will run the Hydra services.
|
||||||
|
|
||||||
|
Having set these environment variables, you can now initialise the
|
||||||
|
database by doing:
|
||||||
|
|
||||||
|
hydra-init
|
||||||
|
|
||||||
|
To create projects, you need to create a user with *admin* privileges.
|
||||||
|
This can be done using the command `hydra-create-user`:
|
||||||
|
|
||||||
|
$ hydra-create-user alice --full-name 'Alice Q. User' \
|
||||||
|
--email-address 'alice@example.org' --password foobar --role admin
|
||||||
|
|
||||||
|
Additional users can be created through the web interface.
|
||||||
|
|
||||||
|
Upgrading
|
||||||
|
---------
|
||||||
|
|
||||||
|
If you\'re upgrading Hydra from a previous version, you should do the
|
||||||
|
following to perform any necessary database schema migrations:
|
||||||
|
|
||||||
|
hydra-init
|
||||||
|
|
||||||
|
Getting Started
|
||||||
|
---------------
|
||||||
|
|
||||||
|
To start the Hydra web server, execute:
|
||||||
|
|
||||||
|
hydra-server
|
||||||
|
|
||||||
|
When the server is started, you can browse to [http://localhost:3000/]()
|
||||||
|
to start configuring your Hydra instance.
|
||||||
|
|
||||||
|
The `hydra-server` command launches the web server. There are two other
|
||||||
|
processes that come into play:
|
||||||
|
|
||||||
|
- The
|
||||||
|
evaluator
|
||||||
|
is responsible for periodically evaluating job sets, checking out
|
||||||
|
their dependencies off their version control systems (VCS), and
|
||||||
|
queueing new builds if the result of the evaluation changed. It is
|
||||||
|
launched by the
|
||||||
|
hydra-evaluator
|
||||||
|
command.
|
||||||
|
- The
|
||||||
|
queue runner
|
||||||
|
launches builds (using Nix) as they are queued by the evaluator,
|
||||||
|
scheduling them onto the configured Nix hosts. It is launched using
|
||||||
|
the
|
||||||
|
hydra-queue-runner
|
||||||
|
command.
|
||||||
|
|
||||||
|
All three processes must be running for Hydra to be fully functional,
|
||||||
|
though it\'s possible to temporarily stop any one of them for
|
||||||
|
maintenance purposes, for instance.
|
||||||
|
|
||||||
|
Serving behind reverse proxy
|
||||||
|
----------------------------
|
||||||
|
|
||||||
|
To serve hydra web server behind reverse proxy like *nginx* or *httpd*
|
||||||
|
some additional configuration must be made.
|
||||||
|
|
||||||
|
Edit your `hydra.conf` file in a similar way to this example:
|
||||||
|
|
||||||
|
using_frontend_proxy 1
|
||||||
|
base_uri example.com
|
||||||
|
|
||||||
|
`base_uri` should be your hydra servers proxied URL. If you are using
|
||||||
|
Hydra nixos module then setting `hydraURL` option should be enough.
|
||||||
|
|
||||||
|
If you want to serve Hydra with a prefix path, for example
|
||||||
|
[http://example.com/hydra]() then you need to configure your reverse
|
||||||
|
proxy to pass `X-Request-Base` to hydra, with prefix path as value. For
|
||||||
|
example if you are using nginx, then use configuration similar to
|
||||||
|
following:
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 433 ssl;
|
||||||
|
server_name example.com;
|
||||||
|
.. other configuration ..
|
||||||
|
location /hydra/ {
|
||||||
|
|
||||||
|
proxy_pass http://127.0.0.1:3000;
|
||||||
|
proxy_redirect http://127.0.0.1:3000 https://example.com/hydra;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-Request-Base /hydra;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Using LDAP as authentication backend (optional)
|
||||||
|
-----------------------------------------------
|
||||||
|
|
||||||
|
Instead of using Hydra\'s built-in user management you can optionally
|
||||||
|
use LDAP to manage roles and users.
|
||||||
|
|
||||||
|
The `hydra-server` accepts the environment variable
|
||||||
|
*HYDRA\_LDAP\_CONFIG*. The value of the variable should point to a valid
|
||||||
|
YAML file containing the Catalyst LDAP configuration. The format of the
|
||||||
|
configuration file is describe in the
|
||||||
|
[*Catalyst::Authentication::Store::LDAP*
|
||||||
|
documentation](https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS).
|
||||||
|
An example is given below.
|
||||||
|
|
||||||
|
Roles can be assigned to users based on their LDAP group membership
|
||||||
|
(*use\_roles: 1* in the below example). For a user to have the role
|
||||||
|
*admin* assigned to them they should be in the group *hydra\_admin*. In
|
||||||
|
general any LDAP group of the form *hydra\_some\_role* (notice the
|
||||||
|
*hydra\_* prefix) will work.
|
||||||
|
|
||||||
|
credential:
|
||||||
|
class: Password
|
||||||
|
password_field: password
|
||||||
|
password_type: self_check
|
||||||
|
store:
|
||||||
|
class: LDAP
|
||||||
|
ldap_server: localhost
|
||||||
|
ldap_server_options.timeout: 30
|
||||||
|
binddn: "cn=root,dc=example"
|
||||||
|
bindpw: notapassword
|
||||||
|
start_tls: 0
|
||||||
|
start_tls_options
|
||||||
|
verify: none
|
||||||
|
user_basedn: "ou=users,dc=example"
|
||||||
|
user_filter: "(&(objectClass=inetOrgPerson)(cn=%s))"
|
||||||
|
user_scope: one
|
||||||
|
user_field: cn
|
||||||
|
user_search_options:
|
||||||
|
deref: always
|
||||||
|
use_roles: 1
|
||||||
|
role_basedn: "ou=groups,dc=example"
|
||||||
|
role_filter: "(&(objectClass=groupOfNames)(member=%s))"
|
||||||
|
role_scope: one
|
||||||
|
role_field: cn
|
||||||
|
role_value: dn
|
||||||
|
role_search_options:
|
||||||
|
deref: always
|
||||||
|
|
||||||
173
doc/manual/src/introduction.md
Normal file
173
doc/manual/src/introduction.md
Normal file
|
|
@ -0,0 +1,173 @@
|
||||||
|
Introduction
|
||||||
|
============
|
||||||
|
|
||||||
|
About Hydra
|
||||||
|
-----------
|
||||||
|
|
||||||
|
Hydra is a tool for continuous integration testing and software release
|
||||||
|
that uses a purely functional language to describe build jobs and their
|
||||||
|
dependencies. Continuous integration is a simple technique to improve
|
||||||
|
the quality of the software development process. An automated system
|
||||||
|
continuously or periodically checks out the source code of a project,
|
||||||
|
builds it, runs tests, and produces reports for the developers. Thus,
|
||||||
|
various errors that might accidentally be committed into the code base
|
||||||
|
are automatically caught. Such a system allows more in-depth testing
|
||||||
|
than what developers could feasibly do manually:
|
||||||
|
|
||||||
|
- Portability testing
|
||||||
|
: The software may need to be built and tested on many different
|
||||||
|
platforms. It is infeasible for each developer to do this before
|
||||||
|
every commit.
|
||||||
|
- Likewise, many projects have very large test sets (e.g., regression
|
||||||
|
tests in a compiler, or stress tests in a DBMS) that can take hours
|
||||||
|
or days to run to completion.
|
||||||
|
- Many kinds of static and dynamic analyses can be performed as part
|
||||||
|
of the tests, such as code coverage runs and static analyses.
|
||||||
|
- It may also be necessary to build many different
|
||||||
|
variants
|
||||||
|
of the software. For instance, it may be necessary to verify that
|
||||||
|
the component builds with various versions of a compiler.
|
||||||
|
- Developers typically use incremental building to test their changes
|
||||||
|
(since a full build may take too long), but this is unreliable with
|
||||||
|
many build management tools (such as Make), i.e., the result of the
|
||||||
|
incremental build might differ from a full build.
|
||||||
|
- It ensures that the software can be built from the sources under
|
||||||
|
revision control. Users of version management systems such as CVS
|
||||||
|
and Subversion often forget to place source files under revision
|
||||||
|
control.
|
||||||
|
- The machines on which the continuous integration system runs ideally
|
||||||
|
provides a clean, well-defined build environment. If this
|
||||||
|
environment is administered through proper SCM techniques, then
|
||||||
|
builds produced by the system can be reproduced. In contrast,
|
||||||
|
developer work environments are typically not under any kind of SCM
|
||||||
|
control.
|
||||||
|
- In large projects, developers often work on a particular component
|
||||||
|
of the project, and do not build and test the composition of those
|
||||||
|
components (again since this is likely to take too long). To prevent
|
||||||
|
the phenomenon of \`\`big bang integration\'\', where components are
|
||||||
|
only tested together near the end of the development process, it is
|
||||||
|
important to test components together as soon as possible (hence
|
||||||
|
continuous integration
|
||||||
|
).
|
||||||
|
- It allows software to be
|
||||||
|
released
|
||||||
|
by automatically creating packages that users can download and
|
||||||
|
install. To do this manually represents an often prohibitive amount
|
||||||
|
of work, as one may want to produce releases for many different
|
||||||
|
platforms: e.g., installers for Windows and Mac OS X, RPM or Debian
|
||||||
|
packages for certain Linux distributions, and so on.
|
||||||
|
|
||||||
|
In its simplest form, a continuous integration tool sits in a loop
|
||||||
|
building and releasing software components from a version management
|
||||||
|
system. For each component, it performs the following tasks:
|
||||||
|
|
||||||
|
- It obtains the latest version of the component\'s source code from
|
||||||
|
the version management system.
|
||||||
|
- It runs the component\'s build process (which presumably includes
|
||||||
|
the execution of the component\'s test set).
|
||||||
|
- It presents the results of the build (such as error logs and
|
||||||
|
releases) to the developers, e.g., by producing a web page.
|
||||||
|
|
||||||
|
Examples of continuous integration tools include Jenkins, CruiseControl
|
||||||
|
Tinderbox, Sisyphus, Anthill and BuildBot. These tools have various
|
||||||
|
limitations.
|
||||||
|
|
||||||
|
- They do not manage the
|
||||||
|
build environment
|
||||||
|
. The build environment consists of the dependencies necessary to
|
||||||
|
perform a build action, e.g., compilers, libraries, etc. Setting up
|
||||||
|
the environment is typically done manually, and without proper SCM
|
||||||
|
control (so it may be hard to reproduce a build at a later time).
|
||||||
|
Manual management of the environment scales poorly in the number of
|
||||||
|
configurations that must be supported. For instance, suppose that we
|
||||||
|
want to build a component that requires a certain compiler X. We
|
||||||
|
then have to go to each machine and install X. If we later need a
|
||||||
|
newer version of X, the process must be repeated all over again. An
|
||||||
|
ever worse problem occurs if there are conflicting, mutually
|
||||||
|
exclusive versions of the dependencies. Thus, simply installing the
|
||||||
|
latest version is not an option. Of course, we can install these
|
||||||
|
components in different directories and manually pass the
|
||||||
|
appropriate paths to the build processes of the various components.
|
||||||
|
But this is a rather tiresome and error-prone process.
|
||||||
|
- They do not easily support
|
||||||
|
variability in software systems
|
||||||
|
. A system may have a great deal of build-time variability: optional
|
||||||
|
functionality, whether to build a debug or production version,
|
||||||
|
different versions of dependencies, and so on. (For instance, the
|
||||||
|
Linux kernel now has over 2,600 build-time configuration switches.)
|
||||||
|
It is therefore important that a continuous integration tool can
|
||||||
|
easily select and test different instances from the configuration
|
||||||
|
space of the system to reveal problems, such as erroneous
|
||||||
|
interactions between features. In a continuous integration setting,
|
||||||
|
it is also useful to test different combinations of versions of
|
||||||
|
subsystems, e.g., the head revision of a component against stable
|
||||||
|
releases of its dependencies, and vice versa, as this can reveal
|
||||||
|
various integration problems.
|
||||||
|
|
||||||
|
*Hydra*, is a continuous integration tool that solves these problems. It
|
||||||
|
is built on top of the [Nix package manager](http://nixos.org/nix/),
|
||||||
|
which has a purely functional language for describing package build
|
||||||
|
actions and their dependencies. This allows the build environment for
|
||||||
|
projects to be produced automatically and deterministically, and
|
||||||
|
variability in components to be expressed naturally using functions; and
|
||||||
|
as such is an ideal fit for a continuous build system.
|
||||||
|
|
||||||
|
About Us
|
||||||
|
--------
|
||||||
|
|
||||||
|
Hydra is the successor of the Nix Buildfarm, which was developed in
|
||||||
|
tandem with the Nix software deployment system. Nix was originally
|
||||||
|
developed at the Department of Information and Computing Sciences,
|
||||||
|
Utrecht University by the TraCE project (2003-2008). The project was
|
||||||
|
funded by the Software Engineering Research Program Jacquard to improve
|
||||||
|
the support for variability in software systems. Funding for the
|
||||||
|
development of Nix and Hydra is now provided by the NIRICT LaQuSo Build
|
||||||
|
Farm project.
|
||||||
|
|
||||||
|
About this Manual
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
This manual tells you how to install the Hydra buildfarm software on
|
||||||
|
your own server and how to operate that server using its web interface.
|
||||||
|
|
||||||
|
License
|
||||||
|
-------
|
||||||
|
|
||||||
|
Hydra is free software: you can redistribute it and/or modify it under
|
||||||
|
the terms of the GNU General Public License as published by the Free
|
||||||
|
Software Foundation, either version 3 of the License, or (at your
|
||||||
|
option) any later version.
|
||||||
|
|
||||||
|
Hydra is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||||
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE. See the [GNU General Public
|
||||||
|
License](http://www.gnu.org/licenses/) for more details.
|
||||||
|
|
||||||
|
Hydra at `nixos.org`
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
The `nixos.org` installation of Hydra runs at
|
||||||
|
[`http://hydra.nixos.org/`](http://hydra.nixos.org/). That installation
|
||||||
|
is used to build software components from the [Nix](http://nixos.org),
|
||||||
|
[NixOS](http://nixos.org/nixos), [GNU](http://www.gnu.org/),
|
||||||
|
[Stratego/XT](http://strategoxt.org), and related projects.
|
||||||
|
|
||||||
|
If you are one of the developers on those projects, it is likely that
|
||||||
|
you will be using the NixOS Hydra server in some way. If you need to
|
||||||
|
administer automatic builds for your project, you should pull the right
|
||||||
|
strings to get an account on the server. This manual will tell you how
|
||||||
|
to set up new projects and build jobs within those projects and write a
|
||||||
|
release.nix file to describe the build process of your project to Hydra.
|
||||||
|
You can skip the next chapter.
|
||||||
|
|
||||||
|
If your project does not yet have automatic builds within the NixOS
|
||||||
|
Hydra server, it may actually be eligible. We are in the process of
|
||||||
|
setting up a large buildfarm that should be able to support open source
|
||||||
|
and academic software projects. Get in touch.
|
||||||
|
|
||||||
|
Hydra on your own buildfarm
|
||||||
|
---------------------------
|
||||||
|
|
||||||
|
If you need to run your own Hydra installation,
|
||||||
|
[installation chapter](installation.md) explains how to download and install the
|
||||||
|
system on your own server.
|
||||||
463
doc/manual/src/projects.md
Normal file
463
doc/manual/src/projects.md
Normal file
|
|
@ -0,0 +1,463 @@
|
||||||
|
Creating and Managing Projects
|
||||||
|
==============================
|
||||||
|
|
||||||
|
Once Hydra is installed and running, the next step is to add projects to
|
||||||
|
the build farm. We follow the example of the [Patchelf
|
||||||
|
project](http://nixos.org/patchelf.html), a software tool written in C
|
||||||
|
and using the GNU Build System (GNU Autoconf and GNU Automake).
|
||||||
|
|
||||||
|
Log in to the web interface of your Hydra installation using the user
|
||||||
|
name and password you inserted in the database (by default, Hydra\'s web
|
||||||
|
server listens on [`localhost:3000`](http://localhost:3000/)). Then
|
||||||
|
follow the \"Create Project\" link to create a new project.
|
||||||
|
|
||||||
|
Project Information
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
A project definition consists of some general information and a set of
|
||||||
|
job sets. The general information identifies a project, its owner, and
|
||||||
|
current state of activity. Here\'s what we fill in for the patchelf
|
||||||
|
project:
|
||||||
|
|
||||||
|
Identifier: patchelf
|
||||||
|
|
||||||
|
The *identifier* is the identity of the project. It is used in URLs and
|
||||||
|
in the names of build results.
|
||||||
|
|
||||||
|
The identifier should be a unique name (it is the primary database key
|
||||||
|
for the project table in the database). If you try to create a project
|
||||||
|
with an already existing identifier you\'d get an error message from the
|
||||||
|
database. So try to create the project after entering just the general
|
||||||
|
information to figure out if you have chosen a unique name. Job sets can
|
||||||
|
be added once the project has been created.
|
||||||
|
|
||||||
|
Display name: Patchelf
|
||||||
|
|
||||||
|
The *display name* is used in menus.
|
||||||
|
|
||||||
|
Description: A tool for modifying ELF binaries
|
||||||
|
|
||||||
|
The *description* is used as short documentation of the nature of the
|
||||||
|
project.
|
||||||
|
|
||||||
|
Owner: eelco
|
||||||
|
|
||||||
|
The *owner* of a project can create and edit job sets.
|
||||||
|
|
||||||
|
Enabled: Yes
|
||||||
|
|
||||||
|
Only if the project is *enabled* are builds performed.
|
||||||
|
|
||||||
|
Once created there should be an entry for the project in the sidebar. Go
|
||||||
|
to the project page for the
|
||||||
|
[Patchelf](http://localhost:3000/project/patchelf) project.
|
||||||
|
|
||||||
|
Job Sets
|
||||||
|
--------
|
||||||
|
|
||||||
|
A project can consist of multiple *job sets* (hereafter *jobsets*),
|
||||||
|
separate tasks that can be built separately, but may depend on each
|
||||||
|
other (without cyclic dependencies, of course). Go to the
|
||||||
|
[Edit](http://localhost:3000/project/patchelf/edit) page of the Patchelf
|
||||||
|
project and \"Add a new jobset\" by providing the following
|
||||||
|
\"Information\":
|
||||||
|
|
||||||
|
Identifier: trunk
|
||||||
|
Description: Trunk
|
||||||
|
Nix expression: release.nix in input patchelfSrc
|
||||||
|
|
||||||
|
This states that in order to build the `trunk` jobset, the Nix
|
||||||
|
expression in the file `release.nix`, which can be obtained from input
|
||||||
|
`patchelfSrc`, should be evaluated. (We\'ll have a look at `release.nix`
|
||||||
|
later.)
|
||||||
|
|
||||||
|
To realize a job we probably need a number of inputs, which can be
|
||||||
|
declared in the table below. As many inputs as required can be added.
|
||||||
|
For patchelf we declare the following inputs.
|
||||||
|
|
||||||
|
patchelfSrc
|
||||||
|
'Git checkout' https://github.com/NixOS/patchelf
|
||||||
|
|
||||||
|
nixpkgs 'Git checkout' https://github.com/NixOS/nixpkgs
|
||||||
|
|
||||||
|
officialRelease Boolean false
|
||||||
|
|
||||||
|
system String value "i686-linux"
|
||||||
|
|
||||||
|
Building Jobs
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Build Recipes
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Build jobs and *build recipes* for a jobset are specified in a text file
|
||||||
|
written in the [Nix language](http://nixos.org/nix/). The recipe is
|
||||||
|
actually called a *Nix expression* in Nix parlance. By convention this
|
||||||
|
file is often called `release.nix`.
|
||||||
|
|
||||||
|
The `release.nix` file is typically kept under version control, and the
|
||||||
|
repository that contains it one of the build inputs of the
|
||||||
|
corresponding--often called `hydraConfig` by convention. The repository
|
||||||
|
for that file and the actual file name are specified on the web
|
||||||
|
interface of Hydra under the `Setup` tab of the jobset\'s overview page,
|
||||||
|
under the `Nix
|
||||||
|
expression` heading. See, for example, the [jobset overview
|
||||||
|
page](http://hydra.nixos.org/jobset/patchelf/trunk) of the PatchELF
|
||||||
|
project, and [the corresponding Nix
|
||||||
|
file](https://github.com/NixOS/patchelf/blob/master/release.nix).
|
||||||
|
|
||||||
|
Knowledge of the Nix language is recommended, but the example below
|
||||||
|
should already give a good idea of how it works:
|
||||||
|
|
||||||
|
let
|
||||||
|
pkgs = import <nixpkgs> {}; ①
|
||||||
|
|
||||||
|
jobs = rec { ②
|
||||||
|
|
||||||
|
tarball = ③
|
||||||
|
pkgs.releaseTools.sourceTarball { ④
|
||||||
|
name = "hello-tarball";
|
||||||
|
src = <hello>; ⑤
|
||||||
|
buildInputs = (with pkgs; [ gettext texLive texinfo ]);
|
||||||
|
};
|
||||||
|
|
||||||
|
build = ⑥
|
||||||
|
{ system ? builtins.currentSystem }: ⑦
|
||||||
|
|
||||||
|
let pkgs = import <nixpkgs> { inherit system; }; in
|
||||||
|
pkgs.releaseTools.nixBuild { ⑧
|
||||||
|
name = "hello";
|
||||||
|
src = jobs.tarball;
|
||||||
|
configureFlags = [ "--disable-silent-rules" ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
jobs ⑨
|
||||||
|
|
||||||
|
|
||||||
|
This file shows what a `release.nix` file for
|
||||||
|
[GNU Hello](http://www.gnu.org/software/hello/) would look like.
|
||||||
|
GNU Hello is representative of many GNU and non-GNU free software
|
||||||
|
projects:
|
||||||
|
|
||||||
|
- it uses the GNU Build System, namely GNU Autoconf, and GNU Automake;
|
||||||
|
for users, it means it can be installed using the
|
||||||
|
usual
|
||||||
|
./configure && make install
|
||||||
|
procedure
|
||||||
|
;
|
||||||
|
- it uses Gettext for internationalization;
|
||||||
|
- it has a Texinfo manual, which can be rendered as PDF with TeX.
|
||||||
|
|
||||||
|
The file defines a jobset consisting of two jobs: `tarball`, and
|
||||||
|
`build`. It contains the following elements (referenced from the figure
|
||||||
|
by numbers):
|
||||||
|
|
||||||
|
1. This defines a variable `pkgs` holding the set of packages provided
|
||||||
|
by [Nixpkgs](http://nixos.org/nixpkgs/).
|
||||||
|
|
||||||
|
Since `nixpkgs` appears in angle brackets, there must be a build
|
||||||
|
input of that name in the Nix search path. In this case, the web
|
||||||
|
interface should show a `nixpkgs` build input, which is a checkout
|
||||||
|
of the Nixpkgs source code repository; Hydra then adds this and
|
||||||
|
other build inputs to the Nix search path when evaluating
|
||||||
|
`release.nix`.
|
||||||
|
|
||||||
|
2. This defines a variable holding the two Hydra jobs--an *attribute
|
||||||
|
set* in Nix.
|
||||||
|
|
||||||
|
3. This is the definition of the first job, named `tarball`. The
|
||||||
|
purpose of this job is to produce a usable source code tarball.
|
||||||
|
|
||||||
|
4. The `tarball` job calls the `sourceTarball` function, which
|
||||||
|
(roughly) runs `autoreconf && ./configure &&
|
||||||
|
make dist` on the checkout. The `buildInputs` attribute
|
||||||
|
specifies additional software dependencies for the job.
|
||||||
|
|
||||||
|
> The package names used in `buildInputs`--e.g., `texLive`--are the
|
||||||
|
> names of the *attributes* corresponding to these packages in
|
||||||
|
> Nixpkgs, specifically in the
|
||||||
|
> [`all-packages.nix`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/all-packages.nix)
|
||||||
|
> file. See the section entitled "Package Naming" in the Nixpkgs
|
||||||
|
> manual for more information.
|
||||||
|
|
||||||
|
5. The `tarball` jobs expects a `hello` build input to be available in
|
||||||
|
the Nix search path. Again, this input is passed by Hydra and is
|
||||||
|
meant to be a checkout of GNU Hello\'s source code repository.
|
||||||
|
|
||||||
|
6. This is the definition of the `build` job, whose purpose is to build
|
||||||
|
Hello from the tarball produced above.
|
||||||
|
|
||||||
|
7. The `build` function takes one parameter, `system`, which should be
|
||||||
|
a string defining the Nix system type--e.g., `"x86_64-linux"`.
|
||||||
|
Additionally, it refers to `jobs.tarball`, seen above.
|
||||||
|
|
||||||
|
Hydra inspects the formal argument list of the function (here, the
|
||||||
|
`system` argument) and passes it the corresponding parameter
|
||||||
|
specified as a build input on Hydra\'s web interface. Here, `system`
|
||||||
|
is passed by Hydra when it calls `build`. Thus, it must be defined
|
||||||
|
as a build input of type string in Hydra, which could take one of
|
||||||
|
several values.
|
||||||
|
|
||||||
|
The question mark after `system` defines the default value for this
|
||||||
|
argument, and is only useful when debugging locally.
|
||||||
|
|
||||||
|
8. The `build` job calls the `nixBuild` function, which unpacks the
|
||||||
|
tarball, then runs `./configure && make
|
||||||
|
&& make check && make install`.
|
||||||
|
|
||||||
|
9. Finally, the set of jobs is returned to Hydra, as a Nix attribute
|
||||||
|
set.
|
||||||
|
|
||||||
|
Building from the Command Line
|
||||||
|
------------------------------
|
||||||
|
|
||||||
|
It is often useful to test a build recipe, for instance before it is
|
||||||
|
actually used by Hydra, when testing changes, or when debugging a build
|
||||||
|
issue. Since build recipes for Hydra jobsets are just plain Nix
|
||||||
|
expressions, they can be evaluated using the standard Nix tools.
|
||||||
|
|
||||||
|
To evaluate the `tarball` jobset of the above example, just
|
||||||
|
run:
|
||||||
|
|
||||||
|
$ nix-build release.nix -A tarball
|
||||||
|
|
||||||
|
However, doing this with the example as is will probably
|
||||||
|
yield an error like this:
|
||||||
|
|
||||||
|
error: user-thrown exception: file `hello' was not found in the Nix search path (add it using $NIX_PATH or -I)
|
||||||
|
|
||||||
|
The error is self-explanatory. Assuming `$HOME/src/hello` points to a
|
||||||
|
checkout of Hello, this can be fixed this way:
|
||||||
|
|
||||||
|
$ nix-build -I ~/src release.nix -A tarball
|
||||||
|
|
||||||
|
Similarly, the `build` jobset can be evaluated:
|
||||||
|
|
||||||
|
$ nix-build -I ~/src release.nix -A build
|
||||||
|
|
||||||
|
The `build` job reuses the result of the `tarball` job, rebuilding it
|
||||||
|
only if it needs to.
|
||||||
|
|
||||||
|
Adding More Jobs
|
||||||
|
----------------
|
||||||
|
|
||||||
|
The example illustrates how to write the most basic
|
||||||
|
jobs, `tarball` and `build`. In practice, much more can be done by using
|
||||||
|
features readily provided by Nixpkgs or by creating new jobs as
|
||||||
|
customizations of existing jobs.
|
||||||
|
|
||||||
|
For instance, test coverage report for projects compiled with GCC can be
|
||||||
|
automatically generated using the `coverageAnalysis` function provided
|
||||||
|
by Nixpkgs instead of `nixBuild`. Back to our GNU Hello example, we can
|
||||||
|
define a `coverage` job that produces an HTML code coverage report
|
||||||
|
directly readable from the corresponding Hydra build page:
|
||||||
|
|
||||||
|
coverage =
|
||||||
|
{ system ? builtins.currentSystem }:
|
||||||
|
|
||||||
|
let pkgs = import nixpkgs { inherit system; }; in
|
||||||
|
pkgs.releaseTools.coverageAnalysis {
|
||||||
|
name = "hello";
|
||||||
|
src = jobs.tarball;
|
||||||
|
configureFlags = [ "--disable-silent-rules" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
As can be seen, the only difference compared to `build` is the use of
|
||||||
|
`coverageAnalysis`.
|
||||||
|
|
||||||
|
Nixpkgs provides many more build tools, including the ability to run
|
||||||
|
build in virtual machines, which can themselves run another GNU/Linux
|
||||||
|
distribution, which allows for the creation of packages for these
|
||||||
|
distributions. Please see [the `pkgs/build-support/release`
|
||||||
|
directory](https://github.com/NixOS/nixpkgs/tree/master/pkgs/build-support/release)
|
||||||
|
of Nixpkgs for more. The NixOS manual also contains information about
|
||||||
|
whole-system testing in virtual machine.
|
||||||
|
|
||||||
|
Now, assume we want to build Hello with an old version of GCC, and with
|
||||||
|
different `configure` flags. A new `build_exotic` job can be written
|
||||||
|
that simply *overrides* the relevant arguments passed to `nixBuild`:
|
||||||
|
|
||||||
|
build_exotic =
|
||||||
|
{ system ? builtins.currentSystem }:
|
||||||
|
|
||||||
|
let
|
||||||
|
pkgs = import nixpkgs { inherit system; };
|
||||||
|
build = jobs.build { inherit system; };
|
||||||
|
in
|
||||||
|
pkgs.lib.overrideDerivation build (attrs: {
|
||||||
|
buildInputs = [ pkgs.gcc33 ];
|
||||||
|
preConfigure = "gcc --version";
|
||||||
|
configureFlags =
|
||||||
|
attrs.configureFlags ++ [ "--disable-nls" ];
|
||||||
|
});
|
||||||
|
|
||||||
|
The `build_exotic` job reuses `build` and overrides some of its
|
||||||
|
arguments: it adds a dependency on GCC 3.3, a pre-configure phase that
|
||||||
|
runs `gcc --version`, and adds the `--disable-nls` configure flags.
|
||||||
|
|
||||||
|
This customization mechanism is very powerful. For instance, it can be
|
||||||
|
used to change the way Hello and *all* its dependencies--including the C
|
||||||
|
library and compiler used to build it--are built. See the Nixpkgs manual
|
||||||
|
for more.
|
||||||
|
|
||||||
|
Declarative projects
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
Hydra supports declaratively configuring a project\'s jobsets. This
|
||||||
|
configuration can be done statically, or generated by a build job.
|
||||||
|
|
||||||
|
> **Note**
|
||||||
|
>
|
||||||
|
> Hydra will treat the project\'s declarative input as a static definition
|
||||||
|
> if and only if the spec file contains a dictionary of dictionaries. If
|
||||||
|
> the value of any key in the spec is not a dictionary, it will treat the
|
||||||
|
> spec as a generated declarative spec.
|
||||||
|
|
||||||
|
### Static, Declarative Projects
|
||||||
|
|
||||||
|
Hydra supports declarative projects, where jobsets are configured from a
|
||||||
|
static JSON document in a repository.
|
||||||
|
|
||||||
|
To configure a static declarative project, take the following steps:
|
||||||
|
|
||||||
|
1. Create a Hydra-fetchable source like a Git repository or local path.
|
||||||
|
|
||||||
|
2. In that source, create a file called `spec.json`, and add the
|
||||||
|
specification for all of the jobsets. Each key is jobset and each
|
||||||
|
value is a jobset\'s specification. For example:
|
||||||
|
|
||||||
|
``` {.json}
|
||||||
|
{
|
||||||
|
"nixpkgs": {
|
||||||
|
"enabled": 1,
|
||||||
|
"hidden": false,
|
||||||
|
"description": "Nixpkgs",
|
||||||
|
"nixexprinput": "nixpkgs",
|
||||||
|
"nixexprpath": "pkgs/top-level/release.nix",
|
||||||
|
"checkinterval": 300,
|
||||||
|
"schedulingshares": 100,
|
||||||
|
"enableemail": false,
|
||||||
|
"emailoverride": "",
|
||||||
|
"keepnr": 3,
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"type": "git",
|
||||||
|
"value": "git://github.com/NixOS/nixpkgs.git master",
|
||||||
|
"emailresponsible": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixos": {
|
||||||
|
"enabled": 1,
|
||||||
|
"hidden": false,
|
||||||
|
"description": "NixOS: Small Evaluation",
|
||||||
|
"nixexprinput": "nixpkgs",
|
||||||
|
"nixexprpath": "nixos/release-small.nix",
|
||||||
|
"checkinterval": 300,
|
||||||
|
"schedulingshares": 100,
|
||||||
|
"enableemail": false,
|
||||||
|
"emailoverride": "",
|
||||||
|
"keepnr": 3,
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"type": "git",
|
||||||
|
"value": "git://github.com/NixOS/nixpkgs.git master",
|
||||||
|
"emailresponsible": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Create a new project, and set the project\'s declarative input type,
|
||||||
|
declarative input value, and declarative spec file to point to the
|
||||||
|
source and JSON file you created in step 2.
|
||||||
|
|
||||||
|
Hydra will create a special jobset named `.jobsets`. When the `.jobsets`
|
||||||
|
jobset is evaluated, this static specification will be used for
|
||||||
|
configuring the rest of the project\'s jobsets.
|
||||||
|
|
||||||
|
### Generated, Declarative Projects
|
||||||
|
|
||||||
|
Hydra also supports generated declarative projects, where jobsets are
|
||||||
|
configured automatically from specification files instead of being
|
||||||
|
managed through the UI. A jobset specification is a JSON object
|
||||||
|
containing the configuration of the jobset, for example:
|
||||||
|
|
||||||
|
``` {.json}
|
||||||
|
{
|
||||||
|
"enabled": 1,
|
||||||
|
"hidden": false,
|
||||||
|
"description": "js",
|
||||||
|
"nixexprinput": "src",
|
||||||
|
"nixexprpath": "release.nix",
|
||||||
|
"checkinterval": 300,
|
||||||
|
"schedulingshares": 100,
|
||||||
|
"enableemail": false,
|
||||||
|
"emailoverride": "",
|
||||||
|
"keepnr": 3,
|
||||||
|
"inputs": {
|
||||||
|
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
|
||||||
|
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
To configure a declarative project, take the following steps:
|
||||||
|
|
||||||
|
1. Create a jobset repository in the normal way (e.g. a git repo with a
|
||||||
|
`release.nix` file, any other needed helper files, and taking any
|
||||||
|
kind of hydra input), but without adding it to the UI. The nix
|
||||||
|
expression of this repository should contain a single job, named
|
||||||
|
`jobsets`. The output of the `jobsets` job should be a JSON file
|
||||||
|
containing an object of jobset specifications. Each member of the
|
||||||
|
object will become a jobset of the project, configured by the
|
||||||
|
corresponding jobset specification.
|
||||||
|
|
||||||
|
2. In some hydra-fetchable source (potentially, but not necessarily,
|
||||||
|
the same repo you created in step 1), create a JSON file containing
|
||||||
|
a jobset specification that points to the jobset repository you
|
||||||
|
created in the first step, specifying any needed inputs
|
||||||
|
(e.g. nixpkgs) as necessary.
|
||||||
|
|
||||||
|
3. In the project creation/edit page, set declarative input type,
|
||||||
|
declarative input value, and declarative spec file to point to the
|
||||||
|
source and JSON file you created in step 2.
|
||||||
|
|
||||||
|
Hydra will create a special jobset named `.jobsets`, which whenever
|
||||||
|
evaluated will go through the steps above in reverse order:
|
||||||
|
|
||||||
|
1. Hydra will fetch the input specified by the declarative input type
|
||||||
|
and value.
|
||||||
|
|
||||||
|
2. Hydra will use the configuration given in the declarative spec file
|
||||||
|
as the jobset configuration for this evaluation. In addition to any
|
||||||
|
inputs specified in the spec file, hydra will also pass the
|
||||||
|
`declInput` argument corresponding to the input fetched in step 1.
|
||||||
|
|
||||||
|
3. As normal, hydra will build the jobs specified in the jobset
|
||||||
|
repository, which in this case is the single `jobsets` job. When
|
||||||
|
that job completes, hydra will read the created jobset
|
||||||
|
specifications and create corresponding jobsets in the project,
|
||||||
|
disabling any jobsets that used to exist but are not present in the
|
||||||
|
current spec.
|
||||||
|
|
||||||
|
Email Notifications
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Hydra can send email notifications when the status of a build changes.
|
||||||
|
This provides immediate feedback to maintainers or committers when a
|
||||||
|
change causes build failures.
|
||||||
|
|
||||||
|
The simplest approach to enable Email Notifications is to use the ssmtp
|
||||||
|
package, which simply hands off the emails to another SMTP server. For
|
||||||
|
details on how to configure ssmtp, see the documentation for the
|
||||||
|
`networking.defaultMailServer` option. To use ssmtp for the Hydra email
|
||||||
|
notifications, add it to the path option of the Hydra services in your
|
||||||
|
`/etc/nixos/configuration.nix` file:
|
||||||
|
|
||||||
|
systemd.services.hydra-queue-runner.path = [ pkgs.ssmtp ];
|
||||||
|
systemd.services.hydra-server.path = [ pkgs.ssmtp ];
|
||||||
|
|
||||||
|
|
@ -241,7 +241,7 @@
|
||||||
buildInputs =
|
buildInputs =
|
||||||
[ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig libpqxx
|
[ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig libpqxx
|
||||||
gitAndTools.topGit mercurial darcs subversion breezy openssl bzip2 libxslt
|
gitAndTools.topGit mercurial darcs subversion breezy openssl bzip2 libxslt
|
||||||
final.nix perlDeps perl
|
final.nix perlDeps perl mdbook
|
||||||
boost
|
boost
|
||||||
postgresql_11
|
postgresql_11
|
||||||
(if lib.versionAtLeast lib.version "20.03pre"
|
(if lib.versionAtLeast lib.version "20.03pre"
|
||||||
|
|
@ -258,8 +258,6 @@
|
||||||
gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused breezy
|
gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused breezy
|
||||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
|
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
|
||||||
|
|
||||||
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
|
|
||||||
|
|
||||||
shellHook = ''
|
shellHook = ''
|
||||||
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
|
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
|
||||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
|
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue