Provide a command ‘hydra-init’ to initialise/upgrade the database

For schema upgrades, hydra-init executes the files
src/sql/upgrade-<N>.sql, each of which upgrades the schema from
version N-1 to N.  The upgrades are wrapped in a transaction.

deps.nix

@@ -2,36 +2,37 @@
 
 with pkgs;
 
-[ perlPackages.CatalystDevel
+[ perlPackages.CatalystAuthenticationStoreDBIxClass
-  perlPackages.CatalystPluginSessionStoreFastMmap
+  perlPackages.CatalystPluginAccessLog
-  perlPackages.CatalystPluginStackTrace
   perlPackages.CatalystPluginAuthorizationRoles
   perlPackages.CatalystPluginSessionStateCookie
-  perlPackages.CatalystPluginAccessLog
+  perlPackages.CatalystPluginSessionStoreFastMmap
-  perlPackages.CatalystAuthenticationStoreDBIxClass
+  perlPackages.CatalystPluginStackTrace
-  perlPackages.CatalystViewTT
   perlPackages.CatalystViewDownload
   perlPackages.CatalystViewJSON
+  perlPackages.CatalystViewTT
   perlPackages.CatalystXScriptServerStarman
-  perlPackages.XMLSimple
+  perlPackages.CryptRandPasswd
-  perlPackages.IPCRun
-  perlPackages.IOCompress
-  perlPackages.Readonly
   perlPackages.DBDPg
   perlPackages.DBDSQLite
-  perlPackages.EmailSender
-  perlPackages.TextTable
-  perlPackages.TextDiff
-  perlPackages.FileSlurp
-  perlPackages.NetTwitterLite
-  perlPackages.PadWalker
   perlPackages.DataDump
-  perlPackages.JSONXS
   perlPackages.DateTime
   perlPackages.DigestSHA1
-  perlPackages.CryptRandPasswd
+  perlPackages.EmailSender
-  perlPackages.TestMore
+  perlPackages.FileSlurp
-  perlPackages.SysHostnameLong
+  perlPackages.IOCompress
+  perlPackages.IPCRun
+  perlPackages.JSONXS
+  perlPackages.NetTwitterLite
+  perlPackages.PadWalker
+  perlPackages.CatalystDevel
+  perlPackages.Readonly
+  perlPackages.SQLSplitStatement
   perlPackages.Starman
+  perlPackages.SysHostnameLong
+  perlPackages.TestMore
+  perlPackages.TextDiff
+  perlPackages.TextTable
+  perlPackages.XMLSimple
   nixUnstable
 ]

doc/manual/installation.xml

@@ -73,21 +73,23 @@
   <section>
     <title>Installation</title>
 
+<!--
     <para>
       Hydra can be installed using Nixpkgs:
 
       <screen>
-nix-env -Ai hydra -f /path/to/nixpkgs</screen>
+nix-env -f /path/to/nixpkgs -iA hydra</screen>
 
       This makes the tools available in your Nix user environment,
       <literal>$HOME/.nix-profile</literal> by default.
     </para>
+-->
 
     <para>
-      Alternatively, the latest development snapshot can be installed
+      The latest development snapshot of Hydra can be installed
       by visiting the URL <link
       xlink:href="http://hydra.nixos.org/view/hydra/unstable"><literal>http://hydra.nixos.org/view/hydra/unstable</literal></link>
-      and use the one-click install available at one of the build
+      and using the one-click install available at one of the build
       pages. You can also install Hydra through the channel by
       performing the following commands:
 
@@ -101,8 +103,10 @@ nix-env -i hydra</screen>
       Command completion should reveal a number of command-line tools from Hydra:
 
           <screen>
-hydra-build      hydra-evaluator     hydra-server
+hydra-build      hydra-init          hydra-update-gc-roots
-hydra-eval-jobs  hydra-queue-runner  hydra-update-gc-roots</screen>
+hydra-eval-jobs  hydra-queue-runner  
+hydra-evaluator  hydra-server
+</screen>
     </para>
   </section>
 
@@ -116,24 +120,46 @@ hydra-eval-jobs  hydra-queue-runner  hydra-update-gc-roots</screen>
 
     <para>
       To setup a PostgreSQL database with <emphasis>hydra</emphasis>
-      as database name and user name, issue the following commands:
+      as database name and user name, issue the following commands on
+      the PostgreSQL server:
 
       <screen>
-createdb hydra
+createuser -S -D -R -P hydra
-echo "CREATE USER hydra WITH PASSWORD '&lt;your-password&gt;' ;" | psql hydra
+createdb -O hydra hydra</screen>
-cat $prefix/share/hydra/sql/hydra-postgresql.sql | psql hydra
-echo "GRANT ALL ON DATABASE hydra TO hydra;" | psql hydra</screen>
 
       Note that <emphasis>$prefix</emphasis> is the location of Hydra
       in the nix store.
     </para>
 
     <para>
-      For SQLite, the following command is all it takes to create the
+      Hydra uses an environment variable to know which database should
-      database:
+      be used, and a variable which point to a location that holds
+      some state. To set these variables for a PostgreSQL database,
+      add the following to the file <filename>~/.profile</filename> of
+      the user running the Hydra services.
 
       <screen>
-cat $prefix/share/hydra/sql/hydra-sqlite.sql | sqlite3 /path/to/hydra.sqlite</screen>
+export HYDRA_DBI="dbi:Pg:dbname=hydra;host=dbserver.example.org;user=hydra;"
+export HYDRA_DATA=/var/lib/hydra</screen>
+
+      You can provide the username and password in the file
+      <filename>~/.pgpass</filename>, e.g.
+
+      <screen>
+dbserver.example.org:*:hydra:hydra:password</screen>
+
+      Make sure that the <emphasis>HYDRA_DATA</emphasis> directory
+      exists and is writable for the user which will run the Hydra
+      services.  For a SQLite database, the
+      <varname>HYDRA_DBI</varname> should be set to something like
+      <literal>dbi:SQLite:/path/to/hydra.sqlite</literal>
+    </para>
+
+    <para>
+      Having set these environment variables, you can now initialise
+      the database by doing:
+      <screen>
+hydra-init</screen>
     </para>
 
     <para>
@@ -148,23 +174,17 @@ echo "INSERT INTO UserRoles(userName, role) values('root', 'admin');" | psql hyd
       /path/to/hydra.sqlite</command>.
     </para>
 
-    <para>
+  </section>
-      Hydra uses an environment variable to know which database should
-      be used, and a variable which point to a location that holds
-      some state. To set these variables for a PostgreSQL database,
-      add the following to the <filename>.profile</filename> of the
-      user running the Hydra services.
 
+  <section>
+    <title>Upgrading</title>
+
+    <para>If you're upgrading Hydra from a previous version, you
+    should do the following to perform any necessary database schema migrations:
       <screen>
-export HYDRA_DBI="dbi:Pg:dbname=hydra;host=localhost;"
+hydra-init</screen>
-export HYDRA_DATA=/var/lib/hydra</screen> 
-
-      Make sure that the <emphasis>HYDRA_DATA</emphasis> directory
-      exists and is writable for the user which will run the Hydra
-      services.  For a SQLite database, the
-      <varname>HYDRA_DBI</varname> should be set to something like
-      <literal>dbi:SQLite:/path/to/hydra.sqlite</literal>
     </para>
+
   </section>
 
   <section>

src/lib/Hydra/Helper/Nix.pm

@@ -8,7 +8,7 @@ use Hydra::Helper::CatalystUtils;
 
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
-    getHydraPath getHydraDBPath openHydraDB getHydraConf txn_do
+    getHydraPath getHydraHome getHydraDBPath openHydraDB getHydraConf txn_do
     registerRoot getGCRootsDir gcRootFor
     getPrimaryBuildsForView
     getPrimaryBuildTotal
@@ -21,6 +21,13 @@ sub getHydraPath {
     return $dir;
 }
 
+
+sub getHydraHome {
+    my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
+    return $dir;
+}
+
+
 sub getHydraConf {
     my $conf = $ENV{"HYDRA_CONFIG"} || (getHydraPath . "/hydra.conf");
     die "The HYDRA_CONFIG file ($conf) does not exist!\n" unless -f $conf;

src/script/hydra-init

@@ -0,0 +1,58 @@
+#! /var/run/current-system/sw/bin/perl -w
+
+use strict;
+use Hydra::Schema;
+use Hydra::Helper::Nix;
+use File::Slurp;
+use SQL::SplitStatement;
+use List::Util qw(max);
+
+my $db = openHydraDB;
+my $dbh = $db->storage->dbh;
+$dbh->{RaiseError} = 1;
+
+my $home = getHydraHome;
+
+my $sql_splitter = SQL::SplitStatement->new;
+
+# Figure out the target schema version.
+my $maxSchemaVersion = max (map { /.*\/upgrade-(\d.*)\.sql/; $1 } (glob "$home/sql/upgrade-[0-9]*.sql")) || 1;
+
+# Check whether the database has been initialised.  If not, load the
+# schema.
+my @tables = $dbh->tables;
+if (! grep { /SchemaVersion/i } @tables) {
+    print STDERR "initialising the Hydra database schema...\n";
+    my $schema = read_file(
+	$dbh->{Driver}->{Name} eq 'SQLite' ? "$home/sql/hydra-sqlite.sql" :
+	$dbh->{Driver}->{Name} eq 'Pg' ? "$home/sql/hydra-postgresql.sql" :
+	die "unsupported database type\n");
+    my @statements = $sql_splitter->split($schema);
+    eval {
+	$dbh->begin_work;
+	$dbh->do($_) foreach @statements;
+	$db->resultset('SchemaVersion')->create({version => $maxSchemaVersion});
+	$dbh->commit;
+    };
+    die "schema initialisation failed: $@\n" if $@;
+    exit 0;
+}
+
+# Get the current schema version.
+my @versions = $db->resultset('SchemaVersion')->all;
+die "couldn't get Hydra schema version!" if scalar @versions != 1;
+my $schemaVersion = $versions[0]->version;
+
+for (my $n = $schemaVersion; $n < $maxSchemaVersion; $n++) {
+    my $m = $n + 1;
+    print STDERR "upgrading Hydra schema from version $n to $m\n";
+    my $schema = read_file("$home/sql/upgrade-$m.sql");
+    my @statements = $sql_splitter->split($schema);
+    eval {
+	$dbh->begin_work;
+	$dbh->do($_) foreach @statements;
+	$db->resultset('SchemaVersion')->update({version => $m});
+	$dbh->commit;
+    };
+    die "schema upgrade failed: $@\n" if $@;
+}

src/sql/Makefile.am

@@ -1,6 +1,6 @@
 EXTRA_DIST = hydra.sql hydra-postgresql.sql hydra-sqlite.sql
 
-sqldir = $(datadir)/hydra/sql
+sqldir = $(libexecdir)/hydra/sql
 nobase_sql_DATA = $(EXTRA_DIST)
 
 hydra-postgresql.sql: hydra.sql

src/sql/hydra.sql

@@ -3,8 +3,6 @@ create table SchemaVersion (
     version       integer not null
 );
 
-insert into SchemaVersion (version) values (1);
-
 
 create table Users (
     userName      text primary key not null,

tests/Makefile.am

@@ -31,7 +31,7 @@ clean :
 check_SCRIPTS = db.sqlite repos
 
 db.sqlite : $(top_srcdir)/src/sql/hydra-sqlite.sql
-	sqlite3 db.sqlite <  $(top_srcdir)/src/sql/hydra-sqlite.sql
+	perl $(top_srcdir)/src/script/hydra-init
 
 repos : dirs git-repo hg-repo svn-repo svn-checkout-repo bzr-repo bzr-checkout-repo