Merge pull request #737 from Ma27/rm-sqlite

Get rid of dependency to SQLite

.gitignore

@@ -32,3 +32,4 @@ Makefile.in
 hydra-config.h
 hydra-config.h.in
 result
+tests/jobs/config.nix

doc/dev-notes.txt

@@ -8,22 +8,23 @@
 
 * Setting the maximum number of concurrent builds per system type:
 
-  $ sqlite3 hydra.sqlite "insert into SystemTypes(system, maxConcurrent) values('i686-linux', 3);"
+  $ psql -d hydra <<< "insert into SystemTypes(system, maxConcurrent) values('i686-linux', 3);"
 
 * Creating a user:
 
-  $ sqlite3 hydra.sqlite "insert into Users(userName, emailAddress, password) values('root', 'e.dolstra@tudelft.nl', '$(echo -n foobar | sha1sum | cut -c1-40)');"
+  $ hydra-create-user root --email-address 'e.dolstra@tudelft.nl' \
+      --password-hash "$(echo -n foobar | sha1sum | cut -c1-40)"
 
   (Replace "foobar" with the desired password.)
 
   To make the user an admin:
 
-  $ sqlite3 hydra.sqlite "insert into UserRoles(userName, role) values('root', 'admin');"
+  $ hydra-create-user root --role admin
 
   To enable a non-admin user to create projects:
-  
+
-  $ sqlite3 hydra.sqlite "insert into UserRoles(userName, role) values('alice', 'create-projects');"
+  $ hydra-create-user root --role create-projects
-  
+
 * Creating a release set:
 
   insert into ReleaseSets(project, name) values('patchelf', 'unstable');

doc/manual/projects.xml

@@ -43,16 +43,11 @@ Identifier: patchelf
       The identifier should be a unique name (it is the primary
       database key for the project table in the database). If you try
       to create a project with an already existing identifier you'd
-      get an error message such as:
+      get an error message from the database.
 
-<screen>
+      So try to create the project after entering just the general
-I'm very sorry, but an error occurred:
+      information to figure out if you have chosen a unique name.
-DBIx::Class::ResultSet::create(): DBI Exception: DBD::SQLite::st execute failed: column name is not unique(19) at dbdimp.c line 402
+      Job sets can be added once the project has been created.
-</screen>
-
-       So try to create the project after entering just the general
-       information to figure out if you have chosen a unique name.
-       Job sets can be added once the project has been created.
 
 <screen>
 Display name: Patchelf

flake.nix

@@ -103,7 +103,7 @@
           src = self;
 
           buildInputs =
-            [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx
+            [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig libpqxx
               gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt
               perlDeps perl final.nix
               postgresql95 # for running the tests
@@ -114,7 +114,7 @@
             ];
 
           hydraPath = lib.makeBinPath (
-            [ sqlite subversion openssh final.nix coreutils findutils pixz
+            [ subversion openssh final.nix coreutils findutils pixz
               gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar
             ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
 

src/lib/Hydra/Controller/API.pm

@@ -214,7 +214,7 @@ sub scmdiff : Path('/api/scmdiff') Args(0) {
 sub triggerJobset {
     my ($self, $c, $jobset, $force) = @_;
     print STDERR "triggering jobset ", $jobset->get_column('project') . ":" . $jobset->name, "\n";
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         $jobset->update({ triggertime => time });
         $jobset->update({ forceeval => 1 }) if $force;
     });

src/lib/Hydra/Controller/Admin.pm

@@ -90,7 +90,7 @@ sub news_submit : Chained('admin') PathPart('news/submit') Args(0) {
 sub news_delete : Chained('admin') PathPart('news/delete') Args(1) {
     my ($self, $c, $id) = @_;
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         my $newsItem = $c->model('DB::NewsItems')->find($id)
           or notFound($c, "Newsitem with id $id doesn't exist.");
         $newsItem->delete;

src/lib/Hydra/Controller/Build.pm

@@ -526,7 +526,7 @@ sub keep : Chained('buildChain') PathPart Args(1) {
         registerRoot $_->path foreach $build->buildoutputs;
     }
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         $build->update({keep => $keep});
     });
 

src/lib/Hydra/Controller/Jobset.pm

@@ -62,7 +62,7 @@ sub jobset_PUT {
     }
 
     if (defined $c->stash->{jobset}) {
-        txn_do($c->model('DB')->schema, sub {
+        $c->model('DB')->schema->txn_do(sub {
             updateJobset($c, $c->stash->{jobset});
         });
 
@@ -74,7 +74,7 @@ sub jobset_PUT {
 
     else {
         my $jobset;
-        txn_do($c->model('DB')->schema, sub {
+        $c->model('DB')->schema->txn_do(sub {
             # Note: $jobsetName is validated in updateProject, which will
             # abort the transaction if the name isn't valid.
             $jobset = $c->stash->{project}->jobsets->create(
@@ -100,7 +100,7 @@ sub jobset_DELETE {
         error($c, "can't modify jobset of declarative project", 403);
     }
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         $c->stash->{jobset}->jobsetevals->delete;
         $c->stash->{jobset}->builds->delete;
         $c->stash->{jobset}->delete;

src/lib/Hydra/Controller/JobsetEval.pm

@@ -146,7 +146,7 @@ sub release : Chained('evalChain') PathPart('release') Args(0) {
 
     my $release;
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
 
         $release = $c->stash->{project}->releases->create(
             { name => $releaseName

src/lib/Hydra/Controller/Project.pm

@@ -41,7 +41,7 @@ sub project_PUT {
     if (defined $c->stash->{project}) {
         requireProjectOwner($c, $c->stash->{project});
 
-        txn_do($c->model('DB')->schema, sub {
+        $c->model('DB')->schema->txn_do(sub {
             updateProject($c, $c->stash->{project});
         });
 
@@ -55,7 +55,7 @@ sub project_PUT {
         requireMayCreateProjects($c);
 
         my $project;
-        txn_do($c->model('DB')->schema, sub {
+        $c->model('DB')->schema->txn_do(sub {
             # Note: $projectName is validated in updateProject,
             # which will abort the transaction if the name isn't
             # valid.  Idem for the owner.
@@ -77,7 +77,7 @@ sub project_DELETE {
 
     requireProjectOwner($c, $c->stash->{project});
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         $c->stash->{project}->jobsetevals->delete;
         $c->stash->{project}->builds->delete;
         $c->stash->{project}->delete;
@@ -198,7 +198,7 @@ sub create_release_submit : Chained('projectChain') PathPart('create-release/sub
     my $releaseName = $c->request->params->{name};
 
     my $release;
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         # Note: $releaseName is validated in updateRelease, which will
         # abort the transaction if the name isn't valid.
         $release = $c->stash->{project}->releases->create(

src/lib/Hydra/Controller/Release.pm

@@ -63,13 +63,13 @@ sub submit : Chained('release') PathPart('submit') Args(0) {
     requireProjectOwner($c, $c->stash->{project});
 
     if (($c->request->params->{action} || "") eq "delete") {
-        txn_do($c->model('DB')->schema, sub {
+        $c->model('DB')->schema->txn_do(sub {
             $c->stash->{release}->delete;
         });
         $c->res->redirect($c->uri_for($c->controller('Project')->action_for('project'),
             [$c->stash->{project}->name]));
     } else {
-        txn_do($c->model('DB')->schema, sub {
+        $c->model('DB')->schema->txn_do(sub {
             updateRelease($c, $c->stash->{release});
         });
         $c->res->redirect($c->uri_for($self->action_for("view"),

src/lib/Hydra/Controller/User.pm

@@ -163,7 +163,7 @@ sub register :Local Args(0) {
     error($c, "Your user name is already taken.")
         if $c->find_user({ username => $userName });
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('DB')->schema->txn_do(sub {
         my $user = $c->model('DB::Users')->create(
             { username => $userName
             , password => "!"
@@ -261,7 +261,7 @@ sub edit_PUT {
         return;
     }
 
-    txn_do($c->model('DB')->schema, sub {
+    $c->model('Db')->schema->txn_do(sub {
         updatePreferences($c, $user);
     });
 

src/lib/Hydra/Helper/AddBuilds.pm

@@ -46,7 +46,7 @@ sub updateDeclarativeJobset {
         $update{$key} = $declSpec->{$key};
         delete $declSpec->{$key};
     }
-    txn_do($db, sub {
+    $db->txn_do(sub {
         my $jobset = $project->jobsets->update_or_create(\%update);
         $jobset->jobsetinputs->delete;
         while ((my $name, my $data) = each %{$declSpec->{"inputs"}}) {
@@ -79,7 +79,7 @@ sub handleDeclarativeJobsetBuild {
         }
 
         my $declSpec = decode_json($declText);
-        txn_do($db, sub {
+        $db->txn_do(sub {
             my @kept = keys %$declSpec;
             push @kept, ".jobsets";
             $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });

src/lib/Hydra/Helper/Nix.pm

@@ -14,7 +14,7 @@ use IPC::Run;
 
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
-    getHydraHome getHydraConfig getBaseUrl txn_do
+    getHydraHome getHydraConfig getBaseUrl
     getSCMCacheDir
     registerRoot getGCRootsDir gcRootFor
     jobsetOverview jobsetOverview_
@@ -61,22 +61,6 @@ sub getBaseUrl {
 }
 
 
-# Awful hack to handle timeouts in SQLite: just retry the transaction.
-# DBD::SQLite *has* a 30 second retry window, but apparently it
-# doesn't work.
-sub txn_do {
-    my ($db, $coderef) = @_;
-    my $res;
-    while (1) {
-        eval {
-            $res = $db->txn_do($coderef);
-        };
-        return $res if !$@;
-        die $@ unless $@ =~ "database is locked";
-    }
-}
-
-
 sub getSCMCacheDir {
     return Hydra::Model::DB::getHydraPath . "/scm" ;
 }
@@ -446,7 +430,7 @@ sub getTotalShares {
 
 sub cancelBuilds($$) {
     my ($db, $builds) = @_;
-    return txn_do($db, sub {
+    return $db->txn_do(sub {
         $builds = $builds->search({ finished => 0 });
         my $n = $builds->count;
         my $time = time();
@@ -473,7 +457,7 @@ sub restartBuilds($$) {
 
     my $nrRestarted = 0;
 
-    txn_do($db, sub {
+    $db->txn_do(sub {
         # Reset the stats for the evals to which the builds belongs.
         # !!! Should do this in a trigger.
         $db->resultset('JobsetEvals')->search(

src/lib/Hydra/Model/DB.pm

@@ -10,11 +10,7 @@ sub getHydraPath {
 }
 
 sub getHydraDBPath {
-    my $db = $ENV{"HYDRA_DBI"};
+    return $ENV{"HYDRA_DBI"} || "dbi:Pg:dbname=hydra;";
-    return $db if defined $db;
-    my $path = getHydraPath . '/hydra.sqlite';
-    #warn "The Hydra database ($path) does not exist!\n" unless -f $path;
-    return "dbi:SQLite:$path";
 }
 
 __PACKAGE__->config(

src/lib/Hydra/Plugin/BazaarInput.pm

@@ -58,7 +58,7 @@ sub fetchInput {
         # FIXME: time window between nix-prefetch-bzr and addTempRoot.
         addTempRoot($storePath);
 
-        txn_do($self->{db}, sub {
+        $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedBazaarInputs')->create(
                 { uri => $uri
                 , revision => $revision

src/lib/Hydra/Plugin/DarcsInput.pm

@@ -77,7 +77,7 @@ sub fetchInput {
         $sha256 = queryPathHash($storePath);
         $sha256 =~ s/sha256://;
 
-        txn_do($self->{db}, sub {
+        $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedDarcsInputs')->update_or_create(
                 { uri => $uri
                 , revision => $revision

src/lib/Hydra/Plugin/GitInput.pm

@@ -218,7 +218,7 @@ sub fetchInput {
         # FIXME: time window between nix-prefetch-git and addTempRoot.
         addTempRoot($storePath);
 
-        txn_do($self->{db}, sub {
+        $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedGitInputs')->update_or_create(
                 { uri => $uri
                 , branch => $branch

src/lib/Hydra/Plugin/MercurialInput.pm

@@ -85,7 +85,7 @@ sub fetchInput {
         # FIXME: time window between nix-prefetch-hg and addTempRoot.
         addTempRoot($storePath);
 
-        txn_do($self->{db}, sub {
+        $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedHgInputs')->update_or_create(
                 { uri => $uri
                 , branch => $branch

src/lib/Hydra/Plugin/PathInput.pm

@@ -54,7 +54,7 @@ sub fetchInput {
         # changes, we get a new "revision", but if it doesn't change
         # (or changes back), we don't get a new "revision".
         if (!defined $cachedInput) {
-            txn_do($self->{db}, sub {
+            $self->{db}->txn_do(sub {
                 $self->{db}->resultset('CachedPathInputs')->update_or_create(
                     { srcpath => $uri
                     , timestamp => $timestamp
@@ -65,7 +65,7 @@ sub fetchInput {
                 });
         } else {
             $timestamp = $cachedInput->timestamp;
-            txn_do($self->{db}, sub {
+            $self->{db}->txn_do(sub {
                 $cachedInput->update({lastseen => time});
             });
         }

src/lib/Hydra/Plugin/SubversionInput.pm

@@ -71,7 +71,7 @@ sub fetchInput {
 
         $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
 
-        txn_do($self->{db}, sub {
+        $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedSubversionInputs')->update_or_create(
                 { uri => $uri
                 , revision => $revision

src/script/hydra-create-user

@@ -54,7 +54,7 @@ die "$0: type must be `hydra' or `google'\n"
 
 my $db = Hydra::Model::DB->new();
 
-txn_do($db, sub {
+$db->txn_do(sub {
     my $user = $db->resultset('Users')->find({ username => $renameFrom // $userName });
     if ($renameFrom) {
         die "$0: user `$renameFrom' does not exist\n" unless $user;

src/script/hydra-eval-jobset

@@ -399,7 +399,7 @@ sub checkBuild {
 
     my $build;
 
-    txn_do($db, sub {
+    $db->txn_do(sub {
         my $job = $jobset->jobs->update_or_create({
             name => $jobName,
             jobset_id => $jobset->id,
@@ -501,7 +501,7 @@ sub setJobsetError {
     my $prevError = $jobset->errormsg;
 
     eval {
-        txn_do($db, sub {
+        $db->txn_do(sub {
             $jobset->update({ errormsg => $errorMsg, errortime => time, fetcherrormsg => undef });
         });
     };
@@ -603,7 +603,7 @@ sub checkJobsetWrapped {
     if ($fetchError) {
         Net::Statsd::increment("hydra.evaluator.failed_checkouts");
         print STDERR $fetchError;
-        txn_do($db, sub {
+        $db->txn_do(sub {
             $jobset->update({ lastcheckedtime => time, fetcherrormsg => $fetchError }) if !$dryRun;
             $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
         });
@@ -619,7 +619,7 @@ sub checkJobsetWrapped {
     if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval && $prevEval->flake eq $flakeRef) {
         print STDERR "  jobset is unchanged, skipping\n";
         Net::Statsd::increment("hydra.evaluator.unchanged_checkouts");
-        txn_do($db, sub {
+        $db->txn_do(sub {
             $jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });
             $db->storage->dbh->do("notify eval_cached, ?", undef, join('\t', $tmpId));
         });
@@ -660,7 +660,7 @@ sub checkJobsetWrapped {
     my $dbStart = clock_gettime(CLOCK_MONOTONIC);
 
     my %buildMap;
-    txn_do($db, sub {
+    $db->txn_do(sub {
 
         my $prevEval = getPrevJobsetEval($db, $jobset, 1);
 
@@ -806,7 +806,7 @@ sub checkJobset {
     my $failed = 0;
     if ($checkError) {
         print STDERR $checkError;
-        txn_do($db, sub {
+        $db->txn_do(sub {
             $jobset->update({lastcheckedtime => time});
             setJobsetError($jobset, $checkError);
             $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));

src/script/hydra-init

@@ -25,9 +25,8 @@ my @tables = $dbh->tables;
 if (! grep { /SchemaVersion/i } @tables) {
     print STDERR "initialising the Hydra database schema...\n";
     my $schema = read_file(
-        $dbh->{Driver}->{Name} eq 'SQLite' ? "$home/sql/hydra-sqlite.sql" :
         $dbh->{Driver}->{Name} eq 'Pg' ? "$home/sql/hydra-postgresql.sql" :
-        die "unsupported database type\n");
+        die "unsupported database type $dbh->{Driver}->{Name}\n");
     my @statements = $sql_splitter->split($schema);
     eval {
         $dbh->begin_work;

src/sql/hydra.sql

@@ -79,14 +79,8 @@ create table Jobsets (
     primary key   (project, name),
     foreign key   (project) references Projects(name) on delete cascade on update cascade,
     constraint    Jobsets_id_unique UNIQUE(id)
-#ifdef SQLITE
-    ,
-    foreign key   (project, name, nixExprInput) references JobsetInputs(project, jobset, name)
-#endif
 );
 
-#ifdef POSTGRESQL
-
 create function notifyJobsetSharesChanged() returns trigger as 'begin notify jobset_shares_changed; return null; end;' language plpgsql;
 create trigger JobsetSharesChanged after update on Jobsets for each row
   when (old.schedulingShares != new.schedulingShares) execute procedure notifyJobsetSharesChanged();
@@ -104,9 +98,6 @@ create trigger JobsetSchedulingChanged after update on Jobsets for each row
         or (old.enabled != new.enabled))
   execute procedure notifyJobsetSchedulingChanged();
 
-#endif
-
-
 create table JobsetRenames (
     project       text not null,
     from_         text not null,
@@ -157,11 +148,7 @@ create table Jobs (
 
 
 create table Builds (
-#ifdef POSTGRESQL
     id            serial primary key not null,
-#else
-    id            integer primary key autoincrement not null,
-#endif
 
     finished      integer not null, -- 0 = scheduled, 1 = finished
 
@@ -244,8 +231,6 @@ create table Builds (
 );
 
 
-#ifdef POSTGRESQL
-
 create function notifyBuildsDeleted() returns trigger as 'begin notify builds_deleted; return null; end;' language plpgsql;
 create trigger BuildsDeleted after delete on Builds execute procedure notifyBuildsDeleted();
 
@@ -261,8 +246,6 @@ create function notifyBuildBumped() returns trigger as 'begin notify builds_bump
 create trigger BuildBumped after update on Builds for each row
   when (old.globalPriority != new.globalPriority) execute procedure notifyBuildBumped();
 
-#endif
-
 
 create table BuildOutputs (
     build         integer not null,
@@ -332,11 +315,7 @@ create table BuildStepOutputs (
 
 -- Inputs of builds.
 create table BuildInputs (
-#ifdef POSTGRESQL
     id            serial primary key not null,
-#else
-    id            integer primary key autoincrement not null,
-#endif
 
     -- Which build this input belongs to.
     build         integer,
@@ -502,11 +481,7 @@ create table ReleaseMembers (
 
 
 create table JobsetEvals (
-#ifdef POSTGRESQL
     id            serial primary key not null,
-#else
-    id            integer primary key autoincrement not null,
-#endif
 
     project       text not null,
     jobset        text not null,
@@ -577,11 +552,7 @@ create table UriRevMapper (
 
 
 create table NewsItems (
-#ifdef POSTGRESQL
     id            serial primary key not null,
-#else
-    id            integer primary key autoincrement not null,
-#endif
     contents      text not null,
     createTime    integer not null,
     author        text not null,
@@ -614,7 +585,6 @@ create table FailedPaths (
     path text primary key not null
 );
 
-#ifdef POSTGRESQL
 
 -- Needed because Postgres doesn't have "ignore duplicate" or upsert
 -- yet.
@@ -622,7 +592,6 @@ create rule IdempotentInsert as on insert to FailedPaths
   where exists (select 1 from FailedPaths where path = new.path)
   do instead nothing;
 
-#endif
 
 
 create table SystemStatus (
@@ -639,7 +608,6 @@ create table NrBuilds (
 
 insert into NrBuilds(what, count) values('finished', 0);
 
-#ifdef POSTGRESQL
 
 create function modifyNrBuildsFinished() returns trigger as $$
   begin
@@ -658,8 +626,6 @@ create trigger NrBuildsFinished after insert or update or delete on Builds
   for each row
   execute procedure modifyNrBuildsFinished();
 
-#endif
-
 
 -- Some indices.
 
@@ -704,7 +670,6 @@ create index IndexJobsetEvalsOnJobsetId on JobsetEvals(project, jobset, id desc)
 
 create index IndexBuildsOnNotificationPendingSince on Builds(notificationPendingSince) where notificationPendingSince is not null;
 
-#ifdef POSTGRESQL
 -- The pg_trgm extension has to be created by a superuser. The NixOS
 -- module creates this extension in the systemd prestart script. We
 -- then ensure the extension has been created before creating the
@@ -721,4 +686,3 @@ exception when others then
     raise warning 'HINT: Temporary provide superuser role to your Hydra Postgresql user and run the script src/sql/upgrade-57.sql';
     raise warning 'The pg_trgm index on builds.drvpath has been skipped (slower complex queries on builds.drvpath)';
 end$$;
-#endif

tests/Makefile.am

@@ -31,9 +31,6 @@ TESTS = \
 
 check_SCRIPTS = repos
 
-db.sqlite: $(top_srcdir)/src/sql/hydra-sqlite.sql
-	$(TESTS_ENVIRONMENT) $(top_srcdir)/src/script/hydra-init
-
 repos: dirs
 
 dirs: