From efcbc08686c3c102827883c8cdc691a08e81f47f Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 10 Apr 2020 18:13:18 +0200 Subject: [PATCH] Get rid of dependency to SQLite SQLite isn't properly supported by Hydra for a few years now[1], but Hydra still depends on it. Apart from a slightly bigger closure this can cause confusion by users since Hydra picks up SQLite rather than PostgreSQL by default if HYDRA_DBI isn't configured properly[2] [1] 78974abb6943533c3d94c7bd3282bf738b7be61c [2] https://logs.nix.samueldr.com/nixos-dev/2020-04-10#3297342; --- .gitignore | 1 + doc/dev-notes.txt | 13 +++++++------ doc/manual/projects.xml | 13 ++++--------- flake.nix | 5 ++--- src/lib/Hydra/Model/DB.pm | 6 +----- src/script/hydra-init | 3 +-- src/sql/hydra.sql | 36 ------------------------------------ tests/Makefile.am | 3 --- 8 files changed, 16 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index 5663a9d6..f0e7b240 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ Makefile.in hydra-config.h hydra-config.h.in result +tests/jobs/config.nix diff --git a/doc/dev-notes.txt b/doc/dev-notes.txt index 601aa852..36ec193f 100644 --- a/doc/dev-notes.txt +++ b/doc/dev-notes.txt @@ -8,22 +8,23 @@ * Setting the maximum number of concurrent builds per system type: - $ sqlite3 hydra.sqlite "insert into SystemTypes(system, maxConcurrent) values('i686-linux', 3);" + $ psql -d hydra <<< "insert into SystemTypes(system, maxConcurrent) values('i686-linux', 3);" * Creating a user: - $ sqlite3 hydra.sqlite "insert into Users(userName, emailAddress, password) values('root', 'e.dolstra@tudelft.nl', '$(echo -n foobar | sha1sum | cut -c1-40)');" + $ hydra-create-user root --email-address 'e.dolstra@tudelft.nl' \ + --password-hash "$(echo -n foobar | sha1sum | cut -c1-40)" (Replace "foobar" with the desired password.) To make the user an admin: - $ sqlite3 hydra.sqlite "insert into UserRoles(userName, role) values('root', 'admin');" + $ hydra-create-user root --role admin To enable a non-admin user to create projects: - - $ sqlite3 hydra.sqlite "insert into UserRoles(userName, role) values('alice', 'create-projects');" - + + $ hydra-create-user root --role create-projects + * Creating a release set: insert into ReleaseSets(project, name) values('patchelf', 'unstable'); diff --git a/doc/manual/projects.xml b/doc/manual/projects.xml index 02b9811d..05791765 100644 --- a/doc/manual/projects.xml +++ b/doc/manual/projects.xml @@ -43,16 +43,11 @@ Identifier: patchelf The identifier should be a unique name (it is the primary database key for the project table in the database). If you try to create a project with an already existing identifier you'd - get an error message such as: + get an error message from the database. - -I'm very sorry, but an error occurred: -DBIx::Class::ResultSet::create(): DBI Exception: DBD::SQLite::st execute failed: column name is not unique(19) at dbdimp.c line 402 - - - So try to create the project after entering just the general - information to figure out if you have chosen a unique name. - Job sets can be added once the project has been created. + So try to create the project after entering just the general + information to figure out if you have chosen a unique name. + Job sets can be added once the project has been created. Display name: Patchelf diff --git a/flake.nix b/flake.nix index 50a60f85..15fc6c75 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,6 @@ CatalystXRoleApplicator CryptRandPasswd DBDPg - DBDSQLite DataDump DateTime DigestSHA1 @@ -103,7 +102,7 @@ src = self; buildInputs = - [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx + [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig libpqxx gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt perlDeps perl final.nix postgresql95 # for running the tests @@ -114,7 +113,7 @@ ]; hydraPath = lib.makeBinPath ( - [ sqlite subversion openssh final.nix coreutils findutils pixz + [ subversion openssh final.nix coreutils findutils pixz gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); diff --git a/src/lib/Hydra/Model/DB.pm b/src/lib/Hydra/Model/DB.pm index c1c2f13e..7514302e 100644 --- a/src/lib/Hydra/Model/DB.pm +++ b/src/lib/Hydra/Model/DB.pm @@ -10,11 +10,7 @@ sub getHydraPath { } sub getHydraDBPath { - my $db = $ENV{"HYDRA_DBI"}; - return $db if defined $db; - my $path = getHydraPath . '/hydra.sqlite'; - #warn "The Hydra database ($path) does not exist!\n" unless -f $path; - return "dbi:SQLite:$path"; + return $ENV{"HYDRA_DBI"} || "dbi:Pg:dbname=hydra;"; } __PACKAGE__->config( diff --git a/src/script/hydra-init b/src/script/hydra-init index 0b564ce6..d9069d4d 100755 --- a/src/script/hydra-init +++ b/src/script/hydra-init @@ -25,9 +25,8 @@ my @tables = $dbh->tables; if (! grep { /SchemaVersion/i } @tables) { print STDERR "initialising the Hydra database schema...\n"; my $schema = read_file( - $dbh->{Driver}->{Name} eq 'SQLite' ? "$home/sql/hydra-sqlite.sql" : $dbh->{Driver}->{Name} eq 'Pg' ? "$home/sql/hydra-postgresql.sql" : - die "unsupported database type\n"); + die "unsupported database type $dbh->{Driver}->{Name}\n"); my @statements = $sql_splitter->split($schema); eval { $dbh->begin_work; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index a5fdc802..d6722597 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -79,14 +79,8 @@ create table Jobsets ( primary key (project, name), foreign key (project) references Projects(name) on delete cascade on update cascade, constraint Jobsets_id_unique UNIQUE(id) -#ifdef SQLITE - , - foreign key (project, name, nixExprInput) references JobsetInputs(project, jobset, name) -#endif ); -#ifdef POSTGRESQL - create function notifyJobsetSharesChanged() returns trigger as 'begin notify jobset_shares_changed; return null; end;' language plpgsql; create trigger JobsetSharesChanged after update on Jobsets for each row when (old.schedulingShares != new.schedulingShares) execute procedure notifyJobsetSharesChanged(); @@ -104,9 +98,6 @@ create trigger JobsetSchedulingChanged after update on Jobsets for each row or (old.enabled != new.enabled)) execute procedure notifyJobsetSchedulingChanged(); -#endif - - create table JobsetRenames ( project text not null, from_ text not null, @@ -157,11 +148,7 @@ create table Jobs ( create table Builds ( -#ifdef POSTGRESQL id serial primary key not null, -#else - id integer primary key autoincrement not null, -#endif finished integer not null, -- 0 = scheduled, 1 = finished @@ -244,8 +231,6 @@ create table Builds ( ); -#ifdef POSTGRESQL - create function notifyBuildsDeleted() returns trigger as 'begin notify builds_deleted; return null; end;' language plpgsql; create trigger BuildsDeleted after delete on Builds execute procedure notifyBuildsDeleted(); @@ -261,8 +246,6 @@ create function notifyBuildBumped() returns trigger as 'begin notify builds_bump create trigger BuildBumped after update on Builds for each row when (old.globalPriority != new.globalPriority) execute procedure notifyBuildBumped(); -#endif - create table BuildOutputs ( build integer not null, @@ -332,11 +315,7 @@ create table BuildStepOutputs ( -- Inputs of builds. create table BuildInputs ( -#ifdef POSTGRESQL id serial primary key not null, -#else - id integer primary key autoincrement not null, -#endif -- Which build this input belongs to. build integer, @@ -502,11 +481,7 @@ create table ReleaseMembers ( create table JobsetEvals ( -#ifdef POSTGRESQL id serial primary key not null, -#else - id integer primary key autoincrement not null, -#endif project text not null, jobset text not null, @@ -577,11 +552,7 @@ create table UriRevMapper ( create table NewsItems ( -#ifdef POSTGRESQL id serial primary key not null, -#else - id integer primary key autoincrement not null, -#endif contents text not null, createTime integer not null, author text not null, @@ -614,7 +585,6 @@ create table FailedPaths ( path text primary key not null ); -#ifdef POSTGRESQL -- Needed because Postgres doesn't have "ignore duplicate" or upsert -- yet. @@ -622,7 +592,6 @@ create rule IdempotentInsert as on insert to FailedPaths where exists (select 1 from FailedPaths where path = new.path) do instead nothing; -#endif create table SystemStatus ( @@ -639,7 +608,6 @@ create table NrBuilds ( insert into NrBuilds(what, count) values('finished', 0); -#ifdef POSTGRESQL create function modifyNrBuildsFinished() returns trigger as $$ begin @@ -658,8 +626,6 @@ create trigger NrBuildsFinished after insert or update or delete on Builds for each row execute procedure modifyNrBuildsFinished(); -#endif - -- Some indices. @@ -704,7 +670,6 @@ create index IndexJobsetEvalsOnJobsetId on JobsetEvals(project, jobset, id desc) create index IndexBuildsOnNotificationPendingSince on Builds(notificationPendingSince) where notificationPendingSince is not null; -#ifdef POSTGRESQL -- The pg_trgm extension has to be created by a superuser. The NixOS -- module creates this extension in the systemd prestart script. We -- then ensure the extension has been created before creating the @@ -721,4 +686,3 @@ exception when others then raise warning 'HINT: Temporary provide superuser role to your Hydra Postgresql user and run the script src/sql/upgrade-57.sql'; raise warning 'The pg_trgm index on builds.drvpath has been skipped (slower complex queries on builds.drvpath)'; end$$; -#endif diff --git a/tests/Makefile.am b/tests/Makefile.am index d3153a55..b0881bce 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -31,9 +31,6 @@ TESTS = \ check_SCRIPTS = repos -db.sqlite: $(top_srcdir)/src/sql/hydra-sqlite.sql - $(TESTS_ENVIRONMENT) $(top_srcdir)/src/script/hydra-init - repos: dirs dirs: