From 6ada4963111908211a4bbcc2ae65f4205cffaa18 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 4 Oct 2021 09:00:01 +0100 Subject: [PATCH 1/6] nix: add (failing) selfreference test for multiple realizations The test illustrates failure in issue #5320. Here derivation and it's built input have identical CA sotre path. As a result we generate extraneout reference to build input: $ make installcheck ... ran test tests/selfref-gc.sh... [PASS] ran test tests/ca/selfref-gc.sh... [FAIL] ... deleting '/tmp/.../tests/ca/selfref-gc/store/iqciq1mpg5hc7p6a52fp2bjxbyc9av0v-selfref-gc' deleting '/tmp/...tests/ca/selfref-gc/store/zh0kwpnirw3qbv6dl1ckr1y0kd5aw6ax-selfref-gc.drv' error: executing SQLite statement 'delete from ValidPaths where path = '/tmp/.../tests/ca/selfref-gc/store/fsjq0k146r85lsh01l0icl30rnhv7z72-selfref-gc';': constraint failed (in '/tmp/.../tests/ca/selfref-gc/var/nix/db/db.sqlite') --- tests/ca/selfref-gc.sh | 11 +++++++++++ tests/local.mk | 1 + tests/selfref-gc.sh | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100755 tests/ca/selfref-gc.sh create mode 100644 tests/selfref-gc.sh diff --git a/tests/ca/selfref-gc.sh b/tests/ca/selfref-gc.sh new file mode 100755 index 000000000..89657f6de --- /dev/null +++ b/tests/ca/selfref-gc.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +requireDaemonNewerThan "2.4pre20210626" + +sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./selfref-gc.sh diff --git a/tests/local.mk b/tests/local.mk index cb869f32e..e3c4ff4eb 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -92,6 +92,7 @@ nix_tests = \ plugins.sh \ build.sh \ ca/nix-run.sh \ + selfref-gc.sh ca/selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ diff --git a/tests/selfref-gc.sh b/tests/selfref-gc.sh new file mode 100644 index 000000000..cf1c4d18e --- /dev/null +++ b/tests/selfref-gc.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore + +nix-build --no-out-link -E ' + with import ./config.nix; + + let d1 = mkDerivation { + name = "selfref-gc"; + outputs = [ "out" ]; + buildCommand = " + echo SELF_REF: $out > $out + "; + }; in + + # the only change from d1 is d1 as an (unused) build input + # to get identical store path in CA. + mkDerivation { + name = "selfref-gc"; + outputs = [ "out" ]; + buildCommand = " + echo UNUSED: ${d1} + echo SELF_REF: $out > $out + "; + } +' + +nix-collect-garbage From 92656da0b953b92228ef4a252d504c07710e4b47 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 3 Nov 2021 16:30:22 +0100 Subject: [PATCH 2/6] Fix the gc with indirect self-references via the realisations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the derivation `foo` depends on `bar`, and they both have the same output path (because they are CA derivations), then this output path will depend both on the realisation of `foo` and of `bar`, which themselves depend on each other. This confuses SQLite which isn’t able to automatically solve this diamond dependency scheme. Help it by adding a trigger to delete all the references between the relevant realisations. Fix #5320 --- src/libstore/ca-specific-schema.sql | 13 +++++++++++++ src/libstore/local-store.cc | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 64cc97fde..d2ea347fb 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -13,6 +13,19 @@ create table if not exists Realisations ( create index if not exists IndexRealisations on Realisations(drvPath, outputName); +-- We can end-up in a weird edge-case where a path depends on itself because +-- it’s an output of a CA derivation, that happens to be the same as one of its +-- dependencies. +-- In that case we have a dependency loop (path -> realisation1 -> realisation2 +-- -> path) that we need to break by removing the dependencies between the +-- realisations +create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference = ( + select id from Realisations where outputPath = old.id + ); + end; + create table if not exists RealisationsRefs ( referrer integer not null, realisationReference integer, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ece5bb5ef..e64917956 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -81,7 +81,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 3; + const int nixCASchemaVersion = 4; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -143,6 +143,19 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) )"); txn.commit(); } + if (curCASchema < 4) { + SQLiteTxn txn(db); + db.exec(R"( + create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference = ( + select id from Realisations where outputPath = old.id + ); + end; + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } From b6e59d71371c85277b3ad3d5fe6352f2462d39e5 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sat, 26 Mar 2022 16:35:06 +0000 Subject: [PATCH 3/6] tests: remove 'ca-references' feature The feature was ctabilized in d589a6aa8a5d0c9f391400d7e0e209106e89c857. --- tests/build-remote-content-addressed-floating.sh | 2 +- tests/ca/common.sh | 2 +- tests/ca/selfref-gc.sh | 2 +- tests/nix-profile.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh index 1f474dde0..e83b42b41 100644 --- a/tests/build-remote-content-addressed-floating.sh +++ b/tests/build-remote-content-addressed-floating.sh @@ -2,7 +2,7 @@ source common.sh file=build-hook-ca-floating.nix -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" CONTENT_ADDRESSED=true diff --git a/tests/ca/common.sh b/tests/ca/common.sh index b9d415863..b104b5a78 100644 --- a/tests/ca/common.sh +++ b/tests/ca/common.sh @@ -1,5 +1,5 @@ source ../common.sh -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" restartDaemon diff --git a/tests/ca/selfref-gc.sh b/tests/ca/selfref-gc.sh index 89657f6de..248778894 100755 --- a/tests/ca/selfref-gc.sh +++ b/tests/ca/selfref-gc.sh @@ -4,7 +4,7 @@ source common.sh requireDaemonNewerThan "2.4pre20210626" -sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf +enableFeatures "ca-derivations nix-command flakes" export NIX_TESTS_CA_BY_DEFAULT=1 cd .. diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index a7a4d4fa2..fad62b993 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -3,7 +3,7 @@ source common.sh clearStore clearProfiles -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" restartDaemon # Make a flake. From 74d6782a6a864eed4c49ab606d130aec1201e584 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 3 Nov 2021 17:52:49 +0100 Subject: [PATCH 4/6] Disable the selfref-gc test when the daemon is too old --- tests/selfref-gc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/selfref-gc.sh b/tests/selfref-gc.sh index cf1c4d18e..3f1f50eea 100644 --- a/tests/selfref-gc.sh +++ b/tests/selfref-gc.sh @@ -1,5 +1,7 @@ source common.sh +requireDaemonNewerThan "2.6.0pre20211215" + clearStore nix-build --no-out-link -E ' From 975b0b52e74168b034ccff23827eff4d626f1c46 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sat, 26 Mar 2022 16:18:51 +0000 Subject: [PATCH 5/6] ca: add sqlite index on `RealisationsRefs(realisationReference)` Without the change any CA deletion triggers linear scan on large RealisationsRefs table: sqlite>.eqp full sqlite> delete from RealisationsRefs where realisationReference IN ( select id from Realisations where outputPath = 1234567890 ); QUERY PLAN |--SCAN RealisationsRefs `--LIST SUBQUERY 1 `--SEARCH Realisations USING COVERING INDEX IndexRealisationsRefsOnOutputPath (outputPath=?) With the change it gets turned into a lookup: sqlite> CREATE INDEX IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); sqlite> delete from RealisationsRefs where realisationReference IN ( select id from Realisations where outputPath = 1234567890 ); QUERY PLAN |--SEARCH RealisationsRefs USING INDEX IndexRealisationsRefsRealisationReference (realisationReference=?) `--LIST SUBQUERY 1 `--SEARCH Realisations USING COVERING INDEX IndexRealisationsRefsOnOutputPath (outputPath=?) --- src/libstore/ca-specific-schema.sql | 2 ++ src/libstore/local-store.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c01dc5a76..4ca91f585 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -32,6 +32,8 @@ create table if not exists RealisationsRefs ( foreign key (referrer) references Realisations(id) on delete cascade, foreign key (realisationReference) references Realisations(id) on delete restrict ); +-- used by deletion trigger +create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); -- used by QueryRealisationReferences create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76524b01c..5cc5c91cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -152,6 +152,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) select id from Realisations where outputPath = old.id ); end; + -- used by deletion trigger + create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); )"); txn.commit(); } From 86d7a11c6b338a73c30476be13a6cac562e309c3 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Nov 2021 11:17:22 +0100 Subject: [PATCH 6/6] Make sure to delete all the realisation refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting just one will only work in the test cases where I didn’t bother creating too many of them :p --- src/libstore/ca-specific-schema.sql | 2 +- src/libstore/local-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index d2ea347fb..c01dc5a76 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -21,7 +21,7 @@ create index if not exists IndexRealisations on Realisations(drvPath, outputName -- realisations create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths begin - delete from RealisationsRefs where realisationReference = ( + delete from RealisationsRefs where realisationReference in ( select id from Realisations where outputPath = old.id ); end; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e64917956..76524b01c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -148,7 +148,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) db.exec(R"( create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths begin - delete from RealisationsRefs where realisationReference = ( + delete from RealisationsRefs where realisationReference in ( select id from Realisations where outputPath = old.id ); end;