From df7dab12911474fe3c6f885a2b404bb4a03711d4 Mon Sep 17 00:00:00 2001 From: Damien Diederen Date: Tue, 15 Jun 2021 10:31:42 +0200 Subject: [PATCH] GitInput: Include deepClone option in the cache key Without this commit, two jobsets using the same repository as input, but different `deepClone` options, end up incorrectly sharing the same "checkout" for a given (`uri`, `branch`, `revision`) tuple. The presence or absence of `.git` is determined by the jobset execution order. This patch adds the missing `isDeepClone` boolean to the cache key. The database upgrade script empties the `CachedGitInputs` table, as we don't know if existing checkouts are deep clones. Unfortunately, this generally forces rebuilds even for correct `deepClone` checkouts, as the binary contents of `.git` are not deterministic. Fixes #510 --- src/lib/Hydra/Plugin/GitInput.pm | 3 ++- src/lib/Hydra/Schema/CachedGitInputs.pm | 15 ++++++++++++--- src/sql/hydra.sql | 3 ++- src/sql/upgrade-76.sql | 12 ++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 src/sql/upgrade-76.sql diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm index 3fbf50f0..848c0dee 100644 --- a/src/lib/Hydra/Plugin/GitInput.pm +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -182,7 +182,7 @@ sub fetchInput { # TODO: Fix case where the branch is reset to a previous commit. my $cachedInput; ($cachedInput) = $self->{db}->resultset('CachedGitInputs')->search( - {uri => $uri, branch => $branch, revision => $revision}, + {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0}, {rows => 1}); addTempRoot($cachedInput->storepath) if defined $cachedInput; @@ -223,6 +223,7 @@ sub fetchInput { { uri => $uri , branch => $branch , revision => $revision + , isdeepclone => defined($deepClone) ? 1 : 0 , sha256hash => $sha256 , storepath => $storePath }); diff --git a/src/lib/Hydra/Schema/CachedGitInputs.pm b/src/lib/Hydra/Schema/CachedGitInputs.pm index d85f4970..4a2b102b 100644 --- a/src/lib/Hydra/Schema/CachedGitInputs.pm +++ b/src/lib/Hydra/Schema/CachedGitInputs.pm @@ -50,6 +50,11 @@ __PACKAGE__->table("cachedgitinputs"); data_type: 'text' is_nullable: 0 +=head2 isdeepclone + + data_type: 'boolean' + is_nullable: 0 + =head2 sha256hash data_type: 'text' @@ -69,6 +74,8 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 0 }, "revision", { data_type => "text", is_nullable => 0 }, + "isdeepclone", + { data_type => "boolean", is_nullable => 0 }, "sha256hash", { data_type => "text", is_nullable => 0 }, "storepath", @@ -85,14 +92,16 @@ __PACKAGE__->add_columns( =item * L +=item * L + =back =cut -__PACKAGE__->set_primary_key("uri", "branch", "revision"); +__PACKAGE__->set_primary_key("uri", "branch", "revision", "isdeepclone"); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:22:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:0sdK9uQZpx869oqS5thRLw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-06-19 17:26:24 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Wr0grsFTaqRlMeM9vpXjrw 1; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index c664d854..d3748bda 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -402,9 +402,10 @@ create table CachedGitInputs ( uri text not null, branch text not null, revision text not null, + isDeepClone boolean not null, sha256hash text not null, storePath text not null, - primary key (uri, branch, revision) + primary key (uri, branch, revision, isDeepClone) ); create table CachedDarcsInputs ( diff --git a/src/sql/upgrade-76.sql b/src/sql/upgrade-76.sql new file mode 100644 index 00000000..0b402b2a --- /dev/null +++ b/src/sql/upgrade-76.sql @@ -0,0 +1,12 @@ +-- We don't know if existing checkouts are deep clones. This will +-- force a new fetch (and most likely trigger a new build for deep +-- clones, as the binary contents of '.git' are not deterministic). +DELETE FROM CachedGitInputs; + +ALTER TABLE CachedGitInputs + ADD COLUMN isDeepClone BOOLEAN NOT NULL; + +ALTER TABLE CachedGitInputs DROP CONSTRAINT cachedgitinputs_pkey; + +ALTER TABLE CachedGitInputs ADD CONSTRAINT cachedgitinputs_pkey + PRIMARY KEY (uri, branch, revision, isDeepClone);