From b13fc7101fb06a65935d145081319145f7fef6f9 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 14 Aug 2023 12:24:26 +0200
Subject: [PATCH] Add positive source filter

Source filtering is a really cool Nix feature that lets us avoid a
lot of rebuilds, which speeds up the iteration cycle a lot in cases
where the relevant source files aren't actually modified.

We used to have a source filter that marked a few files as irrelevant,
but this is the wrong approach, as we have many more files that are
irrelevant. We may call this negative filtering.

This commit switches the source filtering to positive filtering, which
is a lot more robust. Instead of marking which files we don't need
we marked the files that we do need.

It's a superior approach because it is fail safe. Instead of allowing
build performance problems to creep in over time, we require that all
source inputs are declared.

I shouldn't have to explain that declaring inputs is a good practice,
so I'll stop over-explaining here.

I do have to acknowledge that this will cause a build failure when the
filter is incomplete. This is *good*, because it's the only realistic
way we could be reminded of these problems. These events will be
infrequent, so the small cost of extending the filter is worth it,
compared to the hidden cost of longer dev cycles for things like tests,
docker image, etc, etc.

(Also rebuilding Nix for stupid unnecessary reasons makes my blood boil)
---
 flake.nix | 61 +++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/flake.nix b/flake.nix
index bdbf54169..28e581309 100644
--- a/flake.nix
+++ b/flake.nix
@@ -40,6 +40,52 @@
             })
             stdenvs);
 
+      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
+      # Not an "idiomatic" flake input because:
+      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
+      #  - Subflake would download redundant and huge parent flake
+      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
+      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nixpkgs/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
+        fileset;
+
+      baseFiles =
+        # .gitignore has already been processed, so any changes in it are irrelevant
+        # at this point. It is not represented verbatim for test purposes because
+        # that would interfere with repo semantics.
+        fileset.fileFilter (f: f.name != ".gitignore") ./.;
+
+      nixSrc = fileset.toSource {
+        root = ./.;
+        fileset = fileset.intersect baseFiles (
+          fileset.difference
+            (fileset.unions [
+              ./.version
+              ./boehmgc-coroutine-sp-fallback.diff
+              ./bootstrap.sh
+              ./configure.ac
+              ./doc
+              ./local.mk
+              ./m4
+              ./Makefile
+              ./Makefile.config.in
+              ./misc
+              ./mk
+              ./precompiled-headers.h
+              ./src
+              ./tests
+              ./COPYING
+              ./scripts/local.mk
+              (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+              # TODO: do we really need README.md? It doesn't seem used in the build.
+              ./README.md
+            ])
+            (fileset.unions [
+              # Removed file sets
+              ./tests/nixos
+              ./tests/installer
+            ])
+        );
+      };
 
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
@@ -209,7 +255,7 @@
             "-${client.version}-against-${daemon.version}";
         inherit version;
 
-        src = self;
+        src = nixSrc;
 
         VERSION_SUFFIX = versionSuffix;
 
@@ -320,18 +366,11 @@
           };
           let
             canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-
-            sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith {
-              filter = (path: type:
-                let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path);
-                in ! lib.any (re: builtins.match re relPath != null) rxs);
-              src = origSrc;
-            };
           in currentStdenv.mkDerivation (finalAttrs: {
             name = "nix-${version}";
             inherit version;
 
-            src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self;
+            src = nixSrc;
             VERSION_SUFFIX = versionSuffix;
 
             outputs = [ "out" "dev" "doc" ];
@@ -529,7 +568,7 @@
           releaseTools.coverageAnalysis {
             name = "nix-coverage-${version}";
 
-            src = self;
+            src = nixSrc;
 
             configureFlags = testConfigureFlags;
 
@@ -557,7 +596,7 @@
             pname = "nix-internal-api-docs";
             inherit version;
 
-            src = self;
+            src = nixSrc;
 
             configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags;