Deprecate the online flake registries and vendor the default registry

Fixes #183, #110, #116.

The default flake-registry option becomes 'vendored', and refers
to a vendored flake-registry.json file in the install path.

Vendored copy of the flake-registry is from github:NixOS/flake-registry
at commit 9c69f7bd2363e71fe5cd7f608113290c7614dcdd.

Change-Id: I752b81c85ebeaab4e582ac01c239d69d65580f37

doc/manual/rl-next/deprecate-online-flake-registry.md

@@ -0,0 +1,16 @@
+---
+synopsis: "Deprecate the online flake registries and vendor the default registry"
+cls: 1127
+credits: midnightveil
+issues: [fj#183, fj#110, fj#116, 8953, 9087]
+category: Breaking Changes
+---
+
+The online flake registry [https://channels.nixos.org/flake-registry.json](https://channels.nixos.org/flake-registry.json) is not pinned in any way,
+and the targets of the indirections can both update or change entirely at any
+point. Furthermore, it is refetched on every use of a flake reference, even if
+there is a local flake reference, and even if you are offline (which breaks).
+
+For now, we deprecate the (any) online flake registry, and vendor a copy of the
+current online flake registry. This makes it work offline, and ensures that
+it won't change in the future.

misc/flake-registry/flake-registry.json

@@ -0,0 +1,414 @@
+{
+  "flakes": [
+    {
+      "from": {
+        "id": "agda",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "agda",
+        "repo": "agda",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "arion",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "hercules-ci",
+        "repo": "arion",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "blender-bin",
+        "type": "indirect"
+      },
+      "to": {
+        "dir": "blender",
+        "owner": "edolstra",
+        "repo": "nix-warez",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "bundlers",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "bundlers",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "cachix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "cachix",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "composable",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "ComposableFi",
+        "repo": "composable",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "disko",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "dreampkgs",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "dreampkgs",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "dwarffs",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "edolstra",
+        "repo": "dwarffs",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "emacs-overlay",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "emacs-overlay",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "fenix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "flake-parts",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "flake-utils",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "gemini",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "flake-gemini",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "helix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "helix-editor",
+        "repo": "helix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "hercules-ci-agent",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-agent",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "hercules-ci-effects",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-effects",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "home-manager",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "hydra",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "hydra",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "mach-nix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "DavHau",
+        "repo": "mach-nix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nickel",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "tweag",
+        "repo": "nickel",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nimble",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "flake-nimble",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nix-darwin",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "LnL7",
+        "repo": "nix-darwin",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nix-serve",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "edolstra",
+        "repo": "nix-serve",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nixops",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "nixops",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nixos-hardware",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "nixos-hardware",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nixos-homepage",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "nixos-homepage",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nixos-search",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "nixos-search",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "nur",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "NUR",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "patchelf",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "patchelf",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "poetry2nix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-community",
+        "repo": "poetry2nix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "pridefetch",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "SpyHoodle",
+        "repo": "pridefetch",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "sops-nix",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "systems",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    {
+      "from": {
+        "id": "templates",
+        "type": "indirect"
+      },
+      "to": {
+        "owner": "NixOS",
+        "repo": "templates",
+        "type": "github"
+      }
+    }
+  ],
+  "version": 2
+}

misc/flake-registry/meson.build

@@ -0,0 +1,4 @@
+install_data(
+  'flake-registry.json',
+  install_dir : datadir,
+)

misc/meson.build

@@ -3,3 +3,4 @@ subdir('fish')
 subdir('zsh')
 
 subdir('systemd')
+subdir('flake-registry')

package.nix

@@ -313,6 +313,8 @@ stdenv.mkDerivation (finalAttrs: {
     "--suite=check"
     "--print-errorlogs"
   ];
+  # the tests access localhost.
+  __darwinAllowLocalNetworking = true;
 
   # Make sure the internal API docs are already built, because mesonInstallPhase
   # won't let us build them there. They would normally be built in buildPhase,

src/libfetchers/fetch-settings.hh

@@ -71,10 +71,13 @@ struct FetchSettings : public Config
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
 
-    Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
+    Setting<std::string> flakeRegistry{this, "vendored", "flake-registry",
         R"(
           Path or URI of the global flake registry.
 
+          URIs are deprecated. When set to 'vendored', defaults to a vendored
+          copy of https://channels.nixos.org/flake-registry.json.
+
           When empty, disables the global flake registry.
         )",
         {}, true, Xp::Flakes};

src/libfetchers/registry.cc

@@ -16,8 +16,12 @@ std::shared_ptr<Registry> Registry::read(
 {
     auto registry = std::make_shared<Registry>(type);
 
-    if (!pathExists(path))
+    if (!pathExists(path)) {
+        if (type == RegistryType::Global) {
+            warn("cannot read flake registry '%s': path does not exist", path);
+        }
         return std::make_shared<Registry>(type);
+    }
 
     try {
 
@@ -155,9 +159,13 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
         auto path = fetchSettings.flakeRegistry.get();
         if (path == "") {
             return std::make_shared<Registry>(Registry::Global); // empty registry
+        } else if (path == "vendored") {
+            return Registry::read(settings.nixDataDir + "/flake-registry.json", Registry::Global);
         }
 
         if (!path.starts_with("/")) {
+            warn("config option flake-registry referring to a URL is deprecated and will be removed in Lix 3.0; yours is: `%s'", path);
+
             auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");

tests/functional/flakes/flake-registry.sh

@@ -0,0 +1,72 @@
+source ./common.sh
+
+# remove the flake registry from nix.conf, to set to default ("vendored")
+sed -i '/flake-registry/d' "$NIX_CONF_DIR/nix.conf"
+
+# Make sure the vendored registry contains the correct amount.
+[[ $(nix registry list | wc -l) == 37 ]]
+# sanity check, contains the important ones
+nix registry list | grep '^global flake:nixpkgs'
+nix registry list | grep '^global flake:home-manager'
+
+
+# it should work the same if we set to vendored directly.
+echo 'flake-registry = vendored' >> "$NIX_CONF_DIR/nix.conf"
+[[ $(nix registry list | wc -l) == 37 ]]
+# sanity check, contains the important ones
+nix registry list | grep '^global flake:nixpkgs'
+nix registry list | grep '^global flake:home-manager'
+
+
+# the online flake registry should still work, but it is deprecated.
+set -m
+# port 0: auto pick a free port, unbufferred output
+python3 -u -m http.server 0 --bind 127.0.0.1 > server.out &
+# wait for the http server to admit it is working
+while ! grep -qP 'port \d+' server.out ; do
+  echo 'waiting for python http' >&2
+  sleep 0.2
+done
+
+port=$(awk 'match($0,/port ([[:digit:]]+)/, ary) { print ary[1] }' server.out)
+
+sed -i '/flake-registry/d' "$NIX_CONF_DIR/nix.conf"
+echo "flake-registry = http://127.0.0.1:$port/flake-registry.json" >> "$NIX_CONF_DIR/nix.conf"
+cat <<EOF > flake-registry.json
+{
+    "flakes": [
+        {
+        "from": {
+            "type": "indirect",
+            "id": "nixpkgs"
+        },
+        "to": {
+            "type": "github",
+            "owner": "NixOS",
+            "repo": "nixpkgs"
+        }
+        },
+        {
+        "from": {
+            "type": "indirect",
+            "id": "private-flake"
+        },
+        "to": {
+            "type": "github",
+            "owner": "fancy-enterprise",
+            "repo": "private-flake"
+        }
+        }
+    ],
+    "version": 2
+}
+EOF
+
+[[ $(nix registry list | wc -l) == 2 ]]
+nix registry list | grep '^global flake:nixpkgs'
+nix registry list | grep '^global flake:private-flake'
+
+# make sure we have a warning:
+nix registry list 2>&1 | grep "config option flake-registry referring to a URL is deprecated and will be removed"
+
+kill %1

tests/functional/meson.build

@@ -69,6 +69,7 @@ functional_tests_scripts = [
   'flakes/unlocked-override.sh',
   'flakes/absolute-paths.sh',
   'flakes/build-paths.sh',
+  'flakes/flake-registry.sh',
   'flakes/flake-in-submodule.sh',
   'gc.sh',
   'nix-collect-garbage-d.sh',

tests/nixos/github-flakes.nix

@@ -146,6 +146,8 @@ in
           virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ];
           virtualisation.memorySize = 4096;
           nix.settings.substituters = lib.mkForce [ ];
+          # note: URL flake-registries are currently deprecated.
+          nix.settings.flake-registry = "https://channels.nixos.org/flake-registry.json";
           nix.extraOptions = "experimental-features = nix-command flakes";
           networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} =
             [ "channels.nixos.org" "api.github.com" "github.com" ];