From 5932243fcc788f320d66bca9ce60bc84fb70011f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Aug 2022 14:38:27 -0500 Subject: [PATCH 1/5] Support fetching plain files Add supports for new type = "file" locks added in https://github.com/NixOS/nix/commit/5b8c1deb18e0e6fc7a83fb8101cf5fc8dba38843. Unfortunately, the hash provided by the lock file is a recursive hash, while fetchurl wants a flat hash. So we have to use a custom fetchurl function using the builtin:fetchurl builder. Hopefully okay just to add compat for this very useful lock type. --- default.nix | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/default.nix b/default.nix index f8af091..9f98a87 100644 --- a/default.nix +++ b/default.nix @@ -13,6 +13,40 @@ let lockFile = builtins.fromJSON (builtins.readFile lockFilePath); + # Using custom fetchurl function here so that we can specify outputHashMode. + # The hash we get from the lock file is using recursive ingestion even though + # it’s not unpacked. So builtins.fetchurl and import are + # insufficient. + fetchurl = { url, sha256 }: + derivation { + builder = "builtin:fetchurl"; + + name = "source"; + inherit url; + + outputHash = sha256; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + executable = false; + unpack = false; + + system = "builtin"; + + # No need to double the amount of network traffic + preferLocalBuild = true; + + impureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + # To make "nix-prefetch-url" work. + urls = [ url ]; + }; + fetchTree = info: if info.type == "github" then @@ -53,6 +87,7 @@ let ({ inherit (info) url; } // (if info ? narHash then { sha256 = info.narHash; } else {}) ); + narHash = info.narHash; } else if info.type == "gitlab" then { inherit (info) rev narHash lastModified; @@ -63,6 +98,12 @@ let ); shortRev = builtins.substring 0 7 info.rev; } + else if info.type == "file" then + { outPath = fetchurl + ({ inherit (info) url; } + // (if info ? narHash then { sha256 = info.narHash; } else {})); + narHash = info.narHash; + } else # FIXME: add Mercurial, tarball inputs. throw "flake input has unsupported input type '${info.type}'"; -- 2.44.1 From 6e1fbe397a5cc615bafb20cacfcb162c64bde5be Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Aug 2022 15:01:39 -0500 Subject: [PATCH 2/5] Support local files too Add support for local files in addition to http / https. --- default.nix | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/default.nix b/default.nix index 9f98a87..ee455a4 100644 --- a/default.nix +++ b/default.nix @@ -83,10 +83,12 @@ let } else if info.type == "tarball" then { outPath = - fetchTarball - ({ inherit (info) url; } - // (if info ? narHash then { sha256 = info.narHash; } else {}) - ); + if builtins.substring 0 7 info.url == "http://" || builtins.substring 0 8 info.url == "https://" then + fetchTarball + ({ inherit (info) url; } + // (if info ? narHash then { sha256 = info.narHash; } else {}) + ) + else throw "can't support url scheme of flake input with url '${info.url}'"; narHash = info.narHash; } else if info.type == "gitlab" then @@ -99,13 +101,18 @@ let shortRev = builtins.substring 0 7 info.rev; } else if info.type == "file" then - { outPath = fetchurl - ({ inherit (info) url; } - // (if info ? narHash then { sha256 = info.narHash; } else {})); + { outPath = + if builtins.substring 0 7 info.url == "http://" || builtins.substring 0 8 info.url == "https://" then + fetchurl + ({ inherit (info) url; } + // (if info ? narHash then { sha256 = info.narHash; } else {})) + else if builtins.substring 0 7 info.url == "file://" then + builtins.path { path = builtins.substring 7 (-1) info.url; } + else throw "can't support url scheme of flake input with url '${info.url}'"; narHash = info.narHash; } else - # FIXME: add Mercurial, tarball inputs. + # FIXME: add Mercurial input throw "flake input has unsupported input type '${info.type}'"; callFlake4 = flakeSrc: locks: -- 2.44.1 From 0b194c44aa03fc0d23f84e93ee8d98e526658025 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Aug 2022 15:06:04 -0500 Subject: [PATCH 3/5] Add hash to builtins.path calls --- default.nix | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index ee455a4..7d7b467 100644 --- a/default.nix +++ b/default.nix @@ -78,7 +78,10 @@ let } else { }) else if info.type == "path" then - { outPath = builtins.path { path = info.path; }; + { outPath = builtins.path + ({ path = info.path; } + // (if info ? narHash then { sha256 = info.narHash; } else {}) + ); narHash = info.narHash; } else if info.type == "tarball" then @@ -105,9 +108,13 @@ let if builtins.substring 0 7 info.url == "http://" || builtins.substring 0 8 info.url == "https://" then fetchurl ({ inherit (info) url; } - // (if info ? narHash then { sha256 = info.narHash; } else {})) + // (if info ? narHash then { sha256 = info.narHash; } else {}) + ) else if builtins.substring 0 7 info.url == "file://" then - builtins.path { path = builtins.substring 7 (-1) info.url; } + builtins.path + ({ path = builtins.substring 7 (-1) info.url; } + // (if info ? narHash then { sha256 = info.narHash; } else {}) + ) else throw "can't support url scheme of flake input with url '${info.url}'"; narHash = info.narHash; } -- 2.44.1 From b1d2eff40adad79cf8b31a85a76f5becff4b697f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 29 Aug 2022 12:00:24 -0500 Subject: [PATCH 4/5] Discard context on fetchurl derivation so that fetchurl outputs a real path instead of a source, and our hashes match correctly between flake and non-flake. also it looks like fetchTarball can support local files --- default.nix | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/default.nix b/default.nix index 7d7b467..89ec253 100644 --- a/default.nix +++ b/default.nix @@ -18,7 +18,8 @@ let # it’s not unpacked. So builtins.fetchurl and import are # insufficient. fetchurl = { url, sha256 }: - derivation { + # need to hide the fact it’s a derivation so that the output is just a path + builtins.storePath (builtins.unsafeDiscardStringContext (derivation { builder = "builtin:fetchurl"; name = "source"; @@ -45,7 +46,7 @@ let # To make "nix-prefetch-url" work. urls = [ url ]; - }; + }).outPath); fetchTree = info: @@ -86,12 +87,10 @@ let } else if info.type == "tarball" then { outPath = - if builtins.substring 0 7 info.url == "http://" || builtins.substring 0 8 info.url == "https://" then - fetchTarball - ({ inherit (info) url; } - // (if info ? narHash then { sha256 = info.narHash; } else {}) - ) - else throw "can't support url scheme of flake input with url '${info.url}'"; + fetchTarball + ({ inherit (info) url; } + // (if info ? narHash then { sha256 = info.narHash; } else {}) + ); narHash = info.narHash; } else if info.type == "gitlab" then -- 2.44.1 From c1217c6a4e1e47e68988bdcba1403533b7d8dbad Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 29 Aug 2022 13:39:54 -0500 Subject: [PATCH 5/5] Add note --- default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index 89ec253..b6792d0 100644 --- a/default.nix +++ b/default.nix @@ -17,9 +17,10 @@ let # The hash we get from the lock file is using recursive ingestion even though # it’s not unpacked. So builtins.fetchurl and import are # insufficient. + # Note that this will be a derivation and not a path as fetchTarball is, + # causing the hash of this input to be different on flake and non-flake evaluation. fetchurl = { url, sha256 }: - # need to hide the fact it’s a derivation so that the output is just a path - builtins.storePath (builtins.unsafeDiscardStringContext (derivation { + derivation { builder = "builtin:fetchurl"; name = "source"; @@ -46,7 +47,7 @@ let # To make "nix-prefetch-url" work. urls = [ url ]; - }).outPath); + }; fetchTree = info: -- 2.44.1