From b6cc0a704d8c1432e230ff65d4b74ea7114a730b Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 3 Dec 2021 10:53:41 -0500 Subject: [PATCH 1/7] flakes: search up to git or filesystem boundary While parsing a flakeref, upon not finding a flake.nix, search upwards until git or filesystem boundary. --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libexpr/flake/flakeref.cc | 23 +++++++++++++++++++++-- tests/flake-searching.sh | 24 ++++++++++++++++++++++++ tests/local.mk | 1 + 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/flake-searching.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index a6b22dfa7..2826fc8be 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,3 +8,5 @@ * New built-in function: `builtins.groupBy`, with the same functionality as Nixpkgs' `lib.groupBy`, but faster. + +* Nix now searches for a flake.nix up until git or filesystem boundary. diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 29128d789..074727f06 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -117,8 +117,27 @@ std::pair parseFlakeRefWithFragment( if (!S_ISDIR(lstat(path).st_mode)) throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("unable to find a flake before encountering git boundary at '%s'", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } auto flakeRoot = path; std::string subdir; diff --git a/tests/flake-searching.sh b/tests/flake-searching.sh new file mode 100644 index 000000000..82ae66894 --- /dev/null +++ b/tests/flake-searching.sh @@ -0,0 +1,24 @@ +source common.sh + +clearStore + +cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME +cd $TEST_HOME +cat < flake.nix +{ + outputs = a: { + defaultPackage.$system = import ./simple.nix; + packages.$system.test = import ./simple.nix; + }; +} +EOF +mkdir subdir +cd subdir + +for i in "" . "$PWD" .# .#test; do + nix build $i || fail "flake should be found by searching up directories" +done + +for i in "path:$PWD"; do + ! nix build $i || fail "flake should not search up directories when using 'path:'" +done diff --git a/tests/local.mk b/tests/local.mk index 936b72c2a..9277c0b1b 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -47,6 +47,7 @@ nix_tests = \ describe-stores.sh \ flakes.sh \ flake-local-settings.sh \ + flake-searching.sh \ build.sh \ repl.sh ca/repl.sh \ ca/build.sh \ From 2dead2092470f7b0440d34035e19b9d8c80db91e Mon Sep 17 00:00:00 2001 From: tomberek Date: Fri, 14 Jan 2022 09:16:34 -0500 Subject: [PATCH 2/7] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- doc/manual/src/release-notes/rl-next.md | 13 +------------ src/libexpr/flake/flakeref.cc | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 44c6e2c09..49be90f90 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,16 +1,5 @@ # Release X.Y (202?-??-??) - -* Binary cache stores now have a setting `compression-level`. - -* `nix develop` now has a flag `--unpack` to run `unpackPhase`. - -* Lists can now be compared lexicographically using the `<` operator. - -* New built-in function: `builtins.groupBy`, with the same functionality as - Nixpkgs' `lib.groupBy`, but faster. - -* Nix now searches for a flake.nix up until git or filesystem boundary. - +* The Nix cli now searches for a flake.nix up until the root of the current git repository or a filesystem boundary rather than just in the current directory * The TOML parser used by `builtins.fromTOML` has been replaced by [a more compliant one](https://github.com/ToruNiina/toml11). diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index b3f01746e..06f8511b6 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -136,7 +136,7 @@ std::pair parseFlakeRefWithFragment( throw Error("unable to find a flake before encountering git boundary at '%s'", path); else { if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", origPath); } path = dirOf(path); } From e3690ab39382498eaabbd07e696335e17c9f209c Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 14 Jan 2022 17:15:45 +0300 Subject: [PATCH 3/7] Add more tests for flake upward searching --- src/libexpr/flake/flakeref.cc | 37 ++++++++++++++++++----------------- tests/flake-searching.sh | 14 +++++++++++-- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 06f8511b6..930ed9ccd 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -122,27 +122,28 @@ std::pair parseFlakeRefWithFragment( if (isFlake) { - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("unable to find a flake before encountering git boundary at '%s'", path); - else { - if (lstat(path).st_dev != device) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", origPath); + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); } - path = dirOf(path); + if (!found) + throw BadURL("could not find a flake.nix file"); } - if (!found) - throw BadURL("could not find a flake.nix file"); - } + if (!S_ISDIR(lstat(path).st_mode)) throw BadURL("path '%s' is not a flake (because it's not a directory)", path); diff --git a/tests/flake-searching.sh b/tests/flake-searching.sh index 82ae66894..7c2041ce1 100644 --- a/tests/flake-searching.sh +++ b/tests/flake-searching.sh @@ -4,8 +4,11 @@ clearStore cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME cd $TEST_HOME +mkdir -p foo/subdir +echo '{ outputs = _: {}; }' > foo/flake.nix cat < flake.nix { + inputs.foo.url = "$PWD/foo"; outputs = a: { defaultPackage.$system = import ./simple.nix; packages.$system.test = import ./simple.nix; @@ -13,12 +16,19 @@ cat < flake.nix } EOF mkdir subdir -cd subdir +pushd subdir -for i in "" . "$PWD" .# .#test; do +for i in "" . .# .#test ../subdir ../subdir#test "$PWD"; do nix build $i || fail "flake should be found by searching up directories" done for i in "path:$PWD"; do ! nix build $i || fail "flake should not search up directories when using 'path:'" done + +popd + +nix build --override-input foo . || fail "flake should search up directories when not an installable" + +sed "s,$PWD/foo,$PWD/foo/subdir,g" -i flake.nix +! nix build || fail "flake should not search upwards when part of inputs" From f055cc5a0bfc0871d36cc141e14777acd0e214ca Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 14 Jan 2022 17:53:07 +0300 Subject: [PATCH 4/7] Document searching upwards and fix documentation for installables --- src/nix/flake.md | 9 --------- src/nix/nix.md | 30 +++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 3b5812a0a..a8436bcaa 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -137,15 +137,6 @@ Currently the `type` attribute can be one of the following: *path* must be a directory in the file system containing a file named `flake.nix`. - If the directory or any of its parents is a Git repository, then - this is essentially equivalent to `git+file://` (see below), - except that the `dir` parameter is derived automatically. For - example, if `/foo/bar` is a Git repository, then the flake reference - `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`. - - If the directory is not inside a Git repository, then the flake - contents is the entire contents of *path*. - *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is interpreted as relative to the current directory. In this case, it diff --git a/src/nix/nix.md b/src/nix/nix.md index d10de7c01..0756ef0ac 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -57,9 +57,33 @@ the Nix store. Here are the recognised types of installables: These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a flake reference and *attrpath* is an optional attribute path. For more information on flakes, see [the `nix flake` manual - page](./nix3-flake.md). Flake references are most commonly a flake - identifier in the flake registry (e.g. `nixpkgs`) or a path - (e.g. `/path/to/my-flake` or `.`). + page](./nix3-flake.md). Flake references are most commonly a flake + identifier in the flake registry (e.g. `nixpkgs`), or a raw path + (e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL + (e.g. `github:nixos/nixpkgs` or `path:.`) + + When the flake reference is a raw path (a path without any URL + scheme), it is interpreted in the following way: + + - If the supplied path does not contain `flake.nix`, then Nix + searches for a directory containing `flake.nix` upwards of the + supplied path (until a filesystem boundary or a git repository + root). For example, if `/foo/bar/flake.nix` exists, then supplying + `/foo/bar/baz/` will find the directory `/foo/bar/`; + - If `flake.nix` is in a Git repository, then this is essentially + equivalent to `git+file://` (see [the `nix flake` + manual page](./nix3-flake.md)), except that the `dir` parameter is + derived automatically. For example, if `/foo/bar` is a Git + repository and `/foo/bar/baz` contains `flake.nix`, then the flake + reference `/foo/bar/baz` is equivalent to + `git+file:///foo/bar?dir=baz`. Note that it will only include + files indexed by git. In particular, files which are matched by + `.gitignore` will not be available in the flake. If this is + undesireable, specify `path:` explicitly; + - If the directory is not inside a Git repository, then it is + equivalent to `path:` (see [the `nix flake` manual + page](./nix3-flake.md)), which includes the entire contents of the + path. If *attrpath* is omitted, Nix tries some default values; for most subcommands, the default is `defaultPackage.`*system* From b9f5dccdbeb14a229fd14a3211490d83e53f70b8 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 14 Jan 2022 18:03:47 +0300 Subject: [PATCH 5/7] Check that we don't search past a git repo --- tests/flake-searching.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/flake-searching.sh b/tests/flake-searching.sh index 7c2041ce1..6558aff9a 100644 --- a/tests/flake-searching.sh +++ b/tests/flake-searching.sh @@ -18,11 +18,14 @@ EOF mkdir subdir pushd subdir -for i in "" . .# .#test ../subdir ../subdir#test "$PWD"; do +success=("" . .# .#test ../subdir ../subdir#test "$PWD") +failure=("path:$PWD") + +for i in "${success[@]}"; do nix build $i || fail "flake should be found by searching up directories" done -for i in "path:$PWD"; do +for i in "${failure[@]}"; do ! nix build $i || fail "flake should not search up directories when using 'path:'" done @@ -32,3 +35,11 @@ nix build --override-input foo . || fail "flake should search up directories whe sed "s,$PWD/foo,$PWD/foo/subdir,g" -i flake.nix ! nix build || fail "flake should not search upwards when part of inputs" + +pushd subdir +git init +for i in "${success[@]}" "${failure[@]}"; do + ! nix build $i || fail "flake should not search past a git repository" +done +rm -rf .git +popd From 34b66aab009428d3fab05b7530d6d13f1df3b2c9 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Mon, 17 Jan 2022 20:00:04 +0300 Subject: [PATCH 6/7] Update documentation for paths on command line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/nix/nix.md | 54 +++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 0756ef0ac..33942a5ad 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -63,27 +63,41 @@ the Nix store. Here are the recognised types of installables: (e.g. `github:nixos/nixpkgs` or `path:.`) When the flake reference is a raw path (a path without any URL - scheme), it is interpreted in the following way: + scheme), it is interpreted as a `path:` or `git+file:` url in the following + way: + + - If the path is within a Git repository, then the url will be of the form + `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` + where `GIT_REPO_ROOT` is the path to the root of the git repository, + and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory + root) of the closest parent of the given path that contains a `flake.nix` within + the git repository. + If no such directory exists, then Nix will error-out. + + Note that the search will only include files indexed by git. In particular, files + which are matched by `.gitignore` or have never been `git add`-ed will not be + available in the flake. If this is undesireable, specify `path:` explicitly; + + For example, if `/foo/bar` is a git repository with the following structure: + ``` + . + └── baz + ├── blah + │  └── file.txt + └── flake.nix + ``` + + Then `/foo/bar/baz/blah` will resolve to `git+file:///foo/bar?dir=baz` + + - If the supplied path is not a git repository, then the url will have the form + `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent + of the supplied path that contains a `flake.nix` file (within the same file-system). + If no such directory exists, then Nix will error-out. + + For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to + `path:/foo/bar` + - - If the supplied path does not contain `flake.nix`, then Nix - searches for a directory containing `flake.nix` upwards of the - supplied path (until a filesystem boundary or a git repository - root). For example, if `/foo/bar/flake.nix` exists, then supplying - `/foo/bar/baz/` will find the directory `/foo/bar/`; - - If `flake.nix` is in a Git repository, then this is essentially - equivalent to `git+file://` (see [the `nix flake` - manual page](./nix3-flake.md)), except that the `dir` parameter is - derived automatically. For example, if `/foo/bar` is a Git - repository and `/foo/bar/baz` contains `flake.nix`, then the flake - reference `/foo/bar/baz` is equivalent to - `git+file:///foo/bar?dir=baz`. Note that it will only include - files indexed by git. In particular, files which are matched by - `.gitignore` will not be available in the flake. If this is - undesireable, specify `path:` explicitly; - - If the directory is not inside a Git repository, then it is - equivalent to `path:` (see [the `nix flake` manual - page](./nix3-flake.md)), which includes the entire contents of the - path. If *attrpath* is omitted, Nix tries some default values; for most subcommands, the default is `defaultPackage.`*system* From 5753f6efbb46ea172913d03d0b0988546ff4971f Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 18 Jan 2022 10:55:00 +0100 Subject: [PATCH 7/7] Fix the rendering of the example directory tree --- src/nix/nix.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 33942a5ad..2f54c5e2c 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -80,11 +80,11 @@ the Nix store. Here are the recognised types of installables: For example, if `/foo/bar` is a git repository with the following structure: ``` - . - └── baz - ├── blah - │  └── file.txt - └── flake.nix + . + └── baz + ├── blah + │  └── file.txt + └── flake.nix ``` Then `/foo/bar/baz/blah` will resolve to `git+file:///foo/bar?dir=baz`