Merge pull request #8817 from iFreilicht/flake-update-lock-overhaul

Overhaul `nix flake update` and `nix flake lock` UX
This commit is contained in:
Théophane Hufschmitt 2023-10-31 16:19:05 +01:00 committed by GitHub
commit 12a0ae73db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 168 additions and 89 deletions

View file

@ -15,3 +15,17 @@
- `nix-shell` shebang lines now support single-quoted arguments. - `nix-shell` shebang lines now support single-quoted arguments.
- `builtins.fetchTree` is now marked as stable. - `builtins.fetchTree` is now marked as stable.
- The interface for creating and updating lock files has been overhauled:
- [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
It will *never* update existing inputs.
- [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
- Passing no arguments will update all inputs of the current flake, just like it already did.
- Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
- To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
- The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
They are superceded by `nix flake update`.

View file

@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args {
void setEnviron(); void setEnviron();
}; };
void completeFlakeInputPath(
AddCompletions & completions,
ref<EvalState> evalState,
const std::vector<FlakeRef> & flakeRefs,
std::string_view prefix);
void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix); void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
void completeFlakeRefWithFragment( void completeFlakeRefWithFragment(

View file

@ -28,7 +28,7 @@
namespace nix { namespace nix {
static void completeFlakeInputPath( void completeFlakeInputPath(
AddCompletions & completions, AddCompletions & completions,
ref<EvalState> evalState, ref<EvalState> evalState,
const std::vector<FlakeRef> & flakeRefs, const std::vector<FlakeRef> & flakeRefs,
@ -46,13 +46,6 @@ MixFlakeOptions::MixFlakeOptions()
{ {
auto category = "Common flake-related options"; auto category = "Common flake-related options";
addFlag({
.longName = "recreate-lock-file",
.description = "Recreate the flake's lock file from scratch.",
.category = category,
.handler = {&lockFlags.recreateLockFile, true}
});
addFlag({ addFlag({
.longName = "no-update-lock-file", .longName = "no-update-lock-file",
.description = "Do not allow any updates to the flake's lock file.", .description = "Do not allow any updates to the flake's lock file.",
@ -85,19 +78,6 @@ MixFlakeOptions::MixFlakeOptions()
.handler = {&lockFlags.commitLockFile, true} .handler = {&lockFlags.commitLockFile, true}
}); });
addFlag({
.longName = "update-input",
.description = "Update a specific flake input (ignoring its previous entry in the lock file).",
.category = category,
.labels = {"input-path"},
.handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
addFlag({ addFlag({
.longName = "override-input", .longName = "override-input",
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",

View file

@ -447,8 +447,8 @@ LockedFlake lockFlake(
assert(input.ref); assert(input.ref);
/* Do we have an entry in the existing lock file? And we /* Do we have an entry in the existing lock file?
don't have a --update-input flag for this input? */ And the input is not in updateInputs? */
std::shared_ptr<LockedNode> oldLock; std::shared_ptr<LockedNode> oldLock;
updatesUsed.insert(inputPath); updatesUsed.insert(inputPath);
@ -472,9 +472,8 @@ LockedFlake lockFlake(
node->inputs.insert_or_assign(id, childNode); node->inputs.insert_or_assign(id, childNode);
/* If we have an --update-input flag for an input /* If we have this input in updateInputs, then we
of this input, then we must fetch the flake to must fetch the flake to update it. */
update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto mustRefetch = auto mustRefetch =
@ -616,7 +615,7 @@ LockedFlake lockFlake(
for (auto & i : lockFlags.inputUpdates) for (auto & i : lockFlags.inputUpdates)
if (!updatesUsed.count(i)) if (!updatesUsed.count(i))
warn("the flag '--update-input %s' does not match any input", printInputPath(i)); warn("'%s' does not match any input of this flake", printInputPath(i));
/* Check 'follows' inputs. */ /* Check 'follows' inputs. */
newLockFile.check(); newLockFile.check();
@ -651,14 +650,14 @@ LockedFlake lockFlake(
bool lockFileExists = pathExists(outputLockFilePath); bool lockFileExists = pathExists(outputLockFilePath);
auto s = chomp(diff);
if (lockFileExists) { if (lockFileExists) {
auto s = chomp(diff);
if (s.empty()) if (s.empty())
warn("updating lock file '%s'", outputLockFilePath); warn("updating lock file '%s'", outputLockFilePath);
else else
warn("updating lock file '%s':\n%s", outputLockFilePath, s); warn("updating lock file '%s':\n%s", outputLockFilePath, s);
} else } else
warn("creating lock file '%s'", outputLockFilePath); warn("creating lock file '%s': \n%s", outputLockFilePath, s);
std::optional<std::string> commitMessage = std::nullopt; std::optional<std::string> commitMessage = std::nullopt;

View file

@ -255,7 +255,18 @@ bool Args::processArgs(const Strings & args, bool finish)
} }
if (!anyCompleted) if (!anyCompleted)
exp.handler.fun(ss); exp.handler.fun(ss);
expectedArgs.pop_front();
/* Move the list element to the processedArgs. This is almost the same as
`processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
except that it will only adjust the next and prev pointers of the list
elements, meaning the actual contents don't move in memory. This is
critical to prevent invalidating internal pointers! */
processedArgs.splice(
processedArgs.end(),
expectedArgs,
expectedArgs.begin(),
++expectedArgs.begin());
res = true; res = true;
} }

View file

@ -200,13 +200,25 @@ protected:
/** /**
* Queue of expected positional argument forms. * Queue of expected positional argument forms.
* *
* Positional arugment descriptions are inserted on the back. * Positional argument descriptions are inserted on the back.
* *
* As positional arguments are passed, these are popped from the * As positional arguments are passed, these are popped from the
* front, until there are hopefully none left as all args that were * front, until there are hopefully none left as all args that were
* expected in fact were passed. * expected in fact were passed.
*/ */
std::list<ExpectedArg> expectedArgs; std::list<ExpectedArg> expectedArgs;
/**
* List of processed positional argument forms.
*
* All items removed from `expectedArgs` are added here. After all
* arguments were processed, this list should be exactly the same as
* `expectedArgs` was before.
*
* This list is used to extend the lifetime of the argument forms.
* If this is not done, some closures that reference the command
* itself will segfault.
*/
std::list<ExpectedArg> processedArgs;
/** /**
* Process some positional arugments * Process some positional arugments

View file

@ -2,37 +2,39 @@ R""(
# Examples # Examples
* Update the `nixpkgs` and `nix` inputs of the flake in the current * Create the lock file for the flake in the current directory:
directory:
```console ```console
# nix flake lock --update-input nixpkgs --update-input nix # nix flake lock
* Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' warning: creating lock file '/home/myself/repos/testflake/flake.lock':
* Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' • Added input 'nix':
'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
• Added input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
``` ```
* Add missing inputs to the lock file for a flake in a different directory:
```console
# nix flake lock ~/repos/another
warning: updating lock file '/home/myself/repos/another/flake.lock':
• Added input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
```
> **Note**
>
> When trying to refer to a flake in a subdirectory, write `./another`
> instead of `another`.
> Otherwise Nix will try to look up the flake in the registry.
# Description # Description
This command updates the lock file of a flake (`flake.lock`) so that This command adds inputs to the lock file of a flake (`flake.lock`)
it contains a lock for every flake input specified in so that it contains a lock for every flake input specified in
`flake.nix`. Existing lock file entries are not updated unless `flake.nix`. Existing lock file entries are not updated.
required by a flag such as `--update-input`.
Note that every command that operates on a flake will also update the If you want to update existing lock entries, use
lock file if needed, and supports the same flags. Therefore, [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md)
```console
# nix flake lock --update-input nixpkgs
# nix build
```
is equivalent to:
```console
# nix build --update-input nixpkgs
```
Thus, this command is only useful if you want to update the lock file
separately from any other action such as building.
)"" )""

View file

@ -2,33 +2,57 @@ R""(
# Examples # Examples
* Recreate the lock file (i.e. update all inputs) and commit the new * Update all inputs (i.e. recreate the lock file from scratch):
lock file:
```console ```console
# nix flake update --commit-lock-file # nix flake update
* Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' warning: updating lock file '/home/myself/repos/testflake/flake.lock':
* Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' • Updated input 'nix':
'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11)
• Updated input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
→ 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
``` ```
* Update only a single input:
```console
# nix flake update nixpkgs
warning: updating lock file '/home/myself/repos/testflake/flake.lock':
• Updated input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
→ 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
```
* Update only a single input of a flake in a different directory:
```console
# nix flake update nixpkgs --flake ~/repos/another
warning: updating lock file '/home/myself/repos/another/flake.lock':
• Updated input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
→ 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
```
> **Note**
>
> When trying to refer to a flake in a subdirectory, write `./another`
> instead of `another`.
> Otherwise Nix will try to look up the flake in the registry.
# Description # Description
This command recreates the lock file of a flake (`flake.lock`), thus This command updates the inputs in a lock file (`flake.lock`).
updating the lock for every unlocked input (like `nixpkgs`) to its **By default, all inputs are updated**. If the lock file doesn't exist
current version. This is equivalent to passing `--recreate-lock-file` yet, it will be created. If inputs are not in the lock file yet, they will be added.
to any command that operates on a flake. That is,
```console Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs
# nix flake update to update as its positional arguments and operates on the flake in the current directory.
# nix build You can pass a different flake-url with `--flake` to override that default.
```
is equivalent to: The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md)
also creates lock files and adds missing inputs, but is safer as it
```console will never update inputs already in the lock file.
# nix build --recreate-lock-file
```
)"" )""

View file

@ -24,8 +24,10 @@ using namespace nix;
using namespace nix::flake; using namespace nix::flake;
using json = nlohmann::json; using json = nlohmann::json;
struct CmdFlakeUpdate;
class FlakeCommand : virtual Args, public MixFlakeOptions class FlakeCommand : virtual Args, public MixFlakeOptions
{ {
protected:
std::string flakeUrl = "."; std::string flakeUrl = ".";
public: public:
@ -63,6 +65,8 @@ public:
struct CmdFlakeUpdate : FlakeCommand struct CmdFlakeUpdate : FlakeCommand
{ {
public:
std::string description() override std::string description() override
{ {
return "update flake lock file"; return "update flake lock file";
@ -70,9 +74,31 @@ struct CmdFlakeUpdate : FlakeCommand
CmdFlakeUpdate() CmdFlakeUpdate()
{ {
expectedArgs.clear();
addFlag({
.longName="flake",
.description="The flake to operate on. Default is the current directory.",
.labels={"flake-url"},
.handler={&flakeUrl},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(completions, getStore(), prefix);
}}
});
expectArgs({
.label="inputs",
.optional=true,
.handler={[&](std::string inputToUpdate){
auto inputPath = flake::parseInputPath(inputToUpdate);
if (lockFlags.inputUpdates.contains(inputPath))
warn("Input '%s' was specified multiple times. You may have done this by accident.");
lockFlags.inputUpdates.insert(inputPath);
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
/* Remove flags that don't make sense. */ /* Remove flags that don't make sense. */
removeFlag("recreate-lock-file");
removeFlag("update-input");
removeFlag("no-update-lock-file"); removeFlag("no-update-lock-file");
removeFlag("no-write-lock-file"); removeFlag("no-write-lock-file");
} }
@ -87,8 +113,9 @@ struct CmdFlakeUpdate : FlakeCommand
void run(nix::ref<nix::Store> store) override void run(nix::ref<nix::Store> store) override
{ {
settings.tarballTtl = 0; settings.tarballTtl = 0;
auto updateAll = lockFlags.inputUpdates.empty();
lockFlags.recreateLockFile = true; lockFlags.recreateLockFile = updateAll;
lockFlags.writeLockFile = true; lockFlags.writeLockFile = true;
lockFlags.applyNixConfig = true; lockFlags.applyNixConfig = true;

View file

@ -44,15 +44,18 @@ EOF
# Input override completion # Input override completion
[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]]
[[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]]
cd ./foo
[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]]
cd ..
[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]]
## With multiple input flakes ## With multiple input flakes
[[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]]
## With tilde expansion ## With tilde expansion
[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]]
[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]]
[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]]
## Out of order ## Out of order
[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]]
[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]]
# Cli flag completion # Cli flag completion
NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format"

View file

@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo'
sed -i $flakeB/flake.nix -e 's/456/789/' sed -i $flakeB/flake.nix -e 's/456/789/'
git -C $flakeB commit -a -m 'Foo' git -C $flakeB commit -a -m 'Foo'
[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]] nix flake update b --flake $flakeA
[[ $(nix eval $flakeA#foo) = 1912 ]]
# Test list-inputs with circular dependencies # Test list-inputs with circular dependencies
nix flake metadata $flakeA nix flake metadata $flakeA

View file

@ -300,7 +300,7 @@ nix build -o "$TEST_ROOT/result" flake4#xyzzy
nix flake lock "$flake3Dir" nix flake lock "$flake3Dir"
[[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]]
nix flake update "$flake3Dir" --override-flake flake2 nixpkgs nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs
[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]]
# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
@ -437,7 +437,7 @@ cat > "$flake3Dir/flake.nix" <<EOF
} }
EOF EOF
nix flake update "$flake3Dir" nix flake update --flake "$flake3Dir"
[[ $(jq -c .nodes.flake2.inputs.flake1 "$flake3Dir/flake.lock") =~ '["foo"]' ]] [[ $(jq -c .nodes.flake2.inputs.flake1 "$flake3Dir/flake.lock") =~ '["foo"]' ]]
[[ $(jq .nodes.foo.locked.url "$flake3Dir/flake.lock") =~ flake7 ]] [[ $(jq .nodes.foo.locked.url "$flake3Dir/flake.lock") =~ flake7 ]]
@ -480,7 +480,7 @@ nix flake lock "$flake3Dir" --override-input flake2/flake1 flake1/master/$hash1
nix flake lock "$flake3Dir" nix flake lock "$flake3Dir"
[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") = $hash1 ]] [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") = $hash1 ]]
nix flake lock "$flake3Dir" --update-input flake2/flake1 nix flake update flake2/flake1 --flake "$flake3Dir"
[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]] [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]]
# Test 'nix flake metadata --json'. # Test 'nix flake metadata --json'.

View file

@ -77,7 +77,7 @@ git -C $flakeFollowsA add flake.nix flakeB/flake.nix \
nix flake metadata $flakeFollowsA nix flake metadata $flakeFollowsA
nix flake update $flakeFollowsA nix flake update --flake $flakeFollowsA
nix flake lock $flakeFollowsA nix flake lock $flakeFollowsA
@ -228,7 +228,7 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \
flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix
nix flake metadata "$flakeFollowsOverloadA" nix flake metadata "$flakeFollowsOverloadA"
nix flake update "$flakeFollowsOverloadA" nix flake update --flake "$flakeFollowsOverloadA"
nix flake lock "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA"
# Now test follow cycle detection # Now test follow cycle detection