Merge pull request #8817 from iFreilicht/flake-update-lock-overhaul

Overhaul `nix flake update` and `nix flake lock` UX

(cherry picked from commit 12a0ae73db)
Change-Id: Iff3b4f4235ebb1948ec612036b39ab29e4ca22b2

doc/manual/rl-next/nix-flake-update-ux.md

@@ -0,0 +1,17 @@
+---
+synposis: "`Overhaul `nix flake update` and `nix flake lock` UX"
+prs: 8817
+---
+
+The interface for creating and updating lock files has been overhauled:
+
+- [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
+It will *never* update existing inputs.
+
+- [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
+- Passing no arguments will update all inputs of the current flake, just like it already did.
+- Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
+- To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
+
+- The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
+They are superceded by `nix flake update`.

src/libcmd/command.hh

@@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args {
     void setEnviron();
 };
 
+void completeFlakeInputPath(
+    AddCompletions & completions,
+    ref<EvalState> evalState,
+    const std::vector<FlakeRef> & flakeRefs,
+    std::string_view prefix);
+
 void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
 
 void completeFlakeRefWithFragment(

src/libcmd/installables.cc

@@ -28,7 +28,7 @@
 
 namespace nix {
 
-static void completeFlakeInputPath(
+void completeFlakeInputPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
     const std::vector<FlakeRef> & flakeRefs,
@@ -46,13 +46,6 @@ MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
-    addFlag({
-        .longName = "recreate-lock-file",
-        .description = "Recreate the flake's lock file from scratch.",
-        .category = category,
-        .handler = {&lockFlags.recreateLockFile, true}
-    });
-
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -85,19 +78,6 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
-    addFlag({
-        .longName = "update-input",
-        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
-        .category = category,
-        .labels = {"input-path"},
-        .handler = {[&](std::string s) {
-            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
-        }},
-        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
-        }}
-    });
-
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",

src/libexpr/flake/flake.cc

@@ -451,8 +451,8 @@ LockedFlake lockFlake(
 
                     assert(input.ref);
 
-                    /* Do we have an entry in the existing lock file? And we
+                    /* Do we have an entry in the existing lock file?
-                       don't have a --update-input flag for this input? */
+                       And the input is not in updateInputs? */
                     std::shared_ptr<LockedNode> oldLock;
 
                     updatesUsed.insert(inputPath);
@@ -476,9 +476,8 @@ LockedFlake lockFlake(
 
                         node->inputs.insert_or_assign(id, childNode);
 
-                        /* If we have an --update-input flag for an input
+                        /* If we have this input in updateInputs, then we
-                           of this input, then we must fetch the flake to
+                           must fetch the flake to update it. */
-                           update it. */
                         auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
                         auto mustRefetch =
@@ -620,7 +619,7 @@ LockedFlake lockFlake(
 
         for (auto & i : lockFlags.inputUpdates)
             if (!updatesUsed.count(i))
-                warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+                warn("'%s' does not match any input of this flake", printInputPath(i));
 
         /* Check 'follows' inputs. */
         newLockFile.check();
@@ -655,14 +654,14 @@ LockedFlake lockFlake(
 
                             bool lockFileExists = pathExists(outputLockFilePath);
 
+                            auto s = chomp(diff);
                             if (lockFileExists) {
-                                auto s = chomp(diff);
                                 if (s.empty())
                                     warn("updating lock file '%s'", outputLockFilePath);
                                 else
                                     warn("updating lock file '%s':\n%s", outputLockFilePath, s);
                             } else
-                                warn("creating lock file '%s'", outputLockFilePath);
+                                warn("creating lock file '%s':\n%s", outputLockFilePath, s);
 
                             std::optional<std::string> commitMessage = std::nullopt;
 

src/libfetchers/fetchers.hh

@@ -10,7 +10,7 @@
 
 #include <memory>
 
-namespace nix { class Store; class StorePath; }
+namespace nix { class Store; }
 
 namespace nix::fetchers {
 

src/libutil/args.cc

@@ -255,7 +255,18 @@ bool Args::processArgs(const Strings & args, bool finish)
         }
         if (!anyCompleted)
             exp.handler.fun(ss);
-        expectedArgs.pop_front();
+
+        /* Move the list element to the processedArgs. This is almost the same as
+           `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
+           except that it will only adjust the next and prev pointers of the list
+           elements, meaning the actual contents don't move in memory. This is
+           critical to prevent invalidating internal pointers! */
+        processedArgs.splice(
+            processedArgs.end(),
+            expectedArgs,
+            expectedArgs.begin(),
+            ++expectedArgs.begin());
+
         res = true;
     }
 

src/libutil/args.hh

@@ -200,13 +200,25 @@ protected:
     /**
      * Queue of expected positional argument forms.
      *
-     * Positional arugment descriptions are inserted on the back.
+     * Positional argument descriptions are inserted on the back.
      *
      * As positional arguments are passed, these are popped from the
      * front, until there are hopefully none left as all args that were
      * expected in fact were passed.
      */
     std::list<ExpectedArg> expectedArgs;
+    /**
+     * List of processed positional argument forms.
+     *
+     * All items removed from `expectedArgs` are added here. After all
+     * arguments were processed, this list should be exactly the same as
+     * `expectedArgs` was before.
+     *
+     * This list is used to extend the lifetime of the argument forms.
+     * If this is not done, some closures that reference the command
+     * itself will segfault.
+    */
+   std::list<ExpectedArg> processedArgs;
 
     /**
      * Process some positional arugments

src/nix/flake-lock.md

@@ -2,37 +2,39 @@ R""(
 
 # Examples
 
-* Update the `nixpkgs` and `nix` inputs of the flake in the current
+* Create the lock file for the flake in the current directory:
-  directory:
 
   ```console
-  # nix flake lock --update-input nixpkgs --update-input nix
+  # nix flake lock
-  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  warning: creating lock file '/home/myself/repos/testflake/flake.lock':
-  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  • Added input 'nix':
+      'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
+  • Added input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
   ```
 
+* Add missing inputs to the lock file for a flake in a different directory:
+
+  ```console
+  # nix flake lock ~/repos/another
+  warning: updating lock file '/home/myself/repos/another/flake.lock':
+  • Added input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+  ```
+
+  > **Note**
+  >
+  > When trying to refer to a flake in a subdirectory, write `./another`
+  > instead of `another`.
+  > Otherwise Nix will try to look up the flake in the registry.
+
 # Description
 
-This command updates the lock file of a flake (`flake.lock`) so that
+This command adds inputs to the lock file of a flake (`flake.lock`)
-it contains a lock for every flake input specified in
+so that it contains a lock for every flake input specified in
-`flake.nix`. Existing lock file entries are not updated unless
+`flake.nix`. Existing lock file entries are not updated.
-required by a flag such as `--update-input`.
 
-Note that every command that operates on a flake will also update the
+If you want to update existing lock entries, use
-lock file if needed, and supports the same flags. Therefore,
+[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md)
-
-```console
-# nix flake lock --update-input nixpkgs
-# nix build
-```
-
-is equivalent to:
-
-```console
-# nix build --update-input nixpkgs
-```
-
-Thus, this command is only useful if you want to update the lock file
-separately from any other action such as building.
 
 )""

src/nix/flake-update.md

@@ -2,33 +2,57 @@ R""(
 
 # Examples
 
-* Recreate the lock file (i.e. update all inputs) and commit the new
+* Update all inputs (i.e. recreate the lock file from scratch):
-  lock file:
 
   ```console
-  # nix flake update --commit-lock-file
+  # nix flake update
-  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  warning: updating lock file '/home/myself/repos/testflake/flake.lock':
-  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  • Updated input 'nix':
-  …
+      'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
-  warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
+    → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11)
+  • Updated input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+    → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
   ```
 
+* Update only a single input:
+
+  ```console
+  # nix flake update nixpkgs
+  warning: updating lock file '/home/myself/repos/testflake/flake.lock':
+  • Updated input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+    → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
+  ```
+
+* Update only a single input of a flake in a different directory:
+
+  ```console
+  # nix flake update nixpkgs --flake ~/repos/another
+  warning: updating lock file '/home/myself/repos/another/flake.lock':
+  • Updated input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+    → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
+  ```
+
+  > **Note**
+  >
+  > When trying to refer to a flake in a subdirectory, write `./another`
+  > instead of `another`.
+  > Otherwise Nix will try to look up the flake in the registry.
+
 # Description
 
-This command recreates the lock file of a flake (`flake.lock`), thus
+This command updates the inputs in a lock file (`flake.lock`).
-updating the lock for every unlocked input (like `nixpkgs`) to its
+**By default, all inputs are updated**. If the lock file doesn't exist
-current version. This is equivalent to passing `--recreate-lock-file`
+yet, it will be created. If inputs are not in the lock file yet, they will be added.
-to any command that operates on a flake. That is,
 
-```console
+Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs
-# nix flake update
+to update as its positional arguments and operates on the flake in the current directory.
-# nix build
+You can pass a different flake-url with `--flake` to override that default.
-```
 
-is equivalent to:
+The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md)
-
+also creates lock files and adds missing inputs, but is safer as it
-```console
+will never update inputs already in the lock file.
-# nix build --recreate-lock-file
-```
 
 )""

src/nix/flake.cc

@@ -24,8 +24,10 @@ using namespace nix;
 using namespace nix::flake;
 using json = nlohmann::json;
 
+struct CmdFlakeUpdate;
 class FlakeCommand : virtual Args, public MixFlakeOptions
 {
+protected:
     std::string flakeUrl = ".";
 
 public:
@@ -63,6 +65,8 @@ public:
 
 struct CmdFlakeUpdate : FlakeCommand
 {
+public:
+
     std::string description() override
     {
         return "update flake lock file";
@@ -70,9 +74,31 @@ struct CmdFlakeUpdate : FlakeCommand
 
     CmdFlakeUpdate()
     {
+        expectedArgs.clear();
+        addFlag({
+            .longName="flake",
+            .description="The flake to operate on. Default is the current directory.",
+            .labels={"flake-url"},
+            .handler={&flakeUrl},
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+                completeFlakeRef(completions, getStore(), prefix);
+            }}
+        });
+        expectArgs({
+            .label="inputs",
+            .optional=true,
+            .handler={[&](std::string inputToUpdate){
+                auto inputPath = flake::parseInputPath(inputToUpdate);
+                if (lockFlags.inputUpdates.contains(inputPath))
+                    warn("Input '%s' was specified multiple times. You may have done this by accident.");
+                lockFlags.inputUpdates.insert(inputPath);
+            }},
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+                completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+            }}
+        });
+
         /* Remove flags that don't make sense. */
-        removeFlag("recreate-lock-file");
-        removeFlag("update-input");
         removeFlag("no-update-lock-file");
         removeFlag("no-write-lock-file");
     }
@@ -87,8 +113,9 @@ struct CmdFlakeUpdate : FlakeCommand
     void run(nix::ref<nix::Store> store) override
     {
         settings.tarballTtl = 0;
+        auto updateAll = lockFlags.inputUpdates.empty();
 
-        lockFlags.recreateLockFile = true;
+        lockFlags.recreateLockFile = updateAll;
         lockFlags.writeLockFile = true;
         lockFlags.applyNixConfig = true;
 

tests/functional/completions.sh

@@ -44,15 +44,18 @@ EOF
 # Input override completion
 [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]]
 [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]]
+cd ./foo
+[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]]
+cd ..
+[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]]
 ## With multiple input flakes
 [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]]
 ## With tilde expansion
 [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]]
-[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]]
+[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]]
-[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]]
 ## Out of order
-[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]]
+[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]]
-[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]]
+[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]]
 
 # Cli flag completion
 NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format"

tests/functional/flakes/circular.sh

@@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo'
 sed -i $flakeB/flake.nix -e 's/456/789/'
 git -C $flakeB commit -a -m 'Foo'
 
-[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]]
+nix flake update b --flake $flakeA
+[[ $(nix eval $flakeA#foo) = 1912 ]]
 
 # Test list-inputs with circular dependencies
 nix flake metadata $flakeA

tests/functional/flakes/flakes.sh

@@ -302,8 +302,8 @@ nix build -o $TEST_ROOT/result flake4#xyzzy
 nix flake lock $flake3Dir
 [[ -z $(git -C $flake3Dir diff master || echo failed) ]]
 
-nix flake update $flake3Dir --override-flake flake2 nixpkgs
+nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs
-[[ ! -z $(git -C $flake3Dir diff master || echo failed) ]]
+[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]]
 
 # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
 git -C $flake3Dir checkout -b removeXyzzy
@@ -439,9 +439,9 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake update --flake "$flake3Dir"
-[[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]]
+[[ $(jq -c .nodes.flake2.inputs.flake1 "$flake3Dir/flake.lock") =~ '["foo"]' ]]
-[[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
+[[ $(jq .nodes.foo.locked.url "$flake3Dir/flake.lock") =~ flake7 ]]
 
 # Test git+file with bare repo.
 rm -rf $flakeGitBare
@@ -478,12 +478,12 @@ nix flake lock $flake3Dir --override-input flake2/flake1 flake1
 nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]]
 
-# Test --update-input.
 nix flake lock $flake3Dir
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]]
 
-nix flake lock $flake3Dir --update-input flake2/flake1
+# Test updating an individual input of a flake lockfile.
-[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
+nix flake update flake2/flake1 --flake "$flake3Dir"
+[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]]
 
 # Test 'nix flake metadata --json'.
 nix flake metadata $flake3Dir --json | jq .

tests/functional/flakes/follow-paths.sh

@@ -77,7 +77,7 @@ git -C $flakeFollowsA add flake.nix flakeB/flake.nix \
 
 nix flake metadata $flakeFollowsA
 
-nix flake update $flakeFollowsA
+nix flake update --flake $flakeFollowsA
 
 nix flake lock $flakeFollowsA
 
@@ -228,5 +228,5 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \
   flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix
 
 nix flake metadata "$flakeFollowsOverloadA"
-nix flake update "$flakeFollowsOverloadA"
+nix flake update --flake "$flakeFollowsOverloadA"
 nix flake lock "$flakeFollowsOverloadA"