Overhaul completions, redo #6693 (#8131)

As I complained in
https://github.com/NixOS/nix/pull/6784#issuecomment-1421777030 (a
comment on the wrong PR, sorry again!), #6693 introduced a second
completions mechanism to fix a bug. Having two completion mechanisms
isn't so nice.

As @thufschmitt also pointed out, it was a bummer to go from `FlakeRef`
to `std::string` when collecting flake refs. Now it is `FlakeRefs`
again.

The underlying issue that sought to work around was that completion of
arguments not at the end can still benefit from the information from
latter arguments.

To fix this better, we rip out that change and simply defer all
completion processing until after all the (regular, already-complete)
arguments have been passed.

In addition, I noticed the original completion logic used some global
variables. I do not like global variables, because even if they save
lines of code, they also obfuscate the architecture of the code.

I got rid of them  moved them to a new `RootArgs` class, which now has
`parseCmdline` instead of `Args`. The idea is that we have many argument
parsers from subcommands and what-not, but only one root args that owns
the other per actual parsing invocation. The state that was global is
now part of the root args instead.

This did, admittedly, add a bunch of new code. And I do feel bad about
that. So I went and added a lot of API docs to try to at least make the
current state of things clear to the next person.

--

This is needed for RFC 134 (tracking issue #7868). It was very hard to
modularize `Installable` parsing when there were two completion
arguments. I wouldn't go as far as to say it is *easy* now, but at least
it is less hard (and the completions test finally passed).

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
Change-Id: If18cd5be78da4a70635e3fdcac6326dbfeea71a5
(cherry picked from commit 67eb37c1d0de28160cd25376e51d1ec1b1c8305b)

src/libcmd/command.hh

@@ -97,8 +97,6 @@ struct MixFlakeOptions : virtual Args, EvalCommand
 {
     flake::LockFlags lockFlags;
 
-    std::optional<std::string> needsFlakeInputCompletion = {};
-
     MixFlakeOptions();
 
     /**
@@ -109,12 +107,8 @@ struct MixFlakeOptions : virtual Args, EvalCommand
      * command is operating with (presumably specified via some other
      * arguments) so that the completions for these flags can use them.
      */
-    virtual std::vector<std::string> getFlakesForCompletion()
+    virtual std::vector<FlakeRef> getFlakeRefsForCompletion()
     { return {}; }
-
-    void completeFlakeInput(std::string_view prefix);
-
-    void completionHook() override;
 };
 
 struct SourceExprCommand : virtual Args, MixFlakeOptions
@@ -137,7 +131,13 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
     /**
      * Complete an installable from the given prefix.
      */
-    void completeInstallable(std::string_view prefix);
+    void completeInstallable(AddCompletions & completions, std::string_view prefix);
+
+    /**
+     * Convenience wrapper around the underlying function to make setting the
+     * callback easier.
+     */
+    CompleterClosure getCompleteInstallable();
 };
 
 /**
@@ -170,7 +170,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand
 
     bool readFromStdIn = false;
 
-    std::vector<std::string> getFlakesForCompletion() override;
+    std::vector<FlakeRef> getFlakeRefsForCompletion() override;
 
 private:
 
@@ -199,10 +199,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand
 
     void run(ref<Store> store) override;
 
-    std::vector<std::string> getFlakesForCompletion() override
+    std::vector<FlakeRef> getFlakeRefsForCompletion() override;
-    {
-        return {_installable};
-    }
 
 private:
 
@@ -329,9 +326,10 @@ struct MixEnvironment : virtual Args {
     void setEnviron();
 };
 
-void completeFlakeRef(ref<Store> store, std::string_view prefix);
+void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
 
 void completeFlakeRefWithFragment(
+    AddCompletions & completions,
     ref<EvalState> evalState,
     flake::LockFlags lockFlags,
     Strings attrPathPrefixes,

src/libcmd/common-eval-args.cc

@@ -132,8 +132,8 @@ MixEvalArgs::MixEvalArgs()
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }},
-        .completer = {[&](size_t, std::string_view prefix) {
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeRef(openStore(), prefix);
+            completeFlakeRef(completions, openStore(), prefix);
         }}
     });
 

src/libcmd/installables.cc

@@ -28,6 +28,20 @@
 
 namespace nix {
 
+static void completeFlakeInputPath(
+    AddCompletions & completions,
+    ref<EvalState> evalState,
+    const std::vector<FlakeRef> & flakeRefs,
+    std::string_view prefix)
+{
+    for (auto & flakeRef : flakeRefs) {
+        auto flake = flake::getFlake(*evalState, flakeRef, true);
+        for (auto & input : flake.inputs)
+            if (input.first.starts_with(prefix))
+                completions.add(input.first);
+    }
+}
+
 MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
@@ -79,8 +93,8 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string s) {
             lockFlags.inputUpdates.insert(flake::parseInputPath(s));
         }},
-        .completer = {[&](size_t, std::string_view prefix) {
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            needsFlakeInputCompletion = {std::string(prefix)};
+            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
         }}
     });
 
@@ -95,11 +109,12 @@ MixFlakeOptions::MixFlakeOptions()
                 flake::parseInputPath(inputPath),
                 parseFlakeRef(flakeRef, absPath("."), true));
         }},
-        .completer = {[&](size_t n, std::string_view prefix) {
+        .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
-            if (n == 0)
+            if (n == 0) {
-                needsFlakeInputCompletion = {std::string(prefix)};
+                completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
-            else if (n == 1)
+            } else if (n == 1) {
-                completeFlakeRef(getEvalState()->store, prefix);
+                completeFlakeRef(completions, getEvalState()->store, prefix);
+            }
         }}
     });
 
@@ -146,30 +161,12 @@ MixFlakeOptions::MixFlakeOptions()
                 }
             }
         }},
-        .completer = {[&](size_t, std::string_view prefix) {
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeRef(getEvalState()->store, prefix);
+            completeFlakeRef(completions, getEvalState()->store, prefix);
         }}
     });
 }
 
-void MixFlakeOptions::completeFlakeInput(std::string_view prefix)
-{
-    auto evalState = getEvalState();
-    for (auto & flakeRefS : getFlakesForCompletion()) {
-        auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first;
-        auto flake = flake::getFlake(*evalState, flakeRef, true);
-        for (auto & input : flake.inputs)
-            if (input.first.starts_with(prefix))
-                completions->add(input.first);
-    }
-}
-
-void MixFlakeOptions::completionHook()
-{
-    if (auto & prefix = needsFlakeInputCompletion)
-        completeFlakeInput(*prefix);
-}
-
 SourceExprCommand::SourceExprCommand()
 {
     addFlag({
@@ -227,11 +224,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
     };
 }
 
-void SourceExprCommand::completeInstallable(std::string_view prefix)
+Args::CompleterClosure SourceExprCommand::getCompleteInstallable()
+{
+    return [this](AddCompletions & completions, size_t, std::string_view prefix) {
+        completeInstallable(completions, prefix);
+    };
+}
+
+void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix)
 {
     try {
         if (file) {
-            completionType = ctAttrs;
+            completions.setType(AddCompletions::Type::Attrs);
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
@@ -266,14 +270,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
                     std::string name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
-                            completions->add(name);
+                            completions.add(name);
                         else
-                            completions->add(prefix_ + "." + name);
+                            completions.add(prefix_ + "." + name);
                     }
                 }
             }
         } else {
             completeFlakeRefWithFragment(
+                completions,
                 getEvalState(),
                 lockFlags,
                 getDefaultFlakeAttrPathPrefixes(),
@@ -286,6 +291,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
 }
 
 void completeFlakeRefWithFragment(
+    AddCompletions & completions,
     ref<EvalState> evalState,
     flake::LockFlags lockFlags,
     Strings attrPathPrefixes,
@@ -297,9 +303,9 @@ void completeFlakeRefWithFragment(
     try {
         auto hash = prefix.find('#');
         if (hash == std::string::npos) {
-            completeFlakeRef(evalState->store, prefix);
+            completeFlakeRef(completions, evalState->store, prefix);
         } else {
-            completionType = ctAttrs;
+            completions.setType(AddCompletions::Type::Attrs);
 
             auto fragment = prefix.substr(hash + 1);
             std::string prefixRoot = "";
@@ -342,7 +348,7 @@ void completeFlakeRefWithFragment(
                         auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
-                        completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
+                        completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
                     }
                 }
             }
@@ -353,7 +359,7 @@ void completeFlakeRefWithFragment(
                 for (auto & attrPath : defaultFlakeAttrPaths) {
                     auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
                     if (!attr) continue;
-                    completions->add(flakeRefS + "#" + prefixRoot);
+                    completions.add(flakeRefS + "#" + prefixRoot);
                 }
             }
         }
@@ -362,15 +368,15 @@ void completeFlakeRefWithFragment(
     }
 }
 
-void completeFlakeRef(ref<Store> store, std::string_view prefix)
+void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix)
 {
     if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
         return;
 
     if (prefix == "")
-        completions->add(".");
+        completions.add(".");
 
-    completeDir(0, prefix);
+    Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
     for (auto & registry : fetchers::getRegistries(store)) {
@@ -379,10 +385,10 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
             if (!prefix.starts_with("flake:") && from.starts_with("flake:")) {
                 std::string from2(from, 6);
                 if (from2.starts_with(prefix))
-                    completions->add(from2);
+                    completions.add(from2);
             } else {
                 if (from.starts_with(prefix))
-                    completions->add(from);
+                    completions.add(from);
             }
         }
     }
@@ -762,9 +768,7 @@ RawInstallablesCommand::RawInstallablesCommand()
     expectArgs({
         .label = "installables",
         .handler = {&rawInstallables},
-        .completer = {[&](size_t, std::string_view prefix) {
+        .completer = getCompleteInstallable(),
-            completeInstallable(prefix);
-        }}
     });
 }
 
@@ -777,6 +781,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector<std::string> &
     }
 }
 
+std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
+{
+    applyDefaultInstallables(rawInstallables);
+    std::vector<FlakeRef> res;
+    for (auto i : rawInstallables)
+        res.push_back(parseFlakeRefWithFragment(
+            expandTilde(i),
+            absPath(".")).first);
+    return res;
+}
+
 void RawInstallablesCommand::run(ref<Store> store)
 {
     if (readFromStdIn && !isatty(STDIN_FILENO)) {
@@ -790,10 +805,13 @@ void RawInstallablesCommand::run(ref<Store> store)
     run(store, std::move(rawInstallables));
 }
 
-std::vector<std::string> RawInstallablesCommand::getFlakesForCompletion()
+std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
 {
-    applyDefaultInstallables(rawInstallables);
+    return {
-    return rawInstallables;
+        parseFlakeRefWithFragment(
+            expandTilde(_installable),
+            absPath(".")).first
+    };
 }
 
 void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables)
@@ -809,9 +827,7 @@ InstallableCommand::InstallableCommand()
         .label = "installable",
         .optional = true,
         .handler = {&_installable},
-        .completer = {[&](size_t, std::string_view prefix) {
+        .completer = getCompleteInstallable(),
-            completeInstallable(prefix);
-        }}
     });
 }
 

src/libmain/common-args.cc

@@ -1,4 +1,5 @@
 #include "common-args.hh"
+#include "args/root.hh"
 #include "globals.hh"
 #include "loggers.hh"
 
@@ -34,21 +35,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
         .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
         .category = miscCategory,
         .labels = {"name", "value"},
-        .handler = {[](std::string name, std::string value) {
+        .handler = {[this](std::string name, std::string value) {
             try {
                 globalConfig.set(name, value);
             } catch (UsageError & e) {
-                if (!completions)
+                if (!getRoot().completions)
                     warn(e.what());
             }
         }},
-        .completer = [](size_t index, std::string_view prefix) {
+        .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) {
             if (index == 0) {
                 std::map<std::string, Config::SettingInfo> settings;
                 globalConfig.getSettings(settings);
                 for (auto & s : settings)
                     if (s.first.starts_with(prefix))
-                        completions->add(s.first, fmt("Set the `%s` setting.", s.first));
+                        completions.add(s.first, fmt("Set the `%s` setting.", s.first));
             }
         }
     });

src/libmain/shared.hh

@@ -3,6 +3,7 @@
 
 #include "util.hh"
 #include "args.hh"
+#include "args/root.hh"
 #include "common-args.hh"
 #include "path.hh"
 #include "derived-path.hh"
@@ -58,7 +59,7 @@ template<class N> N getIntArg(const std::string & opt,
 }
 
 
-struct LegacyArgs : public MixCommonArgs
+struct LegacyArgs : public MixCommonArgs, public RootArgs
 {
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;
 

src/libutil/args.cc

@@ -1,4 +1,5 @@
 #include "args.hh"
+#include "args/root.hh"
 #include "hash.hh"
 #include "json-utils.hh"
 
@@ -26,6 +27,11 @@ void Args::removeFlag(const std::string & longName)
     longFlags.erase(flag);
 }
 
+void Completions::setType(AddCompletions::Type t)
+{
+    type = t;
+}
+
 void Completions::add(std::string completion, std::string description)
 {
     description = trim(description);
@@ -37,7 +43,7 @@ void Completions::add(std::string completion, std::string description)
         if (needs_ellipsis)
             description.append(" [...]");
     }
-    insert(Completion {
+    completions.insert(Completion {
         .completion = completion,
         .description = description
     });
@@ -46,12 +52,20 @@ void Completions::add(std::string completion, std::string description)
 bool Completion::operator<(const Completion & other) const
 { return completion < other.completion || (completion == other.completion && description < other.description); }
 
-CompletionType completionType = ctNormal;
-std::shared_ptr<Completions> completions;
-
 std::string completionMarker = "___COMPLETE___";
 
-static std::optional<std::string> needsCompletion(std::string_view s)
+RootArgs & Args::getRoot()
+{
+    Args * p = this;
+    while (p->parent)
+        p = p->parent;
+
+    auto * res = dynamic_cast<RootArgs *>(p);
+    assert(res);
+    return *res;
+}
+
+std::optional<std::string> RootArgs::needsCompletion(std::string_view s)
 {
     if (!completions) return {};
     auto i = s.find(completionMarker);
@@ -60,7 +74,7 @@ static std::optional<std::string> needsCompletion(std::string_view s)
     return {};
 }
 
-void Args::parseCmdline(const Strings & _cmdline)
+void RootArgs::parseCmdline(const Strings & _cmdline)
 {
     Strings pendingArgs;
     bool dashDash = false;
@@ -71,7 +85,7 @@ void Args::parseCmdline(const Strings & _cmdline)
         size_t n = std::stoi(*s);
         assert(n > 0 && n <= cmdline.size());
         *std::next(cmdline.begin(), n - 1) += completionMarker;
-        completions = std::make_shared<decltype(completions)::element_type>();
+        completions = std::make_shared<Completions>();
         verbosity = lvlError;
     }
 
@@ -125,17 +139,23 @@ void Args::parseCmdline(const Strings & _cmdline)
     for (auto & f : flagExperimentalFeatures)
         experimentalFeatureSettings.require(f);
 
+    /* Now that all the other args are processed, run the deferred completions.
+     */
+    for (auto d : deferredCompletions)
+        d.completer(*completions, d.n, d.prefix);
 }
 
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     assert(pos != end);
 
+    auto & rootArgs = getRoot();
+
     auto process = [&](const std::string & name, const Flag & flag) -> bool {
         ++pos;
 
         if (auto & f = flag.experimentalFeature)
-            flagExperimentalFeatures.insert(*f);
+            rootArgs.flagExperimentalFeatures.insert(*f);
 
         std::vector<std::string> args;
         bool anyCompleted = false;
@@ -146,10 +166,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
                     "flag '%s' requires %d argument(s), but only %d were given",
                     name, flag.handler.arity, n);
             }
-            if (auto prefix = needsCompletion(*pos)) {
+            if (auto prefix = rootArgs.needsCompletion(*pos)) {
                 anyCompleted = true;
-                if (flag.completer)
+                if (flag.completer) {
-                    flag.completer(n, *prefix);
+                    rootArgs.deferredCompletions.push_back({
+                        .completer = flag.completer,
+                        .n = n,
+                        .prefix = *prefix,
+                    });
+                }
             }
             args.push_back(*pos++);
         }
@@ -159,14 +184,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
     };
 
     if (std::string(*pos, 0, 2) == "--") {
-        if (auto prefix = needsCompletion(*pos)) {
+        if (auto prefix = rootArgs.needsCompletion(*pos)) {
             for (auto & [name, flag] : longFlags) {
                 if (!hiddenCategories.count(flag->category)
                     && name.starts_with(std::string(*prefix, 2)))
                 {
                     if (auto & f = flag->experimentalFeature)
-                        flagExperimentalFeatures.insert(*f);
+                        rootArgs.flagExperimentalFeatures.insert(*f);
-                    completions->add("--" + name, flag->description);
+                    rootArgs.completions->add("--" + name, flag->description);
                 }
             }
             return false;
@@ -183,12 +208,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         return process(std::string("-") + c, *i->second);
     }
 
-    if (auto prefix = needsCompletion(*pos)) {
+    if (auto prefix = rootArgs.needsCompletion(*pos)) {
         if (prefix == "-") {
-            completions->add("--");
+            rootArgs.completions->add("--");
             for (auto & [flagName, flag] : shortFlags)
                 if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature))
-                    completions->add(std::string("-") + flagName, flag->description);
+                    rootArgs.completions->add(std::string("-") + flagName, flag->description);
         }
     }
 
@@ -203,6 +228,8 @@ bool Args::processArgs(const Strings & args, bool finish)
         return true;
     }
 
+    auto & rootArgs = getRoot();
+
     auto & exp = expectedArgs.front();
 
     bool res = false;
@@ -211,15 +238,23 @@ bool Args::processArgs(const Strings & args, bool finish)
         (exp.handler.arity != ArityAny && args.size() == exp.handler.arity))
     {
         std::vector<std::string> ss;
+        bool anyCompleted = false;
         for (const auto &[n, s] : enumerate(args)) {
-            if (auto prefix = needsCompletion(s)) {
+            if (auto prefix = rootArgs.needsCompletion(s)) {
+                anyCompleted = true;
                 ss.push_back(*prefix);
-                if (exp.completer)
+                if (exp.completer) {
-                    exp.completer(n, *prefix);
+                    rootArgs.deferredCompletions.push_back({
+                        .completer = exp.completer,
+                        .n = n,
+                        .prefix = *prefix,
+                    });
+                }
             } else
                 ss.push_back(s);
         }
-        exp.handler.fun(ss);
+        if (!anyCompleted)
+            exp.handler.fun(ss);
         expectedArgs.pop_front();
         res = true;
     }
@@ -271,11 +306,11 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashTypeCompleter(size_t index, std::string_view prefix)
+static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
     for (auto & type : hashTypes)
         if (type.starts_with(prefix))
-            completions->add(type);
+            completions.add(type);
 }
 
 Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
@@ -287,7 +322,7 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
         .handler = {[ht](std::string s) {
             *ht = parseHashType(s);
         }},
-        .completer = hashTypeCompleter
+        .completer = hashTypeCompleter,
     };
 }
 
@@ -300,13 +335,13 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
         .handler = {[oht](std::string s) {
             *oht = std::optional<HashType> { parseHashType(s) };
         }},
-        .completer = hashTypeCompleter
+        .completer = hashTypeCompleter,
     };
 }
 
-static void _completePath(std::string_view prefix, bool onlyDirs)
+static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs)
 {
-    completionType = ctFilenames;
+    completions.setType(Completions::Type::Filenames);
     glob_t globbuf;
     int flags = GLOB_NOESCAPE;
     #ifdef GLOB_ONLYDIR
@@ -320,20 +355,20 @@ static void _completePath(std::string_view prefix, bool onlyDirs)
                 auto st = stat(globbuf.gl_pathv[i]);
                 if (!S_ISDIR(st.st_mode)) continue;
             }
-            completions->add(globbuf.gl_pathv[i]);
+            completions.add(globbuf.gl_pathv[i]);
         }
     }
     globfree(&globbuf);
 }
 
-void completePath(size_t, std::string_view prefix)
+void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix)
 {
-    _completePath(prefix, false);
+    _completePath(completions, prefix, false);
 }
 
-void completeDir(size_t, std::string_view prefix)
+void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix)
 {
-    _completePath(prefix, true);
+    _completePath(completions, prefix, true);
 }
 
 Strings argvToStrings(int argc, char * * argv)
@@ -368,10 +403,10 @@ MultiCommand::MultiCommand(const Commands & commands_)
             command = {s, i->second()};
             command->second->parent = this;
         }},
-        .completer = {[&](size_t, std::string_view prefix) {
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             for (auto & [name, command] : commands)
                 if (name.starts_with(prefix))
-                    completions->add(name);
+                    completions.add(name);
         }}
     });
 
@@ -393,14 +428,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
-void MultiCommand::completionHook()
-{
-    if (command)
-        return command->second->completionHook();
-    else
-        return Args::completionHook();
-}
-
 nlohmann::json MultiCommand::toJSON()
 {
     auto cmds = nlohmann::json::object();

src/libutil/args.hh

@@ -15,16 +15,14 @@ enum HashType : char;
 
 class MultiCommand;
 
+class RootArgs;
+
+class AddCompletions;
+
 class Args
 {
 public:
 
-    /**
-     * Parse the command line, throwing a UsageError if something goes
-     * wrong.
-     */
-    void parseCmdline(const Strings & cmdline);
-
     /**
      * Return a short one-line description of the command.
      */
@@ -123,6 +121,25 @@ protected:
         { }
     };
 
+    /**
+     * The basic function type of the completion callback.
+     *
+     * Used to define `CompleterClosure` and some common case completers
+     * that individual flags/arguments can use.
+     *
+     * The `AddCompletions` that is passed is an interface to the state
+     * stored as part of the root command
+     */
+    typedef void CompleterFun(AddCompletions &, size_t, std::string_view);
+
+    /**
+     * The closure type of the completion callback.
+     *
+     * This is what is actually stored as part of each Flag / Expected
+     * Arg.
+     */
+    typedef std::function<CompleterFun> CompleterClosure;
+
     /**
      * Description of flags / options
      *
@@ -140,7 +157,7 @@ protected:
         std::string category;
         Strings labels;
         Handler handler;
-        std::function<void(size_t, std::string_view)> completer;
+        CompleterClosure completer;
 
         std::optional<ExperimentalFeature> experimentalFeature;
 
@@ -177,7 +194,7 @@ protected:
         std::string label;
         bool optional = false;
         Handler handler;
-        std::function<void(size_t, std::string_view)> completer;
+        CompleterClosure completer;
     };
 
     /**
@@ -211,13 +228,6 @@ protected:
      */
     virtual void initialFlagsProcessed() {}
 
-    /**
-     * Called after the command line has been processed if we need to generate
-     * completions. Useful for commands that need to know the whole command line
-     * in order to know what completions to generate.
-     */
-    virtual void completionHook() { }
-
 public:
 
     void addFlag(Flag && flag);
@@ -252,24 +262,30 @@ public:
         });
     }
 
+    static CompleterFun completePath;
+
+    static CompleterFun completeDir;
+
     virtual nlohmann::json toJSON();
 
     friend class MultiCommand;
 
     /**
      * The parent command, used if this is a subcommand.
+     *
+     * Invariant: An Args with a null parent must also be a RootArgs
+     *
+     * \todo this would probably be better in the CommandClass.
+     * getRoot() could be an abstract method that peels off at most one
+     * layer before recuring.
      */
     MultiCommand * parent = nullptr;
 
-private:
-
     /**
-     * Experimental features needed when parsing args. These are checked
+     * Traverse parent pointers until we find the \ref RootArgs "root
-     * after flag parsing is completed in order to support enabling
+     * arguments" object.
-     * experimental features coming after the flag that needs the
-     * experimental feature.
      */
-    std::set<ExperimentalFeature> flagExperimentalFeatures;
+    RootArgs & getRoot();
 };
 
 /**
@@ -320,8 +336,6 @@ public:
 
     bool processArgs(const Strings & args, bool finish) override;
 
-    void completionHook() override;
-
     nlohmann::json toJSON() override;
 };
 
@@ -333,21 +347,40 @@ struct Completion {
 
     bool operator<(const Completion & other) const;
 };
-class Completions : public std::set<Completion> {
+
+/**
+ * The abstract interface for completions callbacks
+ *
+ * The idea is to restrict the callback so it can only add additional
+ * completions to the collection, or set the completion type. By making
+ * it go through this interface, the callback cannot make any other
+ * changes, or even view the completions / completion type that have
+ * been set so far.
+ */
+class AddCompletions
+{
 public:
-    void add(std::string completion, std::string description = "");
+
+    /**
+     * The type of completion we are collecting.
+     */
+    enum class Type {
+        Normal,
+        Filenames,
+        Attrs,
+    };
+
+    /**
+     * Set the type of the completions being collected
+     *
+     * \todo it should not be possible to change the type after it has been set.
+     */
+    virtual void setType(Type type) = 0;
+
+    /**
+     * Add a single completion to the collection
+     */
+    virtual void add(std::string completion, std::string description = "") = 0;
 };
-extern std::shared_ptr<Completions> completions;
-
-enum CompletionType {
-    ctNormal,
-    ctFilenames,
-    ctAttrs
-};
-extern CompletionType completionType;
-
-void completePath(size_t, std::string_view prefix);
-
-void completeDir(size_t, std::string_view prefix);
 
 }

src/libutil/args/root.hh

@@ -0,0 +1,72 @@
+#pragma once
+
+#include "args.hh"
+
+namespace nix {
+
+/**
+ * The concrete implementation of a collection of completions.
+ *
+ * This is exposed so that the main entry point can print out the
+ * collected completions.
+ */
+struct Completions final : AddCompletions
+{
+    std::set<Completion> completions;
+    Type type = Type::Normal;
+
+    void setType(Type type) override;
+    void add(std::string completion, std::string description = "") override;
+};
+
+/**
+ * The outermost Args object. This is the one we will actually parse a command
+ * line with, whereas the inner ones (if they exists) are subcommands (and this
+ * is also a MultiCommand or something like it).
+ *
+ * This Args contains completions state shared between it and all of its
+ * descendent Args.
+ */
+class RootArgs : virtual public Args
+{
+public:
+    /** Parse the command line, throwing a UsageError if something goes
+     * wrong.
+     */
+    void parseCmdline(const Strings & cmdline);
+
+    std::shared_ptr<Completions> completions;
+
+protected:
+
+    friend class Args;
+
+    /**
+     * A pointer to the completion and its two arguments; a thunk;
+     */
+    struct DeferredCompletion {
+        const CompleterClosure & completer;
+        size_t n;
+        std::string prefix;
+    };
+
+    /**
+     * Completions are run after all args and flags are parsed, so completions
+     * of earlier arguments can benefit from later arguments.
+     */
+    std::vector<DeferredCompletion> deferredCompletions;
+
+    /**
+     * Experimental features needed when parsing args. These are checked
+     * after flag parsing is completed in order to support enabling
+     * experimental features coming after the flag that needs the
+     * experimental feature.
+     */
+    std::set<ExperimentalFeature> flagExperimentalFeatures;
+
+private:
+
+    std::optional<std::string> needsCompletion(std::string_view s);
+};
+
+}

src/libutil/local.mk

@@ -6,8 +6,13 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
+libutil_CXXFLAGS += -I src/libutil
+
 libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
+$(foreach i, $(wildcard $(d)/args/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644)))
+
 ifeq ($(HAVE_LIBCPUID), 1)
 	libutil_LDFLAGS += -lcpuid
 endif

src/nix/bundle.cc

@@ -21,8 +21,8 @@ struct CmdBundle : InstallableValueCommand
             .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler),
             .labels = {"flake-url"},
             .handler = {&bundler},
-            .completer = {[&](size_t, std::string_view prefix) {
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-                completeFlakeRef(getStore(), prefix);
+                completeFlakeRef(completions, getStore(), prefix);
             }}
         });
 

src/nix/flake.cc

@@ -36,8 +36,8 @@ public:
             .label = "flake-url",
             .optional = true,
             .handler = {&flakeUrl},
-            .completer = {[&](size_t, std::string_view prefix) {
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-                completeFlakeRef(getStore(), prefix);
+                completeFlakeRef(completions, getStore(), prefix);
             }}
         });
     }
@@ -52,9 +52,12 @@ public:
         return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags);
     }
 
-    std::vector<std::string> getFlakesForCompletion() override
+    std::vector<FlakeRef> getFlakeRefsForCompletion() override
     {
-        return {flakeUrl};
+        return {
+            // Like getFlakeRef but with expandTilde calld first
+            parseFlakeRef(expandTilde(flakeUrl), absPath("."))
+        };
     }
 };
 
@@ -777,8 +780,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             .description = "The template to use.",
             .labels = {"template"},
             .handler = {&templateUrl},
-            .completer = {[&](size_t, std::string_view prefix) {
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
                 completeFlakeRefWithFragment(
+                    completions,
                     getEvalState(),
                     lockFlags,
                     defaultTemplateAttrPathsPrefixes,

src/nix/main.cc

@@ -1,5 +1,6 @@
 #include <algorithm>
 
+#include "args/root.hh"
 #include "command.hh"
 #include "common-args.hh"
 #include "eval.hh"
@@ -75,7 +76,7 @@ static bool haveInternet()
 
 std::string programPath;
 
-struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
+struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
 {
     bool useNet = true;
     bool refresh = false;
@@ -253,10 +254,7 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
 
 static NixArgs & getNixArgs(Command & cmd)
 {
-    assert(cmd.parent);
+    return dynamic_cast<NixArgs &>(cmd.getRoot());
-    MultiCommand * toplevel = cmd.parent;
-    while (toplevel->parent) toplevel = toplevel->parent;
-    return dynamic_cast<NixArgs &>(*toplevel);
 }
 
 struct CmdHelp : Command
@@ -424,16 +422,16 @@ void mainWrapped(int argc, char * * argv)
 
     Finally printCompletions([&]()
     {
-        if (completions) {
+        if (args.completions) {
-            switch (completionType) {
+            switch (args.completions->type) {
-            case ctNormal:
+            case Completions::Type::Normal:
                 logger->cout("normal"); break;
-            case ctFilenames:
+            case Completions::Type::Filenames:
                 logger->cout("filenames"); break;
-            case ctAttrs:
+            case Completions::Type::Attrs:
                 logger->cout("attrs"); break;
             }
-            for (auto & s : *completions)
+            for (auto & s : args.completions->completions)
                 logger->cout(s.completion + "\t" + trim(s.description));
         }
     });
@@ -441,7 +439,7 @@ void mainWrapped(int argc, char * * argv)
     try {
         args.parseCmdline(argvToStrings(argc, argv));
     } catch (UsageError &) {
-        if (!args.helpRequested && !completions) throw;
+        if (!args.helpRequested && !args.completions) throw;
     }
 
     if (args.helpRequested) {
@@ -458,10 +456,7 @@ void mainWrapped(int argc, char * * argv)
         return;
     }
 
-    if (completions) {
+    if (args.completions) return;
-        args.completionHook();
-        return;
-    }
 
     if (args.showVersion) {
         printVersion(programName);

src/nix/registry.cc

@@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
             .label = "locked",
             .optional = true,
             .handler = {&locked},
-            .completer = {[&](size_t, std::string_view prefix) {
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-                completeFlakeRef(getStore(), prefix);
+                completeFlakeRef(completions, getStore(), prefix);
             }}
         });
     }

src/nix/why-depends.cc

@@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
         expectArgs({
             .label = "package",
             .handler = {&_package},
-            .completer = {[&](size_t, std::string_view prefix) {
+            .completer = getCompleteInstallable(),
-                completeInstallable(prefix);
-            }}
         });
 
         expectArgs({
             .label = "dependency",
             .handler = {&_dependency},
-            .completer = {[&](size_t, std::string_view prefix) {
+            .completer = getCompleteInstallable(),
-                completeInstallable(prefix);
-            }}
         });
 
         addFlag({