From 3be810f5dbfaac8ced55e22fe8939c094e4a57fc Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 31 Oct 2021 13:19:53 -0400 Subject: [PATCH 1/7] bundler: pass drv attrset instead of path --- src/nix/bundle.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index adb5b3e73..9fc2656bb 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -71,6 +71,14 @@ struct CmdBundle : InstallableCommand auto app = installable->toApp(*evalState).resolve(getEvalStore(), store); + auto [progFlakeRef, progName] = parseFlakeRefWithFragment(installable->what(), absPath(".")); + const flake::LockFlags lockFlagsProg{ .writeLockFile = false }; + auto programInstallable = InstallableFlake(this, + evalState, std::move(progFlakeRef), + Strings{progName == "" ? "defaultPackage" : progName}, + Strings({"packages."+settings.thisSystem.get()+".","legacyPackages."+settings.thisSystem.get()+"."}), lockFlagsProg); + auto val = programInstallable.toValue(*evalState).first; + auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; auto bundler = InstallableFlake(this, @@ -80,10 +88,12 @@ struct CmdBundle : InstallableCommand auto attrs = evalState->buildBindings(2); - PathSet context; - for (auto & i : app.context) - context.insert("=" + store->printStorePath(i.path)); - attrs.alloc("program").mkString(app.program, context); + Value & prog = *evalState->allocAttr(*arg, evalState->symbols.create("program")); + evalState->mkAttrs(prog,val->attrs->size()); + for (auto &j : *(val->attrs)){ + prog.attrs->push_back(j); + } + prog.attrs->sort(); attrs.alloc("system").mkString(settings.thisSystem.get()); From c94db0535c4ddd32957fd80d0713797ec73cc70b Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 21 Jan 2022 11:43:11 -0500 Subject: [PATCH 2/7] Refactor bundler API Bundlers now expect to be located at bundlers.. and are a function from derivations to derivations. --- src/nix/bundle.cc | 32 ++++++++++---------------------- src/nix/bundle.md | 41 ++++++++++++++++++++++++++++++++++++----- src/nix/flake.cc | 28 +++++++++++++++++++--------- 3 files changed, 65 insertions(+), 36 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 9fc2656bb..445db5d5f 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -69,39 +69,24 @@ struct CmdBundle : InstallableCommand { auto evalState = getEvalState(); - auto app = installable->toApp(*evalState).resolve(getEvalStore(), store); - auto [progFlakeRef, progName] = parseFlakeRefWithFragment(installable->what(), absPath(".")); const flake::LockFlags lockFlagsProg{ .writeLockFile = false }; auto programInstallable = InstallableFlake(this, evalState, std::move(progFlakeRef), - Strings{progName == "" ? "defaultPackage" : progName}, - Strings({"packages."+settings.thisSystem.get()+".","legacyPackages."+settings.thisSystem.get()+"."}), lockFlagsProg); + Strings{progName == "" ? "defaultApp" : progName}, + Strings(this->getDefaultFlakeAttrPathPrefixes()), + lockFlagsProg); auto val = programInstallable.toValue(*evalState).first; auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; auto bundler = InstallableFlake(this, evalState, std::move(bundlerFlakeRef), - Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, + Strings{bundlerName == "" ? "defaultBundler." + settings.thisSystem.get() : settings.thisSystem.get() + "." + bundlerName}, Strings({"bundlers."}), lockFlags); - auto attrs = evalState->buildBindings(2); - - Value & prog = *evalState->allocAttr(*arg, evalState->symbols.create("program")); - evalState->mkAttrs(prog,val->attrs->size()); - for (auto &j : *(val->attrs)){ - prog.attrs->push_back(j); - } - prog.attrs->sort(); - - attrs.alloc("system").mkString(settings.thisSystem.get()); - auto vRes = evalState->allocValue(); - evalState->callFunction( - *bundler.toValue(*evalState).first, - evalState->allocValue()->mkAttrs(attrs), - *vRes, noPos); + evalState->callFunction(*bundler.toValue(*evalState).first, *val, *vRes, noPos); if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); @@ -123,9 +108,12 @@ struct CmdBundle : InstallableCommand auto outPathS = store->printStorePath(outPath); - if (!outLink) - outLink = baseNameOf(app.program); + if (!outLink) { + auto &attr = vRes->attrs->need(evalState->sName); + outLink = evalState->forceStringNoCtx(*attr.value,*attr.pos); + } + // TODO: will crash if not a localFSStore? store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink)); } }; diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 5e2298376..317b7000b 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -18,12 +18,20 @@ R""( nix (Nix) 2.4pre20201215_e3ddffb ``` +* Bundle a Hello using a specific bundler: + + ```console + # nix bundle --bundler github:NixOS/bundlers#toDockerImage nixpkgs#hello + # docker load < hello-2.10.tar.gz + # docker run hello-2.10:latest hello + Hello, world! + ``` + # Description -`nix bundle` packs the closure of the [Nix app](./nix3-run.md) -*installable* into a single self-extracting executable. See the -[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle) -for more details. +`nix bundle`, by default, packs the closure of the [Nix app](./nix3-run.md) +*installable* into a single self-extracting executable. See the [`nix-bundle` +homepage](https://github.com/matthewbauer/nix-bundle) for more details. > **Note** > @@ -31,6 +39,29 @@ for more details. # Bundler definitions -TODO +If no flake output attribute is given, `nix bundle` tries the following +flake output attributes: + +* `defaultBundler.` + +If an attribute *name* is given, `nix run` tries the following flake +output attributes: + +* `bundler..` + +# Bundlers + +An bundlers is specified by a flake output attribute named +`bundlers..` or `defaultBundler.`. It looks like this: + +```nix +bundlers.x86_64-linux.identity = drv: drv; + +bundlers.x86_64-linux.blender_2_79 = drv: self.packages.x86_64-linux.blender_2_79; + +defaultBundler.x86_64-linux = drv: drv; +``` + +A bundler must be a function that accepts a derivation and returns a derivation. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 97f4d911c..19bf9ab51 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -475,10 +475,7 @@ struct CmdFlakeCheck : FlakeCommand state->forceValue(v, pos); if (!v.isLambda()) throw Error("bundler must be a function"); - if (!v.lambda.fun->formals || - !v.lambda.fun->formals->argNames.count(state->symbols.create("program")) || - !v.lambda.fun->formals->argNames.count(state->symbols.create("system"))) - throw Error("bundler must take formal arguments 'program' and 'system'"); + // TODO: check types of inputs/outputs? } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); reportError(e); @@ -609,14 +606,27 @@ struct CmdFlakeCheck : FlakeCommand *attr.value, *attr.pos); } - else if (name == "defaultBundler") - checkBundler(name, vOutput, pos); + else if (name == "defaultBundler") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkBundler( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } else if (name == "bundlers") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) - checkBundler(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) { + checkBundler( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } } else From 93299efc7c587194de4c88ddcba5d33df3b51bf6 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Tue, 25 Jan 2022 03:39:18 -0500 Subject: [PATCH 3/7] bundler: add tests and change defaults to use a derivation --- src/nix/bundle.cc | 10 +++++----- tests/flake-bundler.sh | 23 +++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 tests/flake-bundler.sh diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 445db5d5f..3d953ebc0 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -51,7 +51,7 @@ struct CmdBundle : InstallableCommand Strings getDefaultFlakeAttrPaths() override { - Strings res{"defaultApp." + settings.thisSystem.get()}; + Strings res{"defaultPackage." + settings.thisSystem.get()}; for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) res.push_back(s); return res; @@ -59,7 +59,7 @@ struct CmdBundle : InstallableCommand Strings getDefaultFlakeAttrPathPrefixes() override { - Strings res{"apps." + settings.thisSystem.get() + "."}; + Strings res{"packages." + settings.thisSystem.get() + "."}; for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) res.push_back(s); return res; @@ -73,7 +73,7 @@ struct CmdBundle : InstallableCommand const flake::LockFlags lockFlagsProg{ .writeLockFile = false }; auto programInstallable = InstallableFlake(this, evalState, std::move(progFlakeRef), - Strings{progName == "" ? "defaultApp" : progName}, + Strings{progName == "" ? "defaultPackage" : progName}, Strings(this->getDefaultFlakeAttrPathPrefixes()), lockFlagsProg); auto val = programInstallable.toValue(*evalState).first; @@ -82,8 +82,8 @@ struct CmdBundle : InstallableCommand const flake::LockFlags lockFlags{ .writeLockFile = false }; auto bundler = InstallableFlake(this, evalState, std::move(bundlerFlakeRef), - Strings{bundlerName == "" ? "defaultBundler." + settings.thisSystem.get() : settings.thisSystem.get() + "." + bundlerName}, - Strings({"bundlers."}), lockFlags); + Strings{bundlerName == "" ? "defaultBundler." + settings.thisSystem.get() : settings.thisSystem.get() + "." + bundlerName, bundlerName}, + Strings({"","bundlers."}), lockFlags); auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *val, *vRes, noPos); diff --git a/tests/flake-bundler.sh b/tests/flake-bundler.sh new file mode 100644 index 000000000..2e52f999d --- /dev/null +++ b/tests/flake-bundler.sh @@ -0,0 +1,23 @@ +source common.sh + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local + +cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME + +cd $TEST_HOME + +cat < flake.nix +{ + outputs = {self}: { + defaultBundler.$system = drv: drv; + defaultPackage.$system = import ./simple.nix; + }; +} +EOF +nix build .# +nix bundle --bundler .# .# +nix bundle --bundler .#defaultBundler.$system .#defaultPackage.$system + +clearStore + diff --git a/tests/local.mk b/tests/local.mk index 93cf20d43..9c3e6bbd3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -48,6 +48,7 @@ nix_tests = \ flakes.sh \ flake-local-settings.sh \ flake-searching.sh \ + flake-bundler.sh \ build.sh \ repl.sh ca/repl.sh \ ca/build.sh \ From dc85e20684c6763330d35f7855e8ac741dfdfa23 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Tue, 25 Jan 2022 03:48:44 -0500 Subject: [PATCH 4/7] bundler: notes and doc update to include bundlers repo --- doc/manual/src/release-notes/rl-next.md | 6 ++++++ src/nix/bundle.md | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 78ae99f4b..e755173c0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,2 +1,8 @@ # Release X.Y (202?-??-??) +* `nix bundle` breaking API change now supports bundlers of the form + `bundler..= derivation: another-derivation;`. This supports + additional functionality to inspect evaluation information during bundling. A + new [repository](https://github.com/NixOS/bundlers) has various bundlers + implemented. + diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 317b7000b..f151eeabc 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -29,9 +29,9 @@ R""( # Description -`nix bundle`, by default, packs the closure of the [Nix app](./nix3-run.md) -*installable* into a single self-extracting executable. See the [`nix-bundle` -homepage](https://github.com/matthewbauer/nix-bundle) for more details. +`nix bundle`, by default, packs the closure of the *installable* into a single +self-extracting executable. See the [`bundlers` +homepage](https://github.com/NixOS/bundlers) for more details. > **Note** > From 4ebc50d92e65e7fd9cf30fb84a0c39a13475a31f Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 28 Jan 2022 09:56:58 -0500 Subject: [PATCH 5/7] bundler: revert default behavior to use defaultApp Bundlers are now responsible for correctly handling their inputs which are no longer constrained to be (Drv->Drv)->Drv->Drv, but can be of type (attrset->Drv)->attrset->Drv. --- src/nix/bundle.cc | 11 ++++++++--- tests/flake-bundler.sh | 9 ++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 3d953ebc0..113ceca33 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -51,7 +51,9 @@ struct CmdBundle : InstallableCommand Strings getDefaultFlakeAttrPaths() override { - Strings res{"defaultPackage." + settings.thisSystem.get()}; + Strings res{ + "defaultApp." + settings.thisSystem.get() + }; for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) res.push_back(s); return res; @@ -59,7 +61,10 @@ struct CmdBundle : InstallableCommand Strings getDefaultFlakeAttrPathPrefixes() override { - Strings res{"packages." + settings.thisSystem.get() + "."}; + Strings res{ + "apps." + settings.thisSystem.get() + "." + + }; for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) res.push_back(s); return res; @@ -73,7 +78,7 @@ struct CmdBundle : InstallableCommand const flake::LockFlags lockFlagsProg{ .writeLockFile = false }; auto programInstallable = InstallableFlake(this, evalState, std::move(progFlakeRef), - Strings{progName == "" ? "defaultPackage" : progName}, + Strings{progName == "" ? "defaultApp" : progName}, Strings(this->getDefaultFlakeAttrPathPrefixes()), lockFlagsProg); auto val = programInstallable.toValue(*evalState).first; diff --git a/tests/flake-bundler.sh b/tests/flake-bundler.sh index 2e52f999d..30e2fbd1d 100644 --- a/tests/flake-bundler.sh +++ b/tests/flake-bundler.sh @@ -10,8 +10,15 @@ cd $TEST_HOME cat < flake.nix { outputs = {self}: { - defaultBundler.$system = drv: drv; + defaultBundler.$system = drv: + if drv?type && drv.type == "derivation" + then drv + else self.defaultPackage.$system; defaultPackage.$system = import ./simple.nix; + defaultApp.$system = { + type = "app"; + program = "\${import ./simple.nix}/hello"; + }; }; } EOF From 73e82ae954be991858053df2a6b7323fe3a82c36 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 28 Jan 2022 10:17:51 -0500 Subject: [PATCH 6/7] bundler: tests various combinations of referring to installables --- tests/flake-bundler.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/flake-bundler.sh b/tests/flake-bundler.sh index 30e2fbd1d..699920f60 100644 --- a/tests/flake-bundler.sh +++ b/tests/flake-bundler.sh @@ -10,10 +10,11 @@ cd $TEST_HOME cat < flake.nix { outputs = {self}: { - defaultBundler.$system = drv: + bundlers.$system.simple = drv: if drv?type && drv.type == "derivation" then drv else self.defaultPackage.$system; + defaultBundler.$system = self.bundlers.$system.simple; defaultPackage.$system = import ./simple.nix; defaultApp.$system = { type = "app"; @@ -24,7 +25,11 @@ cat < flake.nix EOF nix build .# nix bundle --bundler .# .# -nix bundle --bundler .#defaultBundler.$system .#defaultPackage.$system +nix bundle --bundler .#defaultBundler.$system .#defaultPackage.$system +nix bundle --bundler .#bundlers.$system.simple .#defaultPackage.$system + +nix bundle --bundler .#defaultBundler.$system .#defaultApp.$system +nix bundle --bundler .#bundlers.$system.simple .#defaultApp.$system clearStore From 6e5e64fc7485cc12dd98be07e9abcc52d63dfd86 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 28 Jan 2022 10:23:57 -0500 Subject: [PATCH 7/7] bundler: suggested doc fixes --- src/nix/bundle.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix/bundle.md b/src/nix/bundle.md index f151eeabc..a5186a996 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -37,7 +37,7 @@ homepage](https://github.com/NixOS/bundlers) for more details. > > This command only works on Linux. -# Bundler definitions +# Flake output attributes If no flake output attribute is given, `nix bundle` tries the following flake output attributes: @@ -51,7 +51,7 @@ output attributes: # Bundlers -An bundlers is specified by a flake output attribute named +A bundler is specified by a flake output attribute named `bundlers..` or `defaultBundler.`. It looks like this: ```nix @@ -62,6 +62,7 @@ bundlers.x86_64-linux.blender_2_79 = drv: self.packages.x86_64-linux.blender_2_7 defaultBundler.x86_64-linux = drv: drv; ``` -A bundler must be a function that accepts a derivation and returns a derivation. +A bundler must be a function that accepts an arbitrary value (typically a +derivation or app definition) and returns a derivation. )""