puck/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge remote-tracking branch 'origin/master' into libgit2

Browse source
This commit is contained in:
Eelco Dolstra 2023-11-02 16:16:37 +01:00
parent 0c5eac9c45 1e61c007be
commit 39ea46abb1
85 changed files with 971 additions and 576 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/backport.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
fetch-depth: 0 fetch-depth: 0
- name: Create backport PRs - name: Create backport PRs
# should be kept in sync with `version` # should be kept in sync with `version`
uses: zeebe-io/backport-action@v1.4.0 uses: zeebe-io/backport-action@v2.0.0
with: with:
# Config README: https://github.com/zeebe-io/backport-action#backport-action # Config README: https://github.com/zeebe-io/backport-action#backport-action
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}

3
configure.ac
View file

@ -68,6 +68,9 @@ case "$host_os" in
esac esac
ENSURE_NO_GCC_BUG_80431
# Check for pubsetbuf. # Check for pubsetbuf.
AC_MSG_CHECKING([for pubsetbuf]) AC_MSG_CHECKING([for pubsetbuf])
AC_LANG_PUSH(C++) AC_LANG_PUSH(C++)

2
doc/manual/local.mk
View file

@ -103,7 +103,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix $(d)/nix.json: $(bindir)/nix

2
doc/manual/src/SUMMARY.md.in
View file

@ -18,6 +18,8 @@
- [Uninstalling Nix](installation/uninstall.md) - [Uninstalling Nix](installation/uninstall.md)
- [Nix Store](store/index.md) - [Nix Store](store/index.md)
- [File System Object](store/file-system-object.md) - [File System Object](store/file-system-object.md)
- [Store Object](store/store-object.md)
- [Store Path](store/store-path.md)
- [Nix Language](language/index.md) - [Nix Language](language/index.md)
- [Data Types](language/values.md) - [Data Types](language/values.md)
- [Language Constructs](language/constructs.md) - [Language Constructs](language/constructs.md)

2
doc/manual/src/architecture/architecture.md
View file

@ -63,7 +63,7 @@ The command line interface and Nix expressions are what users deal with most.
> The Nix language itself does not have a notion of *packages* or *configurations*. > The Nix language itself does not have a notion of *packages* or *configurations*.
> As far as we are concerned here, the inputs and results of a build plan are just data. > As far as we are concerned here, the inputs and results of a build plan are just data.
Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them.
It can also execute build plans to produce new data, which are made available to the operating system as files. It can also execute build plans to produce new data, which are made available to the operating system as files.
A build plan itself is a series of *build tasks*, together with their build inputs. A build plan itself is a series of *build tasks*, together with their build inputs.

18
doc/manual/src/contributing/documentation.md
View file

@ -162,6 +162,24 @@ Please observe these guidelines to ease reviews:
> This is a note. > This is a note.
``` ```
Highlight examples as such:
````
> **Example**
>
> ```console
> $ nix --version
> ```
````
Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
````
> **Syntax**
>
> *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}`
````
### The `@docroot@` variable ### The `@docroot@` variable
`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.

2
doc/manual/src/contributing/hacking.md
View file

@ -210,7 +210,7 @@ See [supported compilation environments](#compilation-environments) and instruct
To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running:
```console ```console
make clean && bear -- make -j$NIX_BUILD_CORES install make clean && bear -- make -j$NIX_BUILD_CORES default check install
``` ```
Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).

11
doc/manual/src/glossary.md
View file

@ -59,7 +59,7 @@
- [store]{#gloss-store} - [store]{#gloss-store}
A collection of store objects, with operations to manipulate that collection. A collection of store objects, with operations to manipulate that collection.
See [Nix Store] for details. See [Nix store](./store/index.md) for details.
There are many types of stores. There are many types of stores.
See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list.
@ -86,10 +86,13 @@
- [store path]{#gloss-store-path} - [store path]{#gloss-store-path}
The location of a [store object] in the file system, i.e., an The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory.
immediate child of the Nix store directory.
Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` > **Example**
>
> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
See [Store Path](@docroot@/store/store-path.md) for details.
[store path]: #gloss-store-path [store path]: #gloss-store-path

14
doc/manual/src/release-notes/rl-next.md
View file

@ -15,3 +15,17 @@
- `nix-shell` shebang lines now support single-quoted arguments. - `nix-shell` shebang lines now support single-quoted arguments.
- `builtins.fetchTree` is now marked as stable. - `builtins.fetchTree` is now marked as stable.
- The interface for creating and updating lock files has been overhauled:
- [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
It will *never* update existing inputs.
- [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
- Passing no arguments will update all inputs of the current flake, just like it already did.
- Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
- To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
- The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
They are superceded by `nix flake update`.

5
doc/manual/src/store/index.md
View file

@ -1,4 +1,5 @@
# Nix Store # Nix Store
The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches.
There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.

10
doc/manual/src/store/store-object.md Normal file
View file

@ -0,0 +1,10 @@
## Store Object
A Nix store is a collection of *store objects* with *references* between them.
A store object consists of
- A [file system object](./file-system-object.md) as data
- A set of [store paths](./store-path.md) as references to other store objects
Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object):
Once created, they do not change until they are deleted.

69
doc/manual/src/store/store-path.md Normal file
View file

@ -0,0 +1,69 @@
# Store Path
Nix implements references to [store objects](./index.md#store-object) as *store paths*.
Think of a store path as an [opaque], [unique identifier]:
The only way to obtain store path is by adding or building store objects.
A store path will always reference exactly one store object.
[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type
[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier
Store paths are pairs of
- A 20-byte digest for identification
- A symbolic name for people to read
> **Example**
>
> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
> - Name: `firefox-33.1`
To make store objects accessible to operating system processes, stores have to expose store objects through the file system.
A store path is rendered to a file system path as the concatenation of
- [Store directory](#store-directory) (typically `/nix/store`)
- Path separator (`/`)
- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
- Hyphen (`-`)
- Name
> **Example**
>
> ```
> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
> |--------| |------------------------------| |----------|
> store directory digest name
> ```
## Store Directory
Every [Nix store](./index.md) has a store directory.
Not every store can be accessed through the file system.
But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path).
[file system objects]: ./file-system-object.md
This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
> **Note**
>
> The store directory defaults to `/nix/store`, but is in principle arbitrary.
It is important which store a given store object belongs to:
Files in the store object can contain store paths, and processes may read these paths.
Nix can only guarantee referential integrity if store paths do not cross store boundaries.
Therefore one can only copy store objects to a different store if
- The source and target stores' directories match
or
- The store object in question has no references, that is, contains no store paths
One cannot copy a store object to a store with a different store directory.
Instead, it has to be rebuilt, together with all its dependencies.
It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums.

8
flake.lock
View file

@ -34,16 +34,16 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1695283060, "lastModified": 1698876495,
"narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "NixOS", "owner": "NixOS",
"ref": "nixos-23.05-small", "ref": "release-23.05",
"repo": "nixpkgs", "repo": "nixpkgs",
"type": "github" "type": "github"
} }

4
flake.nix
View file

@ -1,7 +1,9 @@
{ {
description = "The purely functional package manager"; description = "The purely functional package manager";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; # FIXME go back to nixos-23.05-small once
# https://github.com/NixOS/nixpkgs/pull/264875 is included.
inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };

64
m4/gcc_bug_80431.m4 Normal file
View file

@ -0,0 +1,64 @@
# Ensure that this bug is not present in the C++ toolchain we are using.
#
# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431
#
# The test program is from that issue, with only a slight modification
# to set an exit status instead of printing strings.
AC_DEFUN([ENSURE_NO_GCC_BUG_80431],
[
AC_MSG_CHECKING([that GCC bug 80431 is fixed])
AC_LANG_PUSH(C++)
AC_RUN_IFELSE(
[AC_LANG_PROGRAM(
[[
#include <cstdio>
static bool a = true;
static bool b = true;
struct Options { };
struct Option
{
Option(Options * options)
{
a = false;
}
~Option()
{
b = false;
}
};
struct MyOptions : Options { };
struct MyOptions2 : virtual MyOptions
{
Option foo{this};
};
]],
[[
{
MyOptions2 opts;
}
return (a << 1) | b;
]])],
[status_80431=0],
[status_80431=$?],
[
# Assume we're bug-free when cross-compiling
])
AC_LANG_POP(C++)
AS_CASE([$status_80431],
[0],[
AC_MSG_RESULT(yes)
],
[2],[
AC_MSG_RESULT(no)
AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug)
],
[
AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring)
])
])

6
src/libcmd/command.hh
View file

@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args {
void setEnviron(); void setEnviron();
}; };
void completeFlakeInputPath(
AddCompletions & completions,
ref<EvalState> evalState,
const std::vector<FlakeRef> & flakeRefs,
std::string_view prefix);
void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix); void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
void completeFlakeRefWithFragment( void completeFlakeRefWithFragment(

22
src/libcmd/installables.cc
View file

@ -28,7 +28,7 @@
namespace nix { namespace nix {
static void completeFlakeInputPath( void completeFlakeInputPath(
AddCompletions & completions, AddCompletions & completions,
ref<EvalState> evalState, ref<EvalState> evalState,
const std::vector<FlakeRef> & flakeRefs, const std::vector<FlakeRef> & flakeRefs,
@ -46,13 +46,6 @@ MixFlakeOptions::MixFlakeOptions()
{ {
auto category = "Common flake-related options"; auto category = "Common flake-related options";
addFlag({
.longName = "recreate-lock-file",
.description = "Recreate the flake's lock file from scratch.",
.category = category,
.handler = {&lockFlags.recreateLockFile, true}
});
addFlag({ addFlag({
.longName = "no-update-lock-file", .longName = "no-update-lock-file",
.description = "Do not allow any updates to the flake's lock file.", .description = "Do not allow any updates to the flake's lock file.",
@ -85,19 +78,6 @@ MixFlakeOptions::MixFlakeOptions()
.handler = {&lockFlags.commitLockFile, true} .handler = {&lockFlags.commitLockFile, true}
}); });
addFlag({
.longName = "update-input",
.description = "Update a specific flake input (ignoring its previous entry in the lock file).",
.category = category,
.labels = {"input-path"},
.handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
addFlag({ addFlag({
.longName = "override-input", .longName = "override-input",
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",

15
src/libexpr/flake/flake.cc
View file

@ -447,8 +447,8 @@ LockedFlake lockFlake(
assert(input.ref); assert(input.ref);
/* Do we have an entry in the existing lock file? And we /* Do we have an entry in the existing lock file?
don't have a --update-input flag for this input? */ And the input is not in updateInputs? */
std::shared_ptr<LockedNode> oldLock; std::shared_ptr<LockedNode> oldLock;
updatesUsed.insert(inputPath); updatesUsed.insert(inputPath);
@ -472,9 +472,8 @@ LockedFlake lockFlake(
node->inputs.insert_or_assign(id, childNode); node->inputs.insert_or_assign(id, childNode);
/* If we have an --update-input flag for an input /* If we have this input in updateInputs, then we
of this input, then we must fetch the flake to must fetch the flake to update it. */
update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto mustRefetch = auto mustRefetch =
@ -616,7 +615,7 @@ LockedFlake lockFlake(
for (auto & i : lockFlags.inputUpdates) for (auto & i : lockFlags.inputUpdates)
if (!updatesUsed.count(i)) if (!updatesUsed.count(i))
warn("the flag '--update-input %s' does not match any input", printInputPath(i)); warn("'%s' does not match any input of this flake", printInputPath(i));
/* Check 'follows' inputs. */ /* Check 'follows' inputs. */
newLockFile.check(); newLockFile.check();
@ -651,14 +650,14 @@ LockedFlake lockFlake(
bool lockFileExists = pathExists(outputLockFilePath); bool lockFileExists = pathExists(outputLockFilePath);
if (lockFileExists) {
auto s = chomp(diff); auto s = chomp(diff);
if (lockFileExists) {
if (s.empty()) if (s.empty())
warn("updating lock file '%s'", outputLockFilePath); warn("updating lock file '%s'", outputLockFilePath);
else else
warn("updating lock file '%s':\n%s", outputLockFilePath, s); warn("updating lock file '%s':\n%s", outputLockFilePath, s);
} else } else
warn("creating lock file '%s'", outputLockFilePath); warn("creating lock file '%s': \n%s", outputLockFilePath, s);
std::optional<std::string> commitMessage = std::nullopt; std::optional<std::string> commitMessage = std::nullopt;

6
src/libexpr/primops.cc
View file

@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
try { try {
auto checked = state.checkSourcePath(path); auto checked = state.checkSourcePath(path);
auto exists = checked.pathExists(); auto st = checked.maybeLstat();
if (exists && mustBeDir) { auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
exists = checked.lstat().type == InputAccessor::tDirectory;
}
v.mkBool(exists); v.mkBool(exists);
} catch (SysError & e) { } catch (SysError & e) {
/* Don't give away info from errors while canonicalising /* Don't give away info from errors while canonicalising

71
src/libfetchers/fetchers.cc
View file

@ -6,12 +6,31 @@
namespace nix::fetchers { namespace nix::fetchers {
std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr; using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme) void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
{ {
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>(); if (!inputSchemes)
inputSchemes->push_back(std::move(inputScheme)); inputSchemes = std::make_unique<InputSchemeMap>();
auto schemeName = inputScheme->schemeName();
if (inputSchemes->count(schemeName) > 0)
throw Error("Input scheme with name %s already registered", schemeName);
inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
}
nlohmann::json dumpRegisterInputSchemeInfo() {
using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : *inputSchemes) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
} }
Input Input::fromURL(const std::string & url, bool requireTree) Input Input::fromURL(const std::string & url, bool requireTree)
@ -34,7 +53,7 @@ static void fixupInput(Input & input)
Input Input::fromURL(const ParsedURL & url, bool requireTree) Input Input::fromURL(const ParsedURL & url, bool requireTree)
{ {
for (auto & inputScheme : *inputSchemes) { for (auto & [_, inputScheme] : *inputSchemes) {
auto res = inputScheme->inputFromURL(url, requireTree); auto res = inputScheme->inputFromURL(url, requireTree);
if (res) { if (res) {
experimentalFeatureSettings.require(inputScheme->experimentalFeature()); experimentalFeatureSettings.require(inputScheme->experimentalFeature());
@ -49,20 +68,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree)
Input Input::fromAttrs(Attrs && attrs) Input Input::fromAttrs(Attrs && attrs)
{ {
for (auto & inputScheme : *inputSchemes) { auto schemeName = ({
auto res = inputScheme->inputFromAttrs(attrs); auto schemeNameOpt = maybeGetStrAttr(attrs, "type");
if (res) { if (!schemeNameOpt)
experimentalFeatureSettings.require(inputScheme->experimentalFeature()); throw Error("'type' attribute to specify input scheme is required but not provided");
res->scheme = inputScheme; *std::move(schemeNameOpt);
fixupInput(*res); });
return std::move(*res);
}
}
auto raw = [&]() {
// Return an input without a scheme; most operations will fail,
// but not all of them. Doing this is to support those other
// operations which are supposed to be robust on
// unknown/uninterpretable inputs.
Input input; Input input;
input.attrs = attrs; input.attrs = attrs;
fixupInput(input); fixupInput(input);
return input; return input;
};
std::shared_ptr<InputScheme> inputScheme = ({
auto i = inputSchemes->find(schemeName);
i == inputSchemes->end() ? nullptr : i->second;
});
if (!inputScheme) return raw();
experimentalFeatureSettings.require(inputScheme->experimentalFeature());
auto allowedAttrs = inputScheme->allowedAttrs();
for (auto & [name, _] : attrs)
if (name != "type" && allowedAttrs.count(name) == 0)
throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
auto res = inputScheme->inputFromAttrs(attrs);
if (!res) return raw();
res->scheme = inputScheme;
fixupInput(*res);
return std::move(*res);
} }
ParsedURL Input::toURL() const ParsedURL Input::toURL() const
@ -325,7 +368,7 @@ std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store,
throw UnimplementedError("InputScheme must implement fetch() or getAccessor()"); throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
} }
std::optional<ExperimentalFeature> InputScheme::experimentalFeature() std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
{ {
return {}; return {};
} }

23
src/libfetchers/fetchers.hh
View file

@ -8,6 +8,7 @@
#include "url.hh" #include "url.hh"
#include <memory> #include <memory>
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; class StorePath; struct InputAccessor; } namespace nix { class Store; class StorePath; struct InputAccessor; }
@ -131,6 +132,24 @@ struct InputScheme
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0; virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
/**
* What is the name of the scheme?
*
* The `type` attribute is used to select which input scheme is
* used, and then the other fields are forwarded to that input
* scheme.
*/
virtual std::string_view schemeName() const = 0;
/**
* Allowed attributes in an attribute set that is converted to an
* input.
*
* `type` is not included from this set, because the `type` field is
parsed first to choose which scheme; `type` is always required.
*/
virtual StringSet allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const; virtual ParsedURL toURL(const Input & input) const;
virtual Input applyOverrides( virtual Input applyOverrides(
@ -155,7 +174,7 @@ struct InputScheme
/** /**
* Is this `InputScheme` part of an experimental feature? * Is this `InputScheme` part of an experimental feature?
*/ */
virtual std::optional<ExperimentalFeature> experimentalFeature(); virtual std::optional<ExperimentalFeature> experimentalFeature() const;
virtual bool isDirect(const Input & input) const virtual bool isDirect(const Input & input) const
{ return true; } { return true; }
@ -163,4 +182,6 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
nlohmann::json dumpRegisterInputSchemeInfo();
} }

4
src/libfetchers/fs-input-accessor.cc
View file

@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
} }
Stat lstat(const CanonPath & path) override std::optional<Stat> maybeLstat(const CanonPath & path) override
{ {
auto absPath = makeAbsPath(path); auto absPath = makeAbsPath(path);
checkAllowed(absPath); checkAllowed(absPath);
return PosixSourceAccessor::lstat(absPath); return PosixSourceAccessor::maybeLstat(absPath);
} }
DirEntries readDirectory(const CanonPath & path) override DirEntries readDirectory(const CanonPath & path) override

6
src/libfetchers/git-utils.cc
View file

@ -402,12 +402,14 @@ struct GitInputAccessor : InputAccessor
return path.isRoot() ? true : (bool) lookup(path); return path.isRoot() ? true : (bool) lookup(path);
} }
Stat lstat(const CanonPath & path) override std::optional<Stat> maybeLstat(const CanonPath & path) override
{ {
if (path.isRoot()) if (path.isRoot())
return Stat { .type = tDirectory }; return Stat { .type = tDirectory };
auto entry = need(path); auto entry = lookup(path);
if (!entry)
return std::nullopt;
auto mode = git_tree_entry_filemode(entry); auto mode = git_tree_entry_filemode(entry);

30
src/libfetchers/git.cc
View file

@ -168,14 +168,32 @@ struct GitInputScheme : InputScheme
return inputFromAttrs(attrs); return inputFromAttrs(attrs);
} }
std::string_view schemeName() const override
{
return "git";
}
StringSet allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"shallow",
"submodules",
"lastModified",
"revCount",
"narHash",
"allRefs",
"name",
"dirtyRev",
"dirtyShortRev",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{ {
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
throw Error("unsupported Git input attribute '%s'", name);
maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "submodules");
maybeGetBoolAttr(attrs, "allRefs"); maybeGetBoolAttr(attrs, "allRefs");

35
src/libfetchers/github.cc
View file

@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
struct GitArchiveInputScheme : InputScheme struct GitArchiveInputScheme : InputScheme
{ {
virtual std::string type() const = 0;
virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0; virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{ {
if (url.scheme != type()) return {}; if (url.scheme != schemeName()) return {};
auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
Input input; Input input;
input.attrs.insert_or_assign("type", type()); input.attrs.insert_or_assign("type", std::string { schemeName() });
input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]); input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme
return input; return input;
} }
StringSet allowedAttrs() const override
{
return {
"owner",
"repo",
"ref",
"rev",
"narHash",
"lastModified",
"host",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{ {
if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner"); getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo"); getStrAttr(attrs, "repo");
@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme
if (ref) path += "/" + *ref; if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
return ParsedURL { return ParsedURL {
.scheme = type(), .scheme = std::string { schemeName() },
.path = path, .path = path,
}; };
} }
@ -220,7 +225,7 @@ struct GitArchiveInputScheme : InputScheme
return {result.storePath, input}; return {result.storePath, input};
} }
std::optional<ExperimentalFeature> experimentalFeature() override std::optional<ExperimentalFeature> experimentalFeature() const override
{ {
return Xp::Flakes; return Xp::Flakes;
} }
@ -228,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme
struct GitHubInputScheme : GitArchiveInputScheme struct GitHubInputScheme : GitArchiveInputScheme
{ {
std::string type() const override { return "github"; } std::string_view schemeName() const override { return "github"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
@ -309,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
struct GitLabInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme
{ {
std::string type() const override { return "gitlab"; } std::string_view schemeName() const override { return "gitlab"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
@ -377,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
struct SourceHutInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme
{ {
std::string type() const override { return "sourcehut"; } std::string_view schemeName() const override { return "sourcehut"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {

23
src/libfetchers/indirect.cc
View file

@ -50,14 +50,23 @@ struct IndirectInputScheme : InputScheme
return input; return input;
} }
std::string_view schemeName() const override
{
return "indirect";
}
StringSet allowedAttrs() const override
{
return {
"id",
"ref",
"rev",
"narHash",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{ {
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
throw Error("unsupported indirect input attribute '%s'", name);
auto id = getStrAttr(attrs, "id"); auto id = getStrAttr(attrs, "id");
if (!std::regex_match(id, flakeRegex)) if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id); throw BadURL("'%s' is not a valid flake ID", id);
@ -93,7 +102,7 @@ struct IndirectInputScheme : InputScheme
throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
} }
std::optional<ExperimentalFeature> experimentalFeature() override std::optional<ExperimentalFeature> experimentalFeature() const override
{ {
return Xp::Flakes; return Xp::Flakes;
} }

4
src/libfetchers/memory-input-accessor.cc
View file

@ -20,12 +20,12 @@ struct MemoryInputAccessorImpl : MemoryInputAccessor
return i != files.end(); return i != files.end();
} }
Stat lstat(const CanonPath & path) override std::optional<Stat> maybeLstat(const CanonPath & path) override
{ {
auto i = files.find(path); auto i = files.find(path);
if (i != files.end()) if (i != files.end())
return Stat { .type = tRegular, .isExecutable = false }; return Stat { .type = tRegular, .isExecutable = false };
throw Error("file '%s' does not exist", path); return std::nullopt;
} }
DirEntries readDirectory(const CanonPath & path) override DirEntries readDirectory(const CanonPath & path) override

23
src/libfetchers/mercurial.cc
View file

@ -69,14 +69,25 @@ struct MercurialInputScheme : InputScheme
return inputFromAttrs(attrs); return inputFromAttrs(attrs);
} }
std::string_view schemeName() const override
{
return "hg";
}
StringSet allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"revCount",
"narHash",
"name",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{ {
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url")); parseURL(getStrAttr(attrs, "url"));
if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (auto ref = maybeGetStrAttr(attrs, "ref")) {

35
src/libfetchers/path.cc
View file

@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme
return input; return input;
} }
std::string_view schemeName() const override
{
return "path";
}
StringSet allowedAttrs() const override
{
return {
"path",
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/
"rev",
"revCount",
"lastModified",
"narHash",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{ {
if (maybeGetStrAttr(attrs, "type") != "path") return {};
getStrAttr(attrs, "path"); getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree
work the same as the repository from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
// checked in Input::fromAttrs
;
else
throw Error("unsupported path input attribute '%s'", name);
Input input; Input input;
input.attrs = attrs; input.attrs = attrs;
return input; return input;
@ -135,7 +142,7 @@ struct PathInputScheme : InputScheme
return {std::move(*storePath), input}; return {std::move(*storePath), input};
} }
std::optional<ExperimentalFeature> experimentalFeature() override std::optional<ExperimentalFeature> experimentalFeature() const override
{ {
return Xp::Flakes; return Xp::Flakes;
} }

34
src/libfetchers/tarball.cc
View file

@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball(
// An input scheme corresponding to a curl-downloadable resource. // An input scheme corresponding to a curl-downloadable resource.
struct CurlInputScheme : InputScheme struct CurlInputScheme : InputScheme
{ {
virtual const std::string inputType() const = 0;
const std::set<std::string> transportUrlSchemes = {"file", "http", "https"}; const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
const bool hasTarballExtension(std::string_view path) const const bool hasTarballExtension(std::string_view path) const
@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme
url.query.erase("rev"); url.query.erase("rev");
url.query.erase("revCount"); url.query.erase("revCount");
input.attrs.insert_or_assign("type", inputType()); input.attrs.insert_or_assign("type", std::string { schemeName() });
input.attrs.insert_or_assign("url", url.to_string()); input.attrs.insert_or_assign("url", url.to_string());
return input; return input;
} }
StringSet allowedAttrs() const override
{
return {
"type",
"url",
"narHash",
"name",
"unpack",
"rev",
"revCount",
"lastModified",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{ {
auto type = maybeGetStrAttr(attrs, "type");
if (type != inputType()) return {};
// FIXME: some of these only apply to TarballInputScheme.
std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
for (auto & [name, value] : attrs)
if (!allowedNames.count(name))
throw Error("unsupported %s input attribute '%s'", *type, name);
Input input; Input input;
input.attrs = attrs; input.attrs = attrs;
@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme
struct FileInputScheme : CurlInputScheme struct FileInputScheme : CurlInputScheme
{ {
const std::string inputType() const override { return "file"; } std::string_view schemeName() const override { return "file"; }
bool isValidURL(const ParsedURL & url, bool requireTree) const override bool isValidURL(const ParsedURL & url, bool requireTree) const override
{ {
auto parsedUrlScheme = parseUrlScheme(url.scheme); auto parsedUrlScheme = parseUrlScheme(url.scheme);
return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
&& (parsedUrlScheme.application && (parsedUrlScheme.application
? parsedUrlScheme.application.value() == inputType() ? parsedUrlScheme.application.value() == schemeName()
: (!requireTree && !hasTarballExtension(url.path))); : (!requireTree && !hasTarballExtension(url.path)));
} }
@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme
struct TarballInputScheme : CurlInputScheme struct TarballInputScheme : CurlInputScheme
{ {
const std::string inputType() const override { return "tarball"; } std::string_view schemeName() const override { return "tarball"; }
bool isValidURL(const ParsedURL & url, bool requireTree) const override bool isValidURL(const ParsedURL & url, bool requireTree) const override
{ {
@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme
return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
&& (parsedUrlScheme.application && (parsedUrlScheme.application
? parsedUrlScheme.application.value() == inputType() ? parsedUrlScheme.application.value() == schemeName()
: (requireTree || hasTarballExtension(url.path))); : (requireTree || hasTarballExtension(url.path)));
} }

4
src/libfetchers/union-input-accessor.cc
View file

@ -28,10 +28,10 @@ struct UnionInputAccessor : InputAccessor
return accessor->pathExists(subpath); return accessor->pathExists(subpath);
} }
Stat lstat(const CanonPath & path) override std::optional<Stat> maybeLstat(const CanonPath & path) override
{ {
auto [accessor, subpath] = resolve(path); auto [accessor, subpath] = resolve(path);
return accessor->lstat(subpath); return accessor->maybeLstat(subpath);
} }
DirEntries readDirectory(const CanonPath & path) override DirEntries readDirectory(const CanonPath & path) override

28
src/libstore/binary-cache-store.cc
View file

@ -2,7 +2,7 @@
#include "binary-cache-store.hh" #include "binary-cache-store.hh"
#include "compression.hh" #include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh" #include "source-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "sync.hh" #include "sync.hh"
@ -143,7 +143,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
write the compressed NAR to disk), into a HashSink (to get the write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */ NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink { htSHA256 }; HashSink fileHashSink { htSHA256 };
std::shared_ptr<FSAccessor> narAccessor; std::shared_ptr<SourceAccessor> narAccessor;
HashSink narHashSink { htSHA256 }; HashSink narHashSink { htSHA256 };
{ {
FdSink fileSink(fdTemp.get()); FdSink fileSink(fdTemp.get());
@ -195,7 +195,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
if (writeNARListing) { if (writeNARListing) {
nlohmann::json j = { nlohmann::json j = {
{"version", 1}, {"version", 1},
{"root", listNar(ref<FSAccessor>(narAccessor), "", true)}, {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
}; };
upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
@ -206,9 +206,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
specify the NAR file and member containing the debug info. */ specify the NAR file and member containing the debug info. */
if (writeDebugInfo) { if (writeDebugInfo) {
std::string buildIdDir = "/lib/debug/.build-id"; CanonPath buildIdDir("lib/debug/.build-id");
if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) {
ThreadPool threadPool(25); ThreadPool threadPool(25);
@ -231,17 +231,17 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
std::regex regex1("^[0-9a-f]{2}$"); std::regex regex1("^[0-9a-f]{2}$");
std::regex regex2("^[0-9a-f]{38}\\.debug$"); std::regex regex2("^[0-9a-f]{38}\\.debug$");
for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + "/" + s1; auto dir = buildIdDir + s1;
if (narAccessor->stat(dir).type != FSAccessor::tDirectory if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory
|| !std::regex_match(s1, regex1)) || !std::regex_match(s1, regex1))
continue; continue;
for (auto & s2 : narAccessor->readDirectory(dir)) { for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
auto debugPath = dir + "/" + s2; auto debugPath = dir + s2;
if (narAccessor->stat(debugPath).type != FSAccessor::tRegular if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
|| !std::regex_match(s2, regex2)) || !std::regex_match(s2, regex2))
continue; continue;
@ -250,7 +250,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
std::string key = "debuginfo/" + buildId; std::string key = "debuginfo/" + buildId;
std::string target = "../" + narInfo->url; std::string target = "../" + narInfo->url;
threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target));
} }
} }
@ -503,9 +503,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
upsertFile(filePath, info.toJSON().dump(), "application/json"); upsertFile(filePath, info.toJSON().dump(), "application/json");
} }
ref<FSAccessor> BinaryCacheStore::getFSAccessor() ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
{ {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, localNarCache);
} }
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)

16
src/libstore/binary-cache-store.hh
View file

@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", const Setting<std::string> compression{this, "xz", "compression",
"NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", const Setting<bool> writeNARListing{this, false, "write-nar-listing",
"Whether to write a JSON file that lists the files in each NAR."}; "Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", const Setting<bool> writeDebugInfo{this, false, "index-debug-info",
R"( R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand fetch debug info on demand
)"}; )"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", const Setting<Path> secretKeyFile{this, "", "secret-key",
"Path to the secret key used to sign the binary cache."}; "Path to the secret key used to sign the binary cache."};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", const Setting<Path> localNarCache{this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", const Setting<bool> parallelCompression{this, false, "parallel-compression",
"Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level", const Setting<int> compressionLevel{this, -1, "compression-level",
R"( R"(
The *preset level* to be used when compressing NARs. The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected. The meaning and accepted values depend on the compression method selected.
@ -148,7 +148,7 @@ public:
void narFromPath(const StorePath & path, Sink & sink) override; void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override; ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override;

3
src/libstore/build/local-derivation-goal.cc
View file

@ -1563,10 +1563,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
Path source = worker.store.Store::toRealPath(path); Path source = worker.store.Store::toRealPath(path);
Path target = chrootRootDir + worker.store.printStorePath(path); Path target = chrootRootDir + worker.store.printStorePath(path);
if (pathExists(target)) if (pathExists(target)) {
// There is a similar debug message in doBind, so only run it in this block to not have double messages. // There is a similar debug message in doBind, so only run it in this block to not have double messages.
debug("bind-mounting %s -> %s", target, source); debug("bind-mounting %s -> %s", target, source);
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
}
/* Bind-mount the path into the sandbox. This requires /* Bind-mount the path into the sandbox. This requires
entering its mount namespace, which is not possible entering its mount namespace, which is not possible

6
src/libstore/daemon.cc
View file

@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
eagerly consume the entire stream it's given, past the eagerly consume the entire stream it's given, past the
length of the Nar. */ length of the Nar. */
TeeSource savedNARSource(from, saved); TeeSource savedNARSource(from, saved);
ParseSink sink; /* null sink; just parse the NAR */ NullParseSink sink; /* just parse the NAR */
parseDump(sink, savedNARSource); parseDump(sink, savedNARSource);
} else { } else {
/* Incrementally parse the NAR file, stripping the /* Incrementally parse the NAR file, stripping the
metadata, and streaming the sole file we expect into metadata, and streaming the sole file we expect into
`saved`. */ `saved`. */
RetrieveRegularNARSink savedRegular { saved }; RegularFileSink savedRegular { saved };
parseDump(savedRegular, from); parseDump(savedRegular, from);
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
} }
@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
source = std::make_unique<TunnelSource>(from, to); source = std::make_unique<TunnelSource>(from, to);
else { else {
TeeSource tee { from, saved }; TeeSource tee { from, saved };
ParseSink ether; NullParseSink ether;
parseDump(ether, tee); parseDump(ether, tee);
source = std::make_unique<StringSource>(saved.s); source = std::make_unique<StringSource>(saved.s);
} }

1
src/libstore/derivations.cc
View file

@ -6,7 +6,6 @@
#include "split.hh" #include "split.hh"
#include "common-protocol.hh" #include "common-protocol.hh"
#include "common-protocol-impl.hh" #include "common-protocol-impl.hh"
#include "fs-accessor.hh"
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>

2
src/libstore/dummy-store.cc
View file

@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
Callback<std::shared_ptr<const Realisation>> callback) noexcept override Callback<std::shared_ptr<const Realisation>> callback) noexcept override
{ callback(nullptr); } { callback(nullptr); }
virtual ref<FSAccessor> getFSAccessor() override virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{ unsupported("getFSAccessor"); } { unsupported("getFSAccessor"); }
}; };

2
src/libstore/export-import.cc
View file

@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
/* Extract the NAR from the source. */ /* Extract the NAR from the source. */
StringSink saved; StringSink saved;
TeeSource tee { source, saved }; TeeSource tee { source, saved };
ParseSink ether; NullParseSink ether;
parseDump(ether, tee); parseDump(ether, tee);
uint32_t magic = readInt(source); uint32_t magic = readInt(source);

52
src/libstore/fs-accessor.hh
View file

@ -1,52 +0,0 @@
#pragma once
///@file
#include "types.hh"
namespace nix {
/**
* An abstract class for accessing a filesystem-like structure, such
* as a (possibly remote) Nix store or the contents of a NAR file.
*/
class FSAccessor
{
public:
enum Type { tMissing, tRegular, tSymlink, tDirectory };
struct Stat
{
Type type = tMissing;
/**
* regular files only
*/
uint64_t fileSize = 0;
/**
* regular files only
*/
bool isExecutable = false; // regular files only
/**
* regular files only
*/
uint64_t narOffset = 0; // regular files only
};
virtual ~FSAccessor() { }
virtual Stat stat(const Path & path) = 0;
virtual StringSet readDirectory(const Path & path) = 0;
/**
* Read a file inside the store.
*
* If `requireValidPath` is set to `true` (the default), the path must be
* inside a valid store path, otherwise it just needs to be physically
* present (but not necessarily properly registered)
*/
virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
virtual std::string readLink(const Path & path) = 0;
};
}

8
src/libstore/legacy-ssh-store.cc
View file

@ -17,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{ {
using CommonSSHStoreConfig::CommonSSHStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."}; "Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent SSH connections."}; "Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; } const std::string name() override { return "SSH Store"; }
@ -38,7 +38,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
// Hack for getting remote build log output. // Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation // the documentation
const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection struct Connection
{ {
@ -363,7 +363,7 @@ public:
void ensurePath(const StorePath & path) override void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); } { unsupported("ensurePath"); }
virtual ref<FSAccessor> getFSAccessor() override virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{ unsupported("getFSAccessor"); } { unsupported("getFSAccessor"); }
/** /**

64
src/libstore/local-fs-store.cc
View file

@ -1,5 +1,5 @@
#include "archive.hh" #include "archive.hh"
#include "fs-accessor.hh" #include "posix-source-accessor.hh"
#include "store-api.hh" #include "store-api.hh"
#include "local-fs-store.hh" #include "local-fs-store.hh"
#include "globals.hh" #include "globals.hh"
@ -13,69 +13,53 @@ LocalFSStore::LocalFSStore(const Params & params)
{ {
} }
struct LocalStoreAccessor : public FSAccessor struct LocalStoreAccessor : PosixSourceAccessor
{ {
ref<LocalFSStore> store; ref<LocalFSStore> store;
bool requireValidPath;
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { } LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
: store(store)
, requireValidPath(requireValidPath)
{ }
Path toRealPath(const Path & path, bool requireValidPath = true) CanonPath toRealPath(const CanonPath & path)
{ {
auto storePath = store->toStorePath(path).first; auto [storePath, rest] = store->toStorePath(path.abs());
if (requireValidPath && !store->isValidPath(storePath)) if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size()); return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest);
} }
FSAccessor::Stat stat(const Path & path) override std::optional<Stat> maybeLstat(const CanonPath & path) override
{ {
auto realPath = toRealPath(path); return PosixSourceAccessor::maybeLstat(toRealPath(path));
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
throw SysError("getting status of '%1%'", path);
} }
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) DirEntries readDirectory(const CanonPath & path) override
throw Error("file '%1%' has unsupported type", path);
return {
S_ISREG(st.st_mode) ? Type::tRegular :
S_ISLNK(st.st_mode) ? Type::tSymlink :
Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
}
StringSet readDirectory(const Path & path) override
{ {
auto realPath = toRealPath(path); return PosixSourceAccessor::readDirectory(toRealPath(path));
auto entries = nix::readDirectory(realPath);
StringSet res;
for (auto & entry : entries)
res.insert(entry.name);
return res;
} }
std::string readFile(const Path & path, bool requireValidPath = true) override void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override
{ {
return nix::readFile(toRealPath(path, requireValidPath)); return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback);
} }
std::string readLink(const Path & path) override std::string readLink(const CanonPath & path) override
{ {
return nix::readLink(toRealPath(path)); return PosixSourceAccessor::readLink(toRealPath(path));
} }
}; };
ref<FSAccessor> LocalFSStore::getFSAccessor() ref<SourceAccessor> LocalFSStore::getFSAccessor(bool requireValidPath)
{ {
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())),
requireValidPath);
} }
void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)

14
src/libstore/local-fs-store.hh
View file

@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes const OptionalPathSetting rootDir{this, std::nullopt,
// it to omit the call to the Setting constructor. Clang works fine
// either way.
const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt,
"root", "root",
"Directory prefixed to all other paths."}; "Directory prefixed to all other paths."};
const PathSetting stateDir{(StoreConfig*) this, const PathSetting stateDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir,
"state", "state",
"Directory where Nix will store state."}; "Directory where Nix will store state."};
const PathSetting logDir{(StoreConfig*) this, const PathSetting logDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir,
"log", "log",
"directory where Nix will store log files."}; "directory where Nix will store log files."};
const PathSetting realStoreDir{(StoreConfig*) this, const PathSetting realStoreDir{this,
rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real",
"Physical path of the Nix store."}; "Physical path of the Nix store."};
}; };
@ -47,7 +43,7 @@ public:
LocalFSStore(const Params & params); LocalFSStore(const Params & params);
void narFromPath(const StorePath & path, Sink & sink) override; void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override; ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
/** /**
* Creates symlink from the `gcRoot` to the `storePath` and * Creates symlink from the `gcRoot` to the `storePath` and

2
src/libstore/local-store.cc
View file

@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
bool narRead = false; bool narRead = false;
Finally cleanup = [&]() { Finally cleanup = [&]() {
if (!narRead) { if (!narRead) {
ParseSink sink; NullParseSink sink;
parseDump(sink, source); parseDump(sink, source);
} }
}; };

4
src/libstore/local-store.hh
View file

@ -40,12 +40,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
{ {
using LocalFSStoreConfig::LocalFSStoreConfig; using LocalFSStoreConfig::LocalFSStoreConfig;
Setting<bool> requireSigs{(StoreConfig*) this, Setting<bool> requireSigs{this,
settings.requireSigs, settings.requireSigs,
"require-sigs", "require-sigs",
"Whether store paths copied into this store should have a trusted signature."}; "Whether store paths copied into this store should have a trusted signature."};
Setting<bool> readOnly{(StoreConfig*) this, Setting<bool> readOnly{this,
false, false,
"read-only", "read-only",
R"( R"(

125
src/libstore/nar-accessor.cc
View file

@ -11,13 +11,7 @@ namespace nix {
struct NarMember struct NarMember
{ {
FSAccessor::Type type = FSAccessor::Type::tMissing; SourceAccessor::Stat stat;
bool isExecutable = false;
/* If this is a regular file, position of the contents of this
file in the NAR. */
uint64_t start = 0, size = 0;
std::string target; std::string target;
@ -25,7 +19,7 @@ struct NarMember
std::map<std::string, NarMember> children; std::map<std::string, NarMember> children;
}; };
struct NarAccessor : public FSAccessor struct NarAccessor : public SourceAccessor
{ {
std::optional<const std::string> nar; std::optional<const std::string> nar;
@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor
acc.root = std::move(member); acc.root = std::move(member);
parents.push(&acc.root); parents.push(&acc.root);
} else { } else {
if (parents.top()->type != FSAccessor::Type::tDirectory) if (parents.top()->stat.type != Type::tDirectory)
throw Error("NAR file missing parent directory of path '%s'", path); throw Error("NAR file missing parent directory of path '%s'", path);
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
parents.push(&result.first->second); parents.push(&result.first->second);
@ -66,12 +60,12 @@ struct NarAccessor : public FSAccessor
void createDirectory(const Path & path) override void createDirectory(const Path & path) override
{ {
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); createMember(path, {Type::tDirectory, false, 0, 0});
} }
void createRegularFile(const Path & path) override void createRegularFile(const Path & path) override
{ {
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); createMember(path, {Type::tRegular, false, 0, 0});
} }
void closeRegularFile() override void closeRegularFile() override
@ -79,14 +73,15 @@ struct NarAccessor : public FSAccessor
void isExecutable() override void isExecutable() override
{ {
parents.top()->isExecutable = true; parents.top()->stat.isExecutable = true;
} }
void preallocateContents(uint64_t size) override void preallocateContents(uint64_t size) override
{ {
assert(size <= std::numeric_limits<uint64_t>::max()); assert(size <= std::numeric_limits<uint64_t>::max());
parents.top()->size = (uint64_t) size; auto & st = parents.top()->stat;
parents.top()->start = pos; st.fileSize = (uint64_t) size;
st.narOffset = pos;
} }
void receiveContents(std::string_view data) override void receiveContents(std::string_view data) override
@ -95,7 +90,9 @@ struct NarAccessor : public FSAccessor
void createSymlink(const Path & path, const std::string & target) override void createSymlink(const Path & path, const std::string & target) override
{ {
createMember(path, createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); NarMember{
.stat = {.type = Type::tSymlink},
.target = target});
} }
size_t read(char * data, size_t len) override size_t read(char * data, size_t len) override
@ -130,18 +127,20 @@ struct NarAccessor : public FSAccessor
std::string type = v["type"]; std::string type = v["type"];
if (type == "directory") { if (type == "directory") {
member.type = FSAccessor::Type::tDirectory; member.stat = {.type = Type::tDirectory};
for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
std::string name = i.key(); std::string name = i.key();
recurse(member.children[name], i.value()); recurse(member.children[name], i.value());
} }
} else if (type == "regular") { } else if (type == "regular") {
member.type = FSAccessor::Type::tRegular; member.stat = {
member.size = v["size"]; .type = Type::tRegular,
member.isExecutable = v.value("executable", false); .fileSize = v["size"],
member.start = v["narOffset"]; .isExecutable = v.value("executable", false),
.narOffset = v["narOffset"]
};
} else if (type == "symlink") { } else if (type == "symlink") {
member.type = FSAccessor::Type::tSymlink; member.stat = {.type = Type::tSymlink};
member.target = v.value("target", ""); member.target = v.value("target", "");
} else return; } else return;
}; };
@ -150,134 +149,122 @@ struct NarAccessor : public FSAccessor
recurse(root, v); recurse(root, v);
} }
NarMember * find(const Path & path) NarMember * find(const CanonPath & path)
{ {
Path canon = path == "" ? "" : canonPath(path);
NarMember * current = &root; NarMember * current = &root;
auto end = path.end();
for (auto it = path.begin(); it != end; ) {
// because it != end, the remaining component is non-empty so we need
// a directory
if (current->type != FSAccessor::Type::tDirectory) return nullptr;
// skip slash (canonPath above ensures that this is always a slash) for (auto & i : path) {
assert(*it == '/'); if (current->stat.type != Type::tDirectory) return nullptr;
it += 1; auto child = current->children.find(std::string(i));
// lookup current component
auto next = std::find(it, end, '/');
auto child = current->children.find(std::string(it, next));
if (child == current->children.end()) return nullptr; if (child == current->children.end()) return nullptr;
current = &child->second; current = &child->second;
it = next;
} }
return current; return current;
} }
NarMember & get(const Path & path) { NarMember & get(const CanonPath & path) {
auto result = find(path); auto result = find(path);
if (result == nullptr) if (!result)
throw Error("NAR file does not contain path '%1%'", path); throw Error("NAR file does not contain path '%1%'", path);
return *result; return *result;
} }
Stat stat(const Path & path) override std::optional<Stat> maybeLstat(const CanonPath & path) override
{ {
auto i = find(path); auto i = find(path);
if (i == nullptr) if (!i)
return {FSAccessor::Type::tMissing, 0, false}; return std::nullopt;
return {i->type, i->size, i->isExecutable, i->start}; return i->stat;
} }
StringSet readDirectory(const Path & path) override DirEntries readDirectory(const CanonPath & path) override
{ {
auto i = get(path); auto i = get(path);
if (i.type != FSAccessor::Type::tDirectory) if (i.stat.type != Type::tDirectory)
throw Error("path '%1%' inside NAR file is not a directory", path); throw Error("path '%1%' inside NAR file is not a directory", path);
StringSet res; DirEntries res;
for (auto & child : i.children) for (auto & child : i.children)
res.insert(child.first); res.insert_or_assign(child.first, std::nullopt);
return res; return res;
} }
std::string readFile(const Path & path, bool requireValidPath = true) override std::string readFile(const CanonPath & path) override
{ {
auto i = get(path); auto i = get(path);
if (i.type != FSAccessor::Type::tRegular) if (i.stat.type != Type::tRegular)
throw Error("path '%1%' inside NAR file is not a regular file", path); throw Error("path '%1%' inside NAR file is not a regular file", path);
if (getNarBytes) return getNarBytes(i.start, i.size); if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize);
assert(nar); assert(nar);
return std::string(*nar, i.start, i.size); return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize);
} }
std::string readLink(const Path & path) override std::string readLink(const CanonPath & path) override
{ {
auto i = get(path); auto i = get(path);
if (i.type != FSAccessor::Type::tSymlink) if (i.stat.type != Type::tSymlink)
throw Error("path '%1%' inside NAR file is not a symlink", path); throw Error("path '%1%' inside NAR file is not a symlink", path);
return i.target; return i.target;
} }
}; };
ref<FSAccessor> makeNarAccessor(std::string && nar) ref<SourceAccessor> makeNarAccessor(std::string && nar)
{ {
return make_ref<NarAccessor>(std::move(nar)); return make_ref<NarAccessor>(std::move(nar));
} }
ref<FSAccessor> makeNarAccessor(Source & source) ref<SourceAccessor> makeNarAccessor(Source & source)
{ {
return make_ref<NarAccessor>(source); return make_ref<NarAccessor>(source);
} }
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, ref<SourceAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes) GetNarBytes getNarBytes)
{ {
return make_ref<NarAccessor>(listing, getNarBytes); return make_ref<NarAccessor>(listing, getNarBytes);
} }
using nlohmann::json; using nlohmann::json;
json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse) json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
{ {
auto st = accessor->stat(path); auto st = accessor->lstat(path);
json obj = json::object(); json obj = json::object();
switch (st.type) { switch (st.type) {
case FSAccessor::Type::tRegular: case SourceAccessor::Type::tRegular:
obj["type"] = "regular"; obj["type"] = "regular";
obj["size"] = st.fileSize; if (st.fileSize)
obj["size"] = *st.fileSize;
if (st.isExecutable) if (st.isExecutable)
obj["executable"] = true; obj["executable"] = true;
if (st.narOffset) if (st.narOffset && *st.narOffset)
obj["narOffset"] = st.narOffset; obj["narOffset"] = *st.narOffset;
break; break;
case FSAccessor::Type::tDirectory: case SourceAccessor::Type::tDirectory:
obj["type"] = "directory"; obj["type"] = "directory";
{ {
obj["entries"] = json::object(); obj["entries"] = json::object();
json &res2 = obj["entries"]; json &res2 = obj["entries"];
for (auto & name : accessor->readDirectory(path)) { for (auto & [name, type] : accessor->readDirectory(path)) {
if (recurse) { if (recurse) {
res2[name] = listNar(accessor, path + "/" + name, true); res2[name] = listNar(accessor, path + name, true);
} else } else
res2[name] = json::object(); res2[name] = json::object();
} }
} }
break; break;
case FSAccessor::Type::tSymlink: case SourceAccessor::Type::tSymlink:
obj["type"] = "symlink"; obj["type"] = "symlink";
obj["target"] = accessor->readLink(path); obj["target"] = accessor->readLink(path);
break; break;
case FSAccessor::Type::tMissing: case SourceAccessor::Type::tMisc:
default: assert(false); // cannot happen for NARs
throw Error("path '%s' does not exist in NAR", path);
} }
return obj; return obj;
} }

11
src/libstore/nar-accessor.hh
View file

@ -1,10 +1,11 @@
#pragma once #pragma once
///@file ///@file
#include "source-accessor.hh"
#include <functional> #include <functional>
#include <nlohmann/json_fwd.hpp> #include <nlohmann/json_fwd.hpp>
#include "fs-accessor.hh"
namespace nix { namespace nix {
@ -14,9 +15,9 @@ struct Source;
* Return an object that provides access to the contents of a NAR * Return an object that provides access to the contents of a NAR
* file. * file.
*/ */
ref<FSAccessor> makeNarAccessor(std::string && nar); ref<SourceAccessor> makeNarAccessor(std::string && nar);
ref<FSAccessor> makeNarAccessor(Source & source); ref<SourceAccessor> makeNarAccessor(Source & source);
/** /**
* Create a NAR accessor from a NAR listing (in the format produced by * Create a NAR accessor from a NAR listing (in the format produced by
@ -26,7 +27,7 @@ ref<FSAccessor> makeNarAccessor(Source & source);
*/ */
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
ref<FSAccessor> makeLazyNarAccessor( ref<SourceAccessor> makeLazyNarAccessor(
const std::string & listing, const std::string & listing,
GetNarBytes getNarBytes); GetNarBytes getNarBytes);
@ -34,6 +35,6 @@ ref<FSAccessor> makeLazyNarAccessor(
* Write a JSON representation of the contents of a NAR (except file * Write a JSON representation of the contents of a NAR (except file
* contents). * contents).
*/ */
nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse); nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
} }

28
src/libstore/remote-fs-accessor.cc
View file

@ -8,8 +8,9 @@
namespace nix { namespace nix {
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, const Path & cacheDir)
: store(store) : store(store)
, requireValidPath(requireValidPath)
, cacheDir(cacheDir) , cacheDir(cacheDir)
{ {
if (cacheDir != "") if (cacheDir != "")
@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin
return fmt("%s/%s.%s", cacheDir, hashPart, ext); return fmt("%s/%s.%s", cacheDir, hashPart, ext);
} }
ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
{ {
if (cacheDir != "") { if (cacheDir != "") {
try { try {
@ -38,7 +39,7 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
if (cacheDir != "") { if (cacheDir != "") {
try { try {
nlohmann::json j = listNar(narAccessor, "", true); nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
writeFile(makeCacheFile(hashPart, "ls"), j.dump()); writeFile(makeCacheFile(hashPart, "ls"), j.dump());
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
@ -48,11 +49,10 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
return narAccessor; return narAccessor;
} }
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
{ {
auto path = canonPath(path_); auto [storePath, restPath_] = store->toStorePath(path.abs());
auto restPath = CanonPath(restPath_);
auto [storePath, restPath] = store->toStorePath(path);
if (requireValidPath && !store->isValidPath(storePath)) if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
@ -63,7 +63,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
std::string listing; std::string listing;
Path cacheFile; Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try { try {
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
@ -101,25 +101,25 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
} }
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)
{ {
auto res = fetch(path); auto res = fetch(path);
return res.first->stat(res.second); return res.first->maybeLstat(res.second);
} }
StringSet RemoteFSAccessor::readDirectory(const Path & path) SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path)
{ {
auto res = fetch(path); auto res = fetch(path);
return res.first->readDirectory(res.second); return res.first->readDirectory(res.second);
} }
std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) std::string RemoteFSAccessor::readFile(const CanonPath & path)
{ {
auto res = fetch(path, requireValidPath); auto res = fetch(path);
return res.first->readFile(res.second); return res.first->readFile(res.second);
} }
std::string RemoteFSAccessor::readLink(const Path & path) std::string RemoteFSAccessor::readLink(const CanonPath & path)
{ {
auto res = fetch(path); auto res = fetch(path);
return res.first->readLink(res.second); return res.first->readLink(res.second);

21
src/libstore/remote-fs-accessor.hh
View file

@ -1,40 +1,43 @@
#pragma once #pragma once
///@file ///@file
#include "fs-accessor.hh" #include "source-accessor.hh"
#include "ref.hh" #include "ref.hh"
#include "store-api.hh" #include "store-api.hh"
namespace nix { namespace nix {
class RemoteFSAccessor : public FSAccessor class RemoteFSAccessor : public SourceAccessor
{ {
ref<Store> store; ref<Store> store;
std::map<std::string, ref<FSAccessor>> nars; std::map<std::string, ref<SourceAccessor>> nars;
bool requireValidPath;
Path cacheDir; Path cacheDir;
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true); std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
friend class BinaryCacheStore; friend class BinaryCacheStore;
Path makeCacheFile(std::string_view hashPart, const std::string & ext); Path makeCacheFile(std::string_view hashPart, const std::string & ext);
ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar); ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
public: public:
RemoteFSAccessor(ref<Store> store, RemoteFSAccessor(ref<Store> store,
bool requireValidPath = true,
const /* FIXME: use std::optional */ Path & cacheDir = ""); const /* FIXME: use std::optional */ Path & cacheDir = "");
Stat stat(const Path & path) override; std::optional<Stat> maybeLstat(const CanonPath & path) override;
StringSet readDirectory(const Path & path) override; DirEntries readDirectory(const CanonPath & path) override;
std::string readFile(const Path & path, bool requireValidPath = true) override; std::string readFile(const CanonPath & path) override;
std::string readLink(const Path & path) override; std::string readLink(const CanonPath & path) override;
}; };
} }

2
src/libstore/remote-store.cc
View file

@ -970,7 +970,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
copyNAR(conn->from, sink); copyNAR(conn->from, sink);
} }
ref<FSAccessor> RemoteStore::getFSAccessor() ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
{ {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
} }

6
src/libstore/remote-store.hh
View file

@ -22,10 +22,10 @@ struct RemoteStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent connections to the Nix daemon."}; "Maximum number of concurrent connections to the Nix daemon."};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, const Setting<unsigned int> maxConnectionAge{this,
std::numeric_limits<unsigned int>::max(), std::numeric_limits<unsigned int>::max(),
"max-connection-age", "max-connection-age",
"Maximum age of a connection before it is closed."}; "Maximum age of a connection before it is closed."};
@ -185,7 +185,7 @@ protected:
friend struct ConnectionHandle; friend struct ConnectionHandle;
virtual ref<FSAccessor> getFSAccessor() override; virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
virtual void narFromPath(const StorePath & path, Sink & sink) override; virtual void narFromPath(const StorePath & path, Sink & sink) override;

18
src/libstore/s3-binary-cache-store.cc
View file

@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{ {
using BinaryCacheStoreConfig::BinaryCacheStoreConfig; using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", const Setting<std::string> profile{this, "", "profile",
R"( R"(
The name of the AWS configuration profile to use. By default The name of the AWS configuration profile to use. By default
Nix will use the `default` profile. Nix will use the `default` profile.
)"}; )"};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region",
R"( R"(
The region of the S3 bucket. If your bucket is not in The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region `us–east-1`, you should always explicitly specify the region
parameter. parameter.
)"}; )"};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", const Setting<std::string> scheme{this, "", "scheme",
R"( R"(
The scheme used for S3 requests, `https` (default) or `http`. This The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't option allows you to disable HTTPS for binary caches which don't
@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
> information. > information.
)"}; )"};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", const Setting<std::string> endpoint{this, "", "endpoint",
R"( R"(
The URL of the endpoint of an S3-compatible service such as MinIO. The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3. Do not specify this setting if you're using Amazon S3.
@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
> addressing instead of virtual host based addressing. > addressing instead of virtual host based addressing.
)"}; )"};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", const Setting<std::string> narinfoCompression{this, "", "narinfo-compression",
"Compression method for `.narinfo` files."}; "Compression method for `.narinfo` files."};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", const Setting<std::string> lsCompression{this, "", "ls-compression",
"Compression method for `.ls` files."}; "Compression method for `.ls` files."};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", const Setting<std::string> logCompression{this, "", "log-compression",
R"( R"(
Compression method for `log/*` files. It is recommended to Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers use a compression method supported by most web browsers
@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
)"}; )"};
const Setting<bool> multipartUpload{ const Setting<bool> multipartUpload{
(StoreConfig*) this, false, "multipart-upload", this, false, "multipart-upload",
"Whether to use multi-part uploads."}; "Whether to use multi-part uploads."};
const Setting<uint64_t> bufferSize{ const Setting<uint64_t> bufferSize{
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", this, 5 * 1024 * 1024, "buffer-size",
"Size (in bytes) of each part in multi-part uploads."}; "Size (in bytes) of each part in multi-part uploads."};
const std::string name() override { return "S3 Binary Cache Store"; } const std::string name() override { return "S3 Binary Cache Store"; }

8
src/libstore/ssh-store-config.hh
View file

@ -9,16 +9,16 @@ struct CommonSSHStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", const Setting<Path> sshKey{this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."}; "Path to the SSH private key used to authenticate to the remote machine."};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", const Setting<std::string> sshPublicHostKey{this, "", "base64-ssh-public-host-key",
"The public host key of the remote machine."}; "The public host key of the remote machine."};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", const Setting<bool> compress{this, false, "compress",
"Whether to enable SSH compression."}; "Whether to enable SSH compression."};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", const Setting<std::string> remoteStore{this, "", "remote-store",
R"( R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to be used on the remote machine. The default is `auto` to be used on the remote machine. The default is `auto`

2
src/libstore/ssh-store.cc
View file

@ -16,7 +16,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
using RemoteStoreConfig::RemoteStoreConfig; using RemoteStoreConfig::RemoteStoreConfig;
using CommonSSHStoreConfig::CommonSSHStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", const Setting<Path> remoteProgram{this, "nix-daemon", "remote-program",
"Path to the `nix-daemon` executable on the remote machine."}; "Path to the `nix-daemon` executable on the remote machine."};
const std::string name() override { return "Experimental SSH Store"; } const std::string name() override { return "Experimental SSH Store"; }

16
src/libstore/store-api.cc
View file

@ -1,5 +1,5 @@
#include "crypto.hh" #include "crypto.hh"
#include "fs-accessor.hh" #include "source-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "derivations.hh" #include "derivations.hh"
#include "store-api.hh" #include "store-api.hh"
@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
/* Note that fileSink and unusualHashTee must be mutually exclusive, since /* Note that fileSink and unusualHashTee must be mutually exclusive, since
they both write to caHashSink. Note that that requisite is currently true they both write to caHashSink. Note that that requisite is currently true
because the former is only used in the flat case. */ because the former is only used in the flat case. */
RetrieveRegularNARSink fileSink { caHashSink }; RegularFileSink fileSink { caHashSink };
TeeSink unusualHashTee { narHashSink, caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink };
auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
information to narSink. */ information to narSink. */
TeeSource tapped { *fileSource, narSink }; TeeSource tapped { *fileSource, narSink };
ParseSink blank; NullParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat auto & parseSink = method == FileIngestionMethod::Flat
? fileSink ? (ParseSink &) fileSink
: blank; : (ParseSink &) blank;
/* The information that flows from tapped (besides being replicated in /* The information that flows from tapped (besides being replicated in
narSink), is now put in parseSink. */ narSink), is now put in parseSink. */
@ -1338,12 +1338,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
return readDerivation(drvPath); return readDerivation(drvPath);
} }
Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath)
{ {
auto accessor = store.getFSAccessor(); auto accessor = store.getFSAccessor(requireValidPath);
try { try {
return parseDerivation(store, return parseDerivation(store,
accessor->readFile(store.printStorePath(drvPath), requireValidPath), accessor->readFile(CanonPath(store.printStorePath(drvPath))),
Derivation::nameFromPath(drvPath)); Derivation::nameFromPath(drvPath));
} catch (FormatError & e) { } catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());

4
src/libstore/store-api.hh
View file

@ -70,7 +70,7 @@ MakeError(InvalidStoreURI, Error);
struct BasicDerivation; struct BasicDerivation;
struct Derivation; struct Derivation;
class FSAccessor; struct SourceAccessor;
class NarInfoDiskCache; class NarInfoDiskCache;
class Store; class Store;
@ -703,7 +703,7 @@ public:
/** /**
* @return An object to access files in the Nix store. * @return An object to access files in the Nix store.
*/ */
virtual ref<FSAccessor> getFSAccessor() = 0; virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) = 0;
/** /**
* Repair the contents of the given path by redownloading it using * Repair the contents of the given path by redownloading it using

4
src/libstore/uds-remote-store.hh
View file

@ -35,8 +35,8 @@ public:
static std::set<std::string> uriSchemes() static std::set<std::string> uriSchemes()
{ return {"unix"}; } { return {"unix"}; }
ref<FSAccessor> getFSAccessor() override ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{ return LocalFSStore::getFSAccessor(); } { return LocalFSStore::getFSAccessor(requireValidPath); }
void narFromPath(const StorePath & path, Sink & sink) override void narFromPath(const StorePath & path, Sink & sink) override
{ LocalFSStore::narFromPath(path, sink); } { LocalFSStore::narFromPath(path, sink); }

8
src/libutil/archive.cc
View file

@ -5,12 +5,6 @@
#include <strings.h> // for strcasecmp #include <strings.h> // for strcasecmp
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <fcntl.h>
#include "archive.hh" #include "archive.hh"
#include "util.hh" #include "util.hh"
#include "config.hh" #include "config.hh"
@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink)
// FIXME: if 'source' is the output of dumpPath() followed by EOF, // FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing. // we should just forward all data directly without parsing.
ParseSink parseSink; /* null sink; just parse the NAR */ NullParseSink parseSink; /* just parse the NAR */
TeeSource wrapper { source, sink }; TeeSource wrapper { source, sink };

27
src/libutil/archive.hh
View file

@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
*/ */
void dumpString(std::string_view s, Sink & sink); void dumpString(std::string_view s, Sink & sink);
/**
* If the NAR archive contains a single file at top-level, then save
* the contents of the file to `s`. Otherwise barf.
*/
struct RetrieveRegularNARSink : ParseSink
{
bool regular = true;
Sink & sink;
RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
void createDirectory(const Path & path) override
{
regular = false;
}
void receiveContents(std::string_view data) override
{
sink(data);
}
void createSymlink(const Path & path, const std::string & target) override
{
regular = false;
}
};
void parseDump(ParseSink & sink, Source & source); void parseDump(ParseSink & sink, Source & source);
void restorePath(const Path & path, Source & source); void restorePath(const Path & path, Source & source);

13
src/libutil/args.cc
View file

@ -255,7 +255,18 @@ bool Args::processArgs(const Strings & args, bool finish)
} }
if (!anyCompleted) if (!anyCompleted)
exp.handler.fun(ss); exp.handler.fun(ss);
expectedArgs.pop_front();
/* Move the list element to the processedArgs. This is almost the same as
`processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
except that it will only adjust the next and prev pointers of the list
elements, meaning the actual contents don't move in memory. This is
critical to prevent invalidating internal pointers! */
processedArgs.splice(
processedArgs.end(),
expectedArgs,
expectedArgs.begin(),
++expectedArgs.begin());
res = true; res = true;
} }

14
src/libutil/args.hh
View file

@ -200,13 +200,25 @@ protected:
/** /**
* Queue of expected positional argument forms. * Queue of expected positional argument forms.
* *
* Positional arugment descriptions are inserted on the back. * Positional argument descriptions are inserted on the back.
* *
* As positional arguments are passed, these are popped from the * As positional arguments are passed, these are popped from the
* front, until there are hopefully none left as all args that were * front, until there are hopefully none left as all args that were
* expected in fact were passed. * expected in fact were passed.
*/ */
std::list<ExpectedArg> expectedArgs; std::list<ExpectedArg> expectedArgs;
/**
* List of processed positional argument forms.
*
* All items removed from `expectedArgs` are added here. After all
* arguments were processed, this list should be exactly the same as
* `expectedArgs` was before.
*
* This list is used to extend the lifetime of the argument forms.
* If this is not done, some closures that reference the command
* itself will segfault.
*/
std::list<ExpectedArg> processedArgs;
/** /**
* Process some positional arugments * Process some positional arugments

5
src/libutil/fmt.hh
View file

@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s)
return s; return s;
} }
inline std::string fmt(std::string_view s)
{
return std::string(s);
}
inline std::string fmt(const char * s) inline std::string fmt(const char * s)
{ {
return s; return s;

48
src/libutil/fs-sink.cc
View file

@ -5,6 +5,54 @@
namespace nix { namespace nix {
void copyRecursive(
SourceAccessor & accessor, const CanonPath & from,
ParseSink & sink, const Path & to)
{
auto stat = accessor.lstat(from);
switch (stat.type) {
case SourceAccessor::tSymlink:
{
sink.createSymlink(to, accessor.readLink(from));
}
case SourceAccessor::tRegular:
{
sink.createRegularFile(to);
if (stat.isExecutable)
sink.isExecutable();
LambdaSink sink2 {
[&](auto d) {
sink.receiveContents(d);
}
};
accessor.readFile(from, sink2, [&](uint64_t size) {
sink.preallocateContents(size);
});
break;
}
case SourceAccessor::tDirectory:
{
sink.createDirectory(to);
for (auto & [name, _] : accessor.readDirectory(from)) {
copyRecursive(
accessor, from + name,
sink, to + "/" + name);
break;
}
}
case SourceAccessor::tMisc:
throw Error("file '%1%' has an unsupported type", from);
default:
abort();
}
}
struct RestoreSinkSettings : Config struct RestoreSinkSettings : Config
{ {
Setting<bool> preallocateContents{this, false, "preallocate-contents", Setting<bool> preallocateContents{this, false, "preallocate-contents",

86
src/libutil/fs-sink.hh
View file

@ -3,6 +3,7 @@
#include "types.hh" #include "types.hh"
#include "serialise.hh" #include "serialise.hh"
#include "source-accessor.hh"
namespace nix { namespace nix {
@ -11,32 +12,93 @@ namespace nix {
*/ */
struct ParseSink struct ParseSink
{ {
virtual void createDirectory(const Path & path) { }; virtual void createDirectory(const Path & path) = 0;
virtual void createRegularFile(const Path & path) { }; virtual void createRegularFile(const Path & path) = 0;
virtual void closeRegularFile() { }; virtual void receiveContents(std::string_view data) = 0;
virtual void isExecutable() { }; virtual void isExecutable() = 0;
virtual void closeRegularFile() = 0;
virtual void createSymlink(const Path & path, const std::string & target) = 0;
/**
* An optimization. By default, do nothing.
*/
virtual void preallocateContents(uint64_t size) { }; virtual void preallocateContents(uint64_t size) { };
virtual void receiveContents(std::string_view data) { };
virtual void createSymlink(const Path & path, const std::string & target) { };
}; };
/**
* Recusively copy file system objects from the source into the sink.
*/
void copyRecursive(
SourceAccessor & accessor, const CanonPath & sourcePath,
ParseSink & sink, const Path & destPath);
/**
* Ignore everything and do nothing
*/
struct NullParseSink : ParseSink
{
void createDirectory(const Path & path) override { }
void receiveContents(std::string_view data) override { }
void createSymlink(const Path & path, const std::string & target) override { }
void createRegularFile(const Path & path) override { }
void closeRegularFile() override { }
void isExecutable() override { }
};
/**
* Write files at the given path
*/
struct RestoreSink : ParseSink struct RestoreSink : ParseSink
{ {
Path dstPath; Path dstPath;
AutoCloseFD fd;
void createDirectory(const Path & path) override; void createDirectory(const Path & path) override;
void createRegularFile(const Path & path) override; void createRegularFile(const Path & path) override;
void closeRegularFile() override;
void isExecutable() override;
void preallocateContents(uint64_t size) override;
void receiveContents(std::string_view data) override; void receiveContents(std::string_view data) override;
void isExecutable() override;
void closeRegularFile() override;
void createSymlink(const Path & path, const std::string & target) override; void createSymlink(const Path & path, const std::string & target) override;
void preallocateContents(uint64_t size) override;
private:
AutoCloseFD fd;
};
/**
* Restore a single file at the top level, passing along
* `receiveContents` to the underlying `Sink`. For anything but a single
* file, set `regular = true` so the caller can fail accordingly.
*/
struct RegularFileSink : ParseSink
{
bool regular = true;
Sink & sink;
RegularFileSink(Sink & sink) : sink(sink) { }
void createDirectory(const Path & path) override
{
regular = false;
}
void receiveContents(std::string_view data) override
{
sink(data);
}
void createSymlink(const Path & path, const std::string & target) override
{
regular = false;
}
void createRegularFile(const Path & path) override { }
void closeRegularFile() override { }
void isExecutable() override { }
}; };
} }

11
src/libutil/posix-source-accessor.cc
View file

@ -44,9 +44,13 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path)
return nix::pathExists(path.abs()); return nix::pathExists(path.abs());
} }
SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
{ {
auto st = nix::lstat(path.abs()); struct stat st;
if (::lstat(path.c_str(), &st)) {
if (errno == ENOENT) return std::nullopt;
throw SysError("getting status of '%s'", showPath(path));
}
mtime = std::max(mtime, st.st_mtime); mtime = std::max(mtime, st.st_mtime);
return Stat { return Stat {
.type = .type =
@ -54,7 +58,8 @@ SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path)
S_ISDIR(st.st_mode) ? tDirectory : S_ISDIR(st.st_mode) ? tDirectory :
S_ISLNK(st.st_mode) ? tSymlink : S_ISLNK(st.st_mode) ? tSymlink :
tMisc, tMisc,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR .fileSize = S_ISREG(st.st_mode) ? std::optional<uint64_t>(st.st_size) : std::nullopt,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR,
}; };
} }

2
src/libutil/posix-source-accessor.hh
View file

@ -22,7 +22,7 @@ struct PosixSourceAccessor : SourceAccessor
bool pathExists(const CanonPath & path) override; bool pathExists(const CanonPath & path) override;
Stat lstat(const CanonPath & path) override; std::optional<Stat> maybeLstat(const CanonPath & path) override;
DirEntries readDirectory(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override;

15
src/libutil/source-accessor.cc
View file

@ -10,6 +10,11 @@ SourceAccessor::SourceAccessor()
{ {
} }
bool SourceAccessor::pathExists(const CanonPath & path)
{
return maybeLstat(path).has_value();
}
std::string SourceAccessor::readFile(const CanonPath & path) std::string SourceAccessor::readFile(const CanonPath & path)
{ {
StringSink sink; StringSink sink;
@ -42,12 +47,12 @@ Hash SourceAccessor::hashPath(
return sink.finish().first; return sink.finish().first;
} }
std::optional<SourceAccessor::Stat> SourceAccessor::maybeLstat(const CanonPath & path) SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
{ {
// FIXME: merge these into one operation. if (auto st = maybeLstat(path))
if (!pathExists(path)) return *st;
return {}; else
return lstat(path); throw Error("path '%s' does not exist", showPath(path));
} }
std::string SourceAccessor::showPath(const CanonPath & path) std::string SourceAccessor::showPath(const CanonPath & path)

26
src/libutil/source-accessor.hh
View file

@ -40,7 +40,7 @@ struct SourceAccessor
Sink & sink, Sink & sink,
std::function<void(uint64_t)> sizeCallback = [](uint64_t size){}); std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
virtual bool pathExists(const CanonPath & path) = 0; virtual bool pathExists(const CanonPath & path);
enum Type { enum Type {
tRegular, tSymlink, tDirectory, tRegular, tSymlink, tDirectory,
@ -57,13 +57,29 @@ struct SourceAccessor
struct Stat struct Stat
{ {
Type type = tMisc; Type type = tMisc;
//uint64_t fileSize = 0; // regular files only
bool isExecutable = false; // regular files only /**
* For regular files only: the size of the file. Not all
* accessors return this since it may be too expensive to
* compute.
*/
std::optional<uint64_t> fileSize;
/**
* For regular files only: whether this is an executable.
*/
bool isExecutable = false;
/**
* For regular files only: the position of the contents of this
* file in the NAR. Only returned by NAR accessors.
*/
std::optional<uint64_t> narOffset;
}; };
virtual Stat lstat(const CanonPath & path) = 0; Stat lstat(const CanonPath & path);
std::optional<Stat> maybeLstat(const CanonPath & path); virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
typedef std::optional<Type> DirEntry; typedef std::optional<Type> DirEntry;

0
src/nix-channel/nix-channel.cc Executable file → Normal file
View file

0
src/nix-copy-closure/nix-copy-closure.cc Executable file → Normal file
View file

1
src/nix/bundle.cc
View file

@ -4,7 +4,6 @@
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "local-fs-store.hh" #include "local-fs-store.hh"
#include "fs-accessor.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
using namespace nix; using namespace nix;

12
src/nix/cat.cc
View file

@ -1,6 +1,5 @@
#include "command.hh" #include "command.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fs-accessor.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
using namespace nix; using namespace nix;
@ -9,15 +8,12 @@ struct MixCat : virtual Args
{ {
std::string path; std::string path;
void cat(ref<FSAccessor> accessor) void cat(ref<SourceAccessor> accessor)
{ {
auto st = accessor->stat(path); auto st = accessor->lstat(CanonPath(path));
if (st.type == FSAccessor::Type::tMissing) if (st.type != SourceAccessor::Type::tRegular)
throw Error("path '%1%' does not exist", path);
if (st.type != FSAccessor::Type::tRegular)
throw Error("path '%1%' is not a regular file", path); throw Error("path '%1%' is not a regular file", path);
writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path)));
writeFull(STDOUT_FILENO, accessor->readFile(path));
} }
}; };

52
src/nix/flake-lock.md
View file

@ -2,37 +2,39 @@ R""(
# Examples # Examples
* Update the `nixpkgs` and `nix` inputs of the flake in the current * Create the lock file for the flake in the current directory:
directory:
```console ```console
# nix flake lock --update-input nixpkgs --update-input nix # nix flake lock
* Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' warning: creating lock file '/home/myself/repos/testflake/flake.lock':
* Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' • Added input 'nix':
'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
• Added input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
``` ```
* Add missing inputs to the lock file for a flake in a different directory:
```console
# nix flake lock ~/repos/another
warning: updating lock file '/home/myself/repos/another/flake.lock':
• Added input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
```
> **Note**
>
> When trying to refer to a flake in a subdirectory, write `./another`
> instead of `another`.
> Otherwise Nix will try to look up the flake in the registry.
# Description # Description
This command updates the lock file of a flake (`flake.lock`) so that This command adds inputs to the lock file of a flake (`flake.lock`)
it contains a lock for every flake input specified in so that it contains a lock for every flake input specified in
`flake.nix`. Existing lock file entries are not updated unless `flake.nix`. Existing lock file entries are not updated.
required by a flag such as `--update-input`.
Note that every command that operates on a flake will also update the If you want to update existing lock entries, use
lock file if needed, and supports the same flags. Therefore, [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md)
```console
# nix flake lock --update-input nixpkgs
# nix build
```
is equivalent to:
```console
# nix build --update-input nixpkgs
```
Thus, this command is only useful if you want to update the lock file
separately from any other action such as building.
)"" )""

64
src/nix/flake-update.md
View file

@ -2,33 +2,57 @@ R""(
# Examples # Examples
* Recreate the lock file (i.e. update all inputs) and commit the new * Update all inputs (i.e. recreate the lock file from scratch):
lock file:
```console ```console
# nix flake update --commit-lock-file # nix flake update
* Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' warning: updating lock file '/home/myself/repos/testflake/flake.lock':
* Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' • Updated input 'nix':
… 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11)
• Updated input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
→ 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
``` ```
* Update only a single input:
```console
# nix flake update nixpkgs
warning: updating lock file '/home/myself/repos/testflake/flake.lock':
• Updated input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
→ 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
```
* Update only a single input of a flake in a different directory:
```console
# nix flake update nixpkgs --flake ~/repos/another
warning: updating lock file '/home/myself/repos/another/flake.lock':
• Updated input 'nixpkgs':
'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
→ 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
```
> **Note**
>
> When trying to refer to a flake in a subdirectory, write `./another`
> instead of `another`.
> Otherwise Nix will try to look up the flake in the registry.
# Description # Description
This command recreates the lock file of a flake (`flake.lock`), thus This command updates the inputs in a lock file (`flake.lock`).
updating the lock for every unlocked input (like `nixpkgs`) to its **By default, all inputs are updated**. If the lock file doesn't exist
current version. This is equivalent to passing `--recreate-lock-file` yet, it will be created. If inputs are not in the lock file yet, they will be added.
to any command that operates on a flake. That is,
```console Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs
# nix flake update to update as its positional arguments and operates on the flake in the current directory.
# nix build You can pass a different flake-url with `--flake` to override that default.
```
is equivalent to: The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md)
also creates lock files and adds missing inputs, but is safer as it
```console will never update inputs already in the lock file.
# nix build --recreate-lock-file
```
)"" )""

33
src/nix/flake.cc
View file

@ -24,8 +24,10 @@ using namespace nix;
using namespace nix::flake; using namespace nix::flake;
using json = nlohmann::json; using json = nlohmann::json;
struct CmdFlakeUpdate;
class FlakeCommand : virtual Args, public MixFlakeOptions class FlakeCommand : virtual Args, public MixFlakeOptions
{ {
protected:
std::string flakeUrl = "."; std::string flakeUrl = ".";
public: public:
@ -63,6 +65,8 @@ public:
struct CmdFlakeUpdate : FlakeCommand struct CmdFlakeUpdate : FlakeCommand
{ {
public:
std::string description() override std::string description() override
{ {
return "update flake lock file"; return "update flake lock file";
@ -70,9 +74,31 @@ struct CmdFlakeUpdate : FlakeCommand
CmdFlakeUpdate() CmdFlakeUpdate()
{ {
expectedArgs.clear();
addFlag({
.longName="flake",
.description="The flake to operate on. Default is the current directory.",
.labels={"flake-url"},
.handler={&flakeUrl},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(completions, getStore(), prefix);
}}
});
expectArgs({
.label="inputs",
.optional=true,
.handler={[&](std::string inputToUpdate){
auto inputPath = flake::parseInputPath(inputToUpdate);
if (lockFlags.inputUpdates.contains(inputPath))
warn("Input '%s' was specified multiple times. You may have done this by accident.");
lockFlags.inputUpdates.insert(inputPath);
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
/* Remove flags that don't make sense. */ /* Remove flags that don't make sense. */
removeFlag("recreate-lock-file");
removeFlag("update-input");
removeFlag("no-update-lock-file"); removeFlag("no-update-lock-file");
removeFlag("no-write-lock-file"); removeFlag("no-write-lock-file");
} }
@ -87,8 +113,9 @@ struct CmdFlakeUpdate : FlakeCommand
void run(nix::ref<nix::Store> store) override void run(nix::ref<nix::Store> store) override
{ {
settings.tarballTtl = 0; settings.tarballTtl = 0;
auto updateAll = lockFlags.inputUpdates.empty();
lockFlags.recreateLockFile = true; lockFlags.recreateLockFile = updateAll;
lockFlags.writeLockFile = true; lockFlags.writeLockFile = true;
lockFlags.applyNixConfig = true; lockFlags.applyNixConfig = true;

48
src/nix/ls.cc
View file

@ -1,6 +1,5 @@
#include "command.hh" #include "command.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fs-accessor.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "common-args.hh" #include "common-args.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -39,61 +38,58 @@ struct MixLs : virtual Args, MixJSON
}); });
} }
void listText(ref<FSAccessor> accessor) void listText(ref<SourceAccessor> accessor)
{ {
std::function<void(const FSAccessor::Stat &, const Path &, const std::string &, bool)> doPath; std::function<void(const SourceAccessor::Stat &, const CanonPath &, std::string_view, bool)> doPath;
auto showFile = [&](const Path & curPath, const std::string & relPath) { auto showFile = [&](const CanonPath & curPath, std::string_view relPath) {
if (verbose) { if (verbose) {
auto st = accessor->stat(curPath); auto st = accessor->lstat(curPath);
std::string tp = std::string tp =
st.type == FSAccessor::Type::tRegular ? st.type == SourceAccessor::Type::tRegular ?
(st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" :
"dr-xr-xr-x"; "dr-xr-xr-x";
auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath);
if (st.type == FSAccessor::Type::tSymlink) if (st.type == SourceAccessor::Type::tSymlink)
line += " -> " + accessor->readLink(curPath); line += " -> " + accessor->readLink(curPath);
logger->cout(line); logger->cout(line);
if (recursive && st.type == FSAccessor::Type::tDirectory) if (recursive && st.type == SourceAccessor::Type::tDirectory)
doPath(st, curPath, relPath, false); doPath(st, curPath, relPath, false);
} else { } else {
logger->cout(relPath); logger->cout(relPath);
if (recursive) { if (recursive) {
auto st = accessor->stat(curPath); auto st = accessor->lstat(curPath);
if (st.type == FSAccessor::Type::tDirectory) if (st.type == SourceAccessor::Type::tDirectory)
doPath(st, curPath, relPath, false); doPath(st, curPath, relPath, false);
} }
} }
}; };
doPath = [&](const FSAccessor::Stat & st, const Path & curPath, doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath,
const std::string & relPath, bool showDirectory) std::string_view relPath, bool showDirectory)
{ {
if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) {
auto names = accessor->readDirectory(curPath); auto names = accessor->readDirectory(curPath);
for (auto & name : names) for (auto & [name, type] : names)
showFile(curPath + "/" + name, relPath + "/" + name); showFile(curPath + name, relPath + "/" + name);
} else } else
showFile(curPath, relPath); showFile(curPath, relPath);
}; };
auto st = accessor->stat(path); auto path2 = CanonPath(path);
if (st.type == FSAccessor::Type::tMissing) auto st = accessor->lstat(path2);
throw Error("path '%1%' does not exist", path); doPath(st, path2,
doPath(st, path, st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""),
st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)),
showDirectory); showDirectory);
} }
void list(ref<FSAccessor> accessor) void list(ref<SourceAccessor> accessor)
{ {
if (path == "/") path = "";
if (json) { if (json) {
if (showDirectory) if (showDirectory)
throw UsageError("'--directory' is useless with '--json'"); throw UsageError("'--directory' is useless with '--json'");
logger->cout("%s", listNar(accessor, path, recursive)); logger->cout("%s", listNar(accessor, CanonPath(path), recursive));
} else } else
listText(accessor); listText(accessor);
} }

1
src/nix/main.cc
View file

@ -188,6 +188,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
j["experimentalFeature"] = storeConfig->experimentalFeature(); j["experimentalFeature"] = storeConfig->experimentalFeature();
} }
res["stores"] = std::move(stores); res["stores"] = std::move(stores);
res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
return res.dump(); return res.dump();
} }

8
src/nix/run.cc
View file

@ -6,7 +6,7 @@
#include "derivations.hh" #include "derivations.hh"
#include "local-store.hh" #include "local-store.hh"
#include "finally.hh" #include "finally.hh"
#include "fs-accessor.hh" #include "source-accessor.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "eval.hh" #include "eval.hh"
#include "build/personality.hh" #include "build/personality.hh"
@ -119,9 +119,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment
if (true) if (true)
unixPath.push_front(store->printStorePath(path) + "/bin"); unixPath.push_front(store->printStorePath(path) + "/bin");
auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages";
if (accessor->stat(propPath).type == FSAccessor::tRegular) { if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) {
for (auto & p : tokenizeString<Paths>(readFile(propPath))) for (auto & p : tokenizeString<Paths>(accessor->readFile(propPath)))
todo.push(store->parseStorePath(p)); todo.push(store->parseStorePath(p));
} }
} }

25
src/nix/why-depends.cc
View file

@ -1,7 +1,7 @@
#include "command.hh" #include "command.hh"
#include "store-api.hh" #include "store-api.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "fs-accessor.hh" #include "source-accessor.hh"
#include "shared.hh" #include "shared.hh"
#include <queue> #include <queue>
@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
struct BailOut { }; struct BailOut { };
printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) {
auto pathS = store->printStorePath(node.path); CanonPath pathS(store->printStorePath(node.path));
assert(node.dist != inf); assert(node.dist != inf);
if (precise) { if (precise) {
@ -183,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
firstPad, firstPad,
node.visited ? "\e[38;5;244m" : "", node.visited ? "\e[38;5;244m" : "",
firstPad != "" ? "→ " : "", firstPad != "" ? "→ " : "",
pathS); pathS.abs());
} }
if (node.path == dependencyPath && !all if (node.path == dependencyPath && !all
@ -210,24 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
contain the reference. */ contain the reference. */
std::map<std::string, Strings> hits; std::map<std::string, Strings> hits;
std::function<void(const Path &)> visitPath; std::function<void(const CanonPath &)> visitPath;
visitPath = [&](const Path & p) { visitPath = [&](const CanonPath & p) {
auto st = accessor->stat(p); auto st = accessor->maybeLstat(p);
assert(st);
auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1);
auto getColour = [&](const std::string & hash) { auto getColour = [&](const std::string & hash) {
return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE;
}; };
if (st.type == FSAccessor::Type::tDirectory) { if (st->type == SourceAccessor::Type::tDirectory) {
auto names = accessor->readDirectory(p); auto names = accessor->readDirectory(p);
for (auto & name : names) for (auto & [name, type] : names)
visitPath(p + "/" + name); visitPath(p + name);
} }
else if (st.type == FSAccessor::Type::tRegular) { else if (st->type == SourceAccessor::Type::tRegular) {
auto contents = accessor->readFile(p); auto contents = accessor->readFile(p);
for (auto & hash : hashes) { for (auto & hash : hashes) {
@ -245,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
} }
} }
else if (st.type == FSAccessor::Type::tSymlink) { else if (st->type == SourceAccessor::Type::tSymlink) {
auto target = accessor->readLink(p); auto target = accessor->readLink(p);
for (auto & hash : hashes) { for (auto & hash : hashes) {

11
tests/functional/completions.sh
View file

@ -44,15 +44,18 @@ EOF
# Input override completion # Input override completion
[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]]
[[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]]
cd ./foo
[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]]
cd ..
[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]]
## With multiple input flakes ## With multiple input flakes
[[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]]
## With tilde expansion ## With tilde expansion
[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]]
[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]]
[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]]
## Out of order ## Out of order
[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]]
[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]]
# Cli flag completion # Cli flag completion
NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format"

3
tests/functional/flakes/circular.sh
View file

@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo'
sed -i $flakeB/flake.nix -e 's/456/789/' sed -i $flakeB/flake.nix -e 's/456/789/'
git -C $flakeB commit -a -m 'Foo' git -C $flakeB commit -a -m 'Foo'
[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]] nix flake update b --flake $flakeA
[[ $(nix eval $flakeA#foo) = 1912 ]]
# Test list-inputs with circular dependencies # Test list-inputs with circular dependencies
nix flake metadata $flakeA nix flake metadata $flakeA

6
tests/functional/flakes/flakes.sh
View file

@ -300,7 +300,7 @@ nix build -o "$TEST_ROOT/result" flake4#xyzzy
nix flake lock "$flake3Dir" nix flake lock "$flake3Dir"
[[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]]
nix flake update "$flake3Dir" --override-flake flake2 nixpkgs nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs
[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]]
# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
@ -437,7 +437,7 @@ cat > "$flake3Dir/flake.nix" <<EOF
} }
EOF EOF
nix flake update "$flake3Dir" nix flake update --flake "$flake3Dir"
[[ $(jq -c .nodes.flake2.inputs.flake1 "$flake3Dir/flake.lock") =~ '["foo"]' ]] [[ $(jq -c .nodes.flake2.inputs.flake1 "$flake3Dir/flake.lock") =~ '["foo"]' ]]
[[ $(jq .nodes.foo.locked.url "$flake3Dir/flake.lock") =~ flake7 ]] [[ $(jq .nodes.foo.locked.url "$flake3Dir/flake.lock") =~ flake7 ]]
@ -480,7 +480,7 @@ nix flake lock "$flake3Dir" --override-input flake2/flake1 flake1/master/$hash1
nix flake lock "$flake3Dir" nix flake lock "$flake3Dir"
[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") = $hash1 ]] [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") = $hash1 ]]
nix flake lock "$flake3Dir" --update-input flake2/flake1 nix flake update flake2/flake1 --flake "$flake3Dir"
[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]] [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]]
# Test 'nix flake metadata --json'. # Test 'nix flake metadata --json'.

4
tests/functional/flakes/follow-paths.sh
View file

@ -77,7 +77,7 @@ git -C $flakeFollowsA add flake.nix flakeB/flake.nix \
nix flake metadata $flakeFollowsA nix flake metadata $flakeFollowsA
nix flake update $flakeFollowsA nix flake update --flake $flakeFollowsA
nix flake lock $flakeFollowsA nix flake lock $flakeFollowsA
@ -228,7 +228,7 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \
flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix
nix flake metadata "$flakeFollowsOverloadA" nix flake metadata "$flakeFollowsOverloadA"
nix flake update "$flakeFollowsOverloadA" nix flake update --flake "$flakeFollowsOverloadA"
nix flake lock "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA"
# Now test follow cycle detection # Now test follow cycle detection

2
tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp
View file

@ -1 +1 @@
error: getting status of '/pwd/lang/fnord': No such file or directory error: path '/pwd/lang/fnord' does not exist

2
tests/functional/lang/eval-fail-nonexist-path.err.exp
View file

@ -1 +1 @@
error: getting status of '/pwd/lang/fnord': No such file or directory error: path '/pwd/lang/fnord' does not exist

8
tests/functional/nar-access.sh
View file

@ -25,6 +25,12 @@ diff -u baz.cat-nar $storePath/foo/baz
nix store cat $storePath/foo/baz > baz.cat-nar nix store cat $storePath/foo/baz > baz.cat-nar
diff -u baz.cat-nar $storePath/foo/baz diff -u baz.cat-nar $storePath/foo/baz
# Check that 'nix store cat' fails on invalid store paths.
invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo"
mv $storePath $invalidPath
expect 1 nix store cat $invalidPath/foo/baz
mv $invalidPath $storePath
# Test --json. # Test --json.
diff -u \ diff -u \
<(nix nar ls --json $narFile / | jq -S) \ <(nix nar ls --json $narFile / | jq -S) \
@ -46,7 +52,7 @@ diff -u \
<(echo '{"type":"regular","size":0}' | jq -S) <(echo '{"type":"regular","size":0}' | jq -S)
# Test missing files. # Test missing files.
expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist'
expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist'
# Test failure to dump. # Test failure to dump.