From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: [PATCH] Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/libstore/filetransfer.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 232 --------------- src/nix/local.mk | 1 - src/nix/prefetch.cc | 352 +++++++++++++++++++++++ src/nix/store-prefetch-file.md | 32 +++ src/nix/store-prefetch-tarball.md | 31 ++ 6 files changed, 416 insertions(+), 234 deletions(-) delete mode 100644 src/nix-prefetch-url/nix-prefetch-url.cc create mode 100644 src/nix/prefetch.cc create mode 100644 src/nix/store-prefetch-file.md create mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc deleted file mode 100644 index 3bdee55a7..000000000 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ /dev/null @@ -1,232 +0,0 @@ -#include "hash.hh" -#include "shared.hh" -#include "filetransfer.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "finally.hh" -#include "../nix/legacy.hh" -#include "progress-bar.hh" -#include "tarfile.hh" - -#include - -#include -#include -#include - -using namespace nix; - - -/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of - mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; - - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); - - Value vMirrors; - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors); - - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error("unknown mirror name '%1%'", mirrorName); - state.forceList(*mirrorList->value); - - if (mirrorList->value->listSize() < 1) - throw Error("mirror URI '%1%' did not expand to anything", uri); - - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} - - -static int main_nix_prefetch_url(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector args; - bool printPath = getEnv("PRINT_PATH") == "1"; - bool fromExpr = false; - string attrPath; - bool unpack = false; - bool executable = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--executable") - executable = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error("cannot figure out file name for '%1%'", uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - std::optional expectedHash; - Hash hash(ht); - std::optional storePath; - if (args.size() == 2) { - expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); - if (store->isValidPath(*storePath)) - hash = *expectedHash; - else - storePath.reset(); - } - - if (!storePath) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - auto mode = 0600; - if (executable) - mode = 0700; - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - FileTransferRequest req(actualUri); - req.decompress = false; - getFileTransfer()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); - storePath = info.path; - assert(info.ca); - hash = getContentAddressHash(*info.ca); - } - - stopProgressBar(); - - if (!printPath) - printInfo("path is '%s'", store->printStorePath(*storePath)); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << store->printStorePath(*storePath) << std::endl; - - return 0; - } -} - -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix/local.mk b/src/nix/local.mk index f37b73384..23c08fc86 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,7 +12,6 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..969299489 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,352 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple prefetchFile( + ref store, + std::string_view url, + std::optional name, + HashType hashType, + std::optional expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional storePath; + std::optional hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetch : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + bool unpack; + std::optional name; + HashType hashType = htSHA256; + std::optional expectedHash; + + CmdStorePrefetch(bool unpack) + : unpack(unpack) + { + addFlag({ + .longName = "name", + .description = "store path name", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdStorePrefetchFile : CmdStorePrefetch +{ + CmdStorePrefetchFile() + : CmdStorePrefetch(false) + { + name = "source"; + + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +struct CmdStorePrefetchTarball : CmdStorePrefetch +{ + CmdStorePrefetchTarball() + : CmdStorePrefetch(true) + { + name = "source"; + } + + std::string description() override + { + return "download and unpack a tarball into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-tarball.md" + ; + } +}; + +static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..1663b847b --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overriden using `--name`. + +)"" diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md new file mode 100644 index 000000000..535d7e022 --- /dev/null +++ b/src/nix/store-prefetch-tarball.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download a tarball and unpack it, unless it already exists in the + Nix store: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ + --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= + ``` + +# Description + +This command downloads a tarball or zip file from *url*, unpacks it, +and adds the unpacked tree to the Nix store. It prints out the +resulting store path and the NAR hash of that store path. + +The name component of the store path defaults to `source`, but this +can be overriden using `--name`. + +)""