diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 82bfeac36..10c9c16bb 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -1,6 +1,6 @@ #include "common-eval-args.hh" #include "shared.hh" -#include "download.hh" +#include "filetransfer.hh" #include "util.hh" #include "eval.hh" #include "fetchers.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dac32b6f5..b91a021b4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "download.hh" +#include "filetransfer.hh" #include "json.hh" #include "function-trace.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6f25f5cf0..b33ab908b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -544,7 +544,7 @@ formal #include #include "eval.hh" -#include "download.hh" +#include "filetransfer.hh" #include "fetchers.hh" #include "store-api.hh" @@ -690,7 +690,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl try { res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; - } catch (DownloadError & e) { + } catch (FileTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9586f71ed..43c58485a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -2,7 +2,7 @@ #include "eval-inline.hh" #include "store-api.hh" #include "fetchers.hh" -#include "download.hh" +#include "filetransfer.hh" #include #include diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ef27eaa76..8675a5a66 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,4 +1,4 @@ -#include "download.hh" +#include "filetransfer.hh" #include "cache.hh" #include "fetchers.hh" #include "globals.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 4c4e5828e..695525b31 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,6 +1,6 @@ #include "fetchers.hh" #include "cache.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "store-api.hh" #include "archive.hh" @@ -36,13 +36,13 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - DownloadRequest request(url); + FileTransferRequest request(url); if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); - DownloadResult res; + FileTransferResult res; try { - res = getDownloader()->download(request); - } catch (DownloadError & e) { + res = getFileTransfer()->download(request); + } catch (FileTransferError & e) { if (cached) { warn("%s; using cached version", e.msg()); return useCached(); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 632982ecd..0febb8dfb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,7 +7,7 @@ #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "download.hh" +#include "filetransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" @@ -361,7 +361,7 @@ public: { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index f6ae5d2e6..486babf14 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh downloader here because we're in + /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ - auto downloader = makeDownloader(); + auto fileTransfer = makeFileTransfer(); auto fetch = [&](const std::string & url) { @@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DownloadRequest request(url); + FileTransferRequest request(url); request.verifyTLS = false; request.decompress = false; auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - downloader->download(std::move(request), *decompressor); + fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/download.cc b/src/libstore/filetransfer.cc similarity index 86% rename from src/libstore/download.cc rename to src/libstore/filetransfer.cc index bc875f653..e9684b3d4 100644 --- a/src/libstore/download.cc +++ b/src/libstore/filetransfer.cc @@ -1,4 +1,4 @@ -#include "download.hh" +#include "filetransfer.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" @@ -27,9 +27,9 @@ using namespace std::string_literals; namespace nix { -DownloadSettings downloadSettings; +FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&downloadSettings); +static GlobalConfig::Register r1(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -39,21 +39,21 @@ std::string resolveUri(const std::string & uri) return uri; } -struct CurlDownloader : public Downloader +struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; std::random_device rd; std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this + struct TransferItem : public std::enable_shared_from_this { - CurlDownloader & downloader; - DownloadRequest request; - DownloadResult result; + curlFileTransfer & fileTransfer; + FileTransferRequest request; + FileTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback callback; + Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -72,12 +72,12 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; - DownloadItem(CurlDownloader & downloader, - const DownloadRequest & request, - Callback && callback) - : downloader(downloader) + TransferItem(curlFileTransfer & fileTransfer, + const FileTransferRequest & request, + Callback && callback) + : fileTransfer(fileTransfer) , request(request) - , act(*logger, lvlTalkative, actDownload, + , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) @@ -102,17 +102,17 @@ struct CurlDownloader : public Downloader requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); } - ~DownloadItem() + ~TransferItem() { if (req) { if (active) - curl_multi_remove_handle(downloader.curlm, req); + curl_multi_remove_handle(fileTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); } catch (...) { ignoreException(); } @@ -156,7 +156,7 @@ struct CurlDownloader : public Downloader static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); + return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } size_t headerCallback(void * contents, size_t size, size_t nmemb) @@ -198,7 +198,7 @@ struct CurlDownloader : public Downloader static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); + return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); } int progressCallback(double dltotal, double dlnow) @@ -213,7 +213,7 @@ struct CurlDownloader : public Downloader static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); + return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -237,7 +237,7 @@ struct CurlDownloader : public Downloader static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { - return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } void init() @@ -248,7 +248,7 @@ struct CurlDownloader : public Downloader if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); @@ -257,19 +257,19 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloadSettings.enableHttp2) + if (fileTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); @@ -297,10 +297,10 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -401,14 +401,14 @@ struct CurlDownloader : public Downloader auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DownloadError(err, + ? FileTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DownloadError(err, + : FileTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); + fileTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -439,12 +439,12 @@ struct CurlDownloader : public Downloader struct State { struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { return i1->embargo > i2->embargo; } }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::priority_queue, std::vector>, EmbargoComparator> incoming; }; Sync state_; @@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; - CurlDownloader() + curlFileTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - downloadSettings.httpConnections.get()); + fileTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); @@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~CurlDownloader() + ~curlFileTransfer() { stopWorkerThread(); @@ -504,7 +504,7 @@ struct CurlDownloader : public Downloader stopWorkerThread(); }); - std::map> items; + std::map> items; bool quit = false; @@ -561,7 +561,7 @@ struct CurlDownloader : public Downloader throw SysError("reading curl wakeup socket"); } - std::vector> incoming; + std::vector> incoming; auto now = std::chrono::steady_clock::now(); { @@ -609,7 +609,7 @@ struct CurlDownloader : public Downloader } } - void enqueueItem(std::shared_ptr item) + void enqueueItem(std::shared_ptr item) { if (item->request.data && !hasPrefix(item->request.uri, "http://") @@ -641,8 +641,8 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DownloadRequest & request, - Callback callback) override + void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,9 +660,9 @@ struct CurlDownloader : public Downloader // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DownloadResult res; + FileTransferResult res; if (!s3Res.data) - throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else @@ -672,26 +672,26 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared(*this, request, std::move(callback))); + enqueueItem(std::make_shared(*this, request, std::move(callback))); } }; -ref getDownloader() +ref getFileTransfer() { - static ref downloader = makeDownloader(); - return downloader; + static ref fileTransfer = makeFileTransfer(); + return fileTransfer; } -ref makeDownloader() +ref makeFileTransfer() { - return make_ref(); + return make_ref(); } -std::future Downloader::enqueueDownload(const DownloadRequest & request) +std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) { - auto promise = std::make_shared>(); - enqueueDownload(request, - {[promise](std::future fut) { + auto promise = std::make_shared>(); + enqueueFileTransfer(request, + {[promise](std::future fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,15 +701,21 @@ std::future Downloader::enqueueDownload(const DownloadRequest & return promise->get_future(); } -DownloadResult Downloader::download(const DownloadRequest & request) +FileTransferResult FileTransfer::download(const FileTransferRequest & request) { - return enqueueDownload(request).get(); + return enqueueFileTransfer(request).get(); } -void Downloader::download(DownloadRequest && request, Sink & sink) +FileTransferResult FileTransfer::upload(const FileTransferRequest & request) +{ + /* Note: this method is the same as download, but helps in readability */ + return enqueueFileTransfer(request).get(); +} + +void FileTransfer::download(FileTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the downloader + that would cause the sink to execute on the fileTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. @@ -755,8 +761,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDownload(request, - {[_state](std::future fut) { + enqueueFileTransfer(request, + {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; try { @@ -801,7 +807,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } - bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/download.hh b/src/libstore/filetransfer.hh similarity index 64% rename from src/libstore/download.hh rename to src/libstore/filetransfer.hh index 28c8a9162..2347f363d 100644 --- a/src/libstore/download.hh +++ b/src/libstore/filetransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DownloadSettings : Config +struct FileTransferSettings : Config { Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,15 +31,15 @@ struct DownloadSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DownloadSettings downloadSettings; +extern FileTransferSettings fileTransferSettings; -struct DownloadRequest +struct FileTransferRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = downloadSettings.tries; + size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -47,7 +47,7 @@ struct DownloadRequest std::string mimeType; std::function dataCallback; - DownloadRequest(const std::string & uri) + FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -56,7 +56,7 @@ struct DownloadRequest } }; -struct DownloadResult +struct FileTransferResult { bool cached = false; std::string etag; @@ -67,40 +67,43 @@ struct DownloadResult class Store; -struct Downloader +struct FileTransfer { - virtual ~Downloader() { } + virtual ~FileTransfer() { } - /* Enqueue a download request, returning a future to the result of - the download. The future may throw a DownloadError + /* Enqueue a data transfer request, returning a future to the result of + the download. The future may throw a FileTransferError exception. */ - virtual void enqueueDownload(const DownloadRequest & request, - Callback callback) = 0; + virtual void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) = 0; - std::future enqueueDownload(const DownloadRequest & request); + std::future enqueueFileTransfer(const FileTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); + FileTransferResult download(const FileTransferRequest & request); + + /* Synchronously upload a file. */ + FileTransferResult upload(const FileTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DownloadRequest && request, Sink & sink); + void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared Downloader object. Using this object is preferred +/* Return a shared FileTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref getDownloader(); +ref getFileTransfer(); -/* Return a new Downloader object. */ -ref makeDownloader(); +/* Return a new FileTransfer object. */ +ref makeFileTransfer(); -class DownloadError : public Error +class FileTransferError : public Error { public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) + FileTransfer::Error error; + FileTransferError(FileTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 011794c62..451a64785 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -85,14 +85,14 @@ protected: checkEnabled(); try { - DownloadRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); request.head = true; - getDownloader()->download(request); + getFileTransfer()->download(request); return true; - } catch (DownloadError & e) { + } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; maybeDisable(); throw; @@ -103,19 +103,19 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DownloadRequest(cacheUri + "/" + path); + auto req = FileTransferRequest(cacheUri + "/" + path); req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDownloader()->download(req); - } catch (DownloadError & e) { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } - DownloadRequest makeRequest(const std::string & path) + FileTransferRequest makeRequest(const std::string & path) { - DownloadRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); return request; } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDownloader()->download(std::move(request), sink); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDownloader()->enqueueDownload(request, - {[callbackPtr, this](std::future result) { + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); callbackPtr->rethrow(); @@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([]( }); } - diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f2e4b63e0..b24e7b7d6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "download.hh" +#include "filetransfer.hh" #include "istringstream_nocopy.hh" #include @@ -132,7 +132,7 @@ ref S3Helper::makeConfig(const string & region return res; } -S3Helper::DownloadResult S3Helper::getObject( +S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); - DownloadResult res; + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index ef5f23d0f..2042bffcf 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DownloadResult + struct FileTransferResult { std::shared_ptr data; unsigned int durationMs; }; - DownloadResult getObject( + FileTransferResult getObject( const std::string & bucketName, const std::string & key); }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index fa5c84a27..bb437cf1c 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg, if (action == "start") { auto type = (ActivityType) json["type"]; - if (trusted || type == actDownload) + if (trusted || type == actFileTransfer) activities.emplace(std::piecewise_construct, std::forward_as_tuple(json["id"]), std::forward_as_tuple(*logger, (Verbosity) json["level"], type, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index beb5e6b64..108b5dcb0 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -17,7 +17,7 @@ typedef enum { typedef enum { actUnknown = 0, actCopyPath = 100, - actDownload = 101, + actFileTransfer = 101, actRealise = 102, actCopyPaths = 103, actBuilds = 104, diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2a9defb4e..abd390414 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "globals.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "../nix/legacy.hh" #include "fetchers.hh" @@ -113,7 +113,7 @@ static void update(const StringSet & channelNames) // Download the channel tarball. try { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); - } catch (DownloadError & e) { + } catch (FileTransferError & e) { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 2b9254659..748554b9c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,6 +1,6 @@ #include "hash.hh" #include "shared.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" @@ -180,9 +180,9 @@ static int _main(int argc, char * * argv) FdSink sink(fd.get()); - DownloadRequest req(actualUri); + FileTransferRequest req(actualUri); req.decompress = false; - getDownloader()->download(std::move(req), sink); + getFileTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ diff --git a/src/nix/main.cc b/src/nix/main.cc index 3b5f5516f..2c64c7476 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,7 +8,7 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "download.hh" +#include "filetransfer.hh" #include "finally.hh" #include @@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv) settings.useSubstitutes = false; if (!settings.tarballTtl.overriden) settings.tarballTtl = std::numeric_limits::max(); - if (!downloadSettings.tries.overriden) - downloadSettings.tries = 0; - if (!downloadSettings.connectTimeout.overriden) - downloadSettings.connectTimeout = 1; + if (!fileTransferSettings.tries.overriden) + fileTransferSettings.tries = 0; + if (!fileTransferSettings.connectTimeout.overriden) + fileTransferSettings.connectTimeout = 1; } if (args.refresh) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 26631416c..adc9b9a5d 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -190,8 +190,8 @@ public: i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); } - if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) - || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) + if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; @@ -416,7 +416,7 @@ public: if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - showActivity(actDownload, "%s MiB DL", "%.1f", MiB); + showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB); { auto s = renderActivity(actOptimiseStore, "%s paths optimised"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c05c29517..4fcc6a721 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "download.hh" +#include "filetransfer.hh" #include "eval.hh" #include "attr-path.hh" #include "names.hh" @@ -138,8 +138,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = DownloadRequest(storePathsUrl); - auto res = getDownloader()->download(req); + auto req = FileTransferRequest(storePathsUrl); + auto res = getFileTransfer()->download(req); auto state = std::make_unique(Strings(), store); auto v = state->allocValue();