From 1fc905ad4c6320d7625d7f9bec06e70531bb0d5f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Apr 2015 12:12:50 +0200 Subject: [PATCH] Move curl stuff into a separate file --- src/libexpr/download.cc | 128 ++++++++++++++++++++++++++++++++++++++++ src/libexpr/download.hh | 16 +++++ src/libexpr/primops.cc | 118 ++---------------------------------- 3 files changed, 150 insertions(+), 112 deletions(-) create mode 100644 src/libexpr/download.cc create mode 100644 src/libexpr/download.hh diff --git a/src/libexpr/download.cc b/src/libexpr/download.cc new file mode 100644 index 000000000..050e8d69c --- /dev/null +++ b/src/libexpr/download.cc @@ -0,0 +1,128 @@ +#include "download.hh" +#include "util.hh" +#include "globals.hh" + +#include + +namespace nix { + +struct Curl +{ + CURL * curl; + string data; + string etag, status, expectedETag; + + struct curl_slist * requestHeaders; + + static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + { + Curl & c(* (Curl *) userp); + size_t realSize = size * nmemb; + c.data.append((char *) contents, realSize); + return realSize; + } + + static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + { + Curl & c(* (Curl *) userp); + size_t realSize = size * nmemb; + string line = string((char *) contents, realSize); + printMsg(lvlVomit, format("got header: %1%") % trim(line)); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + c.etag = ""; + auto ss = tokenizeString>(line, " "); + c.status = ss.size() >= 2 ? ss[1] : ""; + } else { + auto i = line.find(':'); + if (i != string::npos) { + string name = trim(string(line, 0, i)); + if (name == "ETag") { // FIXME: case + c.etag = trim(string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + printMsg(lvlDebug, format("got ETag: %1%") % c.etag); + if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); + return 0; + } + } + } + } + return realSize; + } + + Curl() + { + requestHeaders = 0; + + curl = curl_easy_init(); + if (!curl) throw Error("unable to initialize curl"); + + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); + curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + } + + ~Curl() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const string & expectedETag = "") + { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + data.clear(); + + if (requestHeaders) { + curl_slist_free_all(requestHeaders); + requestHeaders = 0; + } + + if (!expectedETag.empty()) { + this->expectedETag = expectedETag; + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + + CURLcode res = curl_easy_perform(curl); + if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; + if (res != CURLE_OK) + throw Error(format("unable to download ‘%1%’: %2% (%3%)") + % url % curl_easy_strerror(res) % res); + + long httpStatus = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + if (httpStatus == 304) return false; + + return true; + } +}; + + +DownloadResult downloadFile(string url, string expectedETag) +{ + DownloadResult res; + Curl curl; + if (curl.fetch(url, expectedETag)) { + res.cached = false; + res.data = curl.data; + } else + res.cached = true; + res.etag = curl.etag; + return res; +} + +} diff --git a/src/libexpr/download.hh b/src/libexpr/download.hh new file mode 100644 index 000000000..aa4fd5083 --- /dev/null +++ b/src/libexpr/download.hh @@ -0,0 +1,16 @@ +#pragma once + +#include "types.hh" +#include + +namespace nix { + +struct DownloadResult +{ + bool cached; + string data, etag; +}; + +DownloadResult downloadFile(string url, string expectedETag = ""); + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 03a3c8286..33ec26d26 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -9,6 +9,7 @@ #include "json-to-value.hh" #include "names.hh" #include "eval-inline.hh" +#include "download.hh" #include #include @@ -18,8 +19,6 @@ #include #include -#include - namespace nix { @@ -1486,112 +1485,6 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a *************************************************************/ -struct Curl -{ - CURL * curl; - string data; - string etag, status, expectedETag; - - struct curl_slist * requestHeaders; - - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) - { - Curl & c(* (Curl *) userp); - size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); - return realSize; - } - - static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) - { - Curl & c(* (Curl *) userp); - size_t realSize = size * nmemb; - string line = string((char *) contents, realSize); - printMsg(lvlVomit, format("got header: %1%") % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - c.etag = ""; - auto ss = tokenizeString>(line, " "); - c.status = ss.size() >= 2 ? ss[1] : ""; - } else { - auto i = line.find(':'); - if (i != string::npos) { - string name = trim(string(line, 0, i)); - if (name == "ETag") { // FIXME: case - c.etag = trim(string(line, i + 1)); - /* Hack to work around a GitHub bug: it sends - ETags, but ignores If-None-Match. So if we get - the expected ETag on a 200 response, then shut - down the connection because we already have the - data. */ - printMsg(lvlDebug, format("got ETag: %1%") % c.etag); - if (c.etag == c.expectedETag && c.status == "200") { - printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); - return 0; - } - } - } - } - return realSize; - } - - Curl() - { - requestHeaders = 0; - - curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); - - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); - curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); - - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); - } - - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - if (requestHeaders) curl_slist_free_all(requestHeaders); - } - - bool fetch(const string & url, const string & expectedETag = "") - { - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - - data.clear(); - - if (requestHeaders) { - curl_slist_free_all(requestHeaders); - requestHeaders = 0; - } - - if (!expectedETag.empty()) { - this->expectedETag = expectedETag; - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); - } - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); - - CURLcode res = curl_easy_perform(curl); - if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; - if (res != CURLE_OK) - throw Error(format("unable to download ‘%1%’: %2% (%3%)") - % url % curl_easy_strerror(res) % res); - - long httpStatus = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); - if (httpStatus == 304) return false; - - return true; - } -}; - - void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack) { @@ -1662,15 +1555,16 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, printMsg(lvlInfo, format("downloading ‘%1%’...") % url); else printMsg(lvlInfo, format("checking ‘%1%’...") % url); - Curl curl; - if (curl.fetch(url, expectedETag)) - storePath = store->addTextToStore(name, curl.data, PathSet(), state.repair); + auto res = downloadFile(url, expectedETag); + + if (!res.cached) + storePath = store->addTextToStore(name, res.data, PathSet(), state.repair); assert(!storePath.empty()); replaceSymlink(storePath, fileLink); - writeFile(dataFile, url + "\n" + curl.etag + "\n" + int2String(time(0)) + "\n"); + writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n"); } if (unpack) {