diff --git a/default.nix b/default.nix index ea0e68a..a0049e2 100644 --- a/default.nix +++ b/default.nix @@ -16,13 +16,13 @@ stdenv.mkDerivation { name = "nixos-channel-scripts"; buildInputs = with perlPackages; - [ nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc ]; + [ nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json ]; buildCommand = '' mkdir -p $out/bin g++ -g ${./generate-programs-index.cc} -Wall -std=c++11 -o $out/bin/generate-programs-index \ - -I ${nix}/include/nix -lnixmain -lnixexpr -lnixformat -lsqlite3 -lgc + -I ${nix}/include/nix -lnixmain -lnixexpr -lnixformat -lsqlite3 -lgc -I . cp ${./mirror-nixos-branch.pl} $out/bin/mirror-nixos-branch wrapProgram $out/bin/mirror-nixos-branch --set PERL5LIB $PERL5LIB --prefix PATH : ${wget}/bin:${git}/bin:${nix}/bin:${gnutar}/bin:${xz}/bin:$out/bin diff --git a/generate-programs-index.cc b/generate-programs-index.cc index 7383ed0..5d27350 100644 --- a/generate-programs-index.cc +++ b/generate-programs-index.cc @@ -10,10 +10,15 @@ #include "fs-accessor.hh" #include "thread-pool.hh" #include "sqlite.hh" +#include "download.hh" +#include "compression.hh" + +#include #include using namespace nix; +using json = nlohmann::json; static const char * cacheSchema = R"sql( @@ -62,7 +67,9 @@ void mainWrapped(int argc, char * * argv) settings.showTrace = true; auto localStore = openStore(); - auto binaryCache = openStoreAt(argv[3]); + std::string binaryCacheUri = argv[3]; + if (hasSuffix(binaryCacheUri, "/")) binaryCacheUri.pop_back(); + auto binaryCache = openStoreAt(binaryCacheUri); struct CacheState { @@ -172,6 +179,7 @@ void mainWrapped(int argc, char * * argv) auto getFiles = [&](const Path & storePath) { std::map files; + /* Look up the path in the SQLite cache. */ { auto cacheState(cacheState_.lock()); auto useQueryPath(cacheState->queryPath.use()(storePath)); @@ -186,29 +194,42 @@ void mainWrapped(int argc, char * * argv) } } - auto accessor = binaryCache->getFSAccessor(); + /* It's not in the cache, so get the .ls.xz file (which + contains a JSON serialisation of the listing of the NAR + contents) from the binary cache. */ + auto now1 = std::chrono::steady_clock::now(); + DownloadRequest req(binaryCacheUri + "/" + storePathToHash(storePath) + ".ls.xz"); + req.showProgress = DownloadRequest::no; + auto ls = json::parse(*decompress("xz", *getDownloader()->download(req).data)); - /* Get the NAR of the store path and enumerate all files - inside it. */ - std::function recurse; + if (ls.value("version", 0) != 1) + throw Error("NAR index for ā€˜%sā€™ has an unsupported version", storePath); + + std::function recurse; + + recurse = [&](const std::string & relPath, json & v) { + FSAccessor::Stat st; + + std::string type = v["type"]; + + if (type == "directory") { + st.type = FSAccessor::Type::tDirectory; + for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { + std::string name = i.key(); + recurse(relPath.empty() ? name : relPath + "/" + name, i.value()); + } + } else if (type == "regular") { + st.type = FSAccessor::Type::tRegular; + st.fileSize = v["size"]; + st.isExecutable = v.value("executable", false); + } else if (type == "symlink") { + st.type = FSAccessor::Type::tSymlink; + } else return; - recurse = [&](const Path & curPath, - const std::string & relPath) - { - auto st = accessor->stat(curPath); files[relPath] = st; - if (st.type == FSAccessor::Type::tDirectory) { - for (auto & name : accessor->readDirectory(curPath)) - recurse(curPath + "/" + name, relPath.empty() ? name : relPath + "/" + name); - } }; - auto now1 = std::chrono::steady_clock::now(); - recurse(storePath, ""); - auto now2 = std::chrono::steady_clock::now(); - printMsg(lvlInfo, format("processed %s in %d ms") - % storePath - % std::chrono::duration_cast(now2 - now1).count()); + recurse("", ls.at("root")); /* Insert the store path into the database. */ { @@ -232,6 +253,10 @@ void mainWrapped(int argc, char * * argv) txn.commit(); } + auto now2 = std::chrono::steady_clock::now(); + printInfo("processed %s in %d ms", storePath, + std::chrono::duration_cast(now2 - now1).count()); + return files; }; @@ -267,14 +292,13 @@ void mainWrapped(int argc, char * * argv) txn.commit(); } - } catch (InvalidPath & e) { - printMsg(lvlTalkative, format("warning: %s (%s) not in binary cache") % package->attrPath % storePath); - return; + } catch (DownloadError & e) { + printInfo("warning: no listing of %s (%s) in binary cache", package->attrPath, storePath); } }; /* Enqueue work items for each package. */ - ThreadPool threadPool; + ThreadPool threadPool(16); for (auto & i : packagesByPath) threadPool.enqueue(std::bind(doPath, i.first, i.second));