delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Support hydra-build-products on binary cache stores

Browse source
This commit is contained in:
Eelco Dolstra 2016-02-26 14:45:03 +01:00
parent 8e24ad6f0d
commit 02190b0fef
4 changed files with 34 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
src/hydra-queue-runner/build-result.cc
View file

@ -2,26 +2,13 @@
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "regex.hh" #include "regex.hh"
#include "fs-accessor.hh"
using namespace nix; using namespace nix;
static std::tuple<bool, string> secureRead(Path fileName) BuildOutput getBuildOutput(nix::ref<Store> store,
{ nix::ref<nix::FSAccessor> accessor, const Derivation & drv)
auto fail = std::make_tuple(false, "");
if (!pathExists(fileName)) return fail;
try {
/* For security, resolve symlinks. */
fileName = canonPath(fileName, true);
if (!isInStore(fileName)) return fail;
return std::make_tuple(true, readFile(fileName));
} catch (Error & e) { return fail; }
}
BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
{ {
BuildOutput res; BuildOutput res;
@ -41,7 +28,6 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
/* Get build products. */ /* Get build products. */
bool explicitProducts = false; bool explicitProducts = false;
#if 0
Regex regex( Regex regex(
"(([a-zA-Z0-9_-]+)" // type (e.g. "doc") "(([a-zA-Z0-9_-]+)" // type (e.g. "doc")
"[[:space:]]+" "[[:space:]]+"
@ -53,14 +39,16 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
for (auto & output : outputs) { for (auto & output : outputs) {
Path failedFile = output + "/nix-support/failed"; Path failedFile = output + "/nix-support/failed";
if (pathExists(failedFile)) res.failed = true; if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)
res.failed = true;
auto file = secureRead(output + "/nix-support/hydra-build-products"); Path productsFile = output + "/nix-support/hydra-build-products";
if (!std::get<0>(file)) continue; if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)
continue;
explicitProducts = true; explicitProducts = true;
for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) { for (auto & line : tokenizeString<Strings>(accessor->readFile(productsFile), "\n")) {
BuildProduct product; BuildProduct product;
Regex::Subs subs; Regex::Subs subs;
@ -73,32 +61,28 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
/* Ensure that the path exists and points into the Nix /* Ensure that the path exists and points into the Nix
store. */ store. */
// FIXME: should we disallow products referring to other
// store paths, or that are outside the input closure?
if (product.path == "" || product.path[0] != '/') continue; if (product.path == "" || product.path[0] != '/') continue;
try { product.path = canonPath(product.path);
product.path = canonPath(product.path, true); if (!isInStore(product.path)) continue;
} catch (Error & e) { continue; }
if (!isInStore(product.path) || !pathExists(product.path)) continue;
/* FIXME: check that the path is in the input closure auto st = accessor->stat(product.path);
of the build? */ if (st.type == FSAccessor::Type::tMissing) continue;
product.name = product.path == output ? "" : baseNameOf(product.path); product.name = product.path == output ? "" : baseNameOf(product.path);
struct stat st; if (st.type == FSAccessor::Type::tRegular) {
if (stat(product.path.c_str(), &st))
throw SysError(format("getting status of ‘%1%’") % product.path);
if (S_ISREG(st.st_mode)) {
product.isRegular = true; product.isRegular = true;
product.fileSize = st.st_size; product.fileSize = st.fileSize;
product.sha1hash = hashFile(htSHA1, product.path); auto contents = accessor->readFile(product.path);
product.sha256hash = hashFile(htSHA256, product.path); product.sha1hash = hashString(htSHA1, contents);
product.sha256hash = hashString(htSHA256, contents);
} }
res.products.push_back(product); res.products.push_back(product);
} }
} }
#endif
/* If no build products were explicitly declared, then add all /* If no build products were explicitly declared, then add all
outputs as a product of type "nix-build". */ outputs as a product of type "nix-build". */
@ -110,31 +94,29 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
product.subtype = output.first == "out" ? "" : output.first; product.subtype = output.first == "out" ? "" : output.first;
product.name = storePathToName(product.path); product.name = storePathToName(product.path);
#if 0 auto st = accessor->stat(product.path);
struct stat st; if (st.type == FSAccessor::Type::tMissing)
if (stat(product.path.c_str(), &st)) throw Error(format("getting status of ‘%1%’") % product.path);
throw SysError(format("getting status of ‘%1%’") % product.path); if (st.type == FSAccessor::Type::tDirectory)
if (S_ISDIR(st.st_mode))
#endif
res.products.push_back(product); res.products.push_back(product);
} }
} }
#if 0
/* Get the release name from $output/nix-support/hydra-release-name. */ /* Get the release name from $output/nix-support/hydra-release-name. */
for (auto & output : outputs) { for (auto & output : outputs) {
Path p = output + "/nix-support/hydra-release-name"; Path p = output + "/nix-support/hydra-release-name";
if (!pathExists(p)) continue; if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
try { try {
res.releaseName = trim(readFile(p)); res.releaseName = trim(accessor->readFile(p));
} catch (Error & e) { continue; } } catch (Error & e) { continue; }
// FIXME: validate release name // FIXME: validate release name
} }
/* Get metrics. */ /* Get metrics. */
for (auto & output : outputs) { for (auto & output : outputs) {
auto file = secureRead(output + "/nix-support/hydra-metrics"); Path metricsFile = output + "/nix-support/hydra-metrics";
for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) { if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {
auto fields = tokenizeString<std::vector<std::string>>(line); auto fields = tokenizeString<std::vector<std::string>>(line);
if (fields.size() < 2) continue; if (fields.size() < 2) continue;
BuildMetric metric; BuildMetric metric;
@ -144,7 +126,6 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
res.metrics[metric.name] = metric; res.metrics[metric.name] = metric;
} }
} }
#endif
return res; return res;
} }

4
src/hydra-queue-runner/build-result.hh
View file

@ -4,6 +4,7 @@
#include "hash.hh" #include "hash.hh"
#include "derivations.hh" #include "derivations.hh"
#include "store-api.hh"
struct BuildProduct struct BuildProduct
{ {
@ -37,4 +38,5 @@ struct BuildOutput
std::map<std::string, BuildMetric> metrics; std::map<std::string, BuildMetric> metrics;
}; };
BuildOutput getBuildOutput(nix::ref<nix::Store> store, const nix::Derivation & drv); BuildOutput getBuildOutput(nix::ref<nix::Store> store,
nix::ref<nix::FSAccessor> accessor, const nix::Derivation & drv);

2
src/hydra-queue-runner/builder.cc
View file

@ -126,7 +126,7 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
result.errorMsg = e.msg(); result.errorMsg = e.msg();
} }
if (result.success()) res = getBuildOutput(destStore, step->drv); if (result.success()) res = getBuildOutput(destStore, destStore->getFSAccessor(), step->drv);
} }
time_t stepStopTime = time(0); time_t stepStopTime = time(0);

2
src/hydra-queue-runner/queue-monitor.cc
View file

@ -159,7 +159,7 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
all valid. So we mark this as a finished, cached build. */ all valid. So we mark this as a finished, cached build. */
if (!step) { if (!step) {
Derivation drv = readDerivation(build->drvPath); Derivation drv = readDerivation(build->drvPath);
BuildOutput res = getBuildOutput(destStore, drv); BuildOutput res = getBuildOutput(destStore, destStore->getFSAccessor(), drv);
pqxx::work txn(conn); pqxx::work txn(conn);
time_t now = time(0); time_t now = time(0);