Merge remote-tracking branch 'origin/master' into readd-hashed-mirrors

This commit is contained in:
Matthew Bauer 2020-08-06 18:58:59 -05:00
commit 8abc577cc2
50 changed files with 971 additions and 464 deletions

View file

@ -224,7 +224,7 @@ SV * hashString(char * algo, int base32, char * s)
SV * convertHash(char * algo, char * s, int toBase32) SV * convertHash(char * algo, char * s, int toBase32)
PPCODE: PPCODE:
try { try {
Hash h(s, parseHashType(algo)); auto h = Hash::parseAny(s, parseHashType(algo));
string s = h.to_string(toBase32 ? Base32 : Base16, false); string s = h.to_string(toBase32 ? Base32 : Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
@ -285,7 +285,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE: PPCODE:
try { try {
Hash h(hash, parseHashType(algo)); auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name); auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));

View file

@ -170,7 +170,45 @@ static int _main(int argc, char * * argv)
if (rightType && !canBuildLocally) if (rightType && !canBuildLocally)
std::cerr << "# postpone\n"; std::cerr << "# postpone\n";
else else
{
// build the hint template.
string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)";
hintstring += "\n%s available machines:";
hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
for (unsigned int i = 0; i < machines.size(); ++i) {
hintstring += "\n(%s, %s, %s, %s)";
}
// add the template values.
string drvstr;
if (drvPath.has_value())
drvstr = drvPath->to_string();
else
drvstr = "<unknown>";
auto hint = hintformat(hintstring);
hint
% drvstr
% neededSystem
% concatStringsSep<StringSet>(", ", requiredFeatures)
% machines.size();
for (auto & m : machines) {
hint % concatStringsSep<vector<string>>(", ", m.systemTypes)
% m.maxJobs
% concatStringsSep<StringSet>(", ", m.supportedFeatures)
% concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
}
logError({
.name = "Remote build",
.description = "Failed to find a machine for remote build!",
.hint = hint
});
std::cerr << "# decline\n"; std::cerr << "# decline\n";
}
break; break;
} }

View file

@ -345,6 +345,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sStructuredAttrs(symbols.create("__structuredAttrs")) , sStructuredAttrs(symbols.create("__structuredAttrs"))
, sBuilder(symbols.create("builder")) , sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args")) , sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed"))
, sOutputHash(symbols.create("outputHash")) , sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode")) , sOutputHashMode(symbols.create("outputHashMode"))
@ -1259,7 +1260,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
addErrorTrace(e, lambda.pos, "while evaluating %s", addErrorTrace(e, lambda.pos, "while evaluating %s",
(lambda.name.set() (lambda.name.set()
? "'" + (string) lambda.name + "'" ? "'" + (string) lambda.name + "'"
: "anonymous lambdaction")); : "anonymous lambda"));
addErrorTrace(e, pos, "from call site%s", ""); addErrorTrace(e, pos, "from call site%s", "");
throw; throw;
} }

View file

@ -74,6 +74,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString, sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sContentAddressed,
sOutputHash, sOutputHashAlgo, sOutputHashMode, sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations, sRecurseForDerivations,
sDescription, sSelf, sEpsilon; sDescription, sSelf, sEpsilon;

View file

@ -583,6 +583,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context; PathSet context;
bool contentAddressed = false;
std::optional<std::string> outputHash; std::optional<std::string> outputHash;
std::string outputHashAlgo; std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat; auto ingestionMethod = FileIngestionMethod::Flat;
@ -639,9 +640,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (i->value->type == tNull) continue; if (i->value->type == tNull) continue;
} }
if (i->name == state.sContentAddressed) {
settings.requireExperimentalFeature("ca-derivations");
contentAddressed = state.forceBool(*i->value, pos);
}
/* The `args' attribute is special: it supplies the /* The `args' attribute is special: it supplies the
command-line arguments to the builder. */ command-line arguments to the builder. */
if (i->name == state.sArgs) { else if (i->name == state.sArgs) {
state.forceList(*i->value, pos); state.forceList(*i->value, pos);
for (unsigned int n = 0; n < i->value->listSize(); ++n) { for (unsigned int n = 0; n < i->value->listSize(); ++n) {
string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
@ -761,7 +767,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}); });
if (outputHash) { if (outputHash) {
/* Handle fixed-output derivations. */ /* Handle fixed-output derivations.
Ignore `__contentAddressed` because fixed output derivations are
already content addressed. */
if (outputs.size() != 1 || *(outputs.begin()) != "out") if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error({ throw Error({
.hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
@ -774,7 +783,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { drv.outputs.insert_or_assign("out", DerivationOutput {
.output = DerivationOutputFixed { .output = DerivationOutputCAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = ingestionMethod, .method = ingestionMethod,
.hash = std::move(h), .hash = std::move(h),
@ -783,6 +792,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}); });
} }
else if (contentAddressed) {
HashType ht = parseHashType(outputHashAlgo);
for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = hashPlaceholder(i);
drv.outputs.insert_or_assign(i, DerivationOutput {
.output = DerivationOutputCAFloating {
.method = ingestionMethod,
.hashType = std::move(ht),
},
});
}
}
else { else {
/* Compute a hash over the "masked" store derivation, which is /* Compute a hash over the "masked" store derivation, which is
the final one except that in the list of outputs, the the final one except that in the list of outputs, the
@ -800,7 +822,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}); });
} }
Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); // Regular, non-CA derivation should always return a single hash and not
// hash per output.
Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
for (auto & i : outputs) { for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName); auto outPath = state.store->makeOutputPath(i, h, drvName);

View file

@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// be both a revision or a branch/tag name. // be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos); auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex)) if (std::regex_match(value, revRegex))
rev = Hash(value, htSHA1); rev = Hash::parseAny(value, htSHA1);
else else
ref = value; ref = value;
} }

View file

@ -200,9 +200,12 @@ std::string Input::getType() const
std::optional<Hash> Input::getNarHash() const std::optional<Hash> Input::getNarHash() const
{ {
if (auto s = maybeGetStrAttr(attrs, "narHash")) if (auto s = maybeGetStrAttr(attrs, "narHash")) {
// FIXME: require SRI hash. auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
return newHashAllowEmpty(*s, htSHA256); if (hash.type != htSHA256)
throw UsageError("narHash must use SHA-256");
return hash;
}
return {}; return {};
} }
@ -216,7 +219,7 @@ std::optional<std::string> Input::getRef() const
std::optional<Hash> Input::getRev() const std::optional<Hash> Input::getRev() const
{ {
if (auto s = maybeGetStrAttr(attrs, "rev")) if (auto s = maybeGetStrAttr(attrs, "rev"))
return Hash(*s, htSHA1); return Hash::parseAny(*s, htSHA1);
return {}; return {};
} }

View file

@ -121,7 +121,7 @@ struct GitInputScheme : InputScheme
args.push_back(*ref); args.push_back(*ref);
} }
if (input.getRev()) throw Error("cloning a specific revision is not implemented"); if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented");
args.push_back(destDir); args.push_back(destDir);
@ -293,14 +293,14 @@ struct GitInputScheme : InputScheme
if (!input.getRev()) if (!input.getRev())
input.attrs.insert_or_assign("rev", input.attrs.insert_or_assign("rev",
Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl; repoDir = actualUrl;
} else { } else {
if (auto res = getCache()->lookup(store, mutableAttrs)) { if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) { if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev()); input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme
} }
if (!input.getRev()) if (!input.getRev())
input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
} }
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";

View file

@ -29,7 +29,7 @@ struct GitArchiveInputScheme : InputScheme
if (path.size() == 2) { if (path.size() == 2) {
} else if (path.size() == 3) { } else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex)) if (std::regex_match(path[2], revRegex))
rev = Hash(path[2], htSHA1); rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex)) else if (std::regex_match(path[2], refRegex))
ref = path[2]; ref = path[2];
else else
@ -41,7 +41,7 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") { if (name == "rev") {
if (rev) if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url); throw BadURL("URL '%s' contains multiple commit hashes", url.url);
rev = Hash(value, htSHA1); rev = Hash::parseAny(value, htSHA1);
} }
else if (name == "ref") { else if (name == "ref") {
if (!std::regex_match(value, refRegex)) if (!std::regex_match(value, refRegex))
@ -191,7 +191,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
readFile( readFile(
store->toRealPath( store->toRealPath(
downloadFile(store, url, "source", false).storePath))); downloadFile(store, url, "source", false).storePath)));
auto rev = Hash(std::string { json["sha"] }, htSHA1); auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev()); debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev; return rev;
} }
@ -235,7 +235,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
readFile( readFile(
store->toRealPath( store->toRealPath(
downloadFile(store, url, "source", false).storePath))); downloadFile(store, url, "source", false).storePath)));
auto rev = Hash(std::string(json[0]["id"]), htSHA1); auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev()); debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev; return rev;
} }

View file

@ -18,7 +18,7 @@ struct IndirectInputScheme : InputScheme
if (path.size() == 1) { if (path.size() == 1) {
} else if (path.size() == 2) { } else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex)) if (std::regex_match(path[1], revRegex))
rev = Hash(path[1], htSHA1); rev = Hash::parseAny(path[1], htSHA1);
else if (std::regex_match(path[1], refRegex)) else if (std::regex_match(path[1], refRegex))
ref = path[1]; ref = path[1];
else else
@ -29,7 +29,7 @@ struct IndirectInputScheme : InputScheme
ref = path[1]; ref = path[1];
if (!std::regex_match(path[2], revRegex)) if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
rev = Hash(path[2], htSHA1); rev = Hash::parseAny(path[2], htSHA1);
} else } else
throw BadURL("GitHub URL '%s' is invalid", url.url); throw BadURL("GitHub URL '%s' is invalid", url.url);

View file

@ -209,7 +209,7 @@ struct MercurialInputScheme : InputScheme
}); });
if (auto res = getCache()->lookup(store, mutableAttrs)) { if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) { if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev()); input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3); assert(tokens.size() == 3);
input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev()); input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]); auto revCount = std::stoull(tokens[1]);
input.attrs.insert_or_assign("ref", tokens[2]); input.attrs.insert_or_assign("ref", tokens[2]);

View file

@ -169,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
TeeSource teeSource(narSource, *compressionSink); TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource); narAccessor = makeNarAccessor(teeSource);
compressionSink->finish(); compressionSink->finish();
fileSink.flush();
} }
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();
@ -282,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in), std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar"); "application/x-nix-nar");
} else } else
stats.narWriteAverted++; stats.narWriteAverted++;

View file

@ -806,8 +806,8 @@ private:
/* RAII object to delete the chroot directory. */ /* RAII object to delete the chroot directory. */
std::shared_ptr<AutoDelete> autoDelChroot; std::shared_ptr<AutoDelete> autoDelChroot;
/* Whether this is a fixed-output derivation. */ /* The sort of derivation we are building. */
bool fixedOutput; DerivationType derivationType;
/* Whether to run the build in a private network namespace. */ /* Whether to run the build in a private network namespace. */
bool privateNetwork = false; bool privateNetwork = false;
@ -1195,9 +1195,9 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (parsedDrv->contentAddressed()) { if (drv->type() == DerivationType::CAFloating) {
settings.requireExperimentalFeature("ca-derivations"); settings.requireExperimentalFeature("ca-derivations");
throw Error("ca-derivations isn't implemented yet"); throw UnimplementedError("ca-derivations isn't implemented yet");
} }
@ -1392,12 +1392,12 @@ void DerivationGoal::inputsRealised()
debug("added input paths %s", worker.store.showPaths(inputPaths)); debug("added input paths %s", worker.store.showPaths(inputPaths));
/* Is this a fixed-output derivation? */ /* What type of derivation are we building? */
fixedOutput = drv->isFixedOutput(); derivationType = drv->type();
/* Don't repeat fixed-output derivations since they're already /* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/ verified by their output hash.*/
nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build /* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a slot to become available, since we don't need one if there is a
@ -1783,7 +1783,7 @@ void DerivationGoal::buildDone()
st = st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected : statusOk(status) ? BuildResult::OutputRejected :
fixedOutput || diskFull ? BuildResult::TransientFailure : derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure; BuildResult::PermanentFailure;
} }
@ -1996,7 +1996,7 @@ void DerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled) else if (settings.sandboxMode == smDisabled)
useChroot = false; useChroot = false;
else if (settings.sandboxMode == smRelaxed) else if (settings.sandboxMode == smRelaxed)
useChroot = !fixedOutput && !noChroot; useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
} }
if (worker.store.storeDir != worker.store.realStoreDir) { if (worker.store.storeDir != worker.store.realStoreDir) {
@ -2165,7 +2165,7 @@ void DerivationGoal::startBuilder()
"nogroup:x:65534:\n") % sandboxGid).str()); "nogroup:x:65534:\n") % sandboxGid).str());
/* Create /etc/hosts with localhost entry. */ /* Create /etc/hosts with localhost entry. */
if (!fixedOutput) if (!(derivationIsImpure(derivationType)))
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot, /* Make the closure of the inputs available in the chroot,
@ -2373,7 +2373,7 @@ void DerivationGoal::startBuilder()
us. us.
*/ */
if (!fixedOutput) if (!(derivationIsImpure(derivationType)))
privateNetwork = true; privateNetwork = true;
userNamespaceSync.create(); userNamespaceSync.create();
@ -2574,7 +2574,7 @@ void DerivationGoal::initEnv()
derivation, tell the builder, so that for instance `fetchurl' derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment can skip checking the output. On older Nixes, this environment
variable won't be set, so `fetchurl' will do the check. */ variable won't be set, so `fetchurl' will do the check. */
if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
/* *Only* if this is a fixed-output derivation, propagate the /* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the values of the environment variables specified in the
@ -2585,7 +2585,7 @@ void DerivationGoal::initEnv()
to the builder is generally impure, but the output of to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */ already know the cryptographic hash of the output). */
if (fixedOutput) { if (derivationIsImpure(derivationType)) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or(""); env[i] = getEnv(i).value_or("");
} }
@ -3195,7 +3195,7 @@ void DerivationGoal::runChild()
/* Fixed-output derivations typically need to access the /* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so network, so give them access to /etc/resolv.conf and so
on. */ on. */
if (fixedOutput) { if (derivationIsImpure(derivationType)) {
ss.push_back("/etc/resolv.conf"); ss.push_back("/etc/resolv.conf");
// Only use nss functions to resolve hosts and // Only use nss functions to resolve hosts and
@ -3436,7 +3436,7 @@ void DerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
if (fixedOutput) if (derivationIsImpure(derivationType))
sandboxProfile += "(import \"sandbox-network.sb\")\n"; sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Our rwx outputs */ /* Our rwx outputs */
@ -3721,9 +3721,22 @@ void DerivationGoal::registerOutputs()
hash). */ hash). */
std::optional<ContentAddress> ca; std::optional<ContentAddress> ca;
if (fixedOutput) { if (! std::holds_alternative<DerivationOutputInputAddressed>(i.second.output)) {
DerivationOutputCAFloating outputHash;
FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash; std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
assert(false); // Enclosing `if` handles this case in other branch
},
[&](DerivationOutputCAFixed dof) {
outputHash = DerivationOutputCAFloating {
.method = dof.hash.method,
.hashType = dof.hash.hash.type,
};
},
[&](DerivationOutputCAFloating dof) {
outputHash = dof;
},
}, i.second.output);
if (outputHash.method == FileIngestionMethod::Flat) { if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
@ -3737,12 +3750,17 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by /* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */ the derivation to its content-addressed location. */
Hash h2 = outputHash.method == FileIngestionMethod::Recursive Hash h2 = outputHash.method == FileIngestionMethod::Recursive
? hashPath(outputHash.hash.type, actualPath).first ? hashPath(outputHash.hashType, actualPath).first
: hashFile(outputHash.hash.type, actualPath); : hashFile(outputHash.hashType, actualPath);
auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
if (outputHash.hash != h2) { // true if either floating CA, or incorrect fixed hash.
bool needsMove = true;
if (auto p = std::get_if<DerivationOutputCAFixed>(& i.second.output)) {
Hash & h = p->hash.hash;
if (h != h2) {
/* Throw an error after registering the path as /* Throw an error after registering the path as
valid. */ valid. */
@ -3750,9 +3768,15 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr( delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), worker.store.printStorePath(dest),
outputHash.hash.to_string(SRI, true), h.to_string(SRI, true),
h2.to_string(SRI, true))); h2.to_string(SRI, true)));
} else {
// matched the fixed hash, so no move needed.
needsMove = false;
}
}
if (needsMove) {
Path actualDest = worker.store.Store::toRealPath(dest); Path actualDest = worker.store.Store::toRealPath(dest);
if (worker.store.isValidPath(dest)) if (worker.store.isValidPath(dest))
@ -4852,8 +4876,17 @@ void Worker::run(const Goals & _topGoals)
waitForInput(); waitForInput();
else { else {
if (awake.empty() && 0 == settings.maxBuildJobs) if (awake.empty() && 0 == settings.maxBuildJobs)
{
if (getMachines().empty())
throw Error("unable to start any build; either increase '--max-jobs' " throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds"); "or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
else
throw Error("unable to start any build; remote machines may not have "
"all required system features."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
}
assert(!awake.empty()); assert(!awake.empty());
} }
} }

View file

@ -1,4 +1,6 @@
#include "args.hh"
#include "content-address.hh" #include "content-address.hh"
#include "split.hh"
namespace nix { namespace nix {
@ -24,10 +26,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+ hash.to_string(Base32, true); + hash.to_string(Base32, true);
} }
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
std::string renderContentAddress(ContentAddress ca) { std::string renderContentAddress(ContentAddress ca) {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHash th) { [](TextHash th) {
@ -40,38 +38,46 @@ std::string renderContentAddress(ContentAddress ca) {
} }
ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddress parseContentAddress(std::string_view rawCa) {
auto prefixSeparator = rawCa.find(':'); auto rest = rawCa;
if (prefixSeparator != string::npos) {
auto prefix = string(rawCa, 0, prefixSeparator); std::string_view prefix;
{
auto optPrefix = splitPrefixTo(rest, ':');
if (!optPrefix)
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
prefix = *optPrefix;
}
auto parseHashType_ = [&](){
auto hashTypeRaw = splitPrefixTo(rest, ':');
if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
HashType hashType = parseHashType(*hashTypeRaw);
return std::move(hashType);
};
// Switch on prefix
if (prefix == "text") { if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); // No parsing of the method, "text" only support flat.
Hash hash = Hash(string(hashTypeAndHash)); HashType hashType = parseHashType_();
if (hash.type != htSHA256) { if (hashType != htSHA256)
throw Error("parseContentAddress: the text hash should have type SHA256"); throw Error("text content address hash should use %s, but instead uses %s",
} printHashType(htSHA256), printHashType(hashType));
return TextHash { hash }; return TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
};
} else if (prefix == "fixed") { } else if (prefix == "fixed") {
// This has to be an inverse of makeFixedOutputCA // Parse method
auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); auto method = FileIngestionMethod::Flat;
if (methodAndHash.substr(0,2) == "r:") { if (splitPrefix(rest, "r:"))
std::string_view hashRaw = methodAndHash.substr(2,string::npos); method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_();
return FixedOutputHash { return FixedOutputHash {
.method = FileIngestionMethod::Recursive, .method = method,
.hash = Hash(string(hashRaw)), .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
}; };
} else { } else
std::string_view hashRaw = methodAndHash; throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
return FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash(string(hashRaw)),
};
}
} else {
throw Error("parseContentAddress: format not recognized; has to be text or fixed");
}
} else {
throw Error("Not a content address because it lacks an appropriate prefix");
}
}; };
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) { std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {

View file

@ -698,7 +698,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto deriver = readString(from); auto deriver = readString(from);
if (deriver != "") if (deriver != "")
info.deriver = store->parseStorePath(deriver); info.deriver = store->parseStorePath(deriver);
info.narHash = Hash(readString(from), htSHA256); info.narHash = Hash::parseAny(readString(from), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, from); info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate; from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from); info.sigs = readStrings<StringSet>(from);

View file

@ -7,23 +7,54 @@
namespace nix { namespace nix {
// FIXME Put this somewhere? std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) { [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
return doi.path; return { doi.path };
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
}, },
[&](DerivationOutputFixed dof) {
return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
}
}, output); }, output);
} }
bool derivationIsCA(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return true;
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
assert(false);
}
bool derivationIsFixed(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
};
assert(false);
}
bool derivationIsImpure(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
};
assert(false);
}
bool BasicDerivation::isBuiltin() const bool BasicDerivation::isBuiltin() const
{ {
return string(builder, 0, 8) == "builtin:"; return string(builder, 0, 8) == "builtin:";
@ -123,14 +154,22 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
} }
const HashType hashType = parseHashType(hashAlgo); const HashType hashType = parseHashType(hashAlgo);
return DerivationOutput { return hash != ""
.output = DerivationOutputFixed { ? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash(hash, hashType), .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
}, },
} }
}; }
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else } else
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputInputAddressed { .output = DerivationOutputInputAddressed {
@ -278,13 +317,20 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first); s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name))); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) { std::visit(overloaded {
s += ','; printUnquotedString(s, hash->hash.printMethodAlgo()); [&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
} else {
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
} },
[&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
}, i.second.output);
s += ')'; s += ')';
} }
@ -336,60 +382,134 @@ bool isDerivation(const string & fileName)
} }
bool BasicDerivation::isFixedOutput() const DerivationType BasicDerivation::type() const
{ {
return outputs.size() == 1 && std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
outputs.begin()->first == "out" && std::optional<HashType> floatingHashType;
std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output); for (auto & i : outputs) {
std::visit(overloaded {
[&](DerivationOutputInputAddressed _) {
inputAddressedOutputs.insert(i.first);
},
[&](DerivationOutputCAFixed _) {
fixedCAOutputs.insert(i.first);
},
[&](DerivationOutputCAFloating dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
} else {
if (*floatingHashType != dof.hashType)
throw Error("All floating outputs must use the same hash type");
}
},
}, i.second.output);
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
return DerivationType::InputAddressed;
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
return DerivationType::CAFixed;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
return DerivationType::CAFloating;
} else {
throw Error("Can't mix derivation output types");
}
} }
DrvHashes drvHashes; DrvHashes drvHashes;
/* pathDerivationModulo and hashDerivationModulo are mutually recursive
*/
/* Returns the hash of a derivation modulo fixed-output /* Look up the derivation by value and memoize the
subderivations. A fixed-output derivation is a derivation with one `hashDerivationModulo` call.
output (`out') for which an expected hash and hash algorithm are */
specified (using the `outputHash' and `outputHashAlgo' static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
attributes). We don't want changes to such derivations to {
propagate upwards through the dependency graph, changing output auto h = drvHashes.find(drvPath);
paths everywhere. if (h == drvHashes.end()) {
assert(store.isValidPath(drvPath));
// Cache it
h = drvHashes.insert_or_assign(
drvPath,
hashDerivationModulo(
store,
store.readDerivation(drvPath),
false)).first;
}
return h->second;
}
For instance, if we change the url in a call to the `fetchurl' /* See the header for interface details. These are the implementation details.
function, we do not want to rebuild everything depending on it
(after all, (the hash of) the file being downloaded is unchanged).
So the *output paths* should not change. On the other hand, the
*derivation paths* should change to reflect the new dependency
graph.
That's what this function does: it returns a hash which is just the For fixed-output derivations, each hash in the map is not the
hash of the derivation ATerm, except that any input derivation corresponding output's content hash, but a hash of that hash along
paths have been replaced by the result of a recursive call to this with other constant data. The key point is that the value is a pure
function, and that for fixed-output derivations we return a hash of function of the output's contents, and there are no preimage attacks
its output path. */ either spoofing an output's contents for a derivation, or
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) spoofing a derivation for an output's contents.
For regular derivations, it looks up each subderivation from its hash
and recurs. If the subderivation is also regular, it simply
substitutes the derivation path with its hash. If the subderivation
is fixed-output, however, it takes each output hash and pretends it
is a derivation hash producing a single "out" output. This is so we
don't leak the provenance of fixed outputs, reducing pointless cache
misses as the build itself won't know this.
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{ {
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) { switch (drv.type()) {
DerivationOutputs::const_iterator i = drv.outputs.begin(); case DerivationType::CAFloating:
auto hash = std::get<DerivationOutputFixed>(i->second.output); throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
return hashString(htSHA256, "fixed:out:" case DerivationType::CAFixed: {
+ hash.hash.printMethodAlgo() + ":" std::map<std::string, Hash> outputHashes;
+ hash.hash.hash.to_string(Base16, false) + ":" for (const auto & i : drv.outputs) {
+ store.printStorePath(i->second.path(store, drv.name))); auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.path(store, drv.name)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return outputHashes;
}
case DerivationType::InputAddressed:
break;
} }
/* For other derivations, replace the inputs paths with recursive /* For other derivations, replace the inputs paths with recursive
calls to this function.*/ calls to this function. */
std::map<std::string, StringSet> inputs2; std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) { for (auto & i : drv.inputDrvs) {
auto h = drvHashes.find(i.first); const auto & res = pathDerivationModulo(store, i.first);
if (h == drvHashes.end()) { std::visit(overloaded {
assert(store.isValidPath(i.first)); // Regular non-CA derivation, replace derivation
h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, [&](Hash drvHash) {
store.readDerivation(i.first), false)).first; inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes
[&](CaOutputHashes outputHashes) {
std::set<std::string> justOut = { "out" };
for (auto & output : i.second) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
h.to_string(Base16, false),
justOut);
} }
inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); },
}, res);
} }
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@ -431,14 +551,22 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
auto hashType = parseHashType(hashAlgo); auto hashType = parseHashType(hashAlgo);
return DerivationOutput { return hash != ""
.output = DerivationOutputFixed { ? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash(hash, hashType), .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
}, },
} }
}; }
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else } else
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputInputAddressed { .output = DerivationOutputInputAddressed {
@ -498,12 +626,19 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
for (auto & i : drv.outputs) { for (auto & i : drv.outputs) {
out << i.first out << i.first
<< store.printStorePath(i.second.path(store, drv.name)); << store.printStorePath(i.second.path(store, drv.name));
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) { std::visit(overloaded {
out << hash->hash.printMethodAlgo() [&](DerivationOutputInputAddressed doi) {
<< hash->hash.hash.to_string(Base16, false);
} else {
out << "" << ""; out << "" << "";
} },
[&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
}, i.second.output);
} }
writeStorePaths(store, out, drv.inputSrcs); writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args; out << drv.platform << drv.builder << drv.args;

View file

@ -6,6 +6,7 @@
#include "content-address.hh" #include "content-address.hh"
#include <map> #include <map>
#include <variant>
namespace nix { namespace nix {
@ -13,20 +14,46 @@ namespace nix {
/* Abstract syntax of derivations. */ /* Abstract syntax of derivations. */
/* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed struct DerivationOutputInputAddressed
{ {
/* Will need to become `std::optional<StorePath>` once input-addressed
derivations are allowed to depend on cont-addressed derivations */
StorePath path; StorePath path;
}; };
struct DerivationOutputFixed /* Fixed-output derivations, whose output paths are content addressed
according to that fixed output. */
struct DerivationOutputCAFixed
{ {
FixedOutputHash hash; /* hash used for expected hash computation */ FixedOutputHash hash; /* hash used for expected hash computation */
}; };
/* Floating-output derivations, whose output paths are content addressed, but
not fixed, and so are dynamically calculated from whatever the output ends
up being. */
struct DerivationOutputCAFloating
{
/* information used for expected hash computation */
FileIngestionMethod method;
HashType hashType;
};
struct DerivationOutput struct DerivationOutput
{ {
std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output; std::variant<
StorePath path(const Store & store, std::string_view drvName) const; DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -37,6 +64,25 @@ typedef std::map<StorePath, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs; typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
CAFixed,
CAFloating,
};
/* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */
bool derivationIsCA(DerivationType);
/* Is the content of the outputs fixed a-priori via a hash? Never true for
non-CA derivations. */
bool derivationIsFixed(DerivationType);
/* Is the derivation impure and needs to access non-deterministic resources, or
pure and can be sandboxed? Note that whether or not we actually sandbox the
derivation is controlled separately. Never true for non-CA derivations. */
bool derivationIsImpure(DerivationType);
struct BasicDerivation struct BasicDerivation
{ {
DerivationOutputs outputs; /* keyed on symbolic IDs */ DerivationOutputs outputs; /* keyed on symbolic IDs */
@ -53,7 +99,7 @@ struct BasicDerivation
bool isBuiltin() const; bool isBuiltin() const;
/* Return true iff this is a fixed-output derivation. */ /* Return true iff this is a fixed-output derivation. */
bool isFixedOutput() const; DerivationType type() const;
/* Return the output paths of a derivation. */ /* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const; StorePathSet outputPaths(const Store & store) const;
@ -90,10 +136,42 @@ Derivation readDerivation(const Store & store, const Path & drvPath, std::string
// FIXME: remove // FIXME: remove
bool isDerivation(const string & fileName); bool isDerivation(const string & fileName);
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); // known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
typedef std::variant<
Hash, // regular DRV normalized hash
CaOutputHashes
> DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations
expunged.
A fixed-output derivation is a derivation whose outputs have a
specified content hash and hash algorithm. (Currently they must have
exactly one output (`out'), which is specified using the `outputHash'
and `outputHashAlgo' attributes, but the algorithm doesn't assume
this.) We don't want changes to such derivations to propagate upwards
through the dependency graph, changing output paths everywhere.
For instance, if we change the url in a call to the `fetchurl'
function, we do not want to rebuild everything depending on it---after
all, (the hash of) the file being downloaded is unchanged. So the
*output paths* should not change. On the other hand, the *derivation
paths* should change to reflect the new dependency graph.
For fixed-output derivations, this returns a map from the name of
each output to its hash, unique up to the output's contents.
For regular derivations, it returns a single hash of the derivation
ATerm, after subderivations have been likewise expunged from that
derivation.
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/* Memoisation of hashDerivationModulo(). */ /* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, Hash> DrvHashes; typedef std::map<StorePath, DrvHashModulo> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe extern DrvHashes drvHashes; // FIXME: global, not thread-safe

View file

@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from); auto s = readString(conn->from);
info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s}; info->narHash = s.empty() ? std::optional<Hash>{} : Hash::parseAnyPrefixed(s);
info->ca = parseContentAddressOpt(readString(conn->from)); info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from); info->sigs = readStrings<StringSet>(conn->from);
} }

View file

@ -544,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
std::string drvName(drvPath.name()); std::string drvName(drvPath.name());
drvName = string(drvName, 0, drvName.size() - drvExtension.size()); drvName = string(drvName, 0, drvName.size() - drvExtension.size());
auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
{ {
if (actual != expected)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
auto j = drv.env.find(varName); auto j = drv.env.find(varName);
if (j == drv.env.end() || parseStorePath(j->second) != actual) if (j == drv.env.end() || parseStorePath(j->second) != actual)
throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
@ -556,16 +553,34 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
}; };
if (drv.isFixedOutput()) { // Don't need the answer, but do this anyways to assert is proper
DerivationOutputs::const_iterator out = drv.outputs.find("out"); // combination. The code below is more general and naturally allows
if (out == drv.outputs.end()) // combinations that are currently prohibited.
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); drv.type();
}
else { std::optional<Hash> h;
Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) {
for (auto & i : drv.outputs) std::visit(overloaded {
check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first); [&](DerivationOutputInputAddressed doia) {
if (!h) {
// somewhat expensive so we do lazily
auto temp = hashDerivationModulo(*this, drv, true);
h = std::get<Hash>(temp);
}
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
[&](DerivationOutputCAFixed dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
envHasRightPath(path, i.first);
},
[&](DerivationOutputCAFloating _) {
throw UnimplementedError("floating CA output derivations are not yet implemented");
},
}, i.second.output);
} }
} }
@ -640,7 +655,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->id = useQueryPathInfo.getInt(0); info->id = useQueryPathInfo.getInt(0);
try { try {
info->narHash = Hash(useQueryPathInfo.getStr(1)); info->narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
} catch (BadHash & e) { } catch (BadHash & e) {
throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
} }

View file

@ -4,6 +4,7 @@
#include "local-store.hh" #include "local-store.hh"
#include "store-api.hh" #include "store-api.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "topo-sort.hh"
namespace nix { namespace nix {
@ -112,7 +113,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
{ {
auto out = drv.outputs.find("out"); auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) { if (out != drv.outputs.end()) {
if (auto v = std::get_if<DerivationOutputFixed>(&out->second.output)) if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output))
return v->hash; return v->hash;
} }
return std::nullopt; return std::nullopt;
@ -256,41 +257,21 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePaths Store::topoSortPaths(const StorePathSet & paths) StorePaths Store::topoSortPaths(const StorePathSet & paths)
{ {
StorePaths sorted; return topoSort(paths,
StorePathSet visited, parents; {[&](const StorePath & path) {
std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit;
dfsVisit = [&](const StorePath & path, const StorePath * parent) {
if (parents.count(path))
throw BuildError("cycle detected in the references of '%s' from '%s'",
printStorePath(path), printStorePath(*parent));
if (!visited.insert(path).second) return;
parents.insert(path);
StorePathSet references; StorePathSet references;
try { try {
references = queryPathInfo(path)->references; references = queryPathInfo(path)->references;
} catch (InvalidPath &) { } catch (InvalidPath &) {
} }
return references;
for (auto & i : references) }},
/* Don't traverse into paths that don't exist. That can {[&](const StorePath & path, const StorePath & parent) {
happen due to substitutes for non-existent paths. */ return BuildError(
if (i != path && paths.count(i)) "cycle detected in the references of '%s' from '%s'",
dfsVisit(i, &path); printStorePath(path),
printStorePath(parent));
sorted.push_back(path); }});
parents.erase(path);
};
for (auto & i : paths)
dfsVisit(i, nullptr);
std::reverse(sorted.begin(), sorted.end());
return sorted;
} }

View file

@ -193,9 +193,9 @@ public:
narInfo->url = queryNAR.getStr(2); narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3); narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4)) if (!queryNAR.isNull(4))
narInfo->fileHash = Hash(queryNAR.getStr(4)); narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5); narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narHash = Hash::parseAnyPrefixed(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7); narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(StorePath(r)); narInfo->references.insert(StorePath(r));

View file

@ -12,7 +12,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
auto parseHashField = [&](const string & s) { auto parseHashField = [&](const string & s) {
try { try {
return Hash(s); return Hash::parseAnyPrefixed(s);
} catch (BadHash &) { } catch (BadHash &) {
throw corrupt(); throw corrupt();
} }

View file

@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true); return getBoolAttr("allowSubstitutes", true);
} }
bool ParsedDerivation::contentAddressed() const
{
return getBoolAttr("__contentAddressed", false);
}
} }

View file

@ -34,8 +34,6 @@ public:
bool willBuildLocally() const; bool willBuildLocally() const;
bool substitutesAllowed() const; bool substitutesAllowed() const;
bool contentAddressed() const;
}; };
} }

111
src/libstore/path-info.hh Normal file
View file

@ -0,0 +1,111 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include "content-address.hh"
#include <string>
#include <optional>
namespace nix {
class Store;
struct SubstitutablePathInfo
{
std::optional<StorePath> deriver;
StorePathSet references;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
// TODO document this
std::optional<Hash> narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
/* Whether the path is ultimately trusted, that is, it's a
derivation output that was built locally. */
bool ultimate = false;
StringSet sigs; // note: not necessarily verified
/* If non-empty, an assertion that the path is content-addressed,
i.e., that the store path is computed from a cryptographic hash
of the contents of the path, plus some other bits of data like
the "name" part of the path. Such a path doesn't need
signatures, since we don't have to trust anybody's claim that
the path is the output of a particular derivation. (In the
extensional store model, we have to trust that the *contents*
of an output path of a derivation were actually produced by
that derivation. In the intensional model, we have to trust
that a particular output path was produced by a derivation; the
path then implies the contents.)
Ideally, the content-addressability assertion would just be a Boolean,
and the store path would be computed from the name component, narHash
and references. However, we support many types of content addresses.
*/
std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
}
/* Return a fingerprint of the store path to be used in binary
cache signatures. It contains the store path, the base-32
SHA-256 hash of the NAR serialisation of the path, the size of
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
std::string fingerprint(const Store & store) const;
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf() const;
void insertReferencePossiblyToSelf(StorePath && ref);
void setReferencesPossiblyToSelf(StorePathSet && refs);
static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path
is content-addressed. */
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
typedef list<ValidPathInfo> ValidPathInfos;
}

View file

@ -422,7 +422,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
info = std::make_shared<ValidPathInfo>(StorePath(path)); info = std::make_shared<ValidPathInfo>(StorePath(path));
auto deriver = readString(conn->from); auto deriver = readString(conn->from);
if (deriver != "") info->deriver = parseStorePath(deriver); if (deriver != "") info->deriver = parseStorePath(deriver);
info->narHash = Hash(readString(conn->from), htSHA256); info->narHash = Hash::parseAny(readString(conn->from), htSHA256);
info->references = readStorePaths<StorePathSet>(*this, conn->from); info->references = readStorePaths<StorePathSet>(*this, conn->from);
conn->from >> info->registrationTime >> info->narSize; conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {

View file

@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
const std::string & mimeType, const std::string & mimeType,
const std::string & contentEncoding) const std::string & contentEncoding)
{ {
istream->seekg(0, istream->end);
auto size = istream->tellg();
istream->seekg(0, istream->beg);
auto maxThreads = std::thread::hardware_concurrency(); auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -343,10 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count(); .count();
printInfo("uploaded 's3://%s/%s' in %d ms", printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, duration); bucketName, path, size, duration);
stats.putTimeMs += duration; stats.putTimeMs += duration;
stats.putBytes += std::max(size, (decltype(size)) 0);
stats.put++; stats.put++;
} }

View file

@ -19,6 +19,7 @@ public:
struct Stats struct Stats
{ {
std::atomic<uint64_t> put{0}; std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0}; std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0}; std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0}; std::atomic<uint64_t> getBytes{0};

View file

@ -193,10 +193,6 @@ StorePath Store::makeFixedOutputPath(
} }
} }
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const const StorePathSet & references, bool hasSelfReference) const
{ {
@ -876,7 +872,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
if (hashGiven) { if (hashGiven) {
string s; string s;
getline(str, s); getline(str, s);
info.narHash = Hash(s, htSHA256); info.narHash = Hash::parseAny(s, htSHA256);
getline(str, s); getline(str, s);
if (!string2Int(s, info.narSize)) throw Error("number expected"); if (!string2Int(s, info.narSize)) throw Error("number expected");
} }

View file

@ -10,6 +10,7 @@
#include "globals.hh" #include "globals.hh"
#include "config.hh" #include "config.hh"
#include "derivations.hh" #include "derivations.hh"
#include "path-info.hh"
#include <atomic> #include <atomic>
#include <limits> #include <limits>
@ -101,95 +102,6 @@ struct GCResults
}; };
struct SubstitutablePathInfo
{
std::optional<StorePath> deriver;
StorePathSet references;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
// TODO document this
std::optional<Hash> narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
/* Whether the path is ultimately trusted, that is, it's a
derivation output that was built locally. */
bool ultimate = false;
StringSet sigs; // note: not necessarily verified
/* If non-empty, an assertion that the path is content-addressed,
i.e., that the store path is computed from a cryptographic hash
of the contents of the path, plus some other bits of data like
the "name" part of the path. Such a path doesn't need
signatures, since we don't have to trust anybody's claim that
the path is the output of a particular derivation. (In the
extensional store model, we have to trust that the *contents*
of an output path of a derivation were actually produced by
that derivation. In the intensional model, we have to trust
that a particular output path was produced by a derivation; the
path then implies the contents.)
Ideally, the content-addressability assertion would just be a Boolean,
and the store path would be computed from the name component, narHash
and references. However, we support many types of content addresses.
*/
std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
}
/* Return a fingerprint of the store path to be used in binary
cache signatures. It contains the store path, the base-32
SHA-256 hash of the NAR serialisation of the path, the size of
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
std::string fingerprint(const Store & store) const;
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path
is content-addressed. */
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
typedef list<ValidPathInfo> ValidPathInfos;
enum BuildMode { bmNormal, bmRepair, bmCheck }; enum BuildMode { bmNormal, bmRepair, bmCheck };

View file

@ -192,6 +192,7 @@ public:
MakeError(Error, BaseError); MakeError(Error, BaseError);
MakeError(UsageError, Error); MakeError(UsageError, Error);
MakeError(UnimplementedError, Error);
class SysError : public Error class SysError : public Error
{ {

View file

@ -7,6 +7,7 @@
#include "args.hh" #include "args.hh"
#include "hash.hh" #include "hash.hh"
#include "archive.hh" #include "archive.hh"
#include "split.hh"
#include "util.hh" #include "util.hh"
#include <sys/types.h> #include <sys/types.h>
@ -15,6 +16,7 @@
namespace nix { namespace nix {
static size_t regularHashSize(HashType type) { static size_t regularHashSize(HashType type) {
switch (type) { switch (type) {
case htMD5: return md5HashSize; case htMD5: return md5HashSize;
@ -25,10 +27,11 @@ static size_t regularHashSize(HashType type) {
abort(); abort();
} }
std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" }; std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
void Hash::init() Hash::Hash(HashType type) : type(type)
{ {
hashSize = regularHashSize(type); hashSize = regularHashSize(type);
assert(hashSize <= maxHashSize); assert(hashSize <= maxHashSize);
@ -133,57 +136,89 @@ std::string Hash::to_string(Base base, bool includeType) const
return s; return s;
} }
Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash Hash::parseSRI(std::string_view original) {
Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { }
Hash::Hash(std::string_view original, std::optional<HashType> optType)
{
auto rest = original; auto rest = original;
size_t pos = 0; // Parse the has type before the separater, if there was one.
auto hashRaw = splitPrefixTo(rest, '-');
if (!hashRaw)
throw BadHash("hash '%s' is not SRI", original);
HashType parsedType = parseHashType(*hashRaw);
return Hash(rest, parsedType, true);
}
// Mutates the string to eliminate the prefixes when found
static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest) {
bool isSRI = false; bool isSRI = false;
// Parse the has type before the separater, if there was one. // Parse the has type before the separater, if there was one.
std::optional<HashType> optParsedType; std::optional<HashType> optParsedType;
{ {
auto sep = rest.find(':'); auto hashRaw = splitPrefixTo(rest, ':');
if (sep == std::string_view::npos) {
sep = rest.find('-'); if (!hashRaw) {
if (sep != std::string_view::npos) hashRaw = splitPrefixTo(rest, '-');
if (hashRaw)
isSRI = true; isSRI = true;
} }
if (sep != std::string_view::npos) { if (hashRaw)
auto hashRaw = rest.substr(0, sep); optParsedType = parseHashType(*hashRaw);
optParsedType = parseHashType(hashRaw);
rest = rest.substr(sep + 1);
}
} }
return {optParsedType, isSRI};
}
Hash Hash::parseAnyPrefixed(std::string_view original)
{
auto rest = original;
auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
// Either the string or user must provide the type, if they both do they // Either the string or user must provide the type, if they both do they
// must agree. // must agree.
if (!optParsedType && !optType) { if (!optParsedType)
throw BadHash("hash '%s' does not include a type", rest);
return Hash(rest, *optParsedType, isSRI);
}
Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType)
{
auto rest = original;
auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
// Either the string or user must provide the type, if they both do they
// must agree.
if (!optParsedType && !optType)
throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
} else { else if (optParsedType && optType && *optParsedType != *optType)
this->type = optParsedType ? *optParsedType : *optType;
if (optParsedType && optType && *optParsedType != *optType)
throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
}
init(); HashType hashType = optParsedType ? *optParsedType : *optType;
return Hash(rest, hashType, isSRI);
}
Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type)
{
return Hash(s, type, false);
}
Hash::Hash(std::string_view rest, HashType type, bool isSRI)
: Hash(type)
{
if (!isSRI && rest.size() == base16Len()) { if (!isSRI && rest.size() == base16Len()) {
auto parseHexDigit = [&](char c) { auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9') return c - '0'; if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10; if (c >= 'A' && c <= 'F') return c - 'A' + 10;
if (c >= 'a' && c <= 'f') return c - 'a' + 10; if (c >= 'a' && c <= 'f') return c - 'a' + 10;
throw BadHash("invalid base-16 hash '%s'", original); throw BadHash("invalid base-16 hash '%s'", rest);
}; };
for (unsigned int i = 0; i < hashSize; i++) { for (unsigned int i = 0; i < hashSize; i++) {
hash[i] = hash[i] =
parseHexDigit(rest[pos + i * 2]) << 4 parseHexDigit(rest[i * 2]) << 4
| parseHexDigit(rest[pos + i * 2 + 1]); | parseHexDigit(rest[i * 2 + 1]);
} }
} }
@ -195,7 +230,7 @@ Hash::Hash(std::string_view original, std::optional<HashType> optType)
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break; if (base32Chars[digit] == c) break;
if (digit >= 32) if (digit >= 32)
throw BadHash("invalid base-32 hash '%s'", original); throw BadHash("invalid base-32 hash '%s'", rest);
unsigned int b = n * 5; unsigned int b = n * 5;
unsigned int i = b / 8; unsigned int i = b / 8;
unsigned int j = b % 8; unsigned int j = b % 8;
@ -205,7 +240,7 @@ Hash::Hash(std::string_view original, std::optional<HashType> optType)
hash[i + 1] |= digit >> (8 - j); hash[i + 1] |= digit >> (8 - j);
} else { } else {
if (digit >> (8 - j)) if (digit >> (8 - j))
throw BadHash("invalid base-32 hash '%s'", original); throw BadHash("invalid base-32 hash '%s'", rest);
} }
} }
} }
@ -213,7 +248,7 @@ Hash::Hash(std::string_view original, std::optional<HashType> optType)
else if (isSRI || rest.size() == base64Len()) { else if (isSRI || rest.size() == base64Len()) {
auto d = base64Decode(rest); auto d = base64Decode(rest);
if (d.size() != hashSize) if (d.size() != hashSize)
throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original); throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", rest);
assert(hashSize); assert(hashSize);
memcpy(hash, d.data(), hashSize); memcpy(hash, d.data(), hashSize);
} }
@ -231,7 +266,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
return h; return h;
} else } else
return Hash(hashStr, ht); return Hash::parseAny(hashStr, ht);
} }

View file

@ -34,21 +34,31 @@ struct Hash
HashType type; HashType type;
/* Create a zero-filled hash object. */ /* Create a zero-filled hash object. */
Hash(HashType type) : type(type) { init(); }; Hash(HashType type);
/* Initialize the hash from a string representation, in the format /* Parse the hash from a string representation in the format
"[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
Subresource Integrity hash expression). If the 'type' argument Subresource Integrity hash expression). If the 'type' argument
is not present, then the hash type must be specified in the is not present, then the hash type must be specified in the
string. */ string. */
Hash(std::string_view s, std::optional<HashType> type); static Hash parseAny(std::string_view s, std::optional<HashType> type);
// type must be provided
Hash(std::string_view s, HashType type);
// hash type must be part of string
Hash(std::string_view s);
void init(); /* Parse a hash from a string representation like the above, except the
type prefix is mandatory is there is no separate arguement. */
static Hash parseAnyPrefixed(std::string_view s);
/* Parse a plain hash that musst not have any prefix indicating the type.
The type is passed in to disambiguate. */
static Hash parseNonSRIUnprefixed(std::string_view s, HashType type);
static Hash parseSRI(std::string_view original);
private:
/* The type must be provided, the string view must not include <type>
prefix. `isSRI` helps disambigate the various base-* encodings. */
Hash(std::string_view s, HashType type, bool isSRI);
public:
/* Check whether a hash is set. */ /* Check whether a hash is set. */
operator bool () const { return (bool) type; } operator bool () const { return (bool) type; }

33
src/libutil/split.hh Normal file
View file

@ -0,0 +1,33 @@
#pragma once
#include <optional>
#include <string_view>
#include "util.hh"
namespace nix {
// If `separator` is found, we return the portion of the string before the
// separator, and modify the string argument to contain only the part after the
// separator. Otherwise, wer return `std::nullopt`, and we leave the argument
// string alone.
static inline std::optional<std::string_view> splitPrefixTo(std::string_view & string, char separator) {
auto sepInstance = string.find(separator);
if (sepInstance != std::string_view::npos) {
auto prefix = string.substr(0, sepInstance);
string.remove_prefix(sepInstance+1);
return prefix;
}
return std::nullopt;
}
static inline bool splitPrefix(std::string_view & string, std::string_view prefix) {
bool res = hasPrefix(string, prefix);
if (res)
string.remove_prefix(prefix.length());
return res;
}
}

40
src/libutil/topo-sort.hh Normal file
View file

@ -0,0 +1,40 @@
#include "error.hh"
namespace nix {
template<typename T>
std::vector<T> topoSort(std::set<T> items,
std::function<std::set<T>(const T &)> getChildren,
std::function<Error(const T &, const T &)> makeCycleError)
{
std::vector<T> sorted;
std::set<T> visited, parents;
std::function<void(const T & path, const T * parent)> dfsVisit;
dfsVisit = [&](const T & path, const T * parent) {
if (parents.count(path)) throw makeCycleError(path, *parent);
if (!visited.insert(path).second) return;
parents.insert(path);
std::set<T> references = getChildren(path);
for (auto & i : references)
/* Don't traverse into items that don't exist in our starting set. */
if (i != path && items.count(i))
dfsVisit(i, &path);
sorted.push_back(path);
parents.erase(path);
};
for (auto & i : items)
dfsVisit(i, nullptr);
std::reverse(sorted.begin(), sorted.end());
return sorted;
}
}

View file

@ -1450,7 +1450,7 @@ string base64Decode(std::string_view s)
char digit = decode[(unsigned char) c]; char digit = decode[(unsigned char) c];
if (digit == -1) if (digit == -1)
throw Error("invalid character in Base64 string"); throw Error("invalid character in Base64 string: '%c'", c);
bits += 6; bits += 6;
d = d << 6 | digit; d = d << 6 | digit;

View file

@ -601,4 +601,9 @@ constexpr auto enumerate(T && iterable)
} }
// C++17 std::visit boilerplate
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
} }

View file

@ -157,7 +157,7 @@ static int _main(int argc, char * * argv)
Hash hash(ht); Hash hash(ht);
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (args.size() == 2) { if (args.size() == 2) {
expectedHash = Hash(args[1], ht); expectedHash = Hash::parseAny(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))

View file

@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
string hash = *i++; string hash = *i++;
string name = *i++; string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name)));
} }
@ -948,7 +948,7 @@ static void opServe(Strings opFlags, Strings opArgs)
auto deriver = readString(in); auto deriver = readString(in);
if (deriver != "") if (deriver != "")
info.deriver = store->parseStorePath(deriver); info.deriver = store->parseStorePath(deriver);
info.narHash = Hash(readString(in), htSHA256); info.narHash = Hash::parseAny(readString(in), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, in); info.references = readStorePaths<StorePathSet>(*store, in);
in >> info.registrationTime >> info.narSize >> info.ultimate; in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in); info.sigs = readStrings<StringSet>(in);

View file

@ -64,17 +64,24 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
if (dryRun) return; if (dryRun) return;
if (outLink != "") { if (outLink != "")
for (size_t i = 0; i < buildables.size(); ++i) { if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
for (auto & output : buildables[i].outputs) for (size_t i = 0; i < buildables.size(); ++i)
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { std::visit(overloaded {
[&](BuildableOpaque bo) {
std::string symlink = outLink;
if (i) symlink += fmt("-%d", i);
store2->addPermRoot(bo.path, absPath(symlink), true);
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) {
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first); if (output.first != "out") symlink += fmt("-%s", output.first);
store2->addPermRoot(output.second, absPath(symlink), true); store2->addPermRoot(output.second, absPath(symlink), true);
} }
} },
} }, buildables[i]);
updateProfile(buildables); updateProfile(buildables);
} }

View file

@ -128,20 +128,25 @@ void MixProfile::updateProfile(const Buildables & buildables)
{ {
if (!profile) return; if (!profile) return;
std::optional<StorePath> result; std::vector<StorePath> result;
for (auto & buildable : buildables) { for (auto & buildable : buildables) {
for (auto & output : buildable.outputs) { std::visit(overloaded {
if (result) [&](BuildableOpaque bo) {
throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); result.push_back(bo.path);
result = output.second; },
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) {
result.push_back(output.second);
} }
},
}, buildable);
} }
if (!result) if (result.size() != 1)
throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
updateProfile(*result); updateProfile(result[0]);
} }
MixDefaultProfile::MixDefaultProfile() MixDefaultProfile::MixDefaultProfile()

View file

@ -135,7 +135,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drv.env["_outputs_saved"] = drv.env["outputs"]; drv.env["_outputs_saved"] = drv.env["outputs"];
drv.env["outputs"] = "out"; drv.env["outputs"] = "out";
drv.inputSrcs.insert(std::move(getEnvShPath)); drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true); Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
auto shellOutPath = store->makeOutputPath("out", h, drvName); auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
.path = shellOutPath .path = shellOutPath

View file

@ -107,7 +107,7 @@ struct CmdToBase : Command
void run() override void run() override
{ {
for (auto s : args) for (auto s : args)
logger->stdout(Hash(s, ht).to_string(base, base == SRI)); logger->stdout(Hash::parseAny(s, ht).to_string(base, base == SRI));
} }
}; };

View file

@ -308,16 +308,15 @@ struct InstallableStorePath : Installable
for (auto & [name, output] : drv.outputs) for (auto & [name, output] : drv.outputs)
outputs.emplace(name, output.path(*store, drv.name)); outputs.emplace(name, output.path(*store, drv.name));
return { return {
Buildable { BuildableFromDrv {
.drvPath = storePath, .drvPath = storePath,
.outputs = std::move(outputs) .outputs = std::move(outputs)
} }
}; };
} else { } else {
return { return {
Buildable { BuildableOpaque {
.drvPath = {}, .path = storePath,
.outputs = {{"out", storePath}}
} }
}; };
} }
@ -333,32 +332,19 @@ Buildables InstallableValue::toBuildables()
{ {
Buildables res; Buildables res;
StorePathSet drvPaths; std::map<StorePath, OutputPathMap> drvsToOutputs;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) { for (auto & drv : toDerivations()) {
Buildable b{.drvPath = drv.drvPath};
drvPaths.insert(drv.drvPath);
auto outputName = drv.outputName; auto outputName = drv.outputName;
if (outputName == "") if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
b.outputs.emplace(outputName, drv.outPath);
res.push_back(std::move(b));
} }
// Hack to recognize .all: if all drvs have the same drvPath, for (auto & i : drvsToOutputs)
// merge the buildables. res.push_back(BuildableFromDrv { i.first, i.second });
if (drvPaths.size() == 1) {
Buildable b{.drvPath = *drvPaths.begin()};
for (auto & b2 : res)
for (auto & output : b2.outputs)
b.outputs.insert_or_assign(output.first, output.second);
Buildables bs;
bs.push_back(std::move(b));
return bs;
} else
return res; return res;
} }
@ -656,14 +642,17 @@ Buildables build(ref<Store> store, Realise mode,
for (auto & i : installables) { for (auto & i : installables) {
for (auto & b : i->toBuildables()) { for (auto & b : i->toBuildables()) {
if (b.drvPath) { std::visit(overloaded {
[&](BuildableOpaque bo) {
pathsToBuild.push_back({bo.path});
},
[&](BuildableFromDrv bfd) {
StringSet outputNames; StringSet outputNames;
for (auto & output : b.outputs) for (auto & output : bfd.outputs)
outputNames.insert(output.first); outputNames.insert(output.first);
pathsToBuild.push_back({*b.drvPath, outputNames}); pathsToBuild.push_back({bfd.drvPath, outputNames});
} else },
for (auto & output : b.outputs) }, b);
pathsToBuild.push_back({output.second});
buildables.push_back(std::move(b)); buildables.push_back(std::move(b));
} }
} }
@ -684,16 +673,23 @@ StorePathSet toStorePaths(ref<Store> store,
if (operateOn == OperateOn::Output) { if (operateOn == OperateOn::Output) {
for (auto & b : build(store, mode, installables)) for (auto & b : build(store, mode, installables))
for (auto & output : b.outputs) std::visit(overloaded {
[&](BuildableOpaque bo) {
outPaths.insert(bo.path);
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs)
outPaths.insert(output.second); outPaths.insert(output.second);
},
}, b);
} else { } else {
if (mode == Realise::Nothing) if (mode == Realise::Nothing)
settings.readOnlyMode = true; settings.readOnlyMode = true;
for (auto & i : installables) for (auto & i : installables)
for (auto & b : i->toBuildables()) for (auto & b : i->toBuildables())
if (b.drvPath) if (auto bfd = std::get_if<BuildableFromDrv>(&b))
outPaths.insert(*b.drvPath); outPaths.insert(bfd->drvPath);
} }
return outPaths; return outPaths;
@ -717,20 +713,21 @@ StorePathSet toDerivations(ref<Store> store,
StorePathSet drvPaths; StorePathSet drvPaths;
for (auto & i : installables) for (auto & i : installables)
for (auto & b : i->toBuildables()) { for (auto & b : i->toBuildables())
if (!b.drvPath) { std::visit(overloaded {
[&](BuildableOpaque bo) {
if (!useDeriver) if (!useDeriver)
throw Error("argument '%s' did not evaluate to a derivation", i->what()); throw Error("argument '%s' did not evaluate to a derivation", i->what());
for (auto & output : b.outputs) { auto derivers = store->queryValidDerivers(bo.path);
auto derivers = store->queryValidDerivers(output.second);
if (derivers.empty()) if (derivers.empty())
throw Error("'%s' does not have a known deriver", i->what()); throw Error("'%s' does not have a known deriver", i->what());
// FIXME: use all derivers? // FIXME: use all derivers?
drvPaths.insert(*derivers.begin()); drvPaths.insert(*derivers.begin());
} },
} else [&](BuildableFromDrv bfd) {
drvPaths.insert(*b.drvPath); drvPaths.insert(bfd.drvPath);
} },
}, b);
return drvPaths; return drvPaths;
} }

View file

@ -14,12 +14,20 @@ struct SourceExprCommand;
namespace eval_cache { class EvalCache; class AttrCursor; } namespace eval_cache { class EvalCache; class AttrCursor; }
struct Buildable struct BuildableOpaque {
{ StorePath path;
std::optional<StorePath> drvPath; };
struct BuildableFromDrv {
StorePath drvPath;
std::map<std::string, StorePath> outputs; std::map<std::string, StorePath> outputs;
}; };
typedef std::variant<
BuildableOpaque,
BuildableFromDrv
> Buildable;
typedef std::vector<Buildable> Buildables; typedef std::vector<Buildable> Buildables;
struct App struct App

View file

@ -45,11 +45,14 @@ struct CmdLog : InstallableCommand
RunPager pager; RunPager pager;
for (auto & sub : subs) { for (auto & sub : subs) {
auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; auto log = std::visit(overloaded {
for (auto & output : b.outputs) { [&](BuildableOpaque bo) {
if (log) break; return sub->getBuildLog(bo.path);
log = sub->getBuildLog(output.second); },
} [&](BuildableFromDrv bfd) {
return sub->getBuildLog(bfd.drvPath);
},
}, b);
if (!log) continue; if (!log) continue;
stopProgressBar(); stopProgressBar();
printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());

View file

@ -33,12 +33,17 @@ extern "C" {
#include "command.hh" #include "command.hh"
#include "finally.hh" #include "finally.hh"
#if HAVE_BOEHMGC
#define GC_INCLUDE_NEW #define GC_INCLUDE_NEW
#include <gc/gc_cpp.h> #include <gc/gc_cpp.h>
#endif
namespace nix { namespace nix {
struct NixRepl : gc struct NixRepl
#if HAVE_BOEHMGC
: gc
#endif
{ {
string curDir; string curDir;
std::unique_ptr<EvalState> state; std::unique_ptr<EvalState> state;

View file

@ -70,10 +70,18 @@ struct CmdShowDerivation : InstallablesCommand
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first)); auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name))); outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
if (auto hash = std::get_if<DerivationOutputFixed>(&output.second.output)) {
outputObj.attr("hashAlgo", hash->hash.printMethodAlgo()); std::visit(overloaded {
outputObj.attr("hash", hash->hash.hash.to_string(Base16, false)); [&](DerivationOutputInputAddressed doi) {
} },
[&](DerivationOutputCAFixed dof) {
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
}, output.second.output);
} }
} }

View file

@ -218,7 +218,9 @@ outPath=$(nix-build --no-out-link -E '
nix copy --to file://$cacheDir?write-nar-listing=1 $outPath nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]] diff -u \
<(jq -S < $cacheDir/$(basename $outPath).ls) \
<(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S)
# Test debug info index generation. # Test debug info index generation.
@ -234,4 +236,6 @@ outPath=$(nix-build --no-out-link -E '
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]] diff -u \
<(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \
<(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S)

View file

@ -26,12 +26,24 @@ nix cat-store $storePath/foo/baz > baz.cat-nar
diff -u baz.cat-nar $storePath/foo/baz diff -u baz.cat-nar $storePath/foo/baz
# Test --json. # Test --json.
[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] diff -u \
[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]] <(nix ls-nar --json $narFile / | jq -S) \
[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]] <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] diff -u \
[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]] <(nix ls-nar --json -R $narFile /foo | jq -S) \
[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]] <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S)
diff -u \
<(nix ls-nar --json -R $narFile /foo/bar | jq -S) \
<(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S)
diff -u \
<(nix ls-store --json $storePath | jq -S) \
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
diff -u \
<(nix ls-store --json -R $storePath/foo | jq -S) \
<(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S)
diff -u \
<(nix ls-store --json -R $storePath/foo/bar| jq -S) \
<(echo '{"type":"regular","size":0}' | jq -S)
# Test missing files. # Test missing files.
nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'