Merge pull request #3455 from Ericson2314/enum-FileIngestionMethod

Replace some `bool recursive` with a new `FileIngestionMethod` enum
This commit is contained in:
Eelco Dolstra 2020-05-28 13:50:06 +02:00 committed by GitHub
commit de141fcb79
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 152 additions and 111 deletions

View file

@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
SV * addToStore(char * srcPath, int recursive, char * algo) SV * addToStore(char * srcPath, int recursive, char * algo)
PPCODE: PPCODE:
try { try {
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE: PPCODE:
try { try {
Hash h(hash, parseHashType(algo)); Hash h(hash, parseHashType(algo));
auto path = store()->makeFixedOutputPath(recursive, h, name); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -1669,7 +1669,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
else { else {
auto p = settings.readOnlyMode auto p = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p); dstPath = store->printStorePath(p);
srcToStore.insert_or_assign(path, std::move(p)); srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);

View file

@ -562,7 +562,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
std::optional<std::string> outputHash; std::optional<std::string> outputHash;
std::string outputHashAlgo; std::string outputHashAlgo;
bool outputHashRecursive = false; auto ingestionMethod = FileIngestionMethod::Flat;
StringSet outputs; StringSet outputs;
outputs.insert("out"); outputs.insert("out");
@ -573,8 +573,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
vomit("processing attribute '%1%'", key); vomit("processing attribute '%1%'", key);
auto handleHashMode = [&](const std::string & s) { auto handleHashMode = [&](const std::string & s) {
if (s == "recursive") outputHashRecursive = true; if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") outputHashRecursive = false; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
}; };
@ -721,11 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
Hash h(*outputHash, ht); Hash h(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath), drv.outputs.insert_or_assign("out", DerivationOutput {
(outputHashRecursive ? "r:" : "") + printHashType(h.type), std::move(outPath),
h.to_string(Base16, false))); (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
+ printHashType(h.type),
h.to_string(Base16, false),
});
} }
else { else {
@ -1039,7 +1042,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
{ {
const auto path = evalSettings.pureEval && expectedHash ? const auto path = evalSettings.pureEval && expectedHash ?
path_ : path_ :
@ -1070,12 +1073,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
std::optional<StorePath> expectedStorePath; std::optional<StorePath> expectedStorePath;
if (expectedHash) if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name); expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
Path dstPath; Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode dstPath = state.store->printStorePath(settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair)); : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
} else } else
@ -1096,7 +1099,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
if (args[0]->type != tLambda) if (args[0]->type != tLambda)
throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
} }
static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@ -1105,7 +1108,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
Path path; Path path;
string name; string name;
Value * filterFun = nullptr; Value * filterFun = nullptr;
auto recursive = true; auto method = FileIngestionMethod::Recursive;
Hash expectedHash; Hash expectedHash;
for (auto & attr : *args[0]->attrs) { for (auto & attr : *args[0]->attrs) {
@ -1121,7 +1124,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
state.forceValue(*attr.value, pos); state.forceValue(*attr.value, pos);
filterFun = attr.value; filterFun = attr.value;
} else if (n == "recursive") } else if (n == "recursive")
recursive = state.forceBool(*attr.value, *attr.pos); method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
else if (n == "sha256") else if (n == "sha256")
expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else else
@ -1132,7 +1135,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
if (name.empty()) if (name.empty())
name = baseNameOf(path); name = baseNameOf(path);
addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); addPath(state, pos, name, path, filterFun, method, expectedHash, v);
} }

View file

@ -195,7 +195,7 @@ struct GitInput : Input
return files.count(file); return files.count(file);
}; };
auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto tree = Tree { auto tree = Tree {
.actualPath = store->printStorePath(storePath), .actualPath = store->printStorePath(storePath),
@ -347,7 +347,7 @@ struct GitInput : Input
unpackTarfile(*source, tmpDir); unpackTarfile(*source, tmpDir);
} }
auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));

View file

@ -114,7 +114,7 @@ struct MercurialInput : Input
return files.count(file); return files.count(file);
}; };
auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {Tree { return {Tree {
.actualPath = store->printStorePath(storePath), .actualPath = store->printStorePath(storePath),

View file

@ -67,10 +67,10 @@ DownloadFileResult downloadFile(
StringSink sink; StringSink sink;
dumpString(*res.data, sink); dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data); auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(htSHA256, *sink.s); info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(false, hash); info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs); store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }
@ -141,7 +141,7 @@ Tree downloadTarball(
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name; auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime; lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
} }
Attrs infoAttrs({ Attrs infoAttrs({

View file

@ -8,7 +8,7 @@ namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const StorePath TreeInfo::computeStorePath(Store & store) const
{ {
assert(narHash); assert(narHash);
return store.makeFixedOutputPath(true, narHash, "source"); return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
} }
} }

View file

@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
} }
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{ {
// FIXME: some cut&paste from LocalStore::addToStore(). // FIXME: some cut&paste from LocalStore::addToStore().
@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
small files. */ small files. */
StringSink sink; StringSink sink;
Hash h; Hash h;
if (recursive) { if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter); dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s); h = hashString(hashAlgo, *sink.s);
} else { } else {
@ -345,7 +345,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s); h = hashString(hashAlgo, s);
} }
ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); ValidPathInfo info(makeFixedOutputPath(method, h, name));
addToStore(info, sink.s, repair, CheckSigs, nullptr); addToStore(info, sink.s, repair, CheckSigs, nullptr);

View file

@ -79,7 +79,7 @@ public:
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override; PathFilter & filter, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,

View file

@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("queryPathFromHashPart"); } { throw Error("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
{ throw Error("addToStore"); } { throw Error("addToStore"); }
@ -2747,9 +2747,9 @@ struct RestrictedStore : public LocalFSStore
} }
StorePath addToStoreFromDump(const string & dump, const string & name, StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
{ {
auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
goal.addDependency(path); goal.addDependency(path);
return path; return path;
} }
@ -3692,10 +3692,10 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) { if (fixedOutput) {
bool recursive; Hash h; FileIngestionMethod outputHashMode; Hash h;
i.second.parseHashInfo(recursive, h); i.second.parseHashInfo(outputHashMode, h);
if (!recursive) { if (outputHashMode == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
@ -3705,9 +3705,11 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by /* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */ the derivation to its content-addressed location. */
Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); Hash h2 = outputHashMode == FileIngestionMethod::Recursive
? hashPath(h.type, actualPath).first
: hashFile(h.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
if (h != h2) { if (h != h2) {
@ -3736,7 +3738,7 @@ void DerivationGoal::registerOutputs()
else else
assert(worker.store.parseStorePath(path) == dest); assert(worker.store.parseStorePath(path) == dest);
ca = makeFixedOutputCA(recursive, h2); ca = makeFixedOutputCA(outputHashMode, h2);
} }
/* Get rid of all weird permissions. This also checks that /* Get rid of all weird permissions. This also checks that
@ -3957,7 +3959,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);
auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); auto used = recursive
? cloneStorePathSet(getClosure(info.path).first)
: cloneStorePathSet(info.references);
if (recursive && checks.ignoreSelfRefs) if (recursive && checks.ignoreSelfRefs)
used.erase(info.path); used.erase(info.path);

View file

@ -355,20 +355,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case wopAddToStore: { case wopAddToStore: {
bool fixed, recursive;
std::string s, baseName; std::string s, baseName;
from >> baseName >> fixed /* obsolete */ >> recursive >> s; FileIngestionMethod method;
/* Compatibility hack. */ {
if (!fixed) { bool fixed, recursive;
s = "sha256"; from >> baseName >> fixed /* obsolete */ >> recursive >> s;
recursive = true; method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
method = FileIngestionMethod::Recursive;
}
} }
HashType hashAlgo = parseHashType(s); HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from); TeeSource savedNAR(from);
RetrieveRegularNARSink savedRegular; RetrieveRegularNARSink savedRegular;
if (recursive) { if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to /* Get the entire NAR dump from the client and save it to
a string so that we can pass it to a string so that we can pass it to
addToStoreFromDump(). */ addToStoreFromDump(). */
@ -380,7 +384,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
baseName,
method,
hashAlgo);
logger->stopWork(); logger->stopWork();
to << store->printStorePath(path); to << store->printStorePath(path);

View file

@ -9,13 +9,13 @@
namespace nix { namespace nix {
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{ {
recursive = false; recursive = FileIngestionMethod::Flat;
string algo = hashAlgo; string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") { if (string(algo, 0, 2) == "r:") {
recursive = true; recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2); algo = string(algo, 2);
} }

View file

@ -22,7 +22,7 @@ struct DerivationOutput
, hashAlgo(std::move(hashAlgo)) , hashAlgo(std::move(hashAlgo))
, hash(std::move(hash)) , hash(std::move(hash))
{ } { }
void parseHashInfo(bool & recursive, Hash & hash) const; void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;

View file

@ -195,7 +195,7 @@ struct LegacySSHStore : public Store
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); } { unsupported("addToStore"); }

View file

@ -557,10 +557,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end()) if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
bool recursive; Hash h; FileIngestionMethod method; Hash h;
out->second.parseHashInfo(recursive, h); out->second.parseHashInfo(method, h);
check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
} }
else { else {
@ -1043,11 +1043,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
bool recursive, HashType hashAlgo, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{ {
Hash h = hashString(hashAlgo, dump); Hash h = hashString(hashAlgo, dump);
auto dstPath = makeFixedOutputPath(recursive, h, name); auto dstPath = makeFixedOutputPath(method, h, name);
addTempRoot(dstPath); addTempRoot(dstPath);
@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
autoGC(); autoGC();
if (recursive) { if (method == FileIngestionMethod::Recursive) {
StringSource source(dump); StringSource source(dump);
restorePath(realPath, source); restorePath(realPath, source);
} else } else
@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
above (if called with recursive == true and hashAlgo == above (if called with recursive == true and hashAlgo ==
sha256); otherwise, compute it here. */ sha256); otherwise, compute it here. */
HashResult hash; HashResult hash;
if (recursive) { if (method == FileIngestionMethod::Recursive) {
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
hash.second = dump.size(); hash.second = dump.size();
} else } else
@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
ValidPathInfo info(dstPath.clone()); ValidPathInfo info(dstPath.clone());
info.narHash = hash.first; info.narHash = hash.first;
info.narSize = hash.second; info.narSize = hash.second;
info.ca = makeFixedOutputCA(recursive, h); info.ca = makeFixedOutputCA(method, h);
registerValidPath(info); registerValidPath(info);
} }
@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{ {
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
@ -1111,12 +1111,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
method for very large paths, but `copyPath' is mainly used for method for very large paths, but `copyPath' is mainly used for
small files. */ small files. */
StringSink sink; StringSink sink;
if (recursive) if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter); dumpPath(srcPath, sink, filter);
else else
sink.s = make_ref<std::string>(readFile(srcPath)); sink.s = make_ref<std::string>(readFile(srcPath));
return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
} }

View file

@ -149,7 +149,7 @@ public:
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override; PathFilter & filter, RepairFlag repair) override;
/* Like addToStore(), but the contents of the path are contained /* Like addToStore(), but the contents of the path are contained
@ -157,7 +157,7 @@ public:
true) or simply the contents of a regular file (if recursive == true) or simply the contents of a regular file (if recursive ==
false). */ false). */
StorePath addToStoreFromDump(const string & dump, const string & name, StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override; const StorePathSet & references, RepairFlag repair) override;

View file

@ -73,6 +73,11 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";
enum struct FileIngestionMethod : uint8_t {
Flat = false,
Recursive = true
};
struct StorePathWithOutputs struct StorePathWithOutputs
{ {
StorePath path; StorePath path;

View file

@ -484,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{ {
if (repair) throw Error("repairing is not supported when building through the Nix daemon"); if (repair) throw Error("repairing is not supported when building through the Nix daemon");
@ -492,10 +492,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
conn->to << wopAddToStore << name conn->to
<< ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ << wopAddToStore
<< (recursive ? 1 : 0) << name
<< printHashType(hashAlgo); << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
<< (method == FileIngestionMethod::Recursive ? 1 : 0)
<< printHashType(hashAlgo);
try { try {
conn->to.written = 0; conn->to.written = 0;

View file

@ -65,7 +65,7 @@ public:
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,

View file

@ -172,19 +172,22 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath( StorePath Store::makeFixedOutputPath(
bool recursive, FileIngestionMethod recursive,
const Hash & hash, const Hash & hash,
std::string_view name, std::string_view name,
const StorePathSet & references, const StorePathSet & references,
bool hasSelfReference) const bool hasSelfReference) const
{ {
if (hash.type == htSHA256 && recursive) { if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else { } else {
assert(references.empty()); assert(references.empty());
return makeStorePath("output:out", hashString(htSHA256, return makeStorePath("output:out",
"fixed:out:" + (recursive ? (string) "r:" : "") + hashString(htSHA256,
hash.to_string(Base16) + ":"), name); "fixed:out:"
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+ hash.to_string(Base16) + ":"),
name);
} }
} }
@ -201,10 +204,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
{ {
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); Hash h = method == FileIngestionMethod::Recursive
return std::make_pair(makeFixedOutputPath(recursive, h, name), h); ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h);
} }
@ -782,8 +787,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
} }
else if (hasPrefix(ca, "fixed:")) { else if (hasPrefix(ca, "fixed:")) {
bool recursive = ca.compare(6, 2, "r:") == 0; FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
Hash hash(std::string(ca, recursive ? 8 : 6)); Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6));
auto refs = cloneStorePathSet(references); auto refs = cloneStorePathSet(references);
bool hasSelfReference = false; bool hasSelfReference = false;
if (refs.count(path)) { if (refs.count(path)) {
@ -827,9 +832,11 @@ Strings ValidPathInfo::shortRefs() const
} }
std::string makeFixedOutputCA(bool recursive, const Hash & hash) std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
{ {
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); return "fixed:"
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+ hash.to_string();
} }

View file

@ -44,7 +44,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
/* Magic header of exportPath() output (obsolete). */ /* Magic header of exportPath() output (obsolete). */
const uint32_t exportMagic = 0x4558494e; const uint32_t exportMagic = 0x4558494e;
@ -347,7 +346,7 @@ public:
StorePath makeOutputPath(const string & id, StorePath makeOutputPath(const string & id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(bool recursive, StorePath makeFixedOutputPath(FileIngestionMethod method,
const Hash & hash, std::string_view name, const Hash & hash, std::string_view name,
const StorePathSet & references = {}, const StorePathSet & references = {},
bool hasSelfReference = false) const; bool hasSelfReference = false) const;
@ -359,7 +358,7 @@ public:
store path to which srcPath is to be copied. Returns the store store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */ path and the cryptographic hash of the contents of srcPath. */
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive = true, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
/* Preparatory part of addTextToStore(). /* Preparatory part of addTextToStore().
@ -463,12 +462,12 @@ public:
The function object `filter' can be used to exclude files (see The function object `filter' can be used to exclude files (see
libutil/archive.hh). */ libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath, virtual StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
// FIXME: remove? // FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name, virtual StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{ {
throw Error("addToStoreFromDump() is not supported by this store"); throw Error("addToStoreFromDump() is not supported by this store");
} }
@ -851,7 +850,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
/* Compute the content-addressability assertion (ValidPathInfo::ca) /* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */ for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash & hash); std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
/* Split URI into protocol+hierarchy part and its parameter set. */ /* Split URI into protocol+hierarchy part and its parameter set. */

View file

@ -159,7 +159,8 @@ static int _main(int argc, char * * argv)
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (args.size() == 2) { if (args.size() == 2) {
expectedHash = Hash(args[1], ht); expectedHash = Hash(args[1], ht);
storePath = store->makeFixedOutputPath(unpack, expectedHash, name); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = expectedHash; hash = expectedHash;
else else
@ -208,13 +209,15 @@ static int _main(int argc, char * * argv)
if (expectedHash != Hash(ht) && expectedHash != hash) if (expectedHash != Hash(ht) && expectedHash != hash)
throw Error(format("hash mismatch for '%1%'") % uri); throw Error(format("hash mismatch for '%1%'") % uri);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
/* Copy the file to the Nix store. FIXME: if RemoteStore /* Copy the file to the Nix store. FIXME: if RemoteStore
implemented addToStoreFromDump() and downloadFile() implemented addToStoreFromDump() and downloadFile()
supported a sink, we could stream the download directly supported a sink, we could stream the download directly
into the Nix store. */ into the Nix store. */
storePath = store->addToStore(name, tmpFile, unpack, ht); storePath = store->addToStore(name, tmpFile, recursive, ht);
assert(*storePath == store->makeFixedOutputPath(unpack, hash, name)); assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
} }
stopProgressBar(); stopProgressBar();

View file

@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
store. */ store. */
static void opAddFixed(Strings opFlags, Strings opArgs) static void opAddFixed(Strings opFlags, Strings opArgs)
{ {
bool recursive = false; auto recursive = FileIngestionMethod::Flat;
for (auto & i : opFlags) for (auto & i : opFlags)
if (i == "--recursive") recursive = true; if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
else throw UsageError(format("unknown flag '%1%'") % i); else throw UsageError(format("unknown flag '%1%'") % i);
if (opArgs.empty()) if (opArgs.empty())
@ -194,10 +194,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
/* Hack to support caching in `nix-prefetch-url'. */ /* Hack to support caching in `nix-prefetch-url'. */
static void opPrintFixedPath(Strings opFlags, Strings opArgs) static void opPrintFixedPath(Strings opFlags, Strings opArgs)
{ {
bool recursive = false; auto recursive = FileIngestionMethod::Flat;
for (auto i : opFlags) for (auto i : opFlags)
if (i == "--recursive") recursive = true; if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
else throw UsageError(format("unknown flag '%1%'") % i); else throw UsageError(format("unknown flag '%1%'") % i);
if (opArgs.size() != 3) if (opArgs.size() != 3)

View file

@ -45,10 +45,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s); auto narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart)); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
info.narHash = narHash; info.narHash = narHash;
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(true, info.narHash); info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
if (!dryRun) if (!dryRun)
store->addToStore(info, sink.s); store->addToStore(info, sink.s);

View file

@ -9,15 +9,14 @@ using namespace nix;
struct CmdHash : Command struct CmdHash : Command
{ {
enum Mode { mFile, mPath }; FileIngestionMethod mode;
Mode mode;
Base base = SRI; Base base = SRI;
bool truncate = false; bool truncate = false;
HashType ht = htSHA256; HashType ht = htSHA256;
std::vector<std::string> paths; std::vector<std::string> paths;
std::optional<std::string> modulus; std::optional<std::string> modulus;
CmdHash(Mode mode) : mode(mode) CmdHash(FileIngestionMethod mode) : mode(mode)
{ {
mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base64", "print hash in base-64", &base, Base64);
@ -36,9 +35,14 @@ struct CmdHash : Command
std::string description() override std::string description() override
{ {
return mode == mFile const char* d;
? "print cryptographic hash of a regular file" switch (mode) {
: "print cryptographic hash of the NAR serialisation of a path"; case FileIngestionMethod::Flat:
d = "print cryptographic hash of a regular file";
case FileIngestionMethod::Recursive:
d = "print cryptographic hash of the NAR serialisation of a path";
};
return d;
} }
Category category() override { return catUtility; } Category category() override { return catUtility; }
@ -53,10 +57,14 @@ struct CmdHash : Command
else else
hashSink = std::make_unique<HashSink>(ht); hashSink = std::make_unique<HashSink>(ht);
if (mode == mFile) switch (mode) {
case FileIngestionMethod::Flat:
readFile(path, *hashSink); readFile(path, *hashSink);
else break;
case FileIngestionMethod::Recursive:
dumpPath(path, *hashSink); dumpPath(path, *hashSink);
break;
}
Hash h = hashSink->finish().first; Hash h = hashSink->finish().first;
if (truncate && h.hashSize > 20) h = compressHash(h, 20); if (truncate && h.hashSize > 20) h = compressHash(h, 20);
@ -65,8 +73,8 @@ struct CmdHash : Command
} }
}; };
static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(CmdHash::mFile); }); static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(CmdHash::mPath); }); static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
struct CmdToBase : Command struct CmdToBase : Command
{ {
@ -137,7 +145,7 @@ static int compatNixHash(int argc, char * * argv)
}); });
if (op == opHash) { if (op == opHash) {
CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
cmd.ht = ht; cmd.ht = ht;
cmd.base = base32 ? Base32 : Base16; cmd.base = base32 ? Base32 : Base16;
cmd.truncate = truncate; cmd.truncate = truncate;

View file

@ -77,12 +77,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
auto narHash = hashModuloSink.finish().first; auto narHash = hashModuloSink.finish().first;
ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference)); ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference));
info.references = std::move(references); info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path.clone()); if (hasSelfReference) info.references.insert(info.path.clone());
info.narHash = narHash; info.narHash = narHash;
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(true, info.narHash); info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
if (!json) if (!json)
printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));