forked from lix-project/lix
Merge branch 'nar-accessor-tree' of https://github.com/bennofs/nix
This commit is contained in:
commit
edbb105e98
|
@ -2,6 +2,8 @@
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include <stack>
|
||||||
|
#include <algorithm>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
@ -16,16 +18,16 @@ struct NarMember
|
||||||
size_t start, size;
|
size_t start, size;
|
||||||
|
|
||||||
std::string target;
|
std::string target;
|
||||||
|
|
||||||
|
/* If this is a directory, all the children of the directory. */
|
||||||
|
std::map<std::string, NarMember> children;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct NarIndexer : ParseSink, StringSource
|
struct NarIndexer : ParseSink, StringSource
|
||||||
{
|
{
|
||||||
// FIXME: should store this as a tree. Now we're vulnerable to
|
NarMember root;
|
||||||
// O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}).
|
std::stack<NarMember*> parents;
|
||||||
typedef std::map<Path, NarMember> Members;
|
|
||||||
Members members;
|
|
||||||
|
|
||||||
Path currentPath;
|
|
||||||
std::string currentStart;
|
std::string currentStart;
|
||||||
bool isExec = false;
|
bool isExec = false;
|
||||||
|
|
||||||
|
@ -33,28 +35,45 @@ struct NarIndexer : ParseSink, StringSource
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void createMember(const Path & path, NarMember member) {
|
||||||
|
size_t level = std::count(path.begin(), path.end(), '/');
|
||||||
|
while(parents.size() > level) {
|
||||||
|
parents.pop();
|
||||||
|
}
|
||||||
|
|
||||||
|
if(parents.empty()) {
|
||||||
|
root = std::move(member);
|
||||||
|
parents.push(&root);
|
||||||
|
} else {
|
||||||
|
if(parents.top()->type != FSAccessor::Type::tDirectory) {
|
||||||
|
throw Error(format("NAR file missing parent directory of path ‘%1%’") % path);
|
||||||
|
}
|
||||||
|
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
|
||||||
|
parents.push(&result.first->second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void createDirectory(const Path & path) override
|
void createDirectory(const Path & path) override
|
||||||
{
|
{
|
||||||
members.emplace(path,
|
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 });
|
||||||
NarMember{FSAccessor::Type::tDirectory, false, 0, 0});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void createRegularFile(const Path & path) override
|
void createRegularFile(const Path & path) override
|
||||||
{
|
{
|
||||||
currentPath = path;
|
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 });
|
||||||
}
|
}
|
||||||
|
|
||||||
void isExecutable() override
|
void isExecutable() override
|
||||||
{
|
{
|
||||||
isExec = true;
|
parents.top()->isExecutable = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void preallocateContents(unsigned long long size) override
|
void preallocateContents(unsigned long long size) override
|
||||||
{
|
{
|
||||||
currentStart = string(s, pos, 16);
|
currentStart = string(s, pos, 16);
|
||||||
assert(size <= std::numeric_limits<size_t>::max());
|
assert(size <= std::numeric_limits<size_t>::max());
|
||||||
members.emplace(currentPath,
|
parents.top()->size = (size_t)size;
|
||||||
NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size});
|
parents.top()->start = pos;
|
||||||
}
|
}
|
||||||
|
|
||||||
void receiveContents(unsigned char * data, unsigned int len) override
|
void receiveContents(unsigned char * data, unsigned int len) override
|
||||||
|
@ -68,16 +87,42 @@ struct NarIndexer : ParseSink, StringSource
|
||||||
|
|
||||||
void createSymlink(const Path & path, const string & target) override
|
void createSymlink(const Path & path, const string & target) override
|
||||||
{
|
{
|
||||||
members.emplace(path,
|
createMember(path,
|
||||||
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
||||||
}
|
}
|
||||||
|
|
||||||
Members::iterator find(const Path & path)
|
NarMember* find(const Path & path)
|
||||||
{
|
{
|
||||||
auto i = members.find(path);
|
Path canon = path == "" ? "" : canonPath(path);
|
||||||
if (i == members.end())
|
NarMember* current = &root;
|
||||||
|
auto end = path.end();
|
||||||
|
for(auto it = path.begin(); it != end; ) {
|
||||||
|
// because it != end, the remaining component is non-empty so we need
|
||||||
|
// a directory
|
||||||
|
if(current->type != FSAccessor::Type::tDirectory) return nullptr;
|
||||||
|
|
||||||
|
// skip slash (canonPath above ensures that this is always a slash)
|
||||||
|
assert(*it == '/');
|
||||||
|
it += 1;
|
||||||
|
|
||||||
|
// lookup current component
|
||||||
|
auto next = std::find(it, end, '/');
|
||||||
|
auto child = current->children.find(std::string(it, next));
|
||||||
|
if(child == current->children.end()) return nullptr;
|
||||||
|
current = &child->second;
|
||||||
|
|
||||||
|
it = next;
|
||||||
|
}
|
||||||
|
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
NarMember& at(const Path & path) {
|
||||||
|
auto result = find(path);
|
||||||
|
if(result == nullptr) {
|
||||||
throw Error(format("NAR file does not contain path ‘%1%’") % path);
|
throw Error(format("NAR file does not contain path ‘%1%’") % path);
|
||||||
return i;
|
}
|
||||||
|
return *result;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -93,44 +138,41 @@ struct NarAccessor : public FSAccessor
|
||||||
|
|
||||||
Stat stat(const Path & path) override
|
Stat stat(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.members.find(path);
|
auto i = indexer.find(path);
|
||||||
if (i == indexer.members.end())
|
if (i == nullptr)
|
||||||
return {FSAccessor::Type::tMissing, 0, false};
|
return {FSAccessor::Type::tMissing, 0, false};
|
||||||
return {i->second.type, i->second.size, i->second.isExecutable};
|
return {i->type, i->size, i->isExecutable};
|
||||||
}
|
}
|
||||||
|
|
||||||
StringSet readDirectory(const Path & path) override
|
StringSet readDirectory(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.find(path);
|
auto i = indexer.at(path);
|
||||||
|
|
||||||
if (i->second.type != FSAccessor::Type::tDirectory)
|
if (i.type != FSAccessor::Type::tDirectory)
|
||||||
throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path);
|
throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path);
|
||||||
|
|
||||||
++i;
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
while (i != indexer.members.end() && isInDir(i->first, path)) {
|
for(auto&& child : i.children) {
|
||||||
// FIXME: really bad performance.
|
res.insert(child.first);
|
||||||
if (i->first.find('/', path.size() + 1) == std::string::npos)
|
|
||||||
res.insert(std::string(i->first, path.size() + 1));
|
|
||||||
++i;
|
|
||||||
}
|
}
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readFile(const Path & path) override
|
std::string readFile(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.find(path);
|
auto i = indexer.at(path);
|
||||||
if (i->second.type != FSAccessor::Type::tRegular)
|
if (i.type != FSAccessor::Type::tRegular)
|
||||||
throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path);
|
throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path);
|
||||||
return std::string(*nar, i->second.start, i->second.size);
|
return std::string(*nar, i.start, i.size);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readLink(const Path & path) override
|
std::string readLink(const Path & path) override
|
||||||
{
|
{
|
||||||
auto i = indexer.find(path);
|
auto i = indexer.at(path);
|
||||||
if (i->second.type != FSAccessor::Type::tSymlink)
|
if (i.type != FSAccessor::Type::tSymlink)
|
||||||
throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path);
|
throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path);
|
||||||
return i->second.target;
|
return i.target;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,8 @@ nix_tests = \
|
||||||
check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
|
check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
|
||||||
placeholders.sh nix-shell.sh \
|
placeholders.sh nix-shell.sh \
|
||||||
linux-sandbox.sh \
|
linux-sandbox.sh \
|
||||||
build-remote.sh
|
build-remote.sh \
|
||||||
|
nar-index.sh
|
||||||
# parallel.sh
|
# parallel.sh
|
||||||
|
|
||||||
install-tests += $(foreach x, $(nix_tests), tests/$(x))
|
install-tests += $(foreach x, $(nix_tests), tests/$(x))
|
||||||
|
|
23
tests/nar-index.nix
Normal file
23
tests/nar-index.nix
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
with import ./config.nix;
|
||||||
|
|
||||||
|
rec {
|
||||||
|
a = mkDerivation {
|
||||||
|
name = "nar-index-a";
|
||||||
|
builder = builtins.toFile "builder.sh"
|
||||||
|
''
|
||||||
|
mkdir $out
|
||||||
|
mkdir $out/foo
|
||||||
|
touch $out/foo-x
|
||||||
|
touch $out/foo/bar
|
||||||
|
touch $out/foo/baz
|
||||||
|
touch $out/qux
|
||||||
|
mkdir $out/zyx
|
||||||
|
|
||||||
|
cat >$out/foo/data <<EOF
|
||||||
|
lasjdöaxnasd
|
||||||
|
asdom 12398
|
||||||
|
ä"§Æẞ¢«»”alsd
|
||||||
|
EOF
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
23
tests/nar-index.sh
Normal file
23
tests/nar-index.sh
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
echo "building test path"
|
||||||
|
storePath="$(nix-build nar-index.nix -A a --no-out-link)"
|
||||||
|
|
||||||
|
cd "$TEST_ROOT"
|
||||||
|
|
||||||
|
echo "dumping path to nar"
|
||||||
|
narFile="$TEST_ROOT/path.nar"
|
||||||
|
nix-store --dump $storePath > $narFile
|
||||||
|
|
||||||
|
echo "check that find and ls-nar match"
|
||||||
|
( cd $storePath; find . | sort ) > files.find
|
||||||
|
nix ls-nar -R -d $narFile "" | sort > files.ls-nar
|
||||||
|
diff -u files.find files.ls-nar
|
||||||
|
|
||||||
|
echo "check that file contents of data match"
|
||||||
|
nix cat-nar $narFile /foo/data > data.cat-nar
|
||||||
|
diff -u data.cat-nar $storePath/foo/data
|
||||||
|
|
||||||
|
echo "check that file contents of baz match"
|
||||||
|
nix cat-nar $narFile /foo/baz > baz.cat-nar
|
||||||
|
diff -u baz.cat-nar $storePath/foo/baz
|
Loading…
Reference in a new issue