forked from lix-project/lix
e9a4abdb5d
There already existed a smoke test for the link content length, but it appears that there exists some corruptions pernicious enough to replace the file content with zeros, and keeping the same length. --repair-path now goes as far as checking the content of the link, making it true to its name and actually repairing the path for such coruption cases.
298 lines
9.2 KiB
C++
298 lines
9.2 KiB
C++
#include "util.hh"
|
||
#include "local-store.hh"
|
||
#include "globals.hh"
|
||
|
||
#include <cstdlib>
|
||
#include <cstring>
|
||
#include <sys/types.h>
|
||
#include <sys/stat.h>
|
||
#include <unistd.h>
|
||
#include <errno.h>
|
||
#include <stdio.h>
|
||
#include <regex>
|
||
|
||
|
||
namespace nix {
|
||
|
||
|
||
static void makeWritable(const Path & path)
|
||
{
|
||
auto st = lstat(path);
|
||
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
||
throw SysError("changing writability of '%1%'", path);
|
||
}
|
||
|
||
|
||
struct MakeReadOnly
|
||
{
|
||
Path path;
|
||
MakeReadOnly(const Path & path) : path(path) { }
|
||
~MakeReadOnly()
|
||
{
|
||
try {
|
||
/* This will make the path read-only. */
|
||
if (path != "") canonicaliseTimestampAndPermissions(path);
|
||
} catch (...) {
|
||
ignoreException();
|
||
}
|
||
}
|
||
};
|
||
|
||
|
||
LocalStore::InodeHash LocalStore::loadInodeHash()
|
||
{
|
||
debug("loading hash inodes in memory");
|
||
InodeHash inodeHash;
|
||
|
||
AutoCloseDir dir(opendir(linksDir.c_str()));
|
||
if (!dir) throw SysError("opening directory '%1%'", linksDir);
|
||
|
||
struct dirent * dirent;
|
||
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
||
checkInterrupt();
|
||
// We don't care if we hit non-hash files, anything goes
|
||
inodeHash.insert(dirent->d_ino);
|
||
}
|
||
if (errno) throw SysError("reading directory '%1%'", linksDir);
|
||
|
||
printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
|
||
|
||
return inodeHash;
|
||
}
|
||
|
||
|
||
Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
|
||
{
|
||
Strings names;
|
||
|
||
AutoCloseDir dir(opendir(path.c_str()));
|
||
if (!dir) throw SysError("opening directory '%1%'", path);
|
||
|
||
struct dirent * dirent;
|
||
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
||
checkInterrupt();
|
||
|
||
if (inodeHash.count(dirent->d_ino)) {
|
||
debug(format("'%1%' is already linked") % dirent->d_name);
|
||
continue;
|
||
}
|
||
|
||
string name = dirent->d_name;
|
||
if (name == "." || name == "..") continue;
|
||
names.push_back(name);
|
||
}
|
||
if (errno) throw SysError("reading directory '%1%'", path);
|
||
|
||
return names;
|
||
}
|
||
|
||
|
||
void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||
const Path & path, InodeHash & inodeHash, bool repair)
|
||
{
|
||
checkInterrupt();
|
||
|
||
auto st = lstat(path);
|
||
|
||
#if __APPLE__
|
||
/* HFS/macOS has some undocumented security feature disabling hardlinking for
|
||
special files within .app dirs. *.app/Contents/PkgInfo and
|
||
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
|
||
https://github.com/NixOS/nix/issues/1443 for more discussion. */
|
||
|
||
if (std::regex_search(path, std::regex("\\.app/Contents/.+$")))
|
||
{
|
||
debug(format("'%1%' is not allowed to be linked in macOS") % path);
|
||
return;
|
||
}
|
||
#endif
|
||
|
||
if (S_ISDIR(st.st_mode)) {
|
||
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
||
for (auto & i : names)
|
||
optimisePath_(act, stats, path + "/" + i, inodeHash, repair);
|
||
return;
|
||
}
|
||
|
||
/* We can hard link regular files and maybe symlinks. */
|
||
if (!S_ISREG(st.st_mode)
|
||
#if CAN_LINK_SYMLINK
|
||
&& !S_ISLNK(st.st_mode)
|
||
#endif
|
||
) return;
|
||
|
||
/* Sometimes SNAFUs can cause files in the Nix store to be
|
||
modified, in particular when running programs as root under
|
||
NixOS (example: $fontconfig/var/cache being modified). Skip
|
||
those files. FIXME: check the modification time. */
|
||
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
|
||
warn("skipping suspicious writable file '%1%'", path);
|
||
return;
|
||
}
|
||
|
||
/* This can still happen on top-level files. */
|
||
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
|
||
debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2);
|
||
return;
|
||
}
|
||
|
||
/* Hash the file. Note that hashPath() returns the hash over the
|
||
NAR serialisation, which includes the execute bit on the file.
|
||
Thus, executable and non-executable files with the same
|
||
contents *won't* be linked (which is good because otherwise the
|
||
permissions would be screwed up).
|
||
|
||
Also note that if `path' is a symlink, then we're hashing the
|
||
contents of the symlink (i.e. the result of readlink()), not
|
||
the contents of the target (which may not even exist). */
|
||
Hash hash = hashPath(htSHA256, path).first;
|
||
debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true));
|
||
|
||
/* Check if this is a known hash. */
|
||
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
|
||
|
||
/* Maybe delete the link, if it has been corrupted. */
|
||
if (pathExists(linkPath)) {
|
||
auto stLink = lstat(linkPath);
|
||
if (st.st_size != stLink.st_size
|
||
|| (repair && hash != hashPath(htSHA256, linkPath).first))
|
||
{
|
||
// XXX: Consider overwriting linkPath with our valid version.
|
||
warn("removing corrupted link '%s'", linkPath);
|
||
warn("There may be more corrupted paths."
|
||
"\nYou should run `nix-store --verify --check-contents --repair` to fix them all");
|
||
unlink(linkPath.c_str());
|
||
}
|
||
}
|
||
|
||
if (!pathExists(linkPath)) {
|
||
/* Nope, create a hard link in the links directory. */
|
||
if (link(path.c_str(), linkPath.c_str()) == 0) {
|
||
inodeHash.insert(st.st_ino);
|
||
return;
|
||
}
|
||
|
||
switch (errno) {
|
||
case EEXIST:
|
||
/* Fall through if another process created ‘linkPath’ before
|
||
we did. */
|
||
break;
|
||
|
||
case ENOSPC:
|
||
/* On ext4, that probably means the directory index is
|
||
full. When that happens, it's fine to ignore it: we
|
||
just effectively disable deduplication of this
|
||
file. */
|
||
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
|
||
return;
|
||
|
||
default:
|
||
throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
|
||
}
|
||
}
|
||
|
||
/* Yes! We've seen a file with the same contents. Replace the
|
||
current file with a hard link to that file. */
|
||
auto stLink = lstat(linkPath);
|
||
|
||
if (st.st_ino == stLink.st_ino) {
|
||
debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
|
||
return;
|
||
}
|
||
|
||
printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
|
||
|
||
/* Make the containing directory writable, but only if it's not
|
||
the store itself (we don't want or need to mess with its
|
||
permissions). */
|
||
bool mustToggle = dirOf(path) != realStoreDir.get();
|
||
if (mustToggle) makeWritable(dirOf(path));
|
||
|
||
/* When we're done, make the directory read-only again and reset
|
||
its timestamp back to 0. */
|
||
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
|
||
|
||
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
|
||
% realStoreDir % getpid() % random()).str();
|
||
|
||
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
||
if (errno == EMLINK) {
|
||
/* Too many links to the same file (>= 32000 on most file
|
||
systems). This is likely to happen with empty files.
|
||
Just shrug and ignore. */
|
||
if (st.st_size)
|
||
printInfo(format("'%1%' has maximum number of links") % linkPath);
|
||
return;
|
||
}
|
||
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
|
||
}
|
||
|
||
/* Atomically replace the old file with the new hard link. */
|
||
if (rename(tempLink.c_str(), path.c_str()) == -1) {
|
||
if (unlink(tempLink.c_str()) == -1)
|
||
printError("unable to unlink '%1%'", tempLink);
|
||
if (errno == EMLINK) {
|
||
/* Some filesystems generate too many links on the rename,
|
||
rather than on the original link. (Probably it
|
||
temporarily increases the st_nlink field before
|
||
decreasing it again.) */
|
||
debug("'%s' has reached maximum number of links", linkPath);
|
||
return;
|
||
}
|
||
throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
|
||
}
|
||
|
||
stats.filesLinked++;
|
||
stats.bytesFreed += st.st_size;
|
||
stats.blocksFreed += st.st_blocks;
|
||
|
||
if (act)
|
||
act->result(resFileLinked, st.st_size, st.st_blocks);
|
||
}
|
||
|
||
|
||
void LocalStore::optimiseStore(OptimiseStats & stats)
|
||
{
|
||
Activity act(*logger, actOptimiseStore);
|
||
|
||
auto paths = queryAllValidPaths();
|
||
InodeHash inodeHash = loadInodeHash();
|
||
|
||
act.progress(0, paths.size());
|
||
|
||
uint64_t done = 0;
|
||
|
||
for (auto & i : paths) {
|
||
addTempRoot(i);
|
||
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
|
||
{
|
||
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
|
||
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, false);
|
||
}
|
||
done++;
|
||
act.progress(done, paths.size());
|
||
}
|
||
}
|
||
|
||
void LocalStore::optimiseStore()
|
||
{
|
||
OptimiseStats stats;
|
||
|
||
optimiseStore(stats);
|
||
|
||
printInfo("%s freed by hard-linking %d files",
|
||
showBytes(stats.bytesFreed),
|
||
stats.filesLinked);
|
||
}
|
||
|
||
void LocalStore::optimisePath(const Path & path, bool repair)
|
||
{
|
||
OptimiseStats stats;
|
||
InodeHash inodeHash;
|
||
|
||
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash, repair);
|
||
}
|
||
|
||
|
||
}
|