Handle case collisions on case-insensitive systems

When running NixOps under Mac OS X, we need to be able to import store
paths built on Linux into the local Nix store. However, HFS+ is
usually case-insensitive, so if there are directories with file names
that differ only in case, then importing will fail.

The solution is to add a suffix ("~nix~case~hack~<integer>") to
colliding files. For instance, if we have a directory containing
xt_CONNMARK.h and xt_connmark.h, then the latter will be renamed to
"xt_connmark.h~nix~case~hack~1". If a store path is dumped as a NAR,
the suffixes are removed. Thus, importing and exporting via a
case-insensitive Nix store is round-tripping. So when NixOps calls
nix-copy-closure to copy the path to a Linux machine, you get the
original file names back.

Closes #119.

src/libstore/globals.cc

@@ -2,6 +2,7 @@
 
 #include "globals.hh"
 #include "util.hh"
+#include "archive.hh"
 
 #include <map>
 #include <algorithm>
@@ -150,6 +151,7 @@ void Settings::update()
     get(useSshSubstituter, "use-ssh-substituter");
     get(logServers, "log-servers");
     get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
+    get(useCaseHack, "use-case-hack");
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {

src/libutil/archive.cc

@@ -3,6 +3,8 @@
 #include <cerrno>
 #include <algorithm>
 #include <vector>
+#include <map>
+#include <cstring>
 
 #define _XOPEN_SOURCE 600
 #include <sys/types.h>
@@ -18,38 +20,20 @@
 namespace nix {
 
 
+bool useCaseHack =
+#if __APPLE__
+    true;
+#else
+    false;
+#endif
+
 static string archiveVersion1 = "nix-archive-1";
 
+static string caseHackSuffix = "~nix~case~hack~";
 
 PathFilter defaultPathFilter;
 
 
-static void dump(const string & path, Sink & sink, PathFilter & filter);
-
-
-static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
-{
-    Strings names = readDirectory(path);
-    vector<string> names2(names.begin(), names.end());
-    sort(names2.begin(), names2.end());
-
-    for (vector<string>::iterator i = names2.begin();
-         i != names2.end(); ++i)
-    {
-        Path entry = path + "/" + *i;
-        if (filter(entry)) {
-            writeString("entry", sink);
-            writeString("(", sink);
-            writeString("name", sink);
-            writeString(*i, sink);
-            writeString("node", sink);
-            dump(entry, sink, filter);
-            writeString(")", sink);
-        }
-    }
-}
-
-
 static void dumpContents(const Path & path, size_t size,
     Sink & sink)
 {
@@ -94,7 +78,36 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
     else if (S_ISDIR(st.st_mode)) {
         writeString("type", sink);
         writeString("directory", sink);
-        dumpEntries(path, sink, filter);
+
+        /* If we're on a case-insensitive system like Mac OS X, undo
+           the case hack applied by restorePath(). */
+        Strings names = readDirectory(path);
+        std::map<string, string> unhacked;
+        for (auto & i : names)
+            if (useCaseHack) {
+                string name(i);
+                size_t pos = i.find(caseHackSuffix);
+                if (pos != string::npos) {
+                    printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i));
+                    name.erase(pos);
+                }
+                if (unhacked.find(name) != unhacked.end())
+                    throw Error(format("file name collision in between `%1%' and `%2%'")
+                        % (path + "/" + unhacked[name]) % (path + "/" + i));
+                unhacked[name] = i;
+            } else
+                unhacked[i] = i;
+
+        for (auto & i : unhacked)
+            if (filter(path + "/" + i.first)) {
+                writeString("entry", sink);
+                writeString("(", sink);
+                writeString("name", sink);
+                writeString(i.first, sink);
+                writeString("node", sink);
+                dump(path + "/" + i.second, sink, filter);
+                writeString(")", sink);
+            }
     }
 
     else if (S_ISLNK(st.st_mode)) {
@@ -123,6 +136,7 @@ static SerialisationError badArchive(string s)
 }
 
 
+#if 0
 static void skipGeneric(Source & source)
 {
     if (readString(source) == "(") {
@@ -130,37 +144,7 @@ static void skipGeneric(Source & source)
             skipGeneric(source);
     }
 }
-
-
-static void parse(ParseSink & sink, Source & source, const Path & path);
-
-
-
-static void parseEntry(ParseSink & sink, Source & source, const Path & path)
-{
-    string s, name;
-
-    s = readString(source);
-    if (s != "(") throw badArchive("expected open tag");
-
-    while (1) {
-        checkInterrupt();
-
-        s = readString(source);
-
-        if (s == ")") {
-            break;
-        } else if (s == "name") {
-            name = readString(source);
-        } else if (s == "node") {
-            if (s == "") throw badArchive("entry name missing");
-            parse(sink, source, path + "/" + name);
-        } else {
-            throw badArchive("unknown field " + s);
-            skipGeneric(source);
-        }
-    }
-}
+#endif
 
 
 static void parseContents(ParseSink & sink, Source & source, const Path & path)
@@ -185,6 +169,15 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
 }
 
 
+struct CaseInsensitiveCompare
+{
+    bool operator() (const string & a, const string & b) const
+    {
+        return strcasecmp(a.c_str(), b.c_str()) < 0;
+    }
+};
+
+
 static void parse(ParseSink & sink, Source & source, const Path & path)
 {
     string s;
@@ -194,6 +187,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
 
     enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
 
+    std::map<Path, int, CaseInsensitiveCompare> names;
+
     while (1) {
         checkInterrupt();
 
@@ -236,7 +231,35 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
         }
 
         else if (s == "entry" && type == tpDirectory) {
-            parseEntry(sink, source, path);
+            string name;
+
+            s = readString(source);
+            if (s != "(") throw badArchive("expected open tag");
+
+            while (1) {
+                checkInterrupt();
+
+                s = readString(source);
+
+                if (s == ")") {
+                    break;
+                } else if (s == "name") {
+                    name = readString(source);
+                    if (useCaseHack) {
+                        auto i = names.find(name);
+                        if (i != names.end()) {
+                            printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name);
+                            name += caseHackSuffix;
+                            name += int2String(++i->second);
+                        } else
+                            names[name] = 0;
+                    }
+                } else if (s == "node") {
+                    if (s.empty()) throw badArchive("entry name missing");
+                    parse(sink, source, path + "/" + name);
+                } else
+                    throw badArchive("unknown field " + s);
+            }
         }
 
         else if (s == "target" && type == tpSymlink) {
@@ -244,10 +267,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
             sink.createSymlink(path, target);
         }
 
-        else {
+        else
             throw badArchive("unknown field " + s);
-            skipGeneric(source);
-        }
     }
 }
 

src/libutil/archive.hh

@@ -72,4 +72,8 @@ void parseDump(ParseSink & sink, Source & source);
 void restorePath(const Path & path, Source & source);
 
 
+// FIXME: global variables are bad m'kay.
+extern bool useCaseHack;
+
+
 }

tests/case-hack.sh

@@ -0,0 +1,19 @@
+source common.sh
+
+clearStore
+
+rm -rf $TEST_ROOT/case
+
+opts="--option use-case-hack true"
+
+# Check whether restoring and dumping a NAR that contains case
+# collisions is round-tripping, even on a case-insensitive system.
+nix-store $opts  --restore $TEST_ROOT/case < case.nar
+nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar
+cmp case.nar $TEST_ROOT/case.nar
+[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ]
+
+# Check whether we detect true collisions (e.g. those remaining after
+# removal of the suffix).
+touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3"
+! nix-store $opts --dump $TEST_ROOT/case > /dev/null

tests/local.mk

@@ -10,7 +10,7 @@ nix_tests = \
   remote-store.sh export.sh export-graph.sh negative-caching.sh \
   binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
-  binary-cache.sh nix-profile.sh repair.sh dump-db.sh
+  binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))