From ab0bc4999a49efbc8e1c25989662a96e32fa0cc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 22 Nov 2003 18:45:56 +0000 Subject: [PATCH] * Maintain integrity of the substitute and successor mappings when deleting a path in the store. * Allow absolute paths in Nix expressions. * Get nix-prefetch-url to work again. * Various other fixes. --- corepkgs/fetchurl/Makefile.am | 11 ++- .../{fetchurl.sh.in => builder.sh.in} | 0 corepkgs/fetchurl/default.nix | 8 ++ corepkgs/fetchurl/fetchurl.fix | 10 -- scripts/Makefile.am | 2 +- scripts/nix-prefetch-url.in | 28 +++--- scripts/nix-switch.in | 86 ----------------- src/libexpr/nix.sdf | 3 +- src/libexpr/parser.cc | 81 +++++++++------- src/libexpr/parser.hh | 6 ++ src/libstore/normalise.cc | 1 + src/libstore/store.cc | 93 +++++++++++-------- src/libstore/store.hh | 3 - src/nix-instantiate/main.cc | 7 +- substitute.mk | 12 ++- 15 files changed, 152 insertions(+), 199 deletions(-) rename corepkgs/fetchurl/{fetchurl.sh.in => builder.sh.in} (100%) create mode 100644 corepkgs/fetchurl/default.nix delete mode 100644 corepkgs/fetchurl/fetchurl.fix delete mode 100755 scripts/nix-switch.in diff --git a/corepkgs/fetchurl/Makefile.am b/corepkgs/fetchurl/Makefile.am index 0c8f0c939..270bf0142 100644 --- a/corepkgs/fetchurl/Makefile.am +++ b/corepkgs/fetchurl/Makefile.am @@ -1,10 +1,11 @@ -all-local: fetchurl.sh +all-local: builder.sh install-exec-local: - $(INSTALL) -d $(datadir)/fix/fetchurl - $(INSTALL_DATA) fetchurl.fix $(datadir)/fix/fetchurl - $(INSTALL_PROGRAM) fetchurl.sh $(datadir)/fix/fetchurl + $(INSTALL) -d $(datadir)/nix/corepkgs + $(INSTALL) -d $(datadir)/nix/corepkgs/fetchurl + $(INSTALL_DATA) default.nix $(datadir)/nix/corepkgs/fetchurl + $(INSTALL_PROGRAM) builder.sh $(datadir)/nix/corepkgs/fetchurl include ../../substitute.mk -EXTRA_DIST = fetchurl.fix fetchurl.sh.in +EXTRA_DIST = default.nix builder.sh.in diff --git a/corepkgs/fetchurl/fetchurl.sh.in b/corepkgs/fetchurl/builder.sh.in similarity index 100% rename from corepkgs/fetchurl/fetchurl.sh.in rename to corepkgs/fetchurl/builder.sh.in diff --git a/corepkgs/fetchurl/default.nix b/corepkgs/fetchurl/default.nix new file mode 100644 index 000000000..663bba4a3 --- /dev/null +++ b/corepkgs/fetchurl/default.nix @@ -0,0 +1,8 @@ +{system, url, md5}: derivation { + name = baseNameOf (toString url); + system = system; + builder = ./builder.sh; + url = url; + md5 = md5; + id = md5; +} diff --git a/corepkgs/fetchurl/fetchurl.fix b/corepkgs/fetchurl/fetchurl.fix deleted file mode 100644 index 0221b612c..000000000 --- a/corepkgs/fetchurl/fetchurl.fix +++ /dev/null @@ -1,10 +0,0 @@ -Function(["url", "md5"], - Package( - [ ("build", Relative("fetchurl/fetchurl.sh")) - , ("url", Var("url")) - , ("md5", Var("md5")) - , ("name", BaseName(Var("url"))) - , ("id", Var("md5")) - ] - ) -) diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 4a1be7f8f..35bb926af 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -1,4 +1,4 @@ -bin_SCRIPTS = nix-switch nix-collect-garbage \ +bin_SCRIPTS = nix-collect-garbage \ nix-pull nix-push nix-prefetch-url noinst_SCRIPTS = nix-profile.sh diff --git a/scripts/nix-prefetch-url.in b/scripts/nix-prefetch-url.in index 332290d40..0873f5a8d 100644 --- a/scripts/nix-prefetch-url.in +++ b/scripts/nix-prefetch-url.in @@ -22,27 +22,29 @@ print "file has hash $hash\n"; my $out2 = "@prefix@/store/nix-prefetch-url-$hash"; rename $out, $out2; -# Create a Fix expression. -my $fixexpr = - "App(IncludeFix(\"fetchurl/fetchurl.fix\"), " . - "[(\"url\", \"$url\"), (\"md5\", \"$hash\")])"; +# Create a Nix expression. +my $nixexpr = + "(import @datadir@/nix/corepkgs/fetchurl) " . + "{url = $url; md5 = \"$hash\"; system = \"@host@\"}"; + +print "expr: $nixexpr\n"; # Instantiate a Nix expression. -print STDERR "running fix...\n"; -my $pid = open2(\*READ, \*WRITE, "fix -") or die "cannot run fix"; +print STDERR "instantiating Nix expression...\n"; +my $pid = open2(\*READ, \*WRITE, "nix-instantiate -") or die "cannot run nix-instantiate"; -print WRITE $fixexpr; +print WRITE $nixexpr; close WRITE; -my $id = ; -chomp $id; +my $drvpath = ; +chomp $drvpath; waitpid $pid, 0; -$? == 0 or die "fix failed"; +$? == 0 or die "nix-instantiate failed"; # Run Nix. -print STDERR "running nix...\n"; -system "nix --install $id > /dev/null"; -$? == 0 or die "`nix --install' failed"; +print STDERR "realising store expression $drvpath...\n"; +system "nix-store --realise $drvpath > /dev/null"; +$? == 0 or die "realisation failed"; unlink $out2; diff --git a/scripts/nix-switch.in b/scripts/nix-switch.in deleted file mode 100755 index 9fcb598e3..000000000 --- a/scripts/nix-switch.in +++ /dev/null @@ -1,86 +0,0 @@ -#! /usr/bin/perl -w - -use strict; - -my $keep = 0; -my $sourceroot = 1; -my $name = "current"; -my $srcid; - -my $argnr = 0; -while ($argnr < scalar @ARGV) { - my $arg = $ARGV[$argnr++]; - if ($arg eq "--keep") { $keep = 1; } - elsif ($arg eq "--no-source") { $sourceroot = 0; } - elsif ($arg eq "--name") { $name = $ARGV[$argnr++]; } - elsif ($arg =~ /^\//) { $srcid = $arg; } - else { die "unknown argument `$arg'" }; -} - -my $linkdir = "@localstatedir@/nix/links"; - -# Build the specified package, and all its dependencies. -my $nfid = `nix --install $srcid`; -if ($?) { die "`nix --install' failed"; } -chomp $nfid; -die unless $nfid =~ /^\//; - -my $pkgdir = `nix --query --list $nfid`; -if ($?) { die "`nix --query --list' failed"; } -chomp $pkgdir; - -# Figure out a generation number. -opendir(DIR, $linkdir); -my $nr = 0; -foreach my $n (sort(readdir(DIR))) { - next if (!($n =~ /^\d+$/)); - $nr = $n + 1 if ($n >= $nr); -} -closedir(DIR); - -my $link = "$linkdir/$nr"; - -# Create a symlink from $link to $pkgdir. -symlink($pkgdir, $link) or die "cannot create $link: $!"; - -# Store the id of the normal form. This is useful for garbage -# collection and the like. -my $idfile = "$linkdir/$nr.id"; -open ID, "> $idfile" or die "cannot create $idfile"; -print ID "$nfid\n"; -close ID; - -# Optionally store the source id. -if ($sourceroot) { - $idfile = "$linkdir/$nr-src.id"; - open ID, "> $idfile" or die "cannot create $idfile"; - print ID "$srcid\n"; - close ID; -} - -my $current = "$linkdir/$name"; - -# Read the current generation so that we can delete it (if --keep -# wasn't specified). -my $oldlink = readlink($current); - -# Make $link the current generation by pointing $linkdir/current to -# it. The rename() system call is supposed to be essentially atomic -# on Unix. That is, if we have links `current -> X' and `new_current -# -> Y', and we rename new_current to current, a process accessing -# current will see X or Y, but never a file-not-found or other error -# condition. This is sufficient to atomically switch the current link -# tree. - -print "switching $current to $link\n"; - -my $tmplink = "$linkdir/.new_$name"; -symlink($link, $tmplink) or die "cannot create $tmplink"; -rename($tmplink, $current) or die "cannot rename $tmplink"; - -if (!$keep && defined $oldlink) { - print "deleting old $oldlink\n"; - unlink($oldlink) == 1 or print "cannot delete $oldlink\n"; - unlink("$oldlink.id") == 1 or print "cannot delete $oldlink.id\n"; - unlink("$oldlink-src.id"); -} diff --git a/src/libexpr/nix.sdf b/src/libexpr/nix.sdf index 615bdb974..b6bb23ebd 100644 --- a/src/libexpr/nix.sdf +++ b/src/libexpr/nix.sdf @@ -104,6 +104,7 @@ exports "\"" ~[\n\"]* "\"" -> Str PathComp ("/" PathComp)+ -> Path + ("/" PathComp)+ -> Path [a-zA-Z0-9\.\_\-\+]+ -> PathComp "true" -> Bool @@ -184,7 +185,7 @@ exports [0-9] -> Udigit lexical restrictions - Uri -/- [a-zA-Z0-9\-\_\.\!\~\*\'\(\)] + Uri -/- [a-zA-Z0-9\-\_\.\!\~\*\'\(\)\/] %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/src/libexpr/parser.cc b/src/libexpr/parser.cc index e3c863a55..aecfa4348 100644 --- a/src/libexpr/parser.cc +++ b/src/libexpr/parser.cc @@ -66,23 +66,9 @@ struct Cleanup : TermFun }; -Expr parseExprFromFile(Path path) +static Expr parse(const char * text, const string & location, + const Path & basePath) { - assert(path[0] == '/'); - -#if 0 - /* Perhaps this is already an imploded parse tree? */ - Expr e = ATreadFromNamedFile(path.c_str()); - if (e) return e; -#endif - - /* If `path' refers to a directory, append `/default.nix'. */ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); - if (S_ISDIR(st.st_mode)) - path = canonPath(path + "/default.nix"); - /* Initialise the SDF libraries. */ static bool initialised = false; static ATerm parseTable = 0; @@ -113,26 +99,13 @@ Expr parseExprFromFile(Path path) initialised = true; } - /* Read the input file. We can't use SGparseFile() because it's - broken, so we read the input ourselves and call - SGparseString(). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY); - if (fd == -1) throw SysError(format("opening `%1%'") % path); - - if (fstat(fd, &st) == -1) - throw SysError(format("statting `%1%'") % path); - - char text[st.st_size + 1]; - readFull(fd, (unsigned char *) text, st.st_size); - text[st.st_size] = 0; - /* Parse it. */ - ATerm result = SGparseString(lang, "Expr", text); + ATerm result = SGparseString(lang, "Expr", (char *) text); if (!result) - throw SysError(format("parse failed in `%1%'") % path); + throw SysError(format("parse failed in `%1%'") % location); if (SGisParseError(result)) throw Error(format("parse error in `%1%': %2%") - % path % result); + % location % result); /* Implode it. */ PT_ParseTree tree = PT_makeParseTreeFromTerm(result); @@ -155,10 +128,50 @@ Expr parseExprFromFile(Path path) throw Error(format("cannot implode parse tree")); printMsg(lvlVomit, format("imploded parse tree of `%1%': %2%") - % path % imploded); + % location % imploded); /* Finally, clean it up. */ Cleanup cleanup; - cleanup.basePath = dirOf(path); + cleanup.basePath = basePath; return bottomupRewrite(cleanup, imploded); } + + +Expr parseExprFromFile(Path path) +{ + assert(path[0] == '/'); + +#if 0 + /* Perhaps this is already an imploded parse tree? */ + Expr e = ATreadFromNamedFile(path.c_str()); + if (e) return e; +#endif + + /* If `path' refers to a directory, append `/default.nix'. */ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if (S_ISDIR(st.st_mode)) + path = canonPath(path + "/default.nix"); + + /* Read the input file. We can't use SGparseFile() because it's + broken, so we read the input ourselves and call + SGparseString(). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening `%1%'") % path); + + if (fstat(fd, &st) == -1) + throw SysError(format("statting `%1%'") % path); + + char text[st.st_size + 1]; + readFull(fd, (unsigned char *) text, st.st_size); + text[st.st_size] = 0; + + return parse(text, path, dirOf(path)); +} + + +Expr parseExprFromString(const string & s, const Path & basePath) +{ + return parse(s.c_str(), "(string)", basePath); +} diff --git a/src/libexpr/parser.hh b/src/libexpr/parser.hh index 5983ec562..461dae08c 100644 --- a/src/libexpr/parser.hh +++ b/src/libexpr/parser.hh @@ -4,7 +4,13 @@ #include "nixexpr.hh" +/* Parse a Nix expression from the specified file. If `path' refers + to a directory, the "/default.nix" is appended. */ Expr parseExprFromFile(Path path); +/* Parse a Nix expression from the specified string. */ +Expr parseExprFromString(const string & s, + const Path & basePath); + #endif /* !__PARSER_H */ diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc index db85c3b5b..7ef45e292 100644 --- a/src/libstore/normalise.cc +++ b/src/libstore/normalise.cc @@ -82,6 +82,7 @@ Path normaliseStoreExpr(const Path & _nePath, PathSet pending) debug(format("skipping build of expression `%1%', someone beat us to it") % (string) nePath); if (ne.type != StoreExpr::neClosure) abort(); + outputLocks.setDeletion(true); return nePath2; } } diff --git a/src/libstore/store.cc b/src/libstore/store.cc index caaa293a6..4c6b8bbec 100644 --- a/src/libstore/store.cc +++ b/src/libstore/store.cc @@ -243,16 +243,33 @@ bool isValidPath(const Path & path) } -void unregisterValidPath(const Path & _path) +static void invalidatePath(const Path & path, Transaction & txn) { - Path path(canonPath(_path)); - Transaction txn(nixDB); - debug(format("unregistering path `%1%'") % path); nixDB.delPair(txn, dbValidPaths, path); - txn.commit(); + /* Remove any successor mappings to this path (but not *from* + it). */ + Paths revs; + nixDB.queryStrings(txn, dbSuccessorsRev, path, revs); + for (Paths::iterator i = revs.begin(); i != revs.end(); ++i) + nixDB.delPair(txn, dbSuccessors, *i); + nixDB.delPair(txn, dbSuccessorsRev, path); + + /* Remove any substitute mappings to this path. */ + revs.clear(); + nixDB.queryStrings(txn, dbSubstitutesRev, path, revs); + for (Paths::iterator i = revs.begin(); i != revs.end(); ++i) { + Paths subs; + nixDB.queryStrings(txn, dbSubstitutes, *i, subs); + remove(subs.begin(), subs.end(), path); + if (subs.size() > 0) + nixDB.setStrings(txn, dbSubstitutes, *i, subs); + else + nixDB.delPair(txn, dbSubstitutes, *i); + } + nixDB.delPair(txn, dbSubstitutesRev, path); } @@ -289,6 +306,8 @@ Path addToStore(const Path & _srcPath) registerValidPath(txn, dstPath); txn.commit(); } + + outputLock.setDeletion(true); } return dstPath; @@ -310,6 +329,8 @@ void addTextToStore(const Path & dstPath, const string & s) registerValidPath(txn, dstPath); txn.commit(); } + + outputLock.setDeletion(true); } } @@ -321,7 +342,9 @@ void deleteFromStore(const Path & _path) if (!isInPrefix(path, nixStore)) throw Error(format("path `%1%' is not in the store") % path); - unregisterValidPath(path); + Transaction txn(nixDB); + invalidatePath(path, txn); + txn.commit(); deletePath(path); } @@ -332,50 +355,43 @@ void verifyStore() Transaction txn(nixDB); Paths paths; + PathSet validPaths; nixDB.enumTable(txn, dbValidPaths, paths); - for (Paths::iterator i = paths.begin(); - i != paths.end(); i++) + for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) { Path path = *i; if (!pathExists(path)) { debug(format("path `%1%' disappeared") % path); - nixDB.delPair(txn, dbValidPaths, path); - nixDB.delPair(txn, dbSuccessorsRev, path); - nixDB.delPair(txn, dbSubstitutesRev, path); - } + invalidatePath(path, txn); + } else + validPaths.insert(path); } -#if 0 - Strings subs; - nixDB.enumTable(txn, dbSubstitutes, subs); - - for (Strings::iterator i = subs.begin(); - i != subs.end(); i++) - { - FSId srcId = parseHash(*i); - - Strings subIds; - nixDB.queryStrings(txn, dbSubstitutes, srcId, subIds); - - for (Strings::iterator j = subIds.begin(); - j != subIds.end(); ) + Paths sucs; + nixDB.enumTable(txn, dbSuccessors, sucs); + for (Paths::iterator i = sucs.begin(); i != sucs.end(); ++i) { + /* Note that *i itself does not have to be valid, just its + successor. */ + Path sucPath; + if (nixDB.queryString(txn, dbSuccessors, *i, sucPath) && + validPaths.find(sucPath) == validPaths.end()) { - FSId subId = parseHash(*j); - - Strings subPaths; - nixDB.queryStrings(txn, dbId2Paths, subId, subPaths); - if (subPaths.size() == 0) { - debug(format("erasing substitute %1% for %2%") - % (string) subId % (string) srcId); - j = subIds.erase(j); - } else j++; + debug(format("found successor mapping to non-existent path `%1%'") % sucPath); + nixDB.delPair(txn, dbSuccessors, *i); } - - nixDB.setStrings(txn, dbSubstitutes, srcId, subIds); } -#endif + Paths rsucs; + nixDB.enumTable(txn, dbSuccessorsRev, rsucs); + for (Paths::iterator i = rsucs.begin(); i != rsucs.end(); ++i) { + if (validPaths.find(*i) == validPaths.end()) { + debug(format("found reverse successor mapping for non-existent path `%1%'") % *i); + nixDB.delPair(txn, dbSuccessorsRev, *i); + } + } + +#if 0 Paths sucs; nixDB.enumTable(txn, dbSuccessors, sucs); @@ -395,6 +411,7 @@ void verifyStore() nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); } } +#endif txn.commit(); } diff --git a/src/libstore/store.hh b/src/libstore/store.hh index dab3d603f..143cad8db 100644 --- a/src/libstore/store.hh +++ b/src/libstore/store.hh @@ -48,9 +48,6 @@ Paths querySubstitutes(const Path & srcPath); /* Register the validity of a path. */ void registerValidPath(const Transaction & txn, const Path & path); -/* Unregister the validity of a path. */ -void unregisterValidPath(const Path & path); - /* Checks whether a path is valid. */ bool isValidPath(const Path & path); diff --git a/src/nix-instantiate/main.cc b/src/nix-instantiate/main.cc index aa6883ff8..f63789fc4 100644 --- a/src/nix-instantiate/main.cc +++ b/src/nix-instantiate/main.cc @@ -5,6 +5,7 @@ #include "normalise.hh" #include "shared.hh" #include "eval.hh" +#include "parser.hh" #if 0 @@ -29,9 +30,9 @@ static Path searchPath(const Paths & searchDirs, const Path & relPath) static Expr evalStdin(EvalState & state) { startNest(nest, lvlTalkative, format("evaluating standard input")); - Expr e = ATreadFromFile(stdin); - if (!e) - throw Error(format("unable to read a term from stdin")); + string s, s2; + while (getline(cin, s2)) s += s2 + "\n"; + Expr e = parseExprFromString(s, absPath(".")); return evalExpr(state, e); } diff --git a/substitute.mk b/substitute.mk index 8527cf6fd..c575dc0ff 100644 --- a/substitute.mk +++ b/substitute.mk @@ -1,9 +1,11 @@ %: %.in Makefile sed \ - -e s^@prefix\@^$(prefix)^g \ - -e s^@bindir\@^$(bindir)^g \ - -e s^@sysconfdir\@^$(sysconfdir)^g \ - -e s^@localstatedir\@^$(localstatedir)^g \ - -e s^@wget\@^$(wget)^g \ + -e "s^@prefix\@^$(prefix)^g" \ + -e "s^@bindir\@^$(bindir)^g" \ + -e "s^@sysconfdir\@^$(sysconfdir)^g" \ + -e "s^@localstatedir\@^$(localstatedir)^g" \ + -e "s^@datadir\@^$(datadir)^g" \ + -e "s^@host\@^$(host)^g" \ + -e "s^@wget\@^$(wget)^g" \ < $< > $@ || rm $@ chmod +x $@