* The policy-free derivate sharing now *almost* works. :-) For any

hash for which no local expansion is available, Nix can execute a
  `substitute' which should produce a path with such a hash.

  This is policy-free since Nix does not in any way specify how the
  substitute should work, i.e., it's an arbitrary (unnormalised)
  fstate expression.  For example, `nix-pull' registers substitutes
  that fetch Nix archives from the network (through `wget') and unpack
  them, but any other method is possible as well.  This is an
  improvement over the old Nix sharing scheme, which had a policy
  (fetching through `wget') built in.

  The sharing scheme doesn't work completely yet because successors
  from fstate rewriting have to be registered on the receiving side.
  Probably the whole successor stuff can be folded up into the
  substitute mechanism; this would be a nice simplification.
This commit is contained in:
Eelco Dolstra 2003-07-10 15:11:48 +00:00
parent d072485d28
commit 1d1c3691d2
13 changed files with 155 additions and 142 deletions

View file

@ -6,3 +6,5 @@ install-data-local:
$(INSTALL) -d $(datadir)/fix/nar $(INSTALL) -d $(datadir)/fix/nar
$(INSTALL_DATA) nar/nar.fix $(datadir)/fix/nar $(INSTALL_DATA) nar/nar.fix $(datadir)/fix/nar
$(INSTALL_DATA) nar/nar.sh $(datadir)/fix/nar $(INSTALL_DATA) nar/nar.sh $(datadir)/fix/nar
$(INSTALL_DATA) nar/unnar.fix $(datadir)/fix/nar
$(INSTALL_DATA) nar/unnar.sh $(datadir)/fix/nar

8
corepkgs/nar/unnar.fix Normal file
View file

@ -0,0 +1,8 @@
Function(["nar", "name"],
Package(
[ ("name", Var("name"))
, ("build", Relative("nar/unnar.sh"))
, ("nar", Var("nar"))
]
)
)

3
corepkgs/nar/unnar.sh Normal file
View file

@ -0,0 +1,3 @@
#! /bin/sh
/tmp/nix/bin/nix --restore "$out" < $nar || exit 1

View file

@ -5,5 +5,5 @@ install-exec-local:
$(INSTALL) -d $(sysconfdir)/profile.d $(INSTALL) -d $(sysconfdir)/profile.d
$(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh $(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh
$(INSTALL) -d $(sysconfdir)/nix $(INSTALL) -d $(sysconfdir)/nix
# !!! don't overwrite local modifications # !!! don't overwrite local modifications
$(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf $(INSTALL_DATA) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf

View file

@ -1,2 +1,67 @@
#! /usr/bin/perl -w #! /usr/bin/perl -w
my $prefix = $ENV{"NIX"} || "/tmp/nix"; # !!! use prefix
my $etcdir = "$prefix/etc/nix";
my $tmpfile = "$prefix/var/nix/pull.tmp";
my $conffile = "$etcdir/prebuilts.conf";
open CONFFILE, "<$conffile";
while (<CONFFILE>) {
chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1;
print "obtaining list of Nix archives at $url...\n";
system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
if ($?) { die "`wget' failed"; }
open INDEX, "<$tmpfile";
while (<INDEX>) {
# Get all links to prebuilts, that is, file names of the
# form foo-HASH-HASH.tar.bz2.
next unless (/HREF=\"([^\"]*)\"/);
my $fn = $1;
next if $fn =~ /\.\./;
next if $fn =~ /\//;
next unless $fn =~ /([0-9a-z]{32})-([0-9a-z]{32})\.nar/;
my $hash = $2;
print "registering $hash -> $url/$fn\n";
# Construct a Fix expression that fetches and unpacks a
# Nix archive from the network.
my $fetch =
"App(IncludeFix(\"fetchurl/fetchurl.fix\"), " .
"[(\"url\", \"$url/$fn\"), (\"hash\", \"\")])";
my $fixexpr =
"App(IncludeFix(\"nar/unnar.fix\"), " .
"[ (\"nar\", $fetch)" .
", (\"name\", \"fetched-$hash\")" .
"])";
my $fixfile = "/tmp/nix-pull-tmp.fix";
open FIX, ">$fixfile";
print FIX $fixexpr;
close FIX;
# Instantiate a Nix expression from the Fix expression.
my $nhash = `fix $fixfile`;
$? and die "instantiating Nix archive expression";
chomp $nhash;
die unless $nhash =~ /^([0-9a-z]{32})$/;
system "nix --substitute $hash $nhash";
if ($?) { die "`nix --substitute' failed"; }
}
close INDEX;
unlink $tmpfile;
}
}

View file

@ -1,83 +0,0 @@
#! /usr/bin/perl -w
my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
my $etcdir = "$prefix/etc/nix";
my $knowns = "$prefix/var/nix/known-prebuilts";
my $tmpfile = "$prefix/var/nix/prebuilts.tmp";
my $conffile = "$etcdir/prebuilts.conf";
umask 0022;
sub register {
my $fn = shift;
my $url = shift;
return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/;
my $id = $1;
my $pkghash = $2;
my $prebuilthash = $3;
print "$pkghash => $prebuilthash ($id)\n";
system "nix regprebuilt $pkghash $prebuilthash";
if ($?) { die "`nix regprebuilt' failed"; }
if ($url =~ /^\//) {
system "nix regfile $url";
if ($?) { die "`nix regfile' failed"; }
} else {
system "nix regurl $prebuilthash $url";
if ($?) { die "`nix regurl' failed"; }
}
print KNOWNS "$pkghash\n";
}
open KNOWNS, ">$knowns";
open CONFFILE, "<$conffile";
while (<CONFFILE>) {
chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1;
print "obtaining prebuilt list from $url...\n";
if ($url =~ /^\//) {
# It's a local path.
foreach my $fn (glob "$url/*") {
register($fn, $fn);
}
} else {
# It's a URL.
system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
if ($?) { die "`wget' failed"; }
open INDEX, "<$tmpfile";
while (<INDEX>) {
# Get all links to prebuilts, that is, file names of the
# form foo-HASH-HASH.tar.bz2.
next unless (/HREF=\"([^\"]*)\"/);
my $fn = $1;
next if $fn =~ /\.\./;
next if $fn =~ /\//;
register($fn, "$url/$fn");
}
close INDEX;
unlink $tmpfile;
}
}
}
close CONFFILE;
close KNOWNS;

View file

@ -1,44 +0,0 @@
#! /usr/bin/perl -w
my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
my $etcdir = "$prefix/etc/nix";
my $exportdir = "$prefix/var/nix/prebuilts/exports";
my $knowns = "$prefix/var/nix/known-prebuilts";
umask 0022;
# For performance, put the known hashes in an associative array.
my %knowns = ();
open KNOWNS, "<$knowns";
while (<KNOWNS>) {
next unless /([0-9a-z]{32})/;
$knowns{$1} = 1;
}
close KNOWNS;
# For each installed package, check whether a prebuilt is known.
open PKGS, "nix listinst|";
while (<PKGS>) {
chomp;
next unless /([0-9a-z]{32})/;
my $pkghash = $1;
if (!defined $knowns{$1}) {
# No known prebuilt exists for this package; so export it.
print "exporting $pkghash...\n";
system "nix export '$exportdir' $pkghash";
if ($?) { die "`nix export' failed"; }
}
}
close PKGS;
# Push the prebuilts to the server. !!! FIXME
system "rsync -av -e ssh '$exportdir'/ eelco\@losser.st-lab.cs.uu.nl:/home/eelco/public_html/nix-prebuilts/";
# Rerun `nix-pull-prebuilts' to rescan the prebuilt source locations.
print "running nix-pull-prebuilts...";
system "nix-pull-prebuilts";

View file

@ -1,4 +1,2 @@
# A list of URLs or local paths from where we obtain prebuilts. # A list of URLs from where we obtain Nix archives.
/nix/var/nix/prebuilts/imports http://losser.st-lab.cs.uu.nl/~eelco/nix-dist/
/nix/var/nix/prebuilts/exports
http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/

View file

@ -147,6 +147,12 @@ Hash hashTerm(ATerm t)
} }
FState hash2fstate(Hash hash)
{
return ATmake("Include(<str>)", ((string) hash).c_str());
}
ATerm termFromHash(const Hash & hash, string * p) ATerm termFromHash(const Hash & hash, string * p)
{ {
string path = expandHash(hash); string path = expandHash(hash);

View file

@ -85,6 +85,8 @@ Error badTerm(const format & f, ATerm t);
/* Hash an aterm. */ /* Hash an aterm. */
Hash hashTerm(ATerm t); Hash hashTerm(ATerm t);
FState hash2fstate(Hash hash);
/* Read an aterm from disk, given its hash. */ /* Read an aterm from disk, given its hash. */
ATerm termFromHash(const Hash & hash, string * p = 0); ATerm termFromHash(const Hash & hash, string * p = 0);

View file

@ -26,6 +26,8 @@ static ArgType argType = atpUnknown;
--add / -A: copy a path to the Nix store --add / -A: copy a path to the Nix store
--query / -q: query information --query / -q: query information
--substitute: register a substitute expression
--dump: dump a path as a Nix archive --dump: dump a path as a Nix archive
--restore: restore a path from a Nix archive --restore: restore a path from a Nix archive
@ -87,12 +89,6 @@ static Hash argToHash(const string & arg)
} }
static FState hash2fstate(Hash hash)
{
return ATmake("Include(<str>)", ((string) hash).c_str());
}
/* Realise (or install) paths from the given Nix fstate /* Realise (or install) paths from the given Nix fstate
expressions. */ expressions. */
static void opInstall(Strings opFlags, Strings opArgs) static void opInstall(Strings opFlags, Strings opArgs)
@ -187,6 +183,21 @@ static void opQuery(Strings opFlags, Strings opArgs)
} }
static void opSubstitute(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
if (opArgs.size() % 2) throw UsageError("expecting even number of arguments");
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); )
{
Hash srcHash = parseHash(*i++);
Hash subHash = parseHash(*i++);
registerSubstitute(srcHash, subHash);
}
}
/* A sink that writes dump output to stdout. */ /* A sink that writes dump output to stdout. */
struct StdoutSink : DumpSink struct StdoutSink : DumpSink
{ {
@ -277,6 +288,8 @@ void run(Strings args)
op = opAdd; op = opAdd;
else if (arg == "--query" || arg == "-q") else if (arg == "--query" || arg == "-q")
op = opQuery; op = opQuery;
else if (arg == "--substitute")
op = opSubstitute;
else if (arg == "--dump") else if (arg == "--dump")
op = opDump; op = opDump;
else if (arg == "--restore") else if (arg == "--restore")

View file

@ -7,6 +7,7 @@
#include "globals.hh" #include "globals.hh"
#include "db.hh" #include "db.hh"
#include "archive.hh" #include "archive.hh"
#include "fstate.hh"
struct CopySink : DumpSink struct CopySink : DumpSink
@ -83,6 +84,20 @@ void copyPath(string src, string dst)
} }
void registerSubstitute(const Hash & srcHash, const Hash & subHash)
{
Strings subs;
queryListDB(nixDB, dbSubstitutes, srcHash, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++)
if (parseHash(*it) == subHash) return;
subs.push_back(subHash);
setListDB(nixDB, dbSubstitutes, srcHash, subs);
}
Hash registerPath(const string & _path, Hash hash) Hash registerPath(const string & _path, Hash hash)
{ {
string path(canonPath(_path)); string path(canonPath(_path));
@ -139,8 +154,7 @@ string expandHash(const Hash & hash, const string & target,
if (!target.empty() && !isInPrefix(target, prefix)) if (!target.empty() && !isInPrefix(target, prefix))
abort(); abort();
if (!queryListDB(nixDB, dbHash2Paths, hash, paths)) queryListDB(nixDB, dbHash2Paths, hash, paths);
throw Error(format("no paths known with hash `%1%'") % (string) hash);
/* !!! we shouldn't check for staleness by default --- too slow */ /* !!! we shouldn't check for staleness by default --- too slow */
@ -182,7 +196,31 @@ string expandHash(const Hash & hash, const string & target,
} }
} }
throw Error(format("all paths with hash `%1%' are stale") % (string) hash); /* Try to realise the substitutes. */
Strings subs;
queryListDB(nixDB, dbSubstitutes, hash, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
StringSet dummy;
FState nf = realiseFState(hash2fstate(parseHash(*it)), dummy);
string path = fstatePath(nf);
if (hashPath(path) != hash)
throw Error(format("bad substitute in `%1%'") % (string) path);
if (target.empty())
return path; /* !!! prefix */
else {
if (path != target) {
copyPath(path, target);
registerPath(target, hash);
}
return target;
}
}
throw Error(format("cannot expand hash `%1%'") % (string) hash);
} }
@ -193,6 +231,7 @@ void addToStore(string srcPath, string & dstPath, Hash & hash)
hash = hashPath(srcPath); hash = hashPath(srcPath);
try { try {
/* !!! should not use the substitutes! */
dstPath = expandHash(hash, "", nixStore); dstPath = expandHash(hash, "", nixStore);
return; return;
} catch (...) { } catch (...) {

View file

@ -8,8 +8,12 @@
using namespace std; using namespace std;
/* Copy a path recursively. */
void copyPath(string src, string dst); void copyPath(string src, string dst);
/* Register a substitute. */
void registerSubstitute(const Hash & srcHash, const Hash & subHash);
/* Register a path keyed on its hash. */ /* Register a path keyed on its hash. */
Hash registerPath(const string & path, Hash hash = Hash()); Hash registerPath(const string & path, Hash hash = Hash());