forked from lix-project/lix
* Prebuilt package sharing. We allow transparent binary deployment by
sharing package directories (i.e., the result of building a Nix descriptor). `nix-pull-prebuilts' obtains a list of all known prebuilts by consulting the paths and URLs specified in $prefix/etc/nix/prebuilts.conf. The mappings ($pkghash, $prebuilthash) and ($prebuilthash, $location) are registered with Nix so that it can use the prebuilt with hash $prebuilthash when installing a package with hash $pkghash by downloading and unpacking $location. `nix-push-prebuilts' creates prebuilts for all packages for which no prebuilt is known to exist. It can then optionally upload these to the network through rsync. `nix-[pull|push]-prebuilts' just provide a policy. Nix provides the mechanism through the `nix [export|regprebuilt|regurl]' commands.
This commit is contained in:
Eelco Dolstra
parent
0ef4b6d0f8
commit
7dd91d3779
|
|
@ -1,5 +1,9 @@
|
|||
bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage
|
||||
bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage \
|
||||
nix-pull-prebuilts nix-push-prebuilts
|
||||
|
||||
install-exec-local:
|
||||
$(INSTALL) -d $(sysconfdir)/profile.d
|
||||
$(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh
|
||||
$(INSTALL) -d $(sysconfdir)/nix
|
||||
# !!! don't overwrite local modifications
|
||||
$(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
#! /usr/bin/perl -w
|
||||
|
||||
my $dir = shift @ARGV;
|
||||
$dir || die "missing directory";
|
||||
my $url = shift @ARGV;
|
||||
$url || die "missing base url";
|
||||
|
||||
chdir $dir || die "cannot chdir to $dir";
|
||||
|
||||
foreach my $prebuilt (glob("*.tar.bz2")) {
|
||||
|
||||
$prebuilt =~ /-([a-z0-9]+)-([a-z0-9]+).tar.bz2$/
|
||||
|| die "invalid file name: $prebuilt";
|
||||
|
||||
my $pkgHash = $1;
|
||||
my $prebuiltHash = $2;
|
||||
|
||||
print "regprebuilt $pkgHash $prebuiltHash\n";
|
||||
print "regurl $prebuiltHash $url/$prebuilt\n";
|
||||
}
|
||||
69
scripts/nix-pull-prebuilts
Executable file
69
scripts/nix-pull-prebuilts
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
#! /usr/bin/perl -w
|
||||
|
||||
my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
|
||||
my $etcdir = "$prefix/etc/nix";
|
||||
my $knowns = "$prefix/var/nix/known-prebuilts";
|
||||
my $tmpfile = "$prefix/var/nix/prebuilts.tmp";
|
||||
|
||||
my $conffile = "$etcdir/prebuilts.conf";
|
||||
|
||||
sub register {
|
||||
my $fn = shift;
|
||||
return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/;
|
||||
my $id = $1;
|
||||
my $pkghash = $2;
|
||||
my $prebuilthash = $3;
|
||||
print "$pkghash => $prebuilthash ($id)\n";
|
||||
system "nix regprebuilt $pkghash $prebuilthash";
|
||||
if ($?) { die "`nix regprebuilt' failed"; }
|
||||
print KNOWNS "$pkghash\n";
|
||||
}
|
||||
|
||||
open KNOWNS, ">$knowns";
|
||||
|
||||
open CONFFILE, "<$conffile";
|
||||
|
||||
while (<CONFFILE>) {
|
||||
chomp;
|
||||
if (/^\s*(\S+)\s*(\#.*)?$/) {
|
||||
my $url = $1;
|
||||
|
||||
print "obtaining prebuilt list from $url...\n";
|
||||
|
||||
if ($url =~ /^\//) {
|
||||
|
||||
# It's a local path.
|
||||
|
||||
foreach my $fn (glob "$url/*") {
|
||||
register $fn;
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
# It's a URL.
|
||||
|
||||
system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
|
||||
if ($?) { die "`wget' failed"; }
|
||||
|
||||
open INDEX, "<$tmpfile";
|
||||
|
||||
while (<INDEX>) {
|
||||
# Get all links to prebuilts, that is, file names of the
|
||||
# form foo-HASH-HASH.tar.bz2.
|
||||
next unless (/HREF=\"([^\"]*)\"/);
|
||||
my $fn = $1;
|
||||
next if $fn =~ /\.\./;
|
||||
next if $fn =~ /\//;
|
||||
register $fn;
|
||||
}
|
||||
|
||||
close INDEX;
|
||||
|
||||
unlink $tmpfile;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
close CONFFILE;
|
||||
|
||||
close KNOWNS;
|
||||
40
scripts/nix-push-prebuilts
Executable file
40
scripts/nix-push-prebuilts
Executable file
|
|
@ -0,0 +1,40 @@
|
|||
#! /usr/bin/perl -w
|
||||
|
||||
my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
|
||||
my $etcdir = "$prefix/etc/nix";
|
||||
my $exportdir = "$prefix/var/nix/prebuilts/exports";
|
||||
my $knowns = "$prefix/var/nix/known-prebuilts";
|
||||
|
||||
# For performance, put the known hashes in an associative array.
|
||||
my %knowns = ();
|
||||
open KNOWNS, "<$knowns";
|
||||
while (<KNOWNS>) {
|
||||
next unless /([0-9a-z]{32})/;
|
||||
$knowns{$1} = 1;
|
||||
}
|
||||
close KNOWNS;
|
||||
|
||||
# For each installed package, check whether a prebuilt is known.
|
||||
|
||||
open PKGS, "nix listinst|";
|
||||
open KNOWNS, ">>$knowns";
|
||||
|
||||
while (<PKGS>) {
|
||||
chomp;
|
||||
next unless /([0-9a-z]{32})/;
|
||||
my $pkghash = $1;
|
||||
if (!defined $knowns{$1}) {
|
||||
# No known prebuilt exists for this package; so export it.
|
||||
print "exporting $pkghash...\n";
|
||||
system "nix export '$exportdir' $pkghash";
|
||||
if ($?) { die "`nix export' failed"; }
|
||||
print KNOWNS "$pkghash\n";
|
||||
}
|
||||
}
|
||||
|
||||
close KNOWNS;
|
||||
close PKGS;
|
||||
|
||||
# Push the prebuilts to the server. !!! FIXME
|
||||
|
||||
system "rsync -av -e ssh '$exportdir' losser:/home/eelco/public_html/nix-prebuilts/";
|
||||
4
scripts/prebuilts.conf
Normal file
4
scripts/prebuilts.conf
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# A list of URLs or local paths from where we obtain prebuilts.
|
||||
/nix/var/nix/prebuilts/imports
|
||||
/nix/var/nix/prebuilts/exports
|
||||
http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/
|
||||
|
|
@ -13,5 +13,8 @@ install-data-local:
|
|||
$(INSTALL) -d $(localstatedir)/nix/descriptors
|
||||
$(INSTALL) -d $(localstatedir)/nix/sources
|
||||
$(INSTALL) -d $(localstatedir)/nix/links
|
||||
$(INSTALL) -d $(localstatedir)/nix/prebuilts
|
||||
$(INSTALL) -d $(localstatedir)/nix/prebuilts/imports
|
||||
$(INSTALL) -d $(localstatedir)/nix/prebuilts/exports
|
||||
$(INSTALL) -d $(prefix)/pkg
|
||||
$(bindir)/nix init
|
||||
|
|
|
|||
69
src/fix.cc
69
src/fix.cc
|
|
@ -13,7 +13,6 @@ extern "C" {
|
|||
|
||||
|
||||
static string nixDescriptorDir;
|
||||
static string nixSourcesDir;
|
||||
|
||||
|
||||
static bool verbose = false;
|
||||
|
|
@ -33,46 +32,6 @@ void registerFile(string filename)
|
|||
throw Error("cannot register " + filename + " with Nix");
|
||||
}
|
||||
|
||||
|
||||
/* Return the directory part of the given path, i.e., everything
|
||||
before the final `/'. */
|
||||
string dirOf(string s)
|
||||
{
|
||||
unsigned int pos = s.rfind('/');
|
||||
if (pos == string::npos) throw Error("invalid file name");
|
||||
return string(s, 0, pos);
|
||||
}
|
||||
|
||||
|
||||
/* Return the base name of the given path, i.e., everything following
|
||||
the final `/'. */
|
||||
string baseNameOf(string s)
|
||||
{
|
||||
unsigned int pos = s.rfind('/');
|
||||
if (pos == string::npos) throw Error("invalid file name");
|
||||
return string(s, pos + 1);
|
||||
}
|
||||
|
||||
|
||||
/* Download object referenced by the given URL into the sources
|
||||
directory. Return the file name it was downloaded to. */
|
||||
string fetchURL(string url)
|
||||
{
|
||||
string filename = baseNameOf(url);
|
||||
string fullname = nixSourcesDir + "/" + filename;
|
||||
struct stat st;
|
||||
if (stat(fullname.c_str(), &st)) {
|
||||
/* !!! quoting */
|
||||
string shellCmd =
|
||||
"cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\"";
|
||||
int res = system(shellCmd.c_str());
|
||||
if (WEXITSTATUS(res) != 0)
|
||||
throw Error("cannot fetch " + url);
|
||||
}
|
||||
return fullname;
|
||||
}
|
||||
|
||||
|
||||
Error badTerm(const string & msg, ATerm e)
|
||||
{
|
||||
char * s = ATwriteToString(e);
|
||||
|
|
@ -120,7 +79,7 @@ bool evaluateBool(ATerm e, EvalContext ctx)
|
|||
ATerm evaluate(ATerm e, EvalContext ctx)
|
||||
{
|
||||
char * s;
|
||||
ATerm e2;
|
||||
ATerm e2, e3;
|
||||
ATerm eCond, eTrue, eFalse;
|
||||
|
||||
/* Check for normal forms first. */
|
||||
|
|
@ -166,6 +125,7 @@ ATerm evaluate(ATerm e, EvalContext ctx)
|
|||
instantiateDescriptor(filename, ctx).c_str());
|
||||
}
|
||||
|
||||
#if 0
|
||||
/* `Source' copies the specified file to nixSourcesDir, registers
|
||||
it with Nix, and returns the hash of the file. */
|
||||
else if (ATmatch(e, "Source(<term>)", &e2)) {
|
||||
|
|
@ -185,15 +145,29 @@ ATerm evaluate(ATerm e, EvalContext ctx)
|
|||
registerFile(target);
|
||||
return ATmake("File(<str>)", hashFile(target).c_str());
|
||||
}
|
||||
#endif
|
||||
|
||||
/* `Url' fetches a file from the network, caching it in
|
||||
nixSourcesDir and returning the file name. */
|
||||
else if (ATmatch(e, "Url(<term>)", &e2)) {
|
||||
string url = evaluateStr(e2, ctx);
|
||||
/* `Local' registers a file with Nix, and returns the file's
|
||||
hash. */
|
||||
else if (ATmatch(e, "Local(<term>)", &e2)) {
|
||||
string filename = absPath(evaluateStr(e2, ctx), ctx.dir); /* !!! */
|
||||
string hash = hashFile(filename);
|
||||
return ATmake("File(<str>)", hash.c_str());
|
||||
}
|
||||
|
||||
/* `Url' registers a mapping from a hash to an url with Nix, and
|
||||
returns the hash. */
|
||||
else if (ATmatch(e, "Url(<term>, <term>)", &e2, &e3)) {
|
||||
string hash = evaluateStr(e2, ctx);
|
||||
checkHash(hash);
|
||||
string url = evaluateStr(e3, ctx);
|
||||
#if 0
|
||||
if (verbose)
|
||||
cerr << "fetching " << url << endl;
|
||||
string filename = fetchURL(url);
|
||||
return ATmake("Str(<str>)", filename.c_str());
|
||||
#endif
|
||||
/* !!! register */
|
||||
return ATmake("File(<str>)", hash.c_str());
|
||||
}
|
||||
|
||||
/* `If' provides conditional evaluation. */
|
||||
|
|
@ -329,7 +303,6 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
|
|||
if (homeDir) nixHomeDir = homeDir;
|
||||
|
||||
nixDescriptorDir = nixHomeDir + "/var/nix/descriptors";
|
||||
nixSourcesDir = nixHomeDir + "/var/nix/sources";
|
||||
|
||||
for ( ; argCur != argEnd; argCur++) {
|
||||
string arg(*argCur);
|
||||
|
|
|
|||
62
src/nix.cc
62
src/nix.cc
|
|
@ -28,6 +28,9 @@ static string dbInstPkgs = "pkginst";
|
|||
static string dbPrebuilts = "prebuilts";
|
||||
|
||||
|
||||
static string nixSourcesDir;
|
||||
|
||||
|
||||
/* Wrapper classes that ensures that the database is closed upon
|
||||
object destruction. */
|
||||
class Db2 : public Db
|
||||
|
|
@ -435,9 +438,9 @@ void exportPkgs(string outDir,
|
|||
string pkgDir = getPkg(hash);
|
||||
string tmpFile = outDir + "/export_tmp";
|
||||
|
||||
string cmd = "cd " + pkgDir + " && tar cvfj " + tmpFile + " .";
|
||||
string cmd = "cd " + pkgDir + " && tar cfj " + tmpFile + " .";
|
||||
int res = system(cmd.c_str()); // !!! escaping
|
||||
if (WEXITSTATUS(res) != 0)
|
||||
if (!WIFEXITED(res) || WEXITSTATUS(res) != 0)
|
||||
throw Error("cannot tar " + pkgDir);
|
||||
|
||||
string prebuiltHash = hashFile(tmpFile);
|
||||
|
|
@ -458,10 +461,12 @@ void regPrebuilt(string pkgHash, string prebuiltHash)
|
|||
}
|
||||
|
||||
|
||||
void registerFile(string filename)
|
||||
string registerFile(string filename)
|
||||
{
|
||||
filename = absPath(filename);
|
||||
setDB(dbRefs, hashFile(filename), filename);
|
||||
string hash = hashFile(filename);
|
||||
setDB(dbRefs, hash, filename);
|
||||
return hash;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -618,8 +623,46 @@ void printGraph(Strings::iterator first, Strings::iterator last)
|
|||
}
|
||||
|
||||
|
||||
void run(Strings args)
|
||||
/* Download object referenced by the given URL into the sources
|
||||
directory. Return the file name it was downloaded to. */
|
||||
string fetchURL(string url)
|
||||
{
|
||||
string filename = baseNameOf(url);
|
||||
string fullname = nixSourcesDir + "/" + filename;
|
||||
struct stat st;
|
||||
if (stat(fullname.c_str(), &st)) {
|
||||
/* !!! quoting */
|
||||
string shellCmd =
|
||||
"cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\"";
|
||||
int res = system(shellCmd.c_str());
|
||||
if (WEXITSTATUS(res) != 0)
|
||||
throw Error("cannot fetch " + url);
|
||||
}
|
||||
return fullname;
|
||||
}
|
||||
|
||||
|
||||
void fetch(string id)
|
||||
{
|
||||
string fn;
|
||||
|
||||
/* Fetch the object referenced by id. */
|
||||
if (isHash(id)) {
|
||||
throw Error("not implemented");
|
||||
} else {
|
||||
fn = fetchURL(id);
|
||||
}
|
||||
|
||||
/* Register it by hash. */
|
||||
string hash = registerFile(fn);
|
||||
cout << hash << endl;
|
||||
}
|
||||
|
||||
|
||||
void fetch(Strings::iterator first, Strings::iterator last)
|
||||
{
|
||||
for (Strings::iterator it = first; it != last; it++)
|
||||
fetch(*it);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -675,6 +718,11 @@ Subcommands:
|
|||
|
||||
graph HASH...
|
||||
Like closure, but print a dot graph specification.
|
||||
|
||||
fetch ID...
|
||||
Fetch the objects identified by ID and place them in the Nix
|
||||
sources directory. ID can be a hash or URL. Print out the hash
|
||||
of the object.
|
||||
";
|
||||
}
|
||||
|
||||
|
|
@ -686,6 +734,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
|
|||
char * homeDir = getenv(nixHomeDirEnvVar.c_str());
|
||||
if (homeDir) nixHomeDir = homeDir;
|
||||
|
||||
nixSourcesDir = nixHomeDir + "/var/nix/sources";
|
||||
|
||||
/* Parse the global flags. */
|
||||
for ( ; argCur != argEnd; argCur++) {
|
||||
string arg(*argCur);
|
||||
|
|
@ -742,6 +792,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
|
|||
printClosure(argCur, argEnd);
|
||||
} else if (cmd == "graph") {
|
||||
printGraph(argCur, argEnd);
|
||||
} else if (cmd == "fetch") {
|
||||
fetch(argCur, argEnd);
|
||||
} else
|
||||
throw UsageError("unknown command: " + string(cmd));
|
||||
}
|
||||
|
|
|
|||
36
src/util.hh
36
src/util.hh
|
|
@ -85,17 +85,22 @@ string printHash(unsigned char * buf)
|
|||
|
||||
|
||||
/* Verify that a reference is valid (that is, is a MD5 hash code). */
|
||||
void checkHash(const string & s)
|
||||
bool isHash(const string & s)
|
||||
{
|
||||
string err = "invalid reference: " + s;
|
||||
if (s.length() != 32)
|
||||
throw BadRefError(err);
|
||||
if (s.length() != 32) return false;
|
||||
for (int i = 0; i < 32; i++) {
|
||||
char c = s[i];
|
||||
if (!((c >= '0' && c <= '9') ||
|
||||
(c >= 'a' && c <= 'f')))
|
||||
throw BadRefError(err);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void checkHash(const string & s)
|
||||
{
|
||||
if (!isHash(s)) throw BadRefError("invalid reference: " + s);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -113,4 +118,25 @@ string hashFile(string filename)
|
|||
}
|
||||
|
||||
|
||||
|
||||
/* Return the directory part of the given path, i.e., everything
|
||||
before the final `/'. */
|
||||
string dirOf(string s)
|
||||
{
|
||||
unsigned int pos = s.rfind('/');
|
||||
if (pos == string::npos) throw Error("invalid file name");
|
||||
return string(s, 0, pos);
|
||||
}
|
||||
|
||||
|
||||
/* Return the base name of the given path, i.e., everything following
|
||||
the final `/'. */
|
||||
string baseNameOf(string s)
|
||||
{
|
||||
unsigned int pos = s.rfind('/');
|
||||
if (pos == string::npos) throw Error("invalid file name");
|
||||
return string(s, pos + 1);
|
||||
}
|
||||
|
||||
|
||||
#endif /* !__UTIL_H */
|
||||
|
|
|
|||
Loading…
Reference in a new issue