lix-project/lix
20
57
Fork 31
Code Issues 344 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge branch 'nix-channel-c++'

Browse source
This commit is contained in:
Shea Levy 2016-08-31 09:49:24 -04:00
parent aa1ea0d1e4 59124228b3
commit 572aba284a
8 changed files with 304 additions and 233 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -37,7 +37,6 @@ Makefile.config
/scripts/nix-switch
/scripts/nix-collect-garbage
/scripts/nix-prefetch-url
/scripts/nix-channel
/scripts/nix-build
/scripts/nix-copy-closure
/scripts/NixConfig.pm
@ -72,6 +71,9 @@ Makefile.config
# /src/nix-daemon/
/src/nix-daemon/nix-daemon
# /src/nix-channel/
/src/nix-channel/nix-channel
# /src/download-via-ssh/
/src/download-via-ssh/download-via-ssh

1
Makefile
View file

@ -14,6 +14,7 @@ makefiles = \
src/nix-prefetch-url/local.mk \
src/buildenv/local.mk \
src/resolve-system-dependencies/local.mk \
src/nix-channel/local.mk \
perl/local.mk \
scripts/local.mk \
corepkgs/local.mk \

1
scripts/local.mk
View file

@ -1,6 +1,5 @@
nix_bin_scripts := \
$(d)/nix-build \
$(d)/nix-channel \
$(d)/nix-copy-closure \
bin-scripts += $(nix_bin_scripts)

228
scripts/nix-channel.in
View file

@ -1,228 +0,0 @@
#! @perl@ -w @perlFlags@
use utf8;
use strict;
use File::Basename;
use File::Path qw(mkpath);
use Nix::Config;
use Nix::Manifest;
use File::Temp qw(tempdir);
binmode STDERR, ":encoding(utf8)";
Nix::Config::readConfig;
# Turn on caching in nix-prefetch-url.
my $channelCache = "$Nix::Config::stateDir/channel-cache";
mkdir $channelCache, 0755 unless -e $channelCache;
$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache if -W $channelCache;
# Figure out the name of the `.nix-channels' file to use.
my $home = $ENV{"HOME"} or die '$HOME not set\n';
my $channelsList = "$home/.nix-channels";
my $nixDefExpr = "$home/.nix-defexpr";
# Figure out the name of the channels profile.
my $userName = getpwuid($<) || $ENV{"USER"} or die "cannot figure out user name";
my $profile = "$Nix::Config::stateDir/profiles/per-user/$userName/channels";
mkpath(dirname $profile, 0, 0755);
my %channels;
# Reads the list of channels.
sub readChannels {
return if (!-f $channelsList);
open CHANNELS, "<$channelsList" or die "cannot open ‘$channelsList’: $!";
while (<CHANNELS>) {
chomp;
next if /^\s*\#/;
my ($url, $name) = split ' ', $_;
$url =~ s/\/*$//; # remove trailing slashes
$name = basename $url unless defined $name;
$channels{$name} = $url;
}
close CHANNELS;
}
# Writes the list of channels.
sub writeChannels {
open CHANNELS, ">$channelsList" or die "cannot open ‘$channelsList’: $!";
foreach my $name (keys %channels) {
print CHANNELS "$channels{$name} $name\n";
}
close CHANNELS;
}
# Adds a channel.
sub addChannel {
my ($url, $name) = @_;
die "invalid channel URL ‘$url’" unless $url =~ /^(file|http|https):\/\//;
die "invalid channel identifier ‘$name’" unless $name =~ /^[a-zA-Z0-9_][a-zA-Z0-9_\-\.]*$/;
readChannels;
$channels{$name} = $url;
writeChannels;
}
# Remove a channel.
sub removeChannel {
my ($name) = @_;
readChannels;
my $url = $channels{$name};
delete $channels{$name};
writeChannels;
system("$Nix::Config::binDir/nix-env --profile '$profile' -e '$name'") == 0
or die "cannot remove channel ‘$name’\n";
}
# Fetch Nix expressions and binary cache URLs from the subscribed channels.
sub update {
my @channelNames = @_;
readChannels;
# Download each channel.
my $exprs = "";
foreach my $name (keys %channels) {
next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames};
my $url = $channels{$name};
# We want to download the url to a file to see if it's a tarball while also checking if we
# got redirected in the process, so that we can grab the various parts of a nix channel
# definition from a consistent location if the redirect changes mid-download.
my $tmpdir = tempdir( CLEANUP => 1 );
my $filename;
($url, $filename) = `cd $tmpdir && $Nix::Config::curl --silent --write-out '%{url_effective}\n%{filename_effective}' -L '$url' -O`;
chomp $url;
die "$0: unable to check ‘$url’\n" if $? != 0;
# If the URL contains a version number, append it to the name
# attribute (so that "nix-env -q" on the channels profile
# shows something useful).
my $cname = $name;
$cname .= $1 if basename($url) =~ /(-\d.*)$/;
my $path;
my $ret = -1;
if (-e "$tmpdir/$filename" && $filename =~ /\.tar\.(gz|bz2|xz)$/) {
# Get our temporary download into the store.
(my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url 'file://$tmpdir/$filename'`;
chomp $path;
# Try unpacking the expressions to see if they'll be valid for us to process later.
# Like anything in nix, this will cache the result so we don't do it again outside of the loop below.
$ret = system("$Nix::Config::binDir/nix-build --no-out-link -E 'import <nix/unpack-channel.nix> " .
"{ name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }'");
}
# The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
my $extraAttrs = "";
if ($ret != 0) {
# Check if the channel advertises a binary cache.
my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
$extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "
if $? == 0 && $binaryCacheURL ne "";
# Download the channel tarball.
my $fullURL = "$url/nixexprs.tar.xz";
system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or
$fullURL = "$url/nixexprs.tar.bz2";
print STDERR "downloading Nix expressions from ‘$fullURL’...\n";
(my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`;
die "cannot fetch ‘$fullURL’\n" if $? != 0;
chomp $path;
}
# Regardless of where it came from, add the expression representing this channel to accumulated expression
$exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' ";
}
# Unpack the channel tarballs into the Nix store and install them
# into the channels profile.
print STDERR "unpacking channels...\n";
system("$Nix::Config::binDir/nix-env --profile '$profile' " .
"-f '<nix/unpack-channel.nix>' -i -E $exprs --quiet") == 0
or die "cannot unpack the channels";
# Make the channels appear in nix-env.
unlink $nixDefExpr if -l $nixDefExpr; # old-skool ~/.nix-defexpr
mkdir $nixDefExpr or die "cannot create directory ‘$nixDefExpr’" if !-e $nixDefExpr;
my $channelLink = "$nixDefExpr/channels";
unlink $channelLink; # !!! not atomic
symlink($profile, $channelLink) or die "cannot symlink ‘$channelLink’ to ‘$profile’";
}
die "$0: argument expected\n" if scalar @ARGV == 0;
while (scalar @ARGV) {
my $arg = shift @ARGV;
if ($arg eq "--add") {
die "$0: ‘--add’ requires one or two arguments\n" if scalar @ARGV < 1 || scalar @ARGV > 2;
my $url = shift @ARGV;
my $name = shift @ARGV;
unless (defined $name) {
$name = basename $url;
$name =~ s/-unstable//;
$name =~ s/-stable//;
}
addChannel($url, $name);
last;
}
if ($arg eq "--remove") {
die "$0: ‘--remove’ requires one argument\n" if scalar @ARGV != 1;
removeChannel(shift @ARGV);
last;
}
if ($arg eq "--list") {
die "$0: ‘--list’ requires one argument\n" if scalar @ARGV != 0;
readChannels;
foreach my $name (keys %channels) {
print "$name $channels{$name}\n";
}
last;
}
elsif ($arg eq "--update") {
update(@ARGV);
last;
}
elsif ($arg eq "--rollback") {
die "$0: ‘--rollback’ has at most one argument\n" if scalar @ARGV > 1;
my $generation = shift @ARGV;
my @args = ("$Nix::Config::binDir/nix-env", "--profile", $profile);
if (defined $generation) {
die "invalid channel generation number ‘$generation’" unless $generation =~ /^[0-9]+$/;
push @args, "--switch-generation", $generation;
} else {
push @args, "--rollback";
}
system(@args) == 0 or exit 1;
last;
}
elsif ($arg eq "--help") {
exec "man nix-channel" or die;
}
elsif ($arg eq "--version") {
print "nix-channel (Nix) $Nix::Config::version\n";
exit 0;
}
else {
die "unknown argument ‘$arg’; try ‘--help’\n";
}
}

20
src/libstore/download.cc
View file

@ -32,7 +32,7 @@ struct CurlDownloader : public Downloader
{
CURL * curl;
ref<std::string> data;
string etag, status, expectedETag;
string etag, status, expectedETag, effectiveUrl;
struct curl_slist * requestHeaders;
@ -208,6 +208,11 @@ struct CurlDownloader : public Downloader
% url % curl_easy_strerror(res) % res);
}
char *effectiveUrlCStr;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr);
if (effectiveUrlCStr)
effectiveUrl = effectiveUrlCStr;
if (httpStatus == 304) return false;
return true;
@ -225,6 +230,7 @@ struct CurlDownloader : public Downloader
res.data = data;
} else
res.cached = true;
res.effectiveUrl = effectiveUrl;
res.etag = etag;
return res;
} catch (DownloadError & e) {
@ -244,6 +250,12 @@ ref<Downloader> makeDownloader()
}
Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash)
{
string ignored;
return downloadCached(store, url_, unpack, ignored, expectedHash);
}
Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string & effectiveUrl, const Hash & expectedHash)
{
auto url = resolveUri(url_);
@ -281,9 +293,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked;
if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) {
skip = true;
else if (!ss[1].empty()) {
effectiveUrl = url_;
} else if (!ss[1].empty()) {
printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
expectedETag = ss[1];
}
@ -298,6 +311,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
DownloadOptions options;
options.expectedETag = expectedETag;
auto res = download(url, options);
effectiveUrl = res.effectiveUrl;
if (!res.cached) {
ValidPathInfo info;

6
src/libstore/download.hh
View file

@ -21,6 +21,7 @@ struct DownloadResult
{
bool cached;
string etag;
string effectiveUrl;
std::shared_ptr<std::string> data;
};
@ -33,6 +34,11 @@ struct Downloader
Path downloadCached(ref<Store> store, const string & url, bool unpack, string name = "",
const Hash & expectedHash = Hash());
/* Need to overload because can't have an rvalue default value for non-const reference */
Path downloadCached(ref<Store> store, const string & url, bool unpack,
string & effectiveUrl, const Hash & expectedHash = Hash());
enum Error { NotFound, Forbidden, Misc, Transient };
};

7
src/nix-channel/local.mk Normal file
View file

@ -0,0 +1,7 @@
programs += nix-channel
nix-channel_DIR := $(d)
nix-channel_LIBS = libmain libutil libformat libstore
nix-channel_SOURCES := $(d)/nix-channel.cc

270
src/nix-channel/nix-channel.cc Executable file
View file

@ -0,0 +1,270 @@
#include "shared.hh"
#include "globals.hh"
#include "download.hh"
#include <fcntl.h>
#include <regex>
#include "store-api.hh"
#include <pwd.h>
using namespace nix;
typedef std::map<string,string> Channels;
static auto channels = Channels{};
static auto channelsList = Path{};
// Reads the list of channels.
static void readChannels()
{
if (!pathExists(channelsList)) return;
auto channelsFile = readFile(channelsList);
for (const auto & line : tokenizeString<std::vector<string>>(channelsFile, "\n")) {
chomp(line);
if (std::regex_search(line, std::regex("^\\s*\\#")))
continue;
auto split = tokenizeString<std::vector<string>>(line, " ");
auto url = std::regex_replace(split[0], std::regex("/*$"), "");
auto name = split.size() > 1 ? split[1] : baseNameOf(url);
channels[name] = url;
}
}
// Writes the list of channels.
static void writeChannels()
{
auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)};
if (!channelsFD)
throw SysError(format("opening ‘%1%’ for writing") % channelsList);
for (const auto & channel : channels)
writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n");
}
// Adds a channel.
static void addChannel(const string & url, const string & name)
{
if (!regex_search(url, std::regex("^(file|http|https)://")))
throw Error(format("invalid channel URL ‘%1%’") % url);
if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$")))
throw Error(format("invalid channel identifier ‘%1%’") % name);
readChannels();
channels[name] = url;
writeChannels();
}
static auto profile = Path{};
// Remove a channel.
static void removeChannel(const string & name)
{
readChannels();
channels.erase(name);
writeChannels();
runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name });
}
static auto nixDefExpr = Path{};
// Fetch Nix expressions and binary cache URLs from the subscribed channels.
static void update(const StringSet & channelNames)
{
readChannels();
auto store = openStore();
// Download each channel.
auto exprs = Strings{};
for (const auto & channel : channels) {
if (!channelNames.empty() && channelNames.find(channel.first) != channelNames.end())
continue;
auto name = channel.first;
auto url = channel.second;
// We want to download the url to a file to see if it's a tarball while also checking if we
// got redirected in the process, so that we can grab the various parts of a nix channel
// definition from a consistent location if the redirect changes mid-download.
auto effectiveUrl = string{};
auto dl = makeDownloader();
auto filename = dl->downloadCached(store, url, false, effectiveUrl);
url = chomp(std::move(effectiveUrl));
// If the URL contains a version number, append it to the name
// attribute (so that "nix-env -q" on the channels profile
// shows something useful).
auto cname = name;
std::smatch match;
auto urlBase = baseNameOf(url);
if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) {
cname = cname + (string) match[1];
}
auto extraAttrs = string{};
auto unpacked = false;
if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
try {
runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
"{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
unpacked = true;
} catch (ExecError & e) {
}
}
if (!unpacked) {
// The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
// Check if the channel advertises a binary cache.
DownloadOptions opts;
opts.showProgress = DownloadOptions::no;
try {
auto dlRes = dl->download(url + "/binary-cache-url", opts);
extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";";
} catch (DownloadError & e) {
}
// Download the channel tarball.
auto fullURL = url + "/nixexprs.tar.xz";
try {
filename = dl->downloadCached(store, fullURL, false);
} catch (DownloadError & e) {
fullURL = url + "/nixexprs.tar.bz2";
filename = dl->downloadCached(store, fullURL, false);
}
chomp(filename);
}
// Regardless of where it came from, add the expression representing this channel to accumulated expression
exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }");
}
// Unpack the channel tarballs into the Nix store and install them
// into the channels profile.
std::cerr << "unpacking channels...\n";
auto envArgs = Strings{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
for (auto & expr : exprs)
envArgs.push_back(std::move(expr));
envArgs.push_back("--quiet");
runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
// Make the channels appear in nix-env.
struct stat st;
if (lstat(nixDefExpr.c_str(), &st) == 0) {
if (S_ISLNK(st.st_mode))
// old-skool ~/.nix-defexpr
if (unlink(nixDefExpr.c_str()) == -1)
throw SysError(format("unlinking %1%") % nixDefExpr);
} else if (errno != ENOENT) {
throw SysError(format("getting status of %1%") % nixDefExpr);
}
createDirs(nixDefExpr);
auto channelLink = nixDefExpr + "/channels";
replaceSymlink(profile, channelLink);
}
int main(int argc, char ** argv)
{
return handleExceptions(argv[0], [&]() {
initNix();
// Turn on caching in nix-prefetch-url.
auto channelCache = settings.nixStateDir + "/channel-cache";
createDirs(channelCache);
setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1);
// Figure out the name of the `.nix-channels' file to use
auto home = getEnv("HOME");
if (home.empty())
throw Error("$HOME not set");
channelsList = home + "/.nix-channels";
nixDefExpr = home + "/.nix-defexpr";
// Figure out the name of the channels profile.
auto name = string{};
auto pw = getpwuid(getuid());
if (!pw)
name = getEnv("USER", "");
else
name = pw->pw_name;
if (name.empty())
throw Error("cannot figure out user name");
profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels";
createDirs(dirOf(profile));
enum {
cNone,
cAdd,
cRemove,
cList,
cUpdate,
cRollback
} cmd = cNone;
auto args = std::vector<string>{};
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
if (*arg == "--help") {
showManPage("nix-channel");
} else if (*arg == "--version") {
printVersion("nix-channel");
} else if (*arg == "--add") {
cmd = cAdd;
} else if (*arg == "--remove") {
cmd = cRemove;
} else if (*arg == "--list") {
cmd = cList;
} else if (*arg == "--update") {
cmd = cUpdate;
} else if (*arg == "--rollback") {
cmd = cRollback;
} else {
args.push_back(std::move(*arg));
}
return true;
});
switch (cmd) {
case cNone:
throw UsageError("no command specified");
case cAdd:
if (args.size() < 1 || args.size() > 2)
throw UsageError("‘--add’ requires one or two arguments");
{
auto url = args[0];
auto name = string{};
if (args.size() == 2) {
name = args[1];
} else {
name = baseNameOf(url);
name = std::regex_replace(name, std::regex("-unstable$"), "");
name = std::regex_replace(name, std::regex("-stable$"), "");
}
addChannel(url, name);
}
break;
case cRemove:
if (args.size() != 1)
throw UsageError("‘--remove’ requires one argument");
removeChannel(args[0]);
break;
case cList:
if (!args.empty())
throw UsageError("‘--list’ expects no arguments");
readChannels();
for (const auto & channel : channels)
std::cout << channel.first << ' ' << channel.second << '\n';
break;
case cUpdate:
update(StringSet(args.begin(), args.end()));
break;
case cRollback:
if (args.size() > 1)
throw UsageError("‘--rollback’ has at most one argument");
auto envArgs = Strings{"--profile", profile};
if (args.size() == 1) {
envArgs.push_back("--switch-generation");
envArgs.push_back(args[0]);
} else {
envArgs.push_back("--rollback");
}
runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
break;
}
});
}