Merge branch 'nix-channel-c++'

.gitignore

@@ -37,7 +37,6 @@ Makefile.config
 /scripts/nix-switch
 /scripts/nix-collect-garbage
 /scripts/nix-prefetch-url
-/scripts/nix-channel
 /scripts/nix-build
 /scripts/nix-copy-closure
 /scripts/NixConfig.pm
@@ -72,6 +71,9 @@ Makefile.config
 # /src/nix-daemon/
 /src/nix-daemon/nix-daemon
 
+# /src/nix-channel/
+/src/nix-channel/nix-channel
+
 # /src/download-via-ssh/
 /src/download-via-ssh/download-via-ssh
 

Makefile

@@ -14,6 +14,7 @@ makefiles = \
   src/nix-prefetch-url/local.mk \
   src/buildenv/local.mk \
   src/resolve-system-dependencies/local.mk \
+  src/nix-channel/local.mk \
   perl/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \

scripts/local.mk

@@ -1,6 +1,5 @@
 nix_bin_scripts := \
   $(d)/nix-build \
-  $(d)/nix-channel \
   $(d)/nix-copy-closure \
 
 bin-scripts += $(nix_bin_scripts)

scripts/nix-channel.in

@@ -1,228 +0,0 @@
-#! @perl@ -w @perlFlags@
-
-use utf8;
-use strict;
-use File::Basename;
-use File::Path qw(mkpath);
-use Nix::Config;
-use Nix::Manifest;
-use File::Temp qw(tempdir);
-
-binmode STDERR, ":encoding(utf8)";
-
-Nix::Config::readConfig;
-
-
-# Turn on caching in nix-prefetch-url.
-my $channelCache = "$Nix::Config::stateDir/channel-cache";
-mkdir $channelCache, 0755 unless -e $channelCache;
-$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache if -W $channelCache;
-
-# Figure out the name of the `.nix-channels' file to use.
-my $home = $ENV{"HOME"} or die '$HOME not set\n';
-my $channelsList = "$home/.nix-channels";
-my $nixDefExpr = "$home/.nix-defexpr";
-
-# Figure out the name of the channels profile.
-my $userName = getpwuid($<) || $ENV{"USER"} or die "cannot figure out user name";
-my $profile = "$Nix::Config::stateDir/profiles/per-user/$userName/channels";
-mkpath(dirname $profile, 0, 0755);
-
-my %channels;
-
-
-# Reads the list of channels.
-sub readChannels {
-    return if (!-f $channelsList);
-    open CHANNELS, "<$channelsList" or die "cannot open ‘$channelsList’: $!";
-    while (<CHANNELS>) {
-        chomp;
-        next if /^\s*\#/;
-        my ($url, $name) = split ' ', $_;
-        $url =~ s/\/*$//; # remove trailing slashes
-        $name = basename $url unless defined $name;
-        $channels{$name} = $url;
-    }
-    close CHANNELS;
-}
-
-
-# Writes the list of channels.
-sub writeChannels {
-    open CHANNELS, ">$channelsList" or die "cannot open ‘$channelsList’: $!";
-    foreach my $name (keys %channels) {
-        print CHANNELS "$channels{$name} $name\n";
-    }
-    close CHANNELS;
-}
-
-
-# Adds a channel.
-sub addChannel {
-    my ($url, $name) = @_;
-    die "invalid channel URL ‘$url’" unless $url =~ /^(file|http|https):\/\//;
-    die "invalid channel identifier ‘$name’" unless $name =~ /^[a-zA-Z0-9_][a-zA-Z0-9_\-\.]*$/;
-    readChannels;
-    $channels{$name} = $url;
-    writeChannels;
-}
-
-
-# Remove a channel.
-sub removeChannel {
-    my ($name) = @_;
-    readChannels;
-    my $url = $channels{$name};
-    delete $channels{$name};
-    writeChannels;
-
-    system("$Nix::Config::binDir/nix-env --profile '$profile' -e '$name'") == 0
-        or die "cannot remove channel ‘$name’\n";
-}
-
-
-# Fetch Nix expressions and binary cache URLs from the subscribed channels.
-sub update {
-    my @channelNames = @_;
-
-    readChannels;
-
-    # Download each channel.
-    my $exprs = "";
-    foreach my $name (keys %channels) {
-        next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames};
-
-        my $url = $channels{$name};
-
-        # We want to download the url to a file to see if it's a tarball while also checking if we
-        # got redirected in the process, so that we can grab the various parts of a nix channel
-        # definition from a consistent location if the redirect changes mid-download.
-        my $tmpdir = tempdir( CLEANUP => 1 );
-        my $filename;
-        ($url, $filename) = `cd $tmpdir && $Nix::Config::curl --silent --write-out '%{url_effective}\n%{filename_effective}' -L '$url' -O`;
-        chomp $url;
-        die "$0: unable to check ‘$url’\n" if $? != 0;
-
-        # If the URL contains a version number, append it to the name
-        # attribute (so that "nix-env -q" on the channels profile
-        # shows something useful).
-        my $cname = $name;
-        $cname .= $1 if basename($url) =~ /(-\d.*)$/;
-
-        my $path;
-        my $ret = -1;
-        if (-e "$tmpdir/$filename" && $filename =~ /\.tar\.(gz|bz2|xz)$/) {
-            # Get our temporary download into the store.
-            (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url 'file://$tmpdir/$filename'`;
-            chomp $path;
-
-            # Try unpacking the expressions to see if they'll be valid for us to process later.
-            # Like anything in nix, this will cache the result so we don't do it again outside of the loop below.
-            $ret = system("$Nix::Config::binDir/nix-build --no-out-link -E 'import <nix/unpack-channel.nix> " .
-                          "{ name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }'");
-        }
-
-        # The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
-        my $extraAttrs = "";
-        if ($ret != 0) {
-            # Check if the channel advertises a binary cache.
-            my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
-            $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "
-                if $? == 0 && $binaryCacheURL ne "";
-
-            # Download the channel tarball.
-            my $fullURL = "$url/nixexprs.tar.xz";
-            system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or
-                $fullURL = "$url/nixexprs.tar.bz2";
-            print STDERR "downloading Nix expressions from ‘$fullURL’...\n";
-            (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`;
-            die "cannot fetch ‘$fullURL’\n" if $? != 0;
-            chomp $path;
-        }
-
-        # Regardless of where it came from, add the expression representing this channel to accumulated expression
-        $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' ";
-    }
-
-    # Unpack the channel tarballs into the Nix store and install them
-    # into the channels profile.
-    print STDERR "unpacking channels...\n";
-    system("$Nix::Config::binDir/nix-env --profile '$profile' " .
-           "-f '<nix/unpack-channel.nix>' -i -E $exprs --quiet") == 0
-           or die "cannot unpack the channels";
-
-    # Make the channels appear in nix-env.
-    unlink $nixDefExpr if -l $nixDefExpr; # old-skool ~/.nix-defexpr
-    mkdir $nixDefExpr or die "cannot create directory ‘$nixDefExpr’" if !-e $nixDefExpr;
-    my $channelLink = "$nixDefExpr/channels";
-    unlink $channelLink; # !!! not atomic
-    symlink($profile, $channelLink) or die "cannot symlink ‘$channelLink’ to ‘$profile’";
-}
-
-
-die "$0: argument expected\n" if scalar @ARGV == 0;
-
-
-while (scalar @ARGV) {
-    my $arg = shift @ARGV;
-
-    if ($arg eq "--add") {
-        die "$0: ‘--add’ requires one or two arguments\n" if scalar @ARGV < 1 || scalar @ARGV > 2;
-        my $url = shift @ARGV;
-        my $name = shift @ARGV;
-        unless (defined $name) {
-            $name = basename $url;
-            $name =~ s/-unstable//;
-            $name =~ s/-stable//;
-        }
-        addChannel($url, $name);
-        last;
-    }
-
-    if ($arg eq "--remove") {
-        die "$0: ‘--remove’ requires one argument\n" if scalar @ARGV != 1;
-        removeChannel(shift @ARGV);
-        last;
-    }
-
-    if ($arg eq "--list") {
-        die "$0: ‘--list’ requires one argument\n" if scalar @ARGV != 0;
-        readChannels;
-        foreach my $name (keys %channels) {
-            print "$name $channels{$name}\n";
-        }
-        last;
-    }
-
-    elsif ($arg eq "--update") {
-        update(@ARGV);
-        last;
-    }
-
-    elsif ($arg eq "--rollback") {
-        die "$0: ‘--rollback’ has at most one argument\n" if scalar @ARGV > 1;
-        my $generation = shift @ARGV;
-        my @args = ("$Nix::Config::binDir/nix-env", "--profile", $profile);
-        if (defined $generation) {
-            die "invalid channel generation number ‘$generation’" unless $generation =~ /^[0-9]+$/;
-            push @args, "--switch-generation", $generation;
-        } else {
-            push @args, "--rollback";
-        }
-        system(@args) == 0 or exit 1;
-        last;
-    }
-
-    elsif ($arg eq "--help") {
-        exec "man nix-channel" or die;
-    }
-
-    elsif ($arg eq "--version") {
-        print "nix-channel (Nix) $Nix::Config::version\n";
-        exit 0;
-    }
-
-    else {
-        die "unknown argument ‘$arg’; try ‘--help’\n";
-    }
-}

src/libstore/download.cc

@@ -32,7 +32,7 @@ struct CurlDownloader : public Downloader
 {
     CURL * curl;
     ref<std::string> data;
-    string etag, status, expectedETag;
+    string etag, status, expectedETag, effectiveUrl;
 
     struct curl_slist * requestHeaders;
 
@@ -208,6 +208,11 @@ struct CurlDownloader : public Downloader
                     % url % curl_easy_strerror(res) % res);
         }
 
+        char *effectiveUrlCStr;
+        curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr);
+        if (effectiveUrlCStr)
+            effectiveUrl = effectiveUrlCStr;
+
         if (httpStatus == 304) return false;
 
         return true;
@@ -225,6 +230,7 @@ struct CurlDownloader : public Downloader
                     res.data = data;
                 } else
                     res.cached = true;
+                res.effectiveUrl = effectiveUrl;
                 res.etag = etag;
                 return res;
             } catch (DownloadError & e) {
@@ -244,6 +250,12 @@ ref<Downloader> makeDownloader()
 }
 
 Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash)
+{
+    string ignored;
+    return downloadCached(store, url_, unpack, ignored, expectedHash);
+}
+
+Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string & effectiveUrl, const Hash & expectedHash)
 {
     auto url = resolveUri(url_);
 
@@ -281,9 +293,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
             if (ss.size() >= 3 && ss[0] == url) {
                 time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
+                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) {
                     skip = true;
-                else if (!ss[1].empty()) {
+                    effectiveUrl = url_;
+                } else if (!ss[1].empty()) {
                     printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
                     expectedETag = ss[1];
                 }
@@ -298,6 +311,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             DownloadOptions options;
             options.expectedETag = expectedETag;
             auto res = download(url, options);
+            effectiveUrl = res.effectiveUrl;
 
             if (!res.cached) {
                 ValidPathInfo info;

src/libstore/download.hh

@@ -21,6 +21,7 @@ struct DownloadResult
 {
     bool cached;
     string etag;
+    string effectiveUrl;
     std::shared_ptr<std::string> data;
 };
 
@@ -33,6 +34,11 @@ struct Downloader
     Path downloadCached(ref<Store> store, const string & url, bool unpack, string name = "",
         const Hash & expectedHash = Hash());
 
+    /* Need to overload because can't have an rvalue default value for non-const reference */
+
+    Path downloadCached(ref<Store> store, const string & url, bool unpack,
+        string & effectiveUrl, const Hash & expectedHash = Hash());
+
     enum Error { NotFound, Forbidden, Misc, Transient };
 };
 

src/nix-channel/local.mk

@@ -0,0 +1,7 @@
+programs += nix-channel
+
+nix-channel_DIR := $(d)
+
+nix-channel_LIBS = libmain libutil libformat libstore
+
+nix-channel_SOURCES := $(d)/nix-channel.cc

src/nix-channel/nix-channel.cc

@@ -0,0 +1,270 @@
+#include "shared.hh"
+#include "globals.hh"
+#include "download.hh"
+#include <fcntl.h>
+#include <regex>
+#include "store-api.hh"
+#include <pwd.h>
+
+using namespace nix;
+
+typedef std::map<string,string> Channels;
+
+static auto channels = Channels{};
+static auto channelsList = Path{};
+
+// Reads the list of channels.
+static void readChannels()
+{
+    if (!pathExists(channelsList)) return;
+    auto channelsFile = readFile(channelsList);
+
+    for (const auto & line : tokenizeString<std::vector<string>>(channelsFile, "\n")) {
+        chomp(line);
+        if (std::regex_search(line, std::regex("^\\s*\\#")))
+            continue;
+        auto split = tokenizeString<std::vector<string>>(line, " ");
+        auto url = std::regex_replace(split[0], std::regex("/*$"), "");
+        auto name = split.size() > 1 ? split[1] : baseNameOf(url);
+        channels[name] = url;
+    }
+}
+
+// Writes the list of channels.
+static void writeChannels()
+{
+    auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)};
+    if (!channelsFD)
+        throw SysError(format("opening ‘%1%’ for writing") % channelsList);
+    for (const auto & channel : channels)
+        writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n");
+}
+
+// Adds a channel.
+static void addChannel(const string & url, const string & name)
+{
+    if (!regex_search(url, std::regex("^(file|http|https)://")))
+        throw Error(format("invalid channel URL ‘%1%’") % url);
+    if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$")))
+        throw Error(format("invalid channel identifier ‘%1%’") % name);
+    readChannels();
+    channels[name] = url;
+    writeChannels();
+}
+
+static auto profile = Path{};
+
+// Remove a channel.
+static void removeChannel(const string & name)
+{
+    readChannels();
+    channels.erase(name);
+    writeChannels();
+
+    runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name });
+}
+
+static auto nixDefExpr = Path{};
+
+// Fetch Nix expressions and binary cache URLs from the subscribed channels.
+static void update(const StringSet & channelNames)
+{
+    readChannels();
+
+    auto store = openStore();
+
+    // Download each channel.
+    auto exprs = Strings{};
+    for (const auto & channel : channels) {
+        if (!channelNames.empty() && channelNames.find(channel.first) != channelNames.end())
+            continue;
+        auto name = channel.first;
+        auto url = channel.second;
+
+        // We want to download the url to a file to see if it's a tarball while also checking if we
+        // got redirected in the process, so that we can grab the various parts of a nix channel
+        // definition from a consistent location if the redirect changes mid-download.
+        auto effectiveUrl = string{};
+        auto dl = makeDownloader();
+        auto filename = dl->downloadCached(store, url, false, effectiveUrl);
+        url = chomp(std::move(effectiveUrl));
+
+        // If the URL contains a version number, append it to the name
+        // attribute (so that "nix-env -q" on the channels profile
+        // shows something useful).
+        auto cname = name;
+        std::smatch match;
+        auto urlBase = baseNameOf(url);
+        if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) {
+            cname = cname + (string) match[1];
+        }
+
+        auto extraAttrs = string{};
+
+        auto unpacked = false;
+        if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
+            try {
+                runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
+                            "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
+                unpacked = true;
+            } catch (ExecError & e) {
+            }
+        }
+
+        if (!unpacked) {
+            // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
+            // Check if the channel advertises a binary cache.
+            DownloadOptions opts;
+            opts.showProgress = DownloadOptions::no;
+            try {
+                auto dlRes = dl->download(url + "/binary-cache-url", opts);
+                extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";";
+            } catch (DownloadError & e) {
+            }
+
+            // Download the channel tarball.
+            auto fullURL = url + "/nixexprs.tar.xz";
+            try {
+                filename = dl->downloadCached(store, fullURL, false);
+            } catch (DownloadError & e) {
+                fullURL = url + "/nixexprs.tar.bz2";
+                filename = dl->downloadCached(store, fullURL, false);
+            }
+            chomp(filename);
+        }
+
+        // Regardless of where it came from, add the expression representing this channel to accumulated expression
+        exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }");
+    }
+
+    // Unpack the channel tarballs into the Nix store and install them
+    // into the channels profile.
+    std::cerr << "unpacking channels...\n";
+    auto envArgs = Strings{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
+    for (auto & expr : exprs)
+        envArgs.push_back(std::move(expr));
+    envArgs.push_back("--quiet");
+    runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
+
+    // Make the channels appear in nix-env.
+    struct stat st;
+    if (lstat(nixDefExpr.c_str(), &st) == 0) {
+        if (S_ISLNK(st.st_mode))
+            // old-skool ~/.nix-defexpr
+            if (unlink(nixDefExpr.c_str()) == -1)
+                throw SysError(format("unlinking %1%") % nixDefExpr);
+    } else if (errno != ENOENT) {
+        throw SysError(format("getting status of %1%") % nixDefExpr);
+    }
+    createDirs(nixDefExpr);
+    auto channelLink = nixDefExpr + "/channels";
+    replaceSymlink(profile, channelLink);
+}
+
+int main(int argc, char ** argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
+        // Turn on caching in nix-prefetch-url.
+        auto channelCache = settings.nixStateDir + "/channel-cache";
+        createDirs(channelCache);
+        setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1);
+
+        // Figure out the name of the `.nix-channels' file to use
+        auto home = getEnv("HOME");
+        if (home.empty())
+            throw Error("$HOME not set");
+        channelsList = home + "/.nix-channels";
+        nixDefExpr = home + "/.nix-defexpr";
+
+        // Figure out the name of the channels profile.
+        auto name = string{};
+        auto pw = getpwuid(getuid());
+        if (!pw)
+            name = getEnv("USER", "");
+        else
+            name = pw->pw_name;
+        if (name.empty())
+            throw Error("cannot figure out user name");
+        profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels";
+        createDirs(dirOf(profile));
+
+        enum {
+            cNone,
+            cAdd,
+            cRemove,
+            cList,
+            cUpdate,
+            cRollback
+        } cmd = cNone;
+        auto args = std::vector<string>{};
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help") {
+                showManPage("nix-channel");
+            } else if (*arg == "--version") {
+                printVersion("nix-channel");
+            } else if (*arg == "--add") {
+                cmd = cAdd;
+            } else if (*arg == "--remove") {
+                cmd = cRemove;
+            } else if (*arg == "--list") {
+                cmd = cList;
+            } else if (*arg == "--update") {
+                cmd = cUpdate;
+            } else if (*arg == "--rollback") {
+                cmd = cRollback;
+            } else {
+                args.push_back(std::move(*arg));
+            }
+            return true;
+        });
+        switch (cmd) {
+            case cNone:
+                throw UsageError("no command specified");
+            case cAdd:
+                if (args.size() < 1 || args.size() > 2)
+                    throw UsageError("‘--add’ requires one or two arguments");
+                {
+                auto url = args[0];
+                auto name = string{};
+                if (args.size() == 2) {
+                    name = args[1];
+                } else {
+                    name = baseNameOf(url);
+                    name = std::regex_replace(name, std::regex("-unstable$"), "");
+                    name = std::regex_replace(name, std::regex("-stable$"), "");
+                }
+                addChannel(url, name);
+                }
+                break;
+            case cRemove:
+                if (args.size() != 1)
+                    throw UsageError("‘--remove’ requires one argument");
+                removeChannel(args[0]);
+                break;
+            case cList:
+                if (!args.empty())
+                    throw UsageError("‘--list’ expects no arguments");
+                readChannels();
+                for (const auto & channel : channels)
+                    std::cout << channel.first << ' ' << channel.second << '\n';
+                break;
+            case cUpdate:
+                update(StringSet(args.begin(), args.end()));
+                break;
+            case cRollback:
+                if (args.size() > 1)
+                    throw UsageError("‘--rollback’ has at most one argument");
+                auto envArgs = Strings{"--profile", profile};
+                if (args.size() == 1) {
+                    envArgs.push_back("--switch-generation");
+                    envArgs.push_back(args[0]);
+                } else {
+                    envArgs.push_back("--rollback");
+                }
+                runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
+                break;
+        }
+    });
+}