Speed up channel processing

In particular the /pkg action is now O(lg n) instead of O(n) in the
number of packages in the channel, and listing the channel contents
no longer requires calling isValidPath() on all packages.

Derivations (and thus build time dependencies) are no longer included
in the channel, because they're not GC roots.  Thus they could
disappear unexpectedly.

src/lib/Hydra/Base/Controller/ListBuilds.pm

@@ -76,12 +76,10 @@ sub nix : Chained('get_builds') PathPart('channel') CaptureArgs(1) {
     eval {
         if ($channelName eq "latest") {
             $c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest";
-            getChannelData($c, scalar($c->stash->{latestSucceeded}));
+            $c->stash->{channelBuilds} = $c->stash->{latestSucceeded}
+                ->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
+                ->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
         }
-        #elsif ($channelName eq "all") {
-        #    $c->stash->{channelName} = $c->stash->{channelBaseName} . "-all";
-        #    getChannelData($c, scalar($c->stash->{allBuilds}));
-        #}
         else {
             notFound($c, "Unknown channel `$channelName'.");
         }

src/lib/Hydra/Base/Controller/NixChannel.pm

@@ -3,14 +3,45 @@ package Hydra::Base::Controller::NixChannel;
 use strict;
 use warnings;
 use base 'Catalyst::Controller';
+use Nix::Store;
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 
 
+sub getChannelData {
+    my ($c, $checkValidity) = @_;
+
+    my @storePaths = ();
+    foreach my $build ($c->stash->{channelBuilds}->all) {
+        next if $checkValidity && !isValidPath($build->outpath);
+        #if (isValidPath($build->drvpath)) {
+        #    # Adding `drvpath' implies adding `outpath' because of the
+        #    # `--include-outputs' flag passed to `nix-store'.
+        #    push @storePaths, $build->drvpath;
+        #} else {
+        #    push @storePaths, $build->outpath;
+        #}
+        push @storePaths, $build->outpath;
+        my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
+        $c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
+        # Put the system type in the manifest (for top-level paths) as
+        # a hint to the binary patch generator.  (It shouldn't try to
+        # generate patches between builds for different systems.)  It
+        # would be nice if Nix stored this info for every path but it
+        # doesn't.
+        $c->stash->{systemForPath}->{$build->outpath} = $build->system;
+    };
+
+    $c->stash->{storePaths} = [@storePaths];
+}
+
+
 sub closure : Chained('nix') PathPart {
     my ($self, $c) = @_;
     $c->stash->{current_view} = 'NixClosure';
 
+    getChannelData($c, 1);
+
     # !!! quick hack; this is to make HEAD requests return the right
     # MIME type.  This is set in the view as well, but the view isn't
     # called for HEAD requests.  There should be a cleaner solution...
@@ -22,18 +53,23 @@ sub manifest : Chained('nix') PathPart("MANIFEST") Args(0) {
     my ($self, $c) = @_;
     $c->stash->{current_view} = 'NixManifest';
     $c->stash->{narBase} = $c->uri_for($c->controller('Root')->action_for("nar"));
+    getChannelData($c, 1);
 }
 
 
 sub pkg : Chained('nix') PathPart Args(1) {
     my ($self, $c, $pkgName) = @_;
 
-    my $pkg = $c->stash->{nixPkgs}->{$pkgName};
+    if (!$c->stash->{build}) {
+        $pkgName =~ /-(\d+)\.nixpkg$/ or notFound($c, "Bad package name.");
+        $c->stash->{build} = $c->stash->{channelBuilds}->find({ id => $1 })
+            || notFound($c, "No such package in this channel.");
+    }
 
-    notFound($c, "Unknown Nix package `$pkgName'.")
-        unless defined $pkg;
-
-    $c->stash->{build} = $pkg->{build};
+    if (!isValidPath($c->stash->{build}->outpath)) {
+        $c->response->status(410); # "Gone"
+        error($c, "Build " . $c->stash->{build}->id . " is no longer available.");
+    }
 
     $c->stash->{manifestUri} = $c->uri_for($self->action_for("manifest"), $c->req->captures);
 
@@ -46,6 +82,7 @@ sub pkg : Chained('nix') PathPart Args(1) {
 sub nixexprs : Chained('nix') PathPart('nixexprs.tar.bz2') Args(0) {
     my ($self, $c) = @_;
     $c->stash->{current_view} = 'NixExprs';
+    getChannelData($c, 1);
 }
 
 
@@ -65,6 +102,10 @@ sub sortPkgs {
 
 sub channel_contents : Chained('nix') PathPart('') Args(0) {
     my ($self, $c) = @_;
+    # Optimistically assume that none of the packages have been
+    # garbage-collected.  That should be true for the "latest"
+    # channel.
+    getChannelData($c, 0);
     $c->stash->{template} = 'channel-contents.tt';
     $c->stash->{nixPkgs} = [sortPkgs (values %{$c->stash->{nixPkgs}})];
 }

src/lib/Hydra/Controller/Build.pm

@@ -370,10 +370,7 @@ sub nix : Chained('build') PathPart('nix') CaptureArgs(0) {
     notFound($c, "Path " . $build->outpath . " is no longer available.")
         unless isValidPath($build->outpath);
     
-    $c->stash->{storePaths} = [$build->outpath];
-    
-    my $pkgName = $build->nixname . "-" . $build->system;
-    $c->stash->{nixPkgs} = {"${pkgName}.nixpkg" => {build => $build, name => $pkgName}};
+    $c->stash->{channelBuilds} = $c->model('DB::Builds')->search({id => $build->id});
 }
 
 

src/lib/Hydra/Helper/CatalystUtils.pm

@@ -99,37 +99,6 @@ sub getBuildStats {
 }
 
 
-sub getChannelData {
-    my ($c, $builds) = @_;
-
-    my @builds2 = $builds
-	->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
-	->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
-    
-    my @storePaths = ();
-    foreach my $build (@builds2) {
-        next unless isValidPath($build->outpath);
-        if (isValidPath($build->drvpath)) {
-            # Adding `drvpath' implies adding `outpath' because of the
-            # `--include-outputs' flag passed to `nix-store'.
-            push @storePaths, $build->drvpath;
-        } else {
-	    push @storePaths, $build->outpath;
-        }
-        my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
-        $c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
-        # Put the system type in the manifest (for top-level paths) as
-	# a hint to the binary patch generator.  (It shouldn't try to
-	# generate patches between builds for different systems.)  It
-	# would be nice if Nix stored this info for every path but it
-	# doesn't.
-	$c->stash->{systemForPath}->{$build->outpath} = $build->system;
-    };
-
-    $c->stash->{storePaths} = [@storePaths];
-}
-
-
 sub error {
     my ($c, $msg) = @_;
     $c->error($msg);