Speed up channel processing

In particular the /pkg action is now O(lg n) instead of O(n) in the
number of packages in the channel, and listing the channel contents
no longer requires calling isValidPath() on all packages.

Derivations (and thus build time dependencies) are no longer included
in the channel, because they're not GC roots.  Thus they could
disappear unexpectedly.
This commit is contained in:
Eelco Dolstra 2012-03-08 01:17:59 +01:00
parent 9f10c0f9c0
commit 2d1cf73974
4 changed files with 50 additions and 45 deletions

View file

@ -76,12 +76,10 @@ sub nix : Chained('get_builds') PathPart('channel') CaptureArgs(1) {
eval {
if ($channelName eq "latest") {
$c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest";
getChannelData($c, scalar($c->stash->{latestSucceeded}));
$c->stash->{channelBuilds} = $c->stash->{latestSucceeded}
->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
}
#elsif ($channelName eq "all") {
# $c->stash->{channelName} = $c->stash->{channelBaseName} . "-all";
# getChannelData($c, scalar($c->stash->{allBuilds}));
#}
else {
notFound($c, "Unknown channel `$channelName'.");
}

View file

@ -3,14 +3,45 @@ package Hydra::Base::Controller::NixChannel;
use strict;
use warnings;
use base 'Catalyst::Controller';
use Nix::Store;
use Hydra::Helper::Nix;
use Hydra::Helper::CatalystUtils;
sub getChannelData {
my ($c, $checkValidity) = @_;
my @storePaths = ();
foreach my $build ($c->stash->{channelBuilds}->all) {
next if $checkValidity && !isValidPath($build->outpath);
#if (isValidPath($build->drvpath)) {
# # Adding `drvpath' implies adding `outpath' because of the
# # `--include-outputs' flag passed to `nix-store'.
# push @storePaths, $build->drvpath;
#} else {
# push @storePaths, $build->outpath;
#}
push @storePaths, $build->outpath;
my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
$c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
# Put the system type in the manifest (for top-level paths) as
# a hint to the binary patch generator. (It shouldn't try to
# generate patches between builds for different systems.) It
# would be nice if Nix stored this info for every path but it
# doesn't.
$c->stash->{systemForPath}->{$build->outpath} = $build->system;
};
$c->stash->{storePaths} = [@storePaths];
}
sub closure : Chained('nix') PathPart {
my ($self, $c) = @_;
$c->stash->{current_view} = 'NixClosure';
getChannelData($c, 1);
# !!! quick hack; this is to make HEAD requests return the right
# MIME type. This is set in the view as well, but the view isn't
# called for HEAD requests. There should be a cleaner solution...
@ -22,18 +53,23 @@ sub manifest : Chained('nix') PathPart("MANIFEST") Args(0) {
my ($self, $c) = @_;
$c->stash->{current_view} = 'NixManifest';
$c->stash->{narBase} = $c->uri_for($c->controller('Root')->action_for("nar"));
getChannelData($c, 1);
}
sub pkg : Chained('nix') PathPart Args(1) {
my ($self, $c, $pkgName) = @_;
my $pkg = $c->stash->{nixPkgs}->{$pkgName};
if (!$c->stash->{build}) {
$pkgName =~ /-(\d+)\.nixpkg$/ or notFound($c, "Bad package name.");
$c->stash->{build} = $c->stash->{channelBuilds}->find({ id => $1 })
|| notFound($c, "No such package in this channel.");
}
notFound($c, "Unknown Nix package `$pkgName'.")
unless defined $pkg;
$c->stash->{build} = $pkg->{build};
if (!isValidPath($c->stash->{build}->outpath)) {
$c->response->status(410); # "Gone"
error($c, "Build " . $c->stash->{build}->id . " is no longer available.");
}
$c->stash->{manifestUri} = $c->uri_for($self->action_for("manifest"), $c->req->captures);
@ -46,6 +82,7 @@ sub pkg : Chained('nix') PathPart Args(1) {
sub nixexprs : Chained('nix') PathPart('nixexprs.tar.bz2') Args(0) {
my ($self, $c) = @_;
$c->stash->{current_view} = 'NixExprs';
getChannelData($c, 1);
}
@ -65,6 +102,10 @@ sub sortPkgs {
sub channel_contents : Chained('nix') PathPart('') Args(0) {
my ($self, $c) = @_;
# Optimistically assume that none of the packages have been
# garbage-collected. That should be true for the "latest"
# channel.
getChannelData($c, 0);
$c->stash->{template} = 'channel-contents.tt';
$c->stash->{nixPkgs} = [sortPkgs (values %{$c->stash->{nixPkgs}})];
}

View file

@ -370,10 +370,7 @@ sub nix : Chained('build') PathPart('nix') CaptureArgs(0) {
notFound($c, "Path " . $build->outpath . " is no longer available.")
unless isValidPath($build->outpath);
$c->stash->{storePaths} = [$build->outpath];
my $pkgName = $build->nixname . "-" . $build->system;
$c->stash->{nixPkgs} = {"${pkgName}.nixpkg" => {build => $build, name => $pkgName}};
$c->stash->{channelBuilds} = $c->model('DB::Builds')->search({id => $build->id});
}

View file

@ -99,37 +99,6 @@ sub getBuildStats {
}
sub getChannelData {
my ($c, $builds) = @_;
my @builds2 = $builds
->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
my @storePaths = ();
foreach my $build (@builds2) {
next unless isValidPath($build->outpath);
if (isValidPath($build->drvpath)) {
# Adding `drvpath' implies adding `outpath' because of the
# `--include-outputs' flag passed to `nix-store'.
push @storePaths, $build->drvpath;
} else {
push @storePaths, $build->outpath;
}
my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
$c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
# Put the system type in the manifest (for top-level paths) as
# a hint to the binary patch generator. (It shouldn't try to
# generate patches between builds for different systems.) It
# would be nice if Nix stored this info for every path but it
# doesn't.
$c->stash->{systemForPath}->{$build->outpath} = $build->system;
};
$c->stash->{storePaths} = [@storePaths];
}
sub error {
my ($c, $msg) = @_;
$c->error($msg);