From 2d1cf7397469a2adc129b56aeb18b49a0d42aae3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Mar 2012 01:17:59 +0100 Subject: [PATCH] Speed up channel processing In particular the /pkg action is now O(lg n) instead of O(n) in the number of packages in the channel, and listing the channel contents no longer requires calling isValidPath() on all packages. Derivations (and thus build time dependencies) are no longer included in the channel, because they're not GC roots. Thus they could disappear unexpectedly. --- src/lib/Hydra/Base/Controller/ListBuilds.pm | 8 ++-- src/lib/Hydra/Base/Controller/NixChannel.pm | 51 +++++++++++++++++++-- src/lib/Hydra/Controller/Build.pm | 5 +- src/lib/Hydra/Helper/CatalystUtils.pm | 31 ------------- 4 files changed, 50 insertions(+), 45 deletions(-) diff --git a/src/lib/Hydra/Base/Controller/ListBuilds.pm b/src/lib/Hydra/Base/Controller/ListBuilds.pm index 81524cf5..573fe344 100644 --- a/src/lib/Hydra/Base/Controller/ListBuilds.pm +++ b/src/lib/Hydra/Base/Controller/ListBuilds.pm @@ -76,12 +76,10 @@ sub nix : Chained('get_builds') PathPart('channel') CaptureArgs(1) { eval { if ($channelName eq "latest") { $c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest"; - getChannelData($c, scalar($c->stash->{latestSucceeded})); + $c->stash->{channelBuilds} = $c->stash->{latestSucceeded} + ->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')") + ->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] }); } - #elsif ($channelName eq "all") { - # $c->stash->{channelName} = $c->stash->{channelBaseName} . "-all"; - # getChannelData($c, scalar($c->stash->{allBuilds})); - #} else { notFound($c, "Unknown channel `$channelName'."); } diff --git a/src/lib/Hydra/Base/Controller/NixChannel.pm b/src/lib/Hydra/Base/Controller/NixChannel.pm index ec5ba60d..f07dab1e 100644 --- a/src/lib/Hydra/Base/Controller/NixChannel.pm +++ b/src/lib/Hydra/Base/Controller/NixChannel.pm @@ -3,14 +3,45 @@ package Hydra::Base::Controller::NixChannel; use strict; use warnings; use base 'Catalyst::Controller'; +use Nix::Store; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; +sub getChannelData { + my ($c, $checkValidity) = @_; + + my @storePaths = (); + foreach my $build ($c->stash->{channelBuilds}->all) { + next if $checkValidity && !isValidPath($build->outpath); + #if (isValidPath($build->drvpath)) { + # # Adding `drvpath' implies adding `outpath' because of the + # # `--include-outputs' flag passed to `nix-store'. + # push @storePaths, $build->drvpath; + #} else { + # push @storePaths, $build->outpath; + #} + push @storePaths, $build->outpath; + my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id; + $c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName}; + # Put the system type in the manifest (for top-level paths) as + # a hint to the binary patch generator. (It shouldn't try to + # generate patches between builds for different systems.) It + # would be nice if Nix stored this info for every path but it + # doesn't. + $c->stash->{systemForPath}->{$build->outpath} = $build->system; + }; + + $c->stash->{storePaths} = [@storePaths]; +} + + sub closure : Chained('nix') PathPart { my ($self, $c) = @_; $c->stash->{current_view} = 'NixClosure'; + getChannelData($c, 1); + # !!! quick hack; this is to make HEAD requests return the right # MIME type. This is set in the view as well, but the view isn't # called for HEAD requests. There should be a cleaner solution... @@ -22,18 +53,23 @@ sub manifest : Chained('nix') PathPart("MANIFEST") Args(0) { my ($self, $c) = @_; $c->stash->{current_view} = 'NixManifest'; $c->stash->{narBase} = $c->uri_for($c->controller('Root')->action_for("nar")); + getChannelData($c, 1); } sub pkg : Chained('nix') PathPart Args(1) { my ($self, $c, $pkgName) = @_; - my $pkg = $c->stash->{nixPkgs}->{$pkgName}; + if (!$c->stash->{build}) { + $pkgName =~ /-(\d+)\.nixpkg$/ or notFound($c, "Bad package name."); + $c->stash->{build} = $c->stash->{channelBuilds}->find({ id => $1 }) + || notFound($c, "No such package in this channel."); + } - notFound($c, "Unknown Nix package `$pkgName'.") - unless defined $pkg; - - $c->stash->{build} = $pkg->{build}; + if (!isValidPath($c->stash->{build}->outpath)) { + $c->response->status(410); # "Gone" + error($c, "Build " . $c->stash->{build}->id . " is no longer available."); + } $c->stash->{manifestUri} = $c->uri_for($self->action_for("manifest"), $c->req->captures); @@ -46,6 +82,7 @@ sub pkg : Chained('nix') PathPart Args(1) { sub nixexprs : Chained('nix') PathPart('nixexprs.tar.bz2') Args(0) { my ($self, $c) = @_; $c->stash->{current_view} = 'NixExprs'; + getChannelData($c, 1); } @@ -65,6 +102,10 @@ sub sortPkgs { sub channel_contents : Chained('nix') PathPart('') Args(0) { my ($self, $c) = @_; + # Optimistically assume that none of the packages have been + # garbage-collected. That should be true for the "latest" + # channel. + getChannelData($c, 0); $c->stash->{template} = 'channel-contents.tt'; $c->stash->{nixPkgs} = [sortPkgs (values %{$c->stash->{nixPkgs}})]; } diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 1ed6de72..a7824b0e 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -370,10 +370,7 @@ sub nix : Chained('build') PathPart('nix') CaptureArgs(0) { notFound($c, "Path " . $build->outpath . " is no longer available.") unless isValidPath($build->outpath); - $c->stash->{storePaths} = [$build->outpath]; - - my $pkgName = $build->nixname . "-" . $build->system; - $c->stash->{nixPkgs} = {"${pkgName}.nixpkg" => {build => $build, name => $pkgName}}; + $c->stash->{channelBuilds} = $c->model('DB::Builds')->search({id => $build->id}); } diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index 95f77037..da269c93 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -99,37 +99,6 @@ sub getBuildStats { } -sub getChannelData { - my ($c, $builds) = @_; - - my @builds2 = $builds - ->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')") - ->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] }); - - my @storePaths = (); - foreach my $build (@builds2) { - next unless isValidPath($build->outpath); - if (isValidPath($build->drvpath)) { - # Adding `drvpath' implies adding `outpath' because of the - # `--include-outputs' flag passed to `nix-store'. - push @storePaths, $build->drvpath; - } else { - push @storePaths, $build->outpath; - } - my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id; - $c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName}; - # Put the system type in the manifest (for top-level paths) as - # a hint to the binary patch generator. (It shouldn't try to - # generate patches between builds for different systems.) It - # would be nice if Nix stored this info for every path but it - # doesn't. - $c->stash->{systemForPath}->{$build->outpath} = $build->system; - }; - - $c->stash->{storePaths} = [@storePaths]; -} - - sub error { my ($c, $msg) = @_; $c->error($msg);