From 23bf99b8e2eb842da63527797d3c3ae36b622dd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 12 Aug 2016 11:39:07 +0200
Subject: [PATCH] Use pixz instead of bzip2 for compressing nars

According to following two benchmarks:

- https://www.rootusers.com/gzip-vs-bzip2-vs-xz-performance-comparison/
- http://catchchallenger.first-world.info/wiki/Quick_Benchmark:_Gzip_vs_Bzip2_vs_LZMA_vs_XZ_vs_LZ4_vs_LZO

xz has better compression ratio than bzip2 at lowest compression rate.

https://github.com/vasi/pixz has been chosen as it can scale compressing
over multiple cores linearly.

We're using this in snabblab for a month now and it has improved CPU
wise the main Hydra server.
---
 hydra-module.nix              | 1 +
 release.nix                   | 2 +-
 src/lib/Hydra/View/NARInfo.pm | 2 +-
 src/lib/Hydra/View/NixNAR.pm  | 6 ++++--
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 32bb2f75..95574798 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -225,6 +225,7 @@ in
         base_uri ${cfg.hydraURL}
         notification_sender ${cfg.notificationSender}
         max_servers 25
+        compress_num_threads 0
         ${optionalString (cfg.logo != null) ''
           hydra_logo ${cfg.logo}
         ''}
diff --git a/release.nix b/release.nix
index d6e6ae5c..49c1b9a6 100644
--- a/release.nix
+++ b/release.nix
@@ -134,7 +134,7 @@ rec {
         ];
 
       hydraPath = lib.makeBinPath (
-        [ libxslt sqlite subversion openssh nix coreutils findutils
+        [ libxslt sqlite subversion openssh nix coreutils findutils pixz
           gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar
         ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
 
diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm
index 4afe4041..460bd706 100644
--- a/src/lib/Hydra/View/NARInfo.pm
+++ b/src/lib/Hydra/View/NARInfo.pm
@@ -21,7 +21,7 @@ sub process {
     my $info;
     $info .= "StorePath: $storePath\n";
     $info .= "URL: nar/" . basename $storePath. "\n";
-    $info .= "Compression: bzip2\n";
+    $info .= "Compression: xz\n";
     $info .= "NarHash: $narHash\n";
     $info .= "NarSize: $narSize\n";
     $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
diff --git a/src/lib/Hydra/View/NixNAR.pm b/src/lib/Hydra/View/NixNAR.pm
index 4b2160ac..350ee8e9 100644
--- a/src/lib/Hydra/View/NixNAR.pm
+++ b/src/lib/Hydra/View/NixNAR.pm
@@ -7,13 +7,15 @@ use Hydra::Helper::CatalystUtils;
 sub process {
     my ($self, $c) = @_;
 
-    my $storePath = $c->stash->{storePath};
+    my $storePath  = $c->stash->{storePath};
+    my $numThreads = $c->config->{'compress_num_threads'};
+    my $pParam     = ($numThreads > 0) ? "-p$numThreads" : "";
 
     $c->response->content_type('application/x-nix-archive'); # !!! check MIME type
 
     my $fh = new IO::Handle;
 
-    open $fh, "nix-store --dump '$storePath' | bzip2 |";
+    open $fh, "nix-store --dump '$storePath' | pixz -0 $pParam |";
 
     setCacheHeaders($c, 365 * 24 * 60 * 60);