Compare commits

...

1 commit
main ... main

Author SHA1 Message Date
71rd 9b5ac87de7 Stream files from store instead of buffering them
When an artifact is requested from hydra the output is first copied
from the nix store into memory and then sent as a response, delaying
the download and taking up significant amounts of memory.

As reported in https://github.com/NixOS/hydra/issues/1357

Instead of calling a command and blocking while reading in the entire
output, this adds read_into_socket(). the function takes a
command, starting a subprocess with that command, returning a file
descriptor attached to stdout.
This file descriptor is then by responsebuilder of Catalyst to steam
the output directly
2024-08-03 14:30:38 +02:00
2 changed files with 13 additions and 1 deletions

View file

@ -240,7 +240,7 @@ sub serveFile {
# XSS hole. # XSS hole.
$c->response->header('Content-Security-Policy' => 'sandbox allow-scripts'); $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
$c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command", $c->stash->{'plain'} = { data => readIntoSocket(cmd => ["nix", "--experimental-features", "nix-command",
"store", "cat", "--store", getStoreUri(), "$path"]) }; "store", "cat", "--store", getStoreUri(), "$path"]) };
# Detect MIME type. # Detect MIME type.

View file

@ -36,6 +36,7 @@ our @EXPORT = qw(
jobsetOverview jobsetOverview
jobsetOverview_ jobsetOverview_
pathIsInsidePrefix pathIsInsidePrefix
readIntoSocket
readNixFile readNixFile
registerRoot registerRoot
restartBuilds restartBuilds
@ -406,6 +407,17 @@ sub pathIsInsidePrefix {
return $cur; return $cur;
} }
sub readIntoSocket{
my (%args) = @_;
my $sock;
eval {
my $x= join(" ", @{$args{cmd}});
open($sock, "-|", $x) or die q(failed to open socket from command:\n $x);
};
return $sock;
}