hydra-server: Support logs in S3

src/lib/Hydra/Controller/Build.pm

@@ -6,6 +6,7 @@ use warnings;
 use base 'Hydra::Base::Controller::NixChannel';
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
+use File::Basename;
 use File::stat;
 use File::Slurp;
 use Data::Dump qw(dump);
@@ -125,62 +126,41 @@ sub view_nixlog : Chained('buildChain') PathPart('nixlog') {
 
     $c->stash->{step} = $step;
 
-    showLog($c, $mode, $step->busy == 0, $step->drvpath,
-            map { $_->path } $step->buildstepoutputs->all);
+    showLog($c, $mode, $step->busy == 0, $step->drvpath);
 }
 
 
 sub view_log : Chained('buildChain') PathPart('log') {
     my ($self, $c, $mode) = @_;
     showLog($c, $mode, $c->stash->{build}->finished,
-            $c->stash->{build}->drvpath,
-            map { $_->path } $c->stash->{build}->buildoutputs->all);
+            $c->stash->{build}->drvpath);
 }
 
 
 sub showLog {
-    my ($c, $mode, $finished, $drvPath, @outPaths) = @_;
+    my ($c, $mode, $finished, $drvPath) = @_;
     $mode //= "pretty";
 
-    my $logPath = findLog($c, $drvPath, @outPaths);
-
-    notFound($c, "The build log of derivation ‘$drvPath’ is not available.") unless defined $logPath;
-
-    # Don't send logs that we can't stream.
-    my $size = stat($logPath)->size; # FIXME: not so meaningful for compressed logs
-    error($c, "This build log is too big to display ($size bytes).") unless
-        $mode eq "raw"
-        || (($mode eq "tail" || $mode eq "tail-reload") && $logPath !~ /\.bz2$/)
-        || $size < 64 * 1024 * 1024;
+    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]);
 
     if ($mode eq "pretty") {
+        $c->stash->{log_uri} = $log_uri;
         $c->stash->{template} = 'log.tt';
-        $c->stash->{logtext} = logContents($logPath);
     }
 
     elsif ($mode eq "raw") {
-        $c->stash->{logPath} = $logPath;
-        $c->stash->{finished} = $finished;
-        $c->forward('Hydra::View::NixLog');
-    }
-
-    elsif ($mode eq "tail-reload") {
-        my $url = $c->uri_for($c->request->uri->path);
-        $url =~ s/tail-reload/tail/g;
-        $c->stash->{url} = $url;
-        $c->stash->{reload} = !$c->stash->{build}->finished;
-        $c->stash->{title} = "";
-        $c->stash->{contents} = (scalar logContents($logPath, 50)) || " ";
-        $c->stash->{template} = 'plain-reload.tt';
+        $c->res->redirect($log_uri);
     }
 
     elsif ($mode eq "tail") {
-        $c->stash->{'plain'} = { data => (scalar logContents($logPath, 50)) || " " };
-        $c->forward('Hydra::View::Plain');
+        my $lines = 50;
+        $c->stash->{log_uri} = $log_uri . "?tail=$lines";
+        $c->stash->{tail} = $lines;
+        $c->stash->{template} = 'log.tt';
     }
 
     else {
-        error($c, "Unknown log display mode `$mode'.");
+        error($c, "Unknown log display mode '$mode'.");
     }
 }
 

src/lib/Hydra/Controller/Root.pm

@@ -10,6 +10,7 @@ use Digest::SHA1 qw(sha1_hex);
 use Nix::Store;
 use Nix::Config;
 use Encode;
+use File::Basename;
 use JSON;
 
 # Put this controller at top-level.
@@ -434,19 +435,36 @@ sub search :Local Args(0) {
         { order_by => ["id desc"] } ) ];
 }
 
-
-sub log :Local :Args(1) {
-    my ($self, $c, $path) = @_;
-
-    $path = ($ENV{NIX_STORE_DIR} || "/nix/store")."/$path";
-
-    my @outpaths = ($path);
-    my $logPath = findLog($c, $path, @outpaths);
-    notFound($c, "The build log of $path is not available.") unless defined $logPath;
-
+sub serveLogFile {
+    my ($c, $logPath, $tail) = @_;
     $c->stash->{logPath} = $logPath;
+    $c->stash->{tail} = $tail;
     $c->forward('Hydra::View::NixLog');
 }
 
+sub log :Local :Args(1) {
+    my ($self, $c, $drvPath) = @_;
+
+    $drvPath = "/nix/store/$drvPath";
+
+    my $tail = $c->request->params->{"tail"};
+
+    die if defined $tail && $tail !~ /^[0-9]+$/;
+
+    my $logFile = findLog($c, $drvPath);
+
+    if (defined $logFile) {
+        serveLogFile($c, $logFile, $tail);
+        return;
+    }
+
+    my $logPrefix = $c->config->{log_prefix};
+
+    if (defined $logPrefix) {
+        $c->res->redirect($logPrefix . "log/" . basename($drvPath));
+    } else {
+        notFound($c, "The build log of $drvPath is not available.");
+    }
+}
 
 1;

src/lib/Hydra/Helper/Nix.pm

@@ -18,7 +18,7 @@ our @EXPORT = qw(
     getSCMCacheDir
     registerRoot getGCRootsDir gcRootFor
     jobsetOverview jobsetOverview_
-    removeAsciiEscapes getDrvLogPath findLog logContents
+    getDrvLogPath findLog
     getMainOutput
     getEvals getMachines
     pathIsInsidePrefix
@@ -154,9 +154,8 @@ sub getDrvLogPath {
     my ($drvPath) = @_;
     my $base = basename $drvPath;
     my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2);
-    my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
-    my $fn2 = Hydra::Model::DB::getHydraPath . "/build-logs/";
-    for ($fn2 . $bucketed, $fn2 . $bucketed . ".bz2", $fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
+    my $fn = Hydra::Model::DB::getHydraPath . "/build-logs/";
+    for ($fn . $bucketed, $fn . $bucketed . ".bz2") {
         return $_ if -f $_;
     }
     return undef;
@@ -192,27 +191,6 @@ sub findLog {
 }
 
 
-sub logContents {
-    my ($logPath, $tail) = @_;
-    my $cmd;
-    if ($logPath =~ /.bz2$/) {
-        $cmd = "bzip2 -d < $logPath";
-        $cmd = $cmd . " | tail -n $tail" if defined $tail;
-    }
-    else {
-        $cmd = defined $tail ? "tail -$tail $logPath" : "cat $logPath";
-    }
-    return decode("utf-8", `$cmd`);
-}
-
-
-sub removeAsciiEscapes {
-    my ($logtext) = @_;
-    $logtext =~ s/\e\[[0-9]*[A-Za-z]//g;
-    return $logtext;
-}
-
-
 sub getMainOutput {
     my ($build) = @_;
     return

src/lib/Hydra/View/NixLog.pm

@@ -13,10 +13,17 @@ sub process {
 
     my $fh = new IO::Handle;
 
+    my $tail = int($c->stash->{tail} // "0");
+
     if ($logPath =~ /\.bz2$/) {
-        open $fh, "bzip2 -dc < '$logPath' |" or die;
+        my $doTail = $tail ? " tail -n '$tail' |" : "";
+        open $fh, "bzip2 -dc < '$logPath' | $doTail" or die;
     } else {
-        open $fh, "<$logPath" or die;
+        if ($tail) {
+            open $fh, "tail -n '$tail' '$logPath' |" or die;
+        } else {
+            open $fh, "<$logPath" or die;
+        }
     }
     binmode($fh);
 

src/lib/Hydra/View/TT.pm

@@ -13,17 +13,18 @@ __PACKAGE__->config(
 
 sub buildLogExists {
     my ($self, $c, $build) = @_;
+    return 1 if defined $c->config->{log_prefix};
     my @outPaths = map { $_->path } $build->buildoutputs->all;
     return defined findLog($c, $build->drvpath, @outPaths);
 }
 
 sub buildStepLogExists {
     my ($self, $c, $step) = @_;
+    return 1 if defined $c->config->{log_prefix};
     my @outPaths = map { $_->path } $step->buildstepoutputs->all;
     return defined findLog($c, $step->drvpath, @outPaths);
 }
 
-
 sub stripSSHUser {
     my ($self, $c, $name) = @_;
     if ($name =~ /^.*@(.*)$/) {