grahamc/hydra2 - Change EYAA7MJYQFEYESCXEHXOZ7QXIFFLYQYB5C6PIKHKBWS734KEFO2QC

hydra-server: Support logs in S3

Created by Eelco Dolstra on April 5, 2017

EYAA7MJYQFEYESCXEHXOZ7QXIFFLYQYB5C6PIKHKBWS734KEFO2QC

Dependencies

In channels

main

Change contents

Insertion in src/lib/Hydra/Controller/Build.pm at line 9 [10.1]
[10.156]
[11.0]
```
use File::Basename;
```

Replacement in src/lib/Hydra/Controller/Build.pm at line 129 [10.1]

B:BD[10.1443] → [12.0:117]

    showLog($c, $mode, $step->busy == 0, $step->drvpath,
            map { $_->path } $step->buildstepoutputs->all);

[10.1443]

[10.1554]

    showLog($c, $mode, $step->busy == 0, $step->drvpath);

Replacement in src/lib/Hydra/Controller/Build.pm at line 136 [10.1]

B:BD[12.171] → [12.171:281]

            $c->stash->{build}->drvpath,
            map { $_->path } $c->stash->{build}->buildoutputs->all);

[12.171]

[10.1923]

            $c->stash->{build}->drvpath);

Replacement in src/lib/Hydra/Controller/Build.pm at line 141 [10.1]

B:BD[13.302] → [12.282:339]

    my ($c, $mode, $finished, $drvPath, @outPaths) = @_;

[13.302]

[14.0]

    my ($c, $mode, $finished, $drvPath) = @_;

Deletion in src/lib/Hydra/Controller/Build.pm at line 143 [10.1]
∅:D[14.24] → [15.0:1]
∅:D[12.339] → [15.0:1]
∅:D[16.341] → [15.0:1]
∅:D[17.755] → [15.0:1]
B:BD[13.334] → [15.0:1]
B:BD[15.1] → [17.756:808]
```
    my $logPath = findLog($c, $drvPath, @outPaths);
```

Replacement in src/lib/Hydra/Controller/Build.pm at line 144 [10.1]

B:BD[10.1998] → [16.386:492]

    notFound($c, "The build log of derivation ‘$drvPath’ is not available.") unless defined $logPath;

[10.1998]

[18.0]

    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]);

Deletion in src/lib/Hydra/Controller/Build.pm at line 146 [10.1]

B:BD[18.1] → [19.0:345]

∅:D[18.145] → [11.198:199]

∅:D[15.244] → [11.198:199]

∅:D[19.345] → [11.198:199]

∅:D[20.486] → [11.198:199]

∅:D[16.492] → [11.198:199]

B:BD[11.198] → [11.198:199]

    # Don't send logs that we can't stream.
    my $size = stat($logPath)->size; # FIXME: not so meaningful for compressed logs
    error($c, "This build log is too big to display ($size bytes).") unless
        $mode eq "raw"
        || (($mode eq "tail" || $mode eq "tail-reload") && $logPath !~ /\.bz2$/)
        || $size < 64 * 1024 * 1024;

Insertion in src/lib/Hydra/Controller/Build.pm at line 147 [10.1]
[14.54]
[13.637]
```
        $c->stash->{log_uri} = $log_uri;
```
Deletion in src/lib/Hydra/Controller/Build.pm at line 149 [10.1]
B:BD[13.679] → [9.152:206]
```
        $c->stash->{logtext} = logContents($logPath);
```
Replacement in src/lib/Hydra/Controller/Build.pm at line 152 [10.1]
B:BD[13.759] → [19.346:387]
B:BD[19.387] → [12.340:383]
∅:D[12.383] → [19.387:431]
B:BD[19.387] → [19.387:431]
∅:D[21.240] → [22.0:44]
∅:D[19.447] → [22.0:44]
B:BD[15.396] → [22.0:44]
B:BD[22.44] → [7.0:55]
∅:D[7.55] → [23.1206:1244]
B:BD[23.1206] → [23.1206:1244]
∅:D[23.1244] → [22.123:156]
B:BD[22.123] → [22.123:156]
B:BD[22.156] → [24.789:850]
∅:D[24.850] → [22.274:307]
∅:D[25.1522] → [22.274:307]
B:BD[22.274] → [22.274:307]
B:BD[22.307] → [17.1016:1091]
∅:D[26.332] → [22.382:433]
∅:D[16.699] → [22.382:433]
∅:D[17.1091] → [22.382:433]
B:BD[22.382] → [22.382:433]
```
        $c->stash->{logPath} = $logPath;
        $c->stash->{finished} = $finished;
        $c->forward('Hydra::View::NixLog');
    }
    elsif ($mode eq "tail-reload") {
        my $url = $c->uri_for($c->request->uri->path);
        $url =~ s/tail-reload/tail/g;
        $c->stash->{url} = $url;
        $c->stash->{reload} = !$c->stash->{build}->finished;
        $c->stash->{title} = "";
        $c->stash->{contents} = (scalar logContents($logPath, 50)) || " ";
        $c->stash->{template} = 'plain-reload.tt';
```
[13.759]
[13.797]
```
        $c->res->redirect($log_uri);
```

Replacement in src/lib/Hydra/Controller/Build.pm at line 156 [10.1]

B:BD[27.49] → [17.1092:1178]

∅:D[28.79] → [27.119:162]

∅:D[29.80] → [27.119:162]

∅:D[26.416] → [27.119:162]

∅:D[15.483] → [27.119:162]

∅:D[16.786] → [27.119:162]

∅:D[17.1178] → [27.119:162]

B:BD[27.119] → [27.119:162]

        $c->stash->{'plain'} = { data => (scalar logContents($logPath, 50)) || " " };
        $c->forward('Hydra::View::Plain');

[27.49]

[27.162]

        my $lines = 50;
        $c->stash->{log_uri} = $log_uri . "?tail=$lines";
        $c->stash->{tail} = $lines;
        $c->stash->{template} = 'log.tt';

Replacement in src/lib/Hydra/Controller/Build.pm at line 163 [10.1]

B:BD[13.815] → [13.815:871]

        error($c, "Unknown log display mode `$mode'.");

[13.815]

[13.871]

        error($c, "Unknown log display mode '$mode'.");

Insertion in src/lib/Hydra/Controller/Root.pm at line 13 [31.804]
[30.12]
[5.0]
```
use File::Basename;
```

Insertion in src/lib/Hydra/Controller/Root.pm at line 438 [31.804]

[32.286]

[33.0]

sub serveLogFile {
    my ($c, $logPath, $tail) = @_;
    $c->stash->{logPath} = $logPath;
    $c->stash->{tail} = $tail;
    $c->forward('Hydra::View::NixLog');
}

Replacement in src/lib/Hydra/Controller/Root.pm at line 446 [31.804]
B:BD[33.27] → [33.27:59]
```
    my ($self, $c, $path) = @_;
```
[33.27]
[34.418]
```
    my ($self, $c, $drvPath) = @_;
```
Replacement in src/lib/Hydra/Controller/Root.pm at line 448 [31.804]
B:BD[34.419] → [33.60:120]
```
    $path = ($ENV{NIX_STORE_DIR} || "/nix/store")."/$path";
```
[34.419]
[34.419]
```
    $drvPath = "/nix/store/$drvPath";
```

Replacement in src/lib/Hydra/Controller/Root.pm at line 450 [31.804]

B:BD[34.420] → [33.121:284]

    my @outpaths = ($path);
    my $logPath = findLog($c, $path, @outpaths);
    notFound($c, "The build log of $path is not available.") unless defined $logPath;

[34.420]

[33.284]

    my $tail = $c->request->params->{"tail"};

Replacement in src/lib/Hydra/Controller/Root.pm at line 452 [31.804]

B:BD[33.285] → [19.448:525]

∅:D[19.525] → [33.402:404]

B:BD[33.402] → [33.402:404]

    $c->stash->{logPath} = $logPath;
    $c->forward('Hydra::View::NixLog');
}

[33.285]

[33.404]

    die if defined $tail && $tail !~ /^[0-9]+$/;
    my $logFile = findLog($c, $drvPath);
    if (defined $logFile) {
        serveLogFile($c, $logFile, $tail);
        return;
    }
    my $logPrefix = $c->config->{log_prefix};

Insertion in src/lib/Hydra/Controller/Root.pm at line 463 [31.804]

[33.405]

    if (defined $logPrefix) {
        $c->res->redirect($logPrefix . "log/" . basename($drvPath));
    } else {
        notFound($c, "The build log of $drvPath is not available.");
    }
}

Replacement in src/lib/Hydra/Helper/Nix.pm at line 21 [36.436]
B:BD[35.508] → [35.508:565]
```
    removeAsciiEscapes getDrvLogPath findLog logContents
```
[35.508]
[37.844]
```
    getDrvLogPath findLog
```

Replacement in src/lib/Hydra/Helper/Nix.pm at line 157 [36.436]

B:BD[38.65] → [38.65:132]

B:BD[38.132] → [39.14898:14961]

B:BD[39.14961] → [40.3154:3288]

    my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
    my $fn2 = Hydra::Model::DB::getHydraPath . "/build-logs/";
    for ($fn2 . $bucketed, $fn2 . $bucketed . ".bz2", $fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {

[38.65]

[39.15068]

    my $fn = Hydra::Model::DB::getHydraPath . "/build-logs/";
    for ($fn . $bucketed, $fn . $bucketed . ".bz2") {

Deletion in src/lib/Hydra/Helper/Nix.pm at line 191 [36.436]
B:BD[41.3666] → [41.3666:3668]
B:BD[41.3668] → [42.5599:5600]
B:BD[42.5600] → [16.1313:1314]
∅:D[16.1314] → [26.441:459]
B:BD[42.5600] → [26.441:459]
B:BD[26.459] → [17.2046:2077]
∅:D[16.1422] → [26.487:500]
∅:D[17.2077] → [26.487:500]
B:BD[26.487] → [26.487:500]
B:BD[26.500] → [16.1423:1551]
∅:D[16.1551] → [26.624:641]
B:BD[26.624] → [26.624:641]
B:BD[26.641] → [16.1552:1624]
∅:D[16.1624] → [26.707:713]
B:BD[26.707] → [26.707:713]
B:BD[26.713] → [43.174:210]
```
}
sub logContents {
    my ($logPath, $tail) = @_;
    my $cmd;
    if ($logPath =~ /.bz2$/) {
        $cmd = "bzip2 -d < $logPath";
        $cmd = $cmd . " | tail -n $tail" if defined $tail;
    }
    else {
        $cmd = defined $tail ? "tail -$tail $logPath" : "cat $logPath";
    }
    return decode("utf-8", `$cmd`);
```
Deletion in src/lib/Hydra/Helper/Nix.pm at line 194 [36.436]
∅:D[16.1646] → [44.150:199]
B:BD[45.850] → [44.150:199]
B:BD[44.199] → [2.0:41]
∅:D[2.41] → [44.234:257]
B:BD[44.234] → [44.234:257]
∅:D[44.257] → [46.371:372]
B:BD[45.850] → [46.371:372]
B:BD[46.372] → [42.5601:5602]
```
sub removeAsciiEscapes {
    my ($logtext) = @_;
    $logtext =~ s/\e\[[0-9]*[A-Za-z]//g;
    return $logtext;
}
```
Insertion in src/lib/Hydra/View/NixLog.pm at line 15 [19.527]
[19.791]
[19.791]
```
    my $tail = int($c->stash->{tail} // "0");
```

Replacement in src/lib/Hydra/View/NixLog.pm at line 19 [19.527]

B:BD[19.824] → [19.824:877]

        open $fh, "bzip2 -dc < '$logPath' |" or die;

[19.824]

[19.877]

        my $doTail = $tail ? " tail -n '$tail' |" : "";
        open $fh, "bzip2 -dc < '$logPath' | $doTail" or die;

Replacement in src/lib/Hydra/View/NixLog.pm at line 22 [19.527]

B:BD[19.890] → [19.890:928]

        open $fh, "<$logPath" or die;

[19.890]

[19.928]

        if ($tail) {
            open $fh, "tail -n '$tail' '$logPath' |" or die;
        } else {
            open $fh, "<$logPath" or die;
        }

Insertion in src/lib/Hydra/View/TT.pm at line 16 [31.6789]
[17.2211]
[17.2211]
```
    return 1 if defined $c->config->{log_prefix};
```
Insertion in src/lib/Hydra/View/TT.pm at line 23 [31.6789]
[17.2392]
[17.2392]
```
    return 1 if defined $c->config->{log_prefix};
```
Deletion in src/lib/Hydra/View/TT.pm at line 28 [31.6789]
B:BD[4.91] → [4.91:92]

File deletion: plain-reload.tt

BF:BFD[47.1486] → [48.383:422]

BF:BFD[48.422] → [48.1:1]

B:BD[48.1] → [9.278:443]

∅:D[9.443] → [48.38:62]

B:BD[48.38] → [48.38:62]

∅:D[49.81] → [48.62:63]

B:BD[48.62] → [48.62:63]

B:BD[48.63] → [9.444:571]

∅:D[50.183] → [48.93:119]

∅:D[9.571] → [48.93:119]

B:BD[48.93] → [48.93:119]

B:BD[48.119] → [3.0:93]

∅:D[3.93] → [51.0:163]

B:BD[48.119] → [51.0:163]

∅:D[52.74] → [51.163:215]

∅:D[3.120] → [51.163:215]

B:BD[51.163] → [51.163:215]

∅:D[3.139] → [51.215:268]

B:BD[51.215] → [51.215:268]

∅:D[51.268] → [48.287:348]

B:BD[48.287] → [48.287:348]

B:BD[48.348] → [53.6369:6397]

∅:D[54.29] → [48.364:382]

∅:D[53.6397] → [48.364:382]

B:BD[48.364] → [48.364:382]

B:BD[51.215] → [3.121:139]

B:BD[51.163] → [3.94:120]

B:BD[48.62] → [49.0:81]

[% WRAPPER layout.tt title="Log of " _ (step ? " step $step.stepnr of " : "") _ "build ${build.id} of job $build.project.name:$build.jobset.name:$build.job.name" %]
[% PROCESS common.tt %]
<p>Below are the last 50 log lines. The <a href="[% c.uri_for('/build' build.id 'log') %]">full log</a> is also available.</p>
[% IF reload %]
<script>
function scrollDown() {
    $("#contents").scrollTop($("#contents").get(0).scrollHeight);
}
function injectTail() {
    $.ajax({
        url: "[% url %]",
        dataType: "text",
        success: function (tail) {
            $("#contents").text(tail);
        }
    });
}
$(document).ready(function() {
    injectTail();
    setInterval(injectTail, 5000);
});
</script>
[% END %]
<pre class="taillog" id="contents">
[% HTML.escape(contents) %]
</pre>
[% END %]
    scrollDown();
            scrollDown();
[% project = build.project %]
[% jobset = build.jobset %]
[% job = build.job %]

Replacement in src/root/build.tt at line 10 [31.7350]

B:BD[24.2034] → [24.2034:2087]

FOR step IN steps; IF step.busy; busy = 1; END; END;

[24.2034]

[24.2087]

building = 0;
FOR step IN steps;
  IF step.busy;
    busy = 1;
    IF step.drvpath == build.drvpath; building = 1; END;
  END;
END;

Replacement in src/root/build.tt at line 216 [31.7350]

B:BD[55.3656] → [6.0:131]

            [% IF (!isAggregate || !build.ischannel) && build.finished; actualBuild = build.iscachedbuild ? cachedBuild : build %]

[55.3656]

[56.535]

            [% actualBuild = build.iscachedbuild ? cachedBuild : build %]
            [% IF (!isAggregate || !build.ischannel) && build.finished;  %]

Replacement in src/root/build.tt at line 229 [31.7350]

B:BD[57.59] → [6.132:213]

            [% IF (!isAggregate || !build.ischannel) && buildLogExists(build) %]

[57.59]

[58.6492]

            [% IF (!build.finished && building) || (build.finished && (!isAggregate || !build.ischannel) && buildLogExists(build)) %]

Replacement in src/root/build.tt at line 233 [31.7350]

B:BD[58.6532] → [55.3739:4057]

                  <a class="btn btn-mini" href="[% c.uri_for('/build' build.id 'log') %]">pretty</a>
                  <a class="btn btn-mini" href="[% c.uri_for('/build' build.id 'log' 'raw') %]">raw</a>
                  <a class="btn btn-mini" href="[% c.uri_for('/build' build.id 'log' 'tail-reload') %]">tail</a>

[58.6532]

[58.6622]

                  <a class="btn btn-mini" href="[% c.uri_for('/build' actualBuild.id 'log') %]">pretty</a>
                  <a class="btn btn-mini" href="[% c.uri_for('/build' actualBuild.id 'log' 'raw') %]">raw</a>
                  <a class="btn btn-mini" href="[% c.uri_for('/build' actualBuild.id 'log' 'tail') %]">tail</a>

Replacement in src/root/common.tt at line 468 [61.2803]

B:BD[59.11401] → [60.1011:1162]

(<a [% IF inRow %]class="row-link"[% END %] href="[% url %]">log</a>, <a href="[% "$url/raw" %]">raw</a>, <a href="[% "$url/tail-reload" %]">tail</a>)

[59.11401]

[62.210]

(<a [% IF inRow %]class="row-link"[% END %] href="[% url %]">log</a>, <a href="[% "$url/raw" %]">raw</a>, <a href="[% "$url/tail" %]">tail</a>)

Replacement in src/root/log.tt at line 5 [31.11783]

B:BD[63.591] → [64.15317:15419]

  This is the build log of derivation <tt>[% IF step; step.drvpath; ELSE; build.drvpath; END %]</tt>.

[63.591]

[63.687]

  Below
  [% IF tail %]
  are the last lines of
  [% ELSE %]
  is
  [% END %]
  the build log of derivation <tt>[% IF step; step.drvpath; ELSE; build.drvpath; END %]</tt>.

Insertion in src/root/log.tt at line 14 [31.11783]

[63.768]

  [% END %]
  [% IF tail %]
  The <a href="[% step ? c.uri_for('/build' build.id 'nixlog' step.stepnr)
  : c.uri_for('/build' build.id 'log') %]">full log</a> is also available.

Replacement in src/root/log.tt at line 21 [31.11783]

B:BD[31.11896] → [9.207:270]

<pre class="taillog" id="contents">
[% HTML.escape(logtext) %]

[31.11896]

[9.270]

<pre class="log" id="contents">
<em>Loading...</em>

Insertion in src/root/log.tt at line 24 [31.11783]

[9.277]

[31.11963]


<script type="text/javascript">
  $(document).ready(function() {
    requestPlainFile({
      url: "[% HTML.escape(log_uri) %]",
      dataType: "text",
      type: 'GET',
      success: function (log_data) {
        [% IF tail %]
        /* The server may give us a full log (e.g. if the log is in
           S3). So extract the last lines. */
        log_data = log_data.split("\n").slice(-[%tail%]).join("\n");
        [% END %]
        $("#contents").text(log_data);
      },
      error: function () {
        bootbox.alert("The log file is not available.");
        $("#contents").text("(Unavailable)");
      }
    });
  });
</script>

Replacement in src/root/machine-status.tt at line 45 [65.8348]

B:BD[8.788] → [8.788:930]

            <td><a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail-reload') %]">[% step.stepnr %]</a></td>

[8.788]

[66.1149]

            <td><a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a></td>

Replacement in src/root/static/css/hydra.css at line 122 [31.8691]
B:BD[52.78] → [9.572:595]
```
pre.log, pre.taillog {
```
[52.78]
[52.92]
```
pre.log {
```

Insertion in src/root/static/js/common.js at line 121 [68.74]

[67.3699]

};
function requestFile(args) {
    if (!"error" in args) {
        args.error = function(data) {
            json = {};
            try {
                if (data.responseText)
                    json = $.parseJSON(data.responseText);
            } catch (err) {
            }
            if (json.error)
                bootbox.alert(escapeHTML(json.error));
            else if (data.responseText)
                bootbox.alert("Server error: " + escapeHTML(data.responseText));
            else
                bootbox.alert("Unknown server error!");
            if (args.postError) args.postError(data);
        };
    }
    return $.ajax(args);

Replacement in src/root/static/js/common.js at line 146 [68.74]

B:BD[69.2360] → [69.2360:2541]

    args.error = function(data) {
        json = {};
        try {
            if (data.responseText)
                json = $.parseJSON(data.responseText);
        } catch (err) {

[69.2360]

[69.2541]

    requestFile(args);
};
function requestPlainFile(args) {
    args.dataType = 'text';
    /* Remove the X-Requested-With header, which would turn trigger
       CORS checks for this request.
       http://stackoverflow.com/a/24719409/6747243
    */
    args.xhr = function() {
        var xhr = jQuery.ajaxSettings.xhr();
        var setRequestHeader = xhr.setRequestHeader;
        xhr.setRequestHeader = function(name, value) {
            if (name == 'X-Requested-With') return;
            setRequestHeader.call(this, name, value);

Replacement in src/root/static/js/common.js at line 162 [68.74]

B:BD[69.2551] → [69.2551:2575]

B:BD[69.2575] → [67.3733:3784]

∅:D[67.3784] → [69.2614:2650]

B:BD[69.2614] → [69.2614:2650]

B:BD[69.2650] → [67.3785:3862]

∅:D[67.3862] → [69.2715:2780]

B:BD[69.2715] → [69.2715:2780]

B:BD[69.2780] → [70.565:615]

        if (json.error)
            bootbox.alert(escapeHTML(json.error));
        else if (data.responseText)
            bootbox.alert("Server error: " + escapeHTML(data.responseText));
        else
            bootbox.alert("Unknown server error!");
        if (args.postError) args.postError(data);

[69.2551]

[69.2780]

        return xhr;

Replacement in src/root/static/js/common.js at line 164 [68.74]
B:BD[69.2787] → [69.2787:2812]
```
    return $.ajax(args);
```
[69.2787]
[67.3863]
```
    requestFile(args);
```

Replacement in src/root/steps.tt at line 28 [71.1061]

B:BD[71.1943] → [71.1943:2084]

        <td><a class="row-link" href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr 'tail-reload') %]">[% step.stepnr %]</a></td>

[71.1943]

[71.2084]

        <td><a class="row-link" href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a></td>

hydra-server: Support logs in S3

Dependencies

In channels

Change contents

Insertion in src/lib/Hydra/Controller/Build.pm at line 9 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 129 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 136 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 141 [10.1]

Deletion in src/lib/Hydra/Controller/Build.pm at line 143 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 144 [10.1]

Deletion in src/lib/Hydra/Controller/Build.pm at line 146 [10.1]

Insertion in src/lib/Hydra/Controller/Build.pm at line 147 [10.1]

Deletion in src/lib/Hydra/Controller/Build.pm at line 149 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 152 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 156 [10.1]

Replacement in src/lib/Hydra/Controller/Build.pm at line 163 [10.1]

Insertion in src/lib/Hydra/Controller/Root.pm at line 13 [31.804]

Insertion in src/lib/Hydra/Controller/Root.pm at line 438 [31.804]

Replacement in src/lib/Hydra/Controller/Root.pm at line 446 [31.804]

Replacement in src/lib/Hydra/Controller/Root.pm at line 448 [31.804]