ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #730 from NixOS/flake

Browse Source 
Flake support
This commit is contained in:
Eelco Dolstra
2020-04-07 11:18:38 +02:00
committed by GitHub
parent 62e6c65e68 681d84da0f
commit 4cabb37ebd
16 changed files with 609 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/hydra-eval-jobs/hydra-eval-jobs.cc
View File

@ -9,6 +9,8 @@
#include "get-drvs.hh"
#include "globals.hh"
#include "common-eval-args.hh"
#include "flake/flakeref.hh"
#include "flake/flake.hh"
#include "attr-path.hh"
#include "derivations.hh"
@ -28,6 +30,7 @@ static size_t maxMemorySize;
struct MyArgs : MixEvalArgs, MixCommonArgs
{
Path releaseExpr;
bool flake = false;
bool dryRun = false;
MyArgs() : MixCommonArgs("hydra-eval-jobs")
@ -51,6 +54,11 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
.description("don't create store derivations")
.set(&dryRun, true);
mkFlag()
.longName("flake")
.description("build a flake")
.set(&flake, true);
expectArg("expr", &releaseExpr);
}
};
@ -89,7 +97,37 @@ static void worker(
AutoCloseFD & from)
{
Value vTop;
state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop);
if (myArgs.flake) {
using namespace flake;
auto flakeRef = parseFlakeRef(myArgs.releaseExpr);
auto vFlake = state.allocValue();
auto lockedFlake = lockFlake(state, flakeRef,
LockFlags {
.updateLockFile = false,
.useRegistries = false,
.allowMutable = false,
});
callFlake(state, lockedFlake, *vFlake);
auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
state.forceValue(*vOutputs);
auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
if (!aHydraJobs)
aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));
if (!aHydraJobs)
throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);
vTop = *aHydraJobs->value;
} else {
state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop);
}
auto vRoot = state.allocValue();
state.autoCallFunction(autoArgs, vTop, *vRoot);
@ -109,7 +147,7 @@ static void worker(
nlohmann::json reply;
try {
auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot);
auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first;
auto v = state.allocValue();
state.autoCallFunction(autoArgs, *vTmp, *v);
@ -139,23 +177,23 @@ static void worker(
/* If this is an aggregate, then get its constituents. */
auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
if (a && state.forceBool(*(*a)->value, *(*a)->pos)) {
if (a && state.forceBool(*a->value, *a->pos)) {
auto a = v->attrs->get(state.symbols.create("constituents"));
if (!a)
throw EvalError("derivation must have a ‘constituents’ attribute");
PathSet context;
state.coerceToString(*(*a)->pos, *(*a)->value, context, true, false);
state.coerceToString(*a->pos, *a->value, context, true, false);
for (auto & i : context)
if (i.at(0) == '!') {
size_t index = i.find("!", 1);
job["constituents"].push_back(string(i, index + 1));
}
state.forceList(*(*a)->value, *(*a)->pos);
for (unsigned int n = 0; n < (*a)->value->listSize(); ++n) {
auto v = (*a)->value->listElems()[n];
state.forceList(*a->value, *a->pos);
for (unsigned int n = 0; n < a->value->listSize(); ++n) {
auto v = a->value->listElems()[n];
state.forceValue(*v);
if (v->type == tString)
job["namedConstituents"].push_back(state.forceStringNoCtx(*v));
@ -245,6 +283,10 @@ int main(int argc, char * * argv)
to the environment. */
evalSettings.restrictEval = true;
/* When building a flake, use pure evaluation (no access to
'getEnv', 'currentSystem' etc. */
evalSettings.pureEval = myArgs.flake;
if (myArgs.dryRun) settings.readOnlyMode = true;
if (myArgs.releaseExpr == "") throw UsageError("no expression specified");

13
src/hydra-evaluator/hydra-evaluator.cc
View File

@ -103,7 +103,7 @@ struct Evaluator
}
if (evalOne && seen.empty()) {
printError("the specified jobset does not exist");
printError("the specified jobset does not exist or is disabled");
std::_Exit(1);
}
@ -458,14 +458,15 @@ int main(int argc, char * * argv)
return true;
});
if (!args.empty()) {
if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [<project> <jobset>]");
evaluator.evalOne = JobsetName(args[0], args[1]);
}
if (unlock)
evaluator.unlock();
else
else {
if (!args.empty()) {
if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [<project> <jobset>]");
evaluator.evalOne = JobsetName(args[0], args[1]);
}
evaluator.run();
}
});
}

46
src/lib/Hydra/Controller/Jobset.pm
View File

@ -223,7 +223,19 @@ sub updateJobset {
error($c, "Cannot rename jobset to ‘$jobsetName’ since that identifier is already taken.")
if $jobsetName ne $oldName && defined $c->stash->{project}->jobsets->find({ name => $jobsetName });
my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
my $type = int($c->stash->{params}->{"type"}) // 0;
my ($nixExprPath, $nixExprInput);
my $flake;
if ($type == 0) {
($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
} elsif ($type == 1) {
$flake = trim($c->stash->{params}->{"flakeref"});
error($c, "Invalid flake URI ‘$flake’.") if $flake !~ /^[a-zA-Z]/;
} else {
error($c, "Invalid jobset type.");
}
my $enabled = int($c->stash->{params}->{enabled});
die if $enabled < 0 || $enabled > 3;
@ -246,6 +258,8 @@ sub updateJobset {
, checkinterval => $checkinterval
, triggertime => ($enabled && $checkinterval > 0) ? $jobset->triggertime // time() : undef
, schedulingshares => $shares
, type => $type
, flake => $flake
});
$jobset->project->jobsetrenames->search({ from_ => $jobsetName })->delete;
@ -255,23 +269,25 @@ sub updateJobset {
# Set the inputs of this jobset.
$jobset->jobsetinputs->delete;
foreach my $name (keys %{$c->stash->{params}->{inputs}}) {
my $inputData = $c->stash->{params}->{inputs}->{$name};
my $type = $inputData->{type};
my $value = $inputData->{value};
my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0;
if ($type == 0) {
foreach my $name (keys %{$c->stash->{params}->{inputs}}) {
my $inputData = $c->stash->{params}->{inputs}->{$name};
my $type = $inputData->{type};
my $value = $inputData->{value};
my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0;
error($c, "Invalid input name ‘$name’.") unless $name =~ /^[[:alpha:]][\w-]*$/;
error($c, "Invalid input type ‘$type’.") unless defined $c->stash->{inputTypes}->{$type};
error($c, "Invalid input name ‘$name’.") unless $name =~ /^[[:alpha:]][\w-]*$/;
error($c, "Invalid input type ‘$type’.") unless defined $c->stash->{inputTypes}->{$type};
my $input = $jobset->jobsetinputs->create(
{ name => $name,
type => $type,
emailresponsible => $emailresponsible
});
my $input = $jobset->jobsetinputs->create(
{ name => $name,
type => $type,
emailresponsible => $emailresponsible
});
$value = checkInputValue($c, $name, $type, $value);
$input->jobsetinputalts->create({altnr => 0, value => $value});
$value = checkInputValue($c, $name, $type, $value);
$input->jobsetinputalts->create({altnr => 0, value => $value});
}
}
}

2
src/lib/Hydra/Helper/AddBuilds.pm
View File

@ -30,6 +30,8 @@ sub updateDeclarativeJobset {
my @allowed_keys = qw(
enabled
hidden
type
flake
description
nixexprinput
nixexprpath

27
src/root/build.tt
View File

@ -120,7 +120,7 @@ END;
<b class="caret"></b>
</a>
<ul class="dropdown-menu">
[% IF build.nixexprinput %]
[% IF build.nixexprinput || eval.flake %]
<li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li>
[% END %]
[% IF c.user_exists %]
@ -530,18 +530,33 @@ END;
<div class="modal-body">
<p>You can reproduce this build on your own machine by downloading
<a [% HTML.attributes(href => url) %]>a script</a> that checks out
all inputs of the build and then invokes Nix to perform the build.
This script requires that you have Nix on your system.</p>
[% IF eval.flake %]
<p>If you have <a href='https://nixos.org/nix/download.html'>Nix
installed</a>, you can reproduce this build on your own machine by
running the following command:</p>
<pre>
<span class="shell-prompt"># </span>nix build [% HTML.escape(eval.flake) %]#hydraJobs.[% HTML.escape(job.name) %]
</pre>
[% ELSE %]
<p>If you have <a href='https://nixos.org/nix/download.html'>Nix
installed</a>, you can reproduce this build on your own machine by
downloading <a [% HTML.attributes(href => url) %]>a script</a>
that checks out all inputs of the build and then invokes Nix to
perform the build.</p>
<p>To download and execute the script from the command line, run the
following command:</p>
<pre>
<span class="shell-prompt">$ </span>curl <a [% HTML.attributes(href => url) %]>[% HTML.escape(url) %]</a> | bash
<span class="shell-prompt"># </span>curl <a [% HTML.attributes(href => url) %]>[% HTML.escape(url) %]</a> | bash
</pre>
[% END %]
</div>
<div class="modal-footer">

35
src/root/edit-jobset.tt
View File

@ -42,7 +42,7 @@
[% END %]
[% BLOCK renderJobsetInputs %]
<table class="table table-striped table-condensed">
<table class="table table-striped table-condensed show-on-legacy">
<thead>
<tr><th></th><th>Input name</th><th>Type</th><th style="width: 50%">Value</th><th>Notify committers</th></tr>
</thead>
@ -97,6 +97,24 @@
</div>
<div class="control-group">
<label class="control-label">Type</label>
<div class="controls">
<div class="btn-group" data-toggle="buttons-radio">
<input type="hidden" id="type" name="type" value="[% jobset.type %]" />
<button type="button" class="btn" value="1" id="type-flake">Flake</button>
<button type="button" class="btn" value="0" id="type-legacy">Legacy</button>
</div>
</div>
</div>
<div class="control-group show-on-flake">
<label class="control-label">Flake URI</label>
<div class="controls">
<input type="text" class="span3" name="flakeref" [% HTML.attributes(value => jobset.flake) %]/>
</div>
</div>
<div class="control-group show-on-legacy">
<label class="control-label">Nix expression</label>
<div class="controls">
<input type="text" class="span3" name="nixexprpath" [% HTML.attributes(value => jobset.nixexprpath) %]/>
@ -168,6 +186,21 @@
$(document).ready(function() {
var id = 0;
function update() {
if ($("#type").val() == 0) {
$(".show-on-legacy").show();
$(".show-on-flake").hide();
} else {
$(".show-on-legacy").hide();
$(".show-on-flake").show();
}
}
$("#type-flake").click(function() { update(); });
$("#type-legacy").click(function() { update(); });
update();
$(".add-input").click(function() {
var newid = "input-" + id++;
var x = $("#input-template").clone(true).attr("id", "").insertBefore($(this).parents("tr")).show();

2
src/root/edit-project.tt
View File

@ -8,7 +8,7 @@
<div class="control-group">
<div class="controls">
<label class="checkbox">
<input type="checkbox" name="enabled" [% IF project.enabled; 'checked="checked"'; END %]/>Enabled
<input type="checkbox" name="enabled" [% IF create || project.enabled; 'checked="checked"'; END %]/>Enabled
</label>
</div>
<div class="controls">

3
src/root/jobset-eval.tt
View File

@ -18,7 +18,8 @@
</ul>
</div>
<p>This evaluation was performed on [% INCLUDE renderDateTime
<p>This evaluation was performed [% IF eval.flake %]from the flake
<tt>[%HTML.escape(eval.flake)%]</tt>[%END%] on [% INCLUDE renderDateTime
timestamp=eval.timestamp %]. Fetching the dependencies took [%
eval.checkouttime %]s and evaluation took [% eval.evaltime %]s.</p>

12
src/root/jobset.tt
View File

@ -135,6 +135,15 @@
<th>Description:</th>
<td>[% HTML.escape(jobset.description) %]</td>
</tr>
[% IF jobset.type == 1 %]
<tr>
<th>Flake URI:</th>
<td>
<tt>[% HTML.escape(jobset.flake) %]</tt>
</td>
</tr>
[% END %]
[% IF jobset.type == 0 %]
<tr>
<th>Nix expression:</th>
<td>
@ -142,6 +151,7 @@
<tt>[% HTML.escape(jobset.nixexprinput) %]</tt>
</td>
</tr>
[% END %]
<tr>
<th>Check interval:</th>
<td>[% jobset.checkinterval || "<em>disabled</em>" %]</td>
@ -166,7 +176,9 @@
</tr>
</table>
[% IF jobset.type == 0 %]
[% INCLUDE renderJobsetInputs %]
[% END %]
</div>
[% INCLUDE makeLazyTab tabName="tabs-jobs" uri=c.uri_for('/jobset' project.name jobset.name "jobs-tab") %]

46
src/script/hydra-eval-jobset
View File

@ -328,16 +328,25 @@ sub inputsToArgs {
sub evalJobs {
my ($inputInfo, $nixExprInputName, $nixExprPath) = @_;
my ($inputInfo, $nixExprInputName, $nixExprPath, $flakeRef) = @_;
my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
or die "cannot find the input containing the job expression\n";
my @cmd;
my @cmd = ("hydra-eval-jobs",
"<" . $nixExprInputName . "/" . $nixExprPath . ">",
"--gc-roots-dir", getGCRootsDir,
"-j", 1,
inputsToArgs($inputInfo));
if (defined $flakeRef) {
@cmd = ("hydra-eval-jobs",
"--flake", $flakeRef,
"--gc-roots-dir", getGCRootsDir,
"--max-jobs", 1);
} else {
my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
or die "cannot find the input containing the job expression\n";
@cmd = ("hydra-eval-jobs",
"<" . $nixExprInputName . "/" . $nixExprPath . ">",
"--gc-roots-dir", getGCRootsDir,
"--max-jobs", 1,
inputsToArgs($inputInfo));
}
if (defined $ENV{'HYDRA_DEBUG'}) {
sub escape {
@ -356,7 +365,7 @@ sub evalJobs {
print STDERR "$stderr";
return (decode_json($jobsJSON), $nixExprInput);
return decode_json($jobsJSON);
}
@ -374,7 +383,7 @@ sub getPrevJobsetEval {
# Check whether to add the build described by $buildInfo.
sub checkBuild {
my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
my ($db, $jobset, $inputInfo, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
my @outputNames = sort keys %{$buildInfo->{outputs}};
die unless scalar @outputNames;
@ -577,6 +586,16 @@ sub checkJobsetWrapped {
};
my $fetchError = $@;
my $flakeRef = $jobset->flake;
if (defined $flakeRef) {
(my $res, my $json, my $stderr) = captureStdoutStderr(
600, "nix", "flake", "info", "--tarball-ttl", 0, "--json", "--", $flakeRef);
die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
if $res;
$flakeRef = decode_json($json)->{'url'};
}
Net::Statsd::increment("hydra.evaluator.checkouts");
my $checkoutStop = clock_gettime(CLOCK_MONOTONIC);
Net::Statsd::timing("hydra.evaluator.checkout_time", int(($checkoutStop - $checkoutStart) * 1000));
@ -597,7 +616,7 @@ sub checkJobsetWrapped {
my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo));
my $argsHash = sha256_hex("@args");
my $prevEval = getPrevJobsetEval($db, $jobset, 0);
if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval) {
if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval && $prevEval->flake eq $flakeRef) {
print STDERR " jobset is unchanged, skipping\n";
Net::Statsd::increment("hydra.evaluator.unchanged_checkouts");
txn_do($db, sub {
@ -609,7 +628,7 @@ sub checkJobsetWrapped {
# Evaluate the job expression.
my $evalStart = clock_gettime(CLOCK_MONOTONIC);
my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath);
my $jobs = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
my $evalStop = clock_gettime(CLOCK_MONOTONIC);
if ($jobsetsJobset) {
@ -654,7 +673,7 @@ sub checkJobsetWrapped {
foreach my $job (permute(values %{$jobs})) {
next if defined $job->{error};
#print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
checkBuild($db, $jobset, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
}
# Have any builds been added or removed since last time?
@ -669,6 +688,7 @@ sub checkJobsetWrapped {
, evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
, flake => $flakeRef
});
$db->storage->dbh->do("notify eval_added, ?", undef,