ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into flake

Browse Source
This commit is contained in:
Eelco Dolstra
2019-08-15 13:56:00 +02:00
parent 1c9e22e73c c8a4030c5f
commit 2de52d8538
22 changed files with 239 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/script/hydra-eval-jobset
View File

@ -447,7 +447,7 @@ sub checkBuild {
# semantically unnecessary (because they're implied by
# the eval), but they give a factor 1000 speedup on
# the Nixpkgs jobset with PostgreSQL.
{ project => $jobset->project->name, jobset => $jobset->name, job => $jobName,
{ project => $jobset->get_column('project'), jobset => $jobset->name, job => $jobName,
name => $firstOutputName, path => $firstOutputPath },
{ rows => 1, columns => ['id'], join => ['buildoutputs'] });
if (defined $prevBuild) {
@ -498,7 +498,7 @@ sub checkBuild {
$buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath };
$$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id;
print STDERR "added build ${\$build->id} (${\$jobset->project->name}:${\$jobset->name}:$jobName)\n";
print STDERR "added build ${\$build->id} (${\$jobset->get_column('project')}:${\$jobset->name}:$jobName)\n";
});
return $build;
@ -540,7 +540,7 @@ sub sendJobsetErrorNotification() {
return if $jobset->project->owner->emailonerror == 0;
return if $errorMsg eq "";
my $projectName = $jobset->project->name;
my $projectName = $jobset->get_column('project');
my $jobsetName = $jobset->name;
my $body = "Hi,\n"
. "\n"
@ -577,7 +577,7 @@ sub permute {
sub checkJobsetWrapped {
my ($jobset) = @_;
my ($jobset, $tmpId) = @_;
my $project = $jobset->project;
my $jobsetsJobset = length($project->declfile) && $jobset->name eq ".jobsets";
my $inputInfo = {};
@ -626,6 +626,7 @@ sub checkJobsetWrapped {
print STDERR $fetchError;
txn_do($db, sub {
$jobset->update({ lastcheckedtime => time, fetcherrormsg => $fetchError }) if !$dryRun;
$db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
});
return;
}
@ -641,6 +642,7 @@ sub checkJobsetWrapped {
Net::Statsd::increment("hydra.evaluator.unchanged_checkouts");
txn_do($db, sub {
$jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });
$db->storage->dbh->do("notify eval_cached, ?", undef, join('\t', $tmpId));
});
return;
}
@ -710,6 +712,9 @@ sub checkJobsetWrapped {
, flake => $flakeRef
});
$db->storage->dbh->do("notify eval_added, ?", undef,
join('\t', $tmpId, $ev->id));
if ($jobsetChanged) {
# Create JobsetEvalMembers mappings.
while (my ($id, $x) = each %buildMap) {
@ -787,10 +792,6 @@ sub checkJobsetWrapped {
Net::Statsd::increment("hydra.evaluator.evals");
Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
#while (my ($id, $x) = each %buildMap) {
# system("hydra-notify build-queued $id") if $x->{new};
#}
# Store the error messages for jobs that failed to evaluate.
my $msg = "";
foreach my $job (values %{$jobs}) {
@ -808,8 +809,15 @@ sub checkJobset {
my $startTime = clock_gettime(CLOCK_MONOTONIC);
# Add an ID to eval_* notifications so receivers can correlate
# them.
my $tmpId = "${startTime}.$$";
$db->storage->dbh->do("notify eval_started, ?", undef,
join('\t', $tmpId, $jobset->get_column('project'), $jobset->name));
eval {
checkJobsetWrapped($jobset);
checkJobsetWrapped($jobset, $tmpId);
};
my $checkError = $@;
@ -822,6 +830,7 @@ sub checkJobset {
txn_do($db, sub {
$jobset->update({lastcheckedtime => time});
setJobsetError($jobset, $checkError);
$db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
}) if !$dryRun;
$failed = 1;
}

104
src/script/hydra-notify
View File

@ -5,6 +5,7 @@ use utf8;
use Hydra::Plugin;
use Hydra::Helper::Nix;
use Hydra::Helper::AddBuilds;
use IO::Select;
STDERR->autoflush(1);
binmode STDERR, ":encoding(utf8)";
@ -15,20 +16,37 @@ my $db = Hydra::Model::DB->new();
my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);
my $cmd = shift @ARGV or die "Syntax: hydra-notify build-started BUILD | build-finished BUILD-ID [BUILD-IDs...] | step-finished BUILD-ID STEP-NR LOG-PATH\n";
my $dbh = $db->storage->dbh;
my $buildId = shift @ARGV or die;
my $build = $db->resultset('Builds')->find($buildId)
or die "build $buildId does not exist\n";
$dbh->do("listen build_started");
$dbh->do("listen build_finished");
$dbh->do("listen step_finished");
sub buildStarted {
my ($buildId) = @_;
my $build = $db->resultset('Builds')->find($buildId)
or die "build $buildId does not exist\n";
foreach my $plugin (@plugins) {
eval { $plugin->buildStarted($build); };
if ($@) {
print STDERR "$plugin->buildStarted: $@\n";
}
}
}
sub buildFinished {
my ($build, @deps) = @_;
if ($cmd eq "build-finished") {
my $project = $build->project;
my $jobset = $build->jobset;
if (length($project->declfile) && $jobset->name eq ".jobsets" && $build->iscurrent) {
my $jobsetName = $build->get_column('jobset');
if (length($project->declfile) && $jobsetName eq ".jobsets" && $build->iscurrent) {
handleDeclarativeJobsetBuild($db, $project, $build);
}
my @dependents;
foreach my $id (@ARGV) {
foreach my $id (@deps) {
my $dep = $db->resultset('Builds')->find($id)
or die "build $id does not exist\n";
push @dependents, $dep;
@ -40,33 +58,20 @@ if ($cmd eq "build-finished") {
print STDERR "$plugin->buildFinished: $@\n";
}
}
$build->update({ notificationpendingsince => undef });
}
elsif ($cmd eq "build-queued") {
foreach my $plugin (@plugins) {
eval { $plugin->buildQueued($build); };
if ($@) {
print STDERR "$plugin->buildQueued: $@\n";
}
}
}
sub stepFinished {
my ($buildId, $stepNr, $logPath) = @_;
elsif ($cmd eq "build-started") {
foreach my $plugin (@plugins) {
eval { $plugin->buildStarted($build); };
if ($@) {
print STDERR "$plugin->buildStarted: $@\n";
}
}
}
my $build = $db->resultset('Builds')->find($buildId)
or die "build $buildId does not exist\n";
elsif ($cmd eq "step-finished") {
die if scalar @ARGV < 2;
my $stepNr = shift @ARGV;
my $step = $build->buildsteps->find({stepnr => $stepNr})
or die "step $stepNr does not exist\n";
my $logPath = shift @ARGV;
$logPath = undef if $logPath eq "";
$logPath = undef if $logPath eq "-";
foreach my $plugin (@plugins) {
eval { $plugin->stepFinished($step, $logPath); };
@ -76,6 +81,43 @@ elsif ($cmd eq "step-finished") {
}
}
else {
die "unknown action ‘$cmd’";
# Process builds that finished while hydra-notify wasn't running.
for my $build ($db->resultset('Builds')->search(
{ notificationpendingsince => { '!=', undef } }))
{
my $buildId = $build->id;
print STDERR "sending notifications for build ${\$buildId}...\n";
buildFinished($build);
}
# Process incoming notifications.
my $fd = $dbh->func("getfd");
my $sel = IO::Select->new($fd);
while (1) {
$sel->can_read;
while (my $notify = $dbh->func("pg_notifies")) {
my ($channelName, $pid, $payload) = @$notify;
#print STDERR "got '$channelName' from $pid: $payload\n";
my @payload = split /\t/, $payload;
eval {
if ($channelName eq "build_started") {
buildStarted(int($payload[0]));
} elsif ($channelName eq "build_finished") {
my $buildId = int($payload[0]);
my $build = $db->resultset('Builds')->find($buildId)
or die "build $buildId does not exist\n";
buildFinished($build, @payload[1..$#payload]);
} elsif ($channelName eq "step_finished") {
stepFinished(int($payload[0]), int($payload[1]));
}
};
if ($@) {
print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
}
}
}