2015-08-17 14:18:07 +02:00
|
|
|
#! /usr/bin/env perl
|
2015-06-23 00:14:49 +02:00
|
|
|
|
|
|
|
use strict;
|
|
|
|
use utf8;
|
|
|
|
use Hydra::Plugin;
|
|
|
|
use Hydra::Helper::Nix;
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
use Hydra::Helper::AddBuilds;
|
2019-08-09 19:11:38 +02:00
|
|
|
use IO::Select;
|
2015-06-23 00:14:49 +02:00
|
|
|
|
|
|
|
STDERR->autoflush(1);
|
|
|
|
binmode STDERR, ":encoding(utf8)";
|
|
|
|
|
|
|
|
my $config = getHydraConfig();
|
|
|
|
|
|
|
|
my $db = Hydra::Model::DB->new();
|
|
|
|
|
|
|
|
my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);
|
|
|
|
|
2019-08-09 19:11:38 +02:00
|
|
|
my $dbh = $db->storage->dbh;
|
2015-06-23 00:14:49 +02:00
|
|
|
|
2019-08-09 19:11:38 +02:00
|
|
|
$dbh->do("listen build_started");
|
|
|
|
$dbh->do("listen build_finished");
|
|
|
|
$dbh->do("listen step_finished");
|
|
|
|
|
|
|
|
sub buildStarted {
|
|
|
|
my ($buildId) = @_;
|
|
|
|
|
|
|
|
my $build = $db->resultset('Builds')->find($buildId)
|
|
|
|
or die "build $buildId does not exist\n";
|
|
|
|
|
|
|
|
foreach my $plugin (@plugins) {
|
Improve handling of Perl's block eval errors
Taken from `Perl::Critic`:
A common idiom in perl for dealing with possible errors is to use `eval`
followed by a check of `$@`/`$EVAL_ERROR`:
eval {
...
};
if ($EVAL_ERROR) {
...
}
There's a problem with this: the value of `$EVAL_ERROR` (`$@`) can change
between the end of the `eval` and the `if` statement. The issue are object
destructors:
package Foo;
...
sub DESTROY {
...
eval { ... };
...
}
package main;
eval {
my $foo = Foo->new();
...
};
if ($EVAL_ERROR) {
...
}
Assuming there are no other references to `$foo` created, when the
`eval` block in `main` is exited, `Foo::DESTROY()` will be invoked,
regardless of whether the `eval` finished normally or not. If the `eval`
in `main` fails, but the `eval` in `Foo::DESTROY()` succeeds, then
`$EVAL_ERROR` will be empty by the time that the `if` is executed.
Additional issues arise if you depend upon the exact contents of
`$EVAL_ERROR` and both `eval`s fail, because the messages from both will
be concatenated.
Even if there isn't an `eval` directly in the `DESTROY()` method code,
it may invoke code that does use `eval` or otherwise affects
`$EVAL_ERROR`.
The solution is to ensure that, upon normal exit, an `eval` returns a
true value and to test that value:
# Constructors are no problem.
my $object = eval { Class->new() };
# To cover the possiblity that an operation may correctly return a
# false value, end the block with "1":
if ( eval { something(); 1 } ) {
...
}
eval {
...
1;
}
or do {
# Error handling here
};
Unfortunately, you can't use the `defined` function to test the result;
`eval` returns an empty string on failure.
Various modules have been written to take some of the pain out of
properly localizing and checking `$@`/`$EVAL_ERROR`. For example:
use Try::Tiny;
try {
...
} catch {
# Error handling here;
# The exception is in $_/$ARG, not $@/$EVAL_ERROR.
}; # Note semicolon.
"But we don't use DESTROY() anywhere in our code!" you say. That may be
the case, but do any of the third-party modules you use have them? What
about any you may use in the future or updated versions of the ones you
already use?
2020-05-26 10:56:24 +02:00
|
|
|
eval {
|
|
|
|
$plugin->buildStarted($build);
|
|
|
|
1;
|
|
|
|
} or do {
|
|
|
|
print STDERR "error with $plugin->buildStarted: $@\n";
|
2019-08-09 19:11:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
sub buildFinished {
|
|
|
|
my ($build, @deps) = @_;
|
2016-05-27 14:32:48 +02:00
|
|
|
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
my $project = $build->project;
|
2019-08-13 17:19:40 +02:00
|
|
|
my $jobsetName = $build->get_column('jobset');
|
|
|
|
if (length($project->declfile) && $jobsetName eq ".jobsets" && $build->iscurrent) {
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
handleDeclarativeJobsetBuild($db, $project, $build);
|
|
|
|
}
|
2019-08-09 19:11:38 +02:00
|
|
|
|
2015-06-23 00:14:49 +02:00
|
|
|
my @dependents;
|
2019-08-09 19:11:38 +02:00
|
|
|
foreach my $id (@deps) {
|
2015-06-23 00:14:49 +02:00
|
|
|
my $dep = $db->resultset('Builds')->find($id)
|
|
|
|
or die "build $id does not exist\n";
|
|
|
|
push @dependents, $dep;
|
|
|
|
}
|
2016-05-27 14:32:48 +02:00
|
|
|
|
|
|
|
foreach my $plugin (@plugins) {
|
Improve handling of Perl's block eval errors
Taken from `Perl::Critic`:
A common idiom in perl for dealing with possible errors is to use `eval`
followed by a check of `$@`/`$EVAL_ERROR`:
eval {
...
};
if ($EVAL_ERROR) {
...
}
There's a problem with this: the value of `$EVAL_ERROR` (`$@`) can change
between the end of the `eval` and the `if` statement. The issue are object
destructors:
package Foo;
...
sub DESTROY {
...
eval { ... };
...
}
package main;
eval {
my $foo = Foo->new();
...
};
if ($EVAL_ERROR) {
...
}
Assuming there are no other references to `$foo` created, when the
`eval` block in `main` is exited, `Foo::DESTROY()` will be invoked,
regardless of whether the `eval` finished normally or not. If the `eval`
in `main` fails, but the `eval` in `Foo::DESTROY()` succeeds, then
`$EVAL_ERROR` will be empty by the time that the `if` is executed.
Additional issues arise if you depend upon the exact contents of
`$EVAL_ERROR` and both `eval`s fail, because the messages from both will
be concatenated.
Even if there isn't an `eval` directly in the `DESTROY()` method code,
it may invoke code that does use `eval` or otherwise affects
`$EVAL_ERROR`.
The solution is to ensure that, upon normal exit, an `eval` returns a
true value and to test that value:
# Constructors are no problem.
my $object = eval { Class->new() };
# To cover the possiblity that an operation may correctly return a
# false value, end the block with "1":
if ( eval { something(); 1 } ) {
...
}
eval {
...
1;
}
or do {
# Error handling here
};
Unfortunately, you can't use the `defined` function to test the result;
`eval` returns an empty string on failure.
Various modules have been written to take some of the pain out of
properly localizing and checking `$@`/`$EVAL_ERROR`. For example:
use Try::Tiny;
try {
...
} catch {
# Error handling here;
# The exception is in $_/$ARG, not $@/$EVAL_ERROR.
}; # Note semicolon.
"But we don't use DESTROY() anywhere in our code!" you say. That may be
the case, but do any of the third-party modules you use have them? What
about any you may use in the future or updated versions of the ones you
already use?
2020-05-26 10:56:24 +02:00
|
|
|
eval {
|
|
|
|
$plugin->buildFinished($build, [@dependents]);
|
|
|
|
1;
|
|
|
|
} or do {
|
|
|
|
print STDERR "error with $plugin->buildFinished: $@\n";
|
2016-05-27 14:32:48 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-28 10:45:41 +02:00
|
|
|
# We have to iterate through all dependents as well, and if they are finished
|
|
|
|
# to mark their notificationpendingsince.
|
|
|
|
# Otherwise, the dependent builds will remain with notificationpendingsince set
|
|
|
|
# until hydra-notify is started, as buildFinished is never emitted for them.
|
|
|
|
foreach my $b ($build, @dependents) {
|
|
|
|
$b->update({ notificationpendingsince => undef }) if $b->finished;
|
|
|
|
}
|
2017-05-24 09:45:31 -04:00
|
|
|
}
|
|
|
|
|
2019-08-09 19:11:38 +02:00
|
|
|
sub stepFinished {
|
|
|
|
my ($buildId, $stepNr, $logPath) = @_;
|
|
|
|
|
|
|
|
my $build = $db->resultset('Builds')->find($buildId)
|
|
|
|
or die "build $buildId does not exist\n";
|
2016-05-27 14:32:48 +02:00
|
|
|
|
|
|
|
my $step = $build->buildsteps->find({stepnr => $stepNr})
|
|
|
|
or die "step $stepNr does not exist\n";
|
2019-08-09 19:11:38 +02:00
|
|
|
|
|
|
|
$logPath = undef if $logPath eq "-";
|
2016-05-27 14:32:48 +02:00
|
|
|
|
|
|
|
foreach my $plugin (@plugins) {
|
Improve handling of Perl's block eval errors
Taken from `Perl::Critic`:
A common idiom in perl for dealing with possible errors is to use `eval`
followed by a check of `$@`/`$EVAL_ERROR`:
eval {
...
};
if ($EVAL_ERROR) {
...
}
There's a problem with this: the value of `$EVAL_ERROR` (`$@`) can change
between the end of the `eval` and the `if` statement. The issue are object
destructors:
package Foo;
...
sub DESTROY {
...
eval { ... };
...
}
package main;
eval {
my $foo = Foo->new();
...
};
if ($EVAL_ERROR) {
...
}
Assuming there are no other references to `$foo` created, when the
`eval` block in `main` is exited, `Foo::DESTROY()` will be invoked,
regardless of whether the `eval` finished normally or not. If the `eval`
in `main` fails, but the `eval` in `Foo::DESTROY()` succeeds, then
`$EVAL_ERROR` will be empty by the time that the `if` is executed.
Additional issues arise if you depend upon the exact contents of
`$EVAL_ERROR` and both `eval`s fail, because the messages from both will
be concatenated.
Even if there isn't an `eval` directly in the `DESTROY()` method code,
it may invoke code that does use `eval` or otherwise affects
`$EVAL_ERROR`.
The solution is to ensure that, upon normal exit, an `eval` returns a
true value and to test that value:
# Constructors are no problem.
my $object = eval { Class->new() };
# To cover the possiblity that an operation may correctly return a
# false value, end the block with "1":
if ( eval { something(); 1 } ) {
...
}
eval {
...
1;
}
or do {
# Error handling here
};
Unfortunately, you can't use the `defined` function to test the result;
`eval` returns an empty string on failure.
Various modules have been written to take some of the pain out of
properly localizing and checking `$@`/`$EVAL_ERROR`. For example:
use Try::Tiny;
try {
...
} catch {
# Error handling here;
# The exception is in $_/$ARG, not $@/$EVAL_ERROR.
}; # Note semicolon.
"But we don't use DESTROY() anywhere in our code!" you say. That may be
the case, but do any of the third-party modules you use have them? What
about any you may use in the future or updated versions of the ones you
already use?
2020-05-26 10:56:24 +02:00
|
|
|
eval {
|
|
|
|
$plugin->stepFinished($step, $logPath);
|
|
|
|
1;
|
|
|
|
} or do {
|
|
|
|
print STDERR "error with $plugin->stepFinished: $@\n";
|
2016-05-27 14:32:48 +02:00
|
|
|
}
|
|
|
|
}
|
2015-06-23 00:14:49 +02:00
|
|
|
}
|
|
|
|
|
2019-08-09 19:11:38 +02:00
|
|
|
# Process builds that finished while hydra-notify wasn't running.
|
|
|
|
for my $build ($db->resultset('Builds')->search(
|
|
|
|
{ notificationpendingsince => { '!=', undef } }))
|
|
|
|
{
|
|
|
|
my $buildId = $build->id;
|
|
|
|
print STDERR "sending notifications for build ${\$buildId}...\n";
|
|
|
|
buildFinished($build);
|
|
|
|
}
|
|
|
|
|
|
|
|
# Process incoming notifications.
|
|
|
|
my $fd = $dbh->func("getfd");
|
|
|
|
my $sel = IO::Select->new($fd);
|
|
|
|
|
|
|
|
while (1) {
|
|
|
|
$sel->can_read;
|
2019-08-12 17:28:21 +02:00
|
|
|
|
|
|
|
while (my $notify = $dbh->func("pg_notifies")) {
|
|
|
|
|
|
|
|
my ($channelName, $pid, $payload) = @$notify;
|
|
|
|
#print STDERR "got '$channelName' from $pid: $payload\n";
|
|
|
|
|
|
|
|
my @payload = split /\t/, $payload;
|
|
|
|
|
|
|
|
eval {
|
|
|
|
if ($channelName eq "build_started") {
|
|
|
|
buildStarted(int($payload[0]));
|
|
|
|
} elsif ($channelName eq "build_finished") {
|
|
|
|
my $buildId = int($payload[0]);
|
|
|
|
my $build = $db->resultset('Builds')->find($buildId)
|
|
|
|
or die "build $buildId does not exist\n";
|
|
|
|
buildFinished($build, @payload[1..$#payload]);
|
|
|
|
} elsif ($channelName eq "step_finished") {
|
|
|
|
stepFinished(int($payload[0]), int($payload[1]));
|
|
|
|
}
|
Improve handling of Perl's block eval errors
Taken from `Perl::Critic`:
A common idiom in perl for dealing with possible errors is to use `eval`
followed by a check of `$@`/`$EVAL_ERROR`:
eval {
...
};
if ($EVAL_ERROR) {
...
}
There's a problem with this: the value of `$EVAL_ERROR` (`$@`) can change
between the end of the `eval` and the `if` statement. The issue are object
destructors:
package Foo;
...
sub DESTROY {
...
eval { ... };
...
}
package main;
eval {
my $foo = Foo->new();
...
};
if ($EVAL_ERROR) {
...
}
Assuming there are no other references to `$foo` created, when the
`eval` block in `main` is exited, `Foo::DESTROY()` will be invoked,
regardless of whether the `eval` finished normally or not. If the `eval`
in `main` fails, but the `eval` in `Foo::DESTROY()` succeeds, then
`$EVAL_ERROR` will be empty by the time that the `if` is executed.
Additional issues arise if you depend upon the exact contents of
`$EVAL_ERROR` and both `eval`s fail, because the messages from both will
be concatenated.
Even if there isn't an `eval` directly in the `DESTROY()` method code,
it may invoke code that does use `eval` or otherwise affects
`$EVAL_ERROR`.
The solution is to ensure that, upon normal exit, an `eval` returns a
true value and to test that value:
# Constructors are no problem.
my $object = eval { Class->new() };
# To cover the possiblity that an operation may correctly return a
# false value, end the block with "1":
if ( eval { something(); 1 } ) {
...
}
eval {
...
1;
}
or do {
# Error handling here
};
Unfortunately, you can't use the `defined` function to test the result;
`eval` returns an empty string on failure.
Various modules have been written to take some of the pain out of
properly localizing and checking `$@`/`$EVAL_ERROR`. For example:
use Try::Tiny;
try {
...
} catch {
# Error handling here;
# The exception is in $_/$ARG, not $@/$EVAL_ERROR.
}; # Note semicolon.
"But we don't use DESTROY() anywhere in our code!" you say. That may be
the case, but do any of the third-party modules you use have them? What
about any you may use in the future or updated versions of the ones you
already use?
2020-05-26 10:56:24 +02:00
|
|
|
1;
|
|
|
|
} or do {
|
2019-08-12 17:28:21 +02:00
|
|
|
print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
|
2019-08-09 19:11:38 +02:00
|
|
|
}
|
|
|
|
}
|
2015-06-23 00:14:49 +02:00
|
|
|
}
|