hydra/src/script/hydra-notify

#! /usr/bin/env perl

use strict;
use utf8;
use Getopt::Long;
use HTTP::Server::PSGI;
use Hydra::Event;
use Hydra::Event::BuildFinished;
use Hydra::Helper::AddBuilds;
use Hydra::Helper::Nix;
use Hydra::Plugin;
use Hydra::PostgresListener;
use Parallel::ForkManager;
use Prometheus::Tiny::Shared;
use Time::HiRes qw( gettimeofday tv_interval );

STDERR->autoflush(1);
STDOUT->autoflush(1);
binmode STDERR, ":encoding(utf8)";

my $config = getHydraConfig();

my $prom = Prometheus::Tiny::Shared->new;
# Note: It is very important to pre-declare any metrics before using them.
# Add a new declaration for any new metrics you create. See:
# https://metacpan.org/pod/Prometheus::Tiny#declare
$prom->declare(
    "notify_plugin_executions",
    type => "counter",
    help => "Number of times each plugin has been called by channel."
);
$prom->declare(
    "notify_plugin_runtime",
    type => "histogram",
    help => "Number of seconds spent executing each plugin by channel."
);
$prom->declare(
    "notify_plugin_success",
    type => "counter",
    help => "Number of successful executions of this plugin on this channel."
);
$prom->declare(
    "notify_plugin_error",
    type => "counter",
    help => "Number of failed executions of this plugin on this channel."
);
$prom->declare(
    "event_loop_iterations",
    type => "counter",
    help => "Number of iterations through the event loop. Incremented at the start of the event loop."
);
$prom->declare(
    "event_received",
    type => "counter",
    help => "Timestamp of the last time a new event was received."
);
$prom->declare(
    "notify_event",
    type => "counter",
    help => "Number of events received on the given channel."
);
$prom->declare(
    "notify_event_error",
    type => "counter",
    help => "Number of events received that were unprocessable by channel."
);

my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config);
if (defined($promCfg)) {
    my $fork_manager = Parallel::ForkManager->new(1);
    $fork_manager->start_child("metrics_exporter", sub {
        my $server = HTTP::Server::PSGI->new(
            host => $promCfg->{"listen_address"},
            port => $promCfg->{"port"},
            timeout => 1,
        );

        $server->run($prom->psgi);
    });
}

my $queued_only;

GetOptions(
    "queued-only" => \$queued_only
) or exit 1;


my $db = Hydra::Model::DB->new();

my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);

my $dbh = $db->storage->dbh;

my $listener = Hydra::PostgresListener->new($dbh);
$listener->subscribe("build_started");
$listener->subscribe("build_finished");
$listener->subscribe("step_finished");

sub runPluginsForEvent {
    my ($event) = @_;

    my $channelName = $event->{'channel_name'};

    foreach my $plugin (@plugins) {
        $prom->inc("notify_plugin_executions", { channel => $channelName, plugin => ref $plugin });
        eval {
            my $startTime = [gettimeofday()];
            $event->execute($db, $plugin);

            $prom->histogram_observe("notify_plugin_runtime", tv_interval($startTime), { channel => $channelName, plugin => ref $plugin });
            $prom->inc("notify_plugin_success", { channel => $channelName, plugin => ref $plugin });
            1;
        } or do {
            $prom->inc("notify_plugin_error", { channel => $channelName, plugin => ref $plugin });
            print STDERR "error running $event->{'channel_name'} hooks: $@\n";
        }
    }
}

# Process builds that finished while hydra-notify wasn't running.
for my $build ($db->resultset('Builds')->search(
                   { notificationpendingsince => { '!=', undef } }))
{
    print STDERR "sending notifications for build ${\$build->id}...\n";


    my $event = Hydra::Event::BuildFinished->new($build->id);
    runPluginsForEvent($event);
}


# Process incoming notifications.
while (!$queued_only) {
    $prom->inc("event_loop_iterations");
    my $messages = $listener->block_for_messages();
    while (my $message = $messages->()) {
        $prom->set("event_received", time());
        my $channelName = $message->{"channel"};
        my $pid = $message->{"pid"};
        my $payload = $message->{"payload"};

        $prom->inc("notify_event", { channel => $channelName });

        eval {
            my $event = Hydra::Event->new_event($channelName, $message->{"payload"});
            runPluginsForEvent($event);

            1;
        } or do {
            $prom->inc("notify_event_error", { channel => $channelName });
            print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
        }
    }
}