ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

TaskDispatcher: Support re-queueing tasks, and dropping tasks after 100 failures.

Browse Source
This commit is contained in:
Graham Christensen
2021-08-26 17:00:39 -04:00
parent d0b0fc21b3
commit b0055a23df
2 changed files with 249 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
t/TaskDispatcher.t
View File

@ -60,6 +60,31 @@ sub make_failing_event {
return $mock_event;
}
sub make_fake_record {
my %attrs = @_;
my $record = {
"attempts" => $attrs{"attempts"} || 0,
"requeued" => 0,
"deleted" => 0
};
my $mock_record = mock_obj $record => (
add => [
"delete" => sub {
my ($self, $db, $plugin) = @_;
$self->{"deleted"} = 1;
},
"requeue" => sub {
my ($self, $db, $plugin) = @_;
$self->{"requeued"} = 1;
}
]
);
return $mock_record;
}
subtest "dispatch_event" => sub {
subtest "every plugin gets called once, even if it fails all of them." => sub {
my @plugins = [make_noop_plugin("bogus-1"), make_noop_plugin("bogus-2")];
@ -104,6 +129,7 @@ subtest "dispatch_task" => sub {
"Just bogus-1 should be executed."
);
};
subtest "a task with an invalid plugin is not fatal" => sub {
my $bogus_plugin = make_noop_plugin("bogus-1");
my @plugins = [$bogus_plugin, make_noop_plugin("bogus-2")];
@ -116,6 +142,80 @@ subtest "dispatch_task" => sub {
is(@{$event->{"called_with"}}, 0, "No plugins are called");
};
subtest "a failed run without a record saves the task for later" => sub {
my $db = "bogus db";
my $record = make_fake_record();
my $bogus_plugin = make_noop_plugin("bogus-1");
my $task = {
"event" => make_failing_event("fail-event"),
"plugin_name" => ref $bogus_plugin,
"record" => undef,
};
my $save_hook_called = 0;
my $dispatcher = Hydra::TaskDispatcher->new($db, $prometheus, [$bogus_plugin],
sub {
$save_hook_called = 1;
}
);
$dispatcher->dispatch_task($task);
is($save_hook_called, 1, "The record was requeued with the store hook.");
};
subtest "a successful run from a record deletes the record" => sub {
my $db = "bogus db";
my $record = make_fake_record();
my $bogus_plugin = make_noop_plugin("bogus-1");
my $task = {
"event" => make_fake_event("success-event"),
"plugin_name" => ref $bogus_plugin,
"record" => $record,
};
my $dispatcher = Hydra::TaskDispatcher->new($db, $prometheus, [$bogus_plugin]);
$dispatcher->dispatch_task($task);
is($record->{"deleted"}, 1, "The record was deleted.");
};
subtest "a failed run from a record re-queues the task" => sub {
my $db = "bogus db";
my $record = make_fake_record();
my $bogus_plugin = make_noop_plugin("bogus-1");
my $task = {
"event" => make_failing_event("fail-event"),
"plugin_name" => ref $bogus_plugin,
"record" => $record,
};
my $dispatcher = Hydra::TaskDispatcher->new($db, $prometheus, [$bogus_plugin]);
$dispatcher->dispatch_task($task);
is($record->{"requeued"}, 1, "The record was requeued.");
};
subtest "a failed run from a record with a lot of attempts deletes the task" => sub {
my $db = "bogus db";
my $record = make_fake_record(attempts => 101);
my $bogus_plugin = make_noop_plugin("bogus-1");
my $task = {
"event" => make_failing_event("fail-event"),
"plugin_name" => ref $bogus_plugin,
"record" => $record,
};
my $dispatcher = Hydra::TaskDispatcher->new($db, $prometheus, [$bogus_plugin]);
$dispatcher->dispatch_task($task);
is($record->{"deleted"}, 1, "The record was deleted.");
};
};