ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #1103 from DeterminateSystems/runcommand/dynamic

Browse Source 
Dynamic RunCommand
This commit is contained in:
Graham Christensen
2022-04-19 10:09:47 -04:00
committed by GitHub
parent c44d9d9e91 01aedc92ed
commit 5c90edd19f
29 changed files with 1233 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/lib/Hydra/Controller/Build.pm
View File

@ -38,6 +38,17 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
$c->stash->{jobset} = $c->stash->{build}->jobset;
$c->stash->{job} = $c->stash->{build}->job;
$c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})];
$c->stash->{runcommandlogProblem} = undef;
if ($c->stash->{job} =~ qr/^runCommandHook\..*/) {
if (!$c->config->{dynamicruncommand}->{enable}) {
$c->stash->{runcommandlogProblem} = "disabled-server";
} elsif (!$c->stash->{project}->enable_dynamic_run_command) {
$c->stash->{runcommandlogProblem} = "disabled-project";
} elsif (!$c->stash->{jobset}->enable_dynamic_run_command) {
$c->stash->{runcommandlogProblem} = "disabled-jobset";
}
}
}

9
src/lib/Hydra/Controller/Jobset.pm
View File

@ -261,6 +261,14 @@ sub updateJobset {
my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($c->config->{dynamicruncommand}->{enable}
&& $jobset->project->enable_dynamic_run_command))
{
badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
}
$jobset->update(
{ name => $jobsetName
, description => trim($c->stash->{params}->{"description"})
@ -268,6 +276,7 @@ sub updateJobset {
, nixexprinput => $nixExprInput
, enabled => $enabled
, enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
, enable_dynamic_run_command => $enable_dynamic_run_command
, emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))

6
src/lib/Hydra/Controller/Project.pm
View File

@ -149,6 +149,11 @@ sub updateProject {
my $displayName = trim $c->stash->{params}->{displayname};
error($c, "You must specify a display name.") if $displayName eq "";
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
badRequest($c, "Dynamic RunCommand is not enabled by the server.");
}
$project->update(
{ name => $projectName
, displayname => $displayName
@ -157,6 +162,7 @@ sub updateProject {
, enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, owner => $owner
, enable_dynamic_run_command => $enable_dynamic_run_command
, declfile => trim($c->stash->{params}->{declarative}->{file})
, decltype => trim($c->stash->{params}->{declarative}->{type})
, declvalue => trim($c->stash->{params}->{declarative}->{value})

45
src/lib/Hydra/Helper/AddBuilds.pm
View File

@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
our @ISA = qw(Exporter);
our @EXPORT = qw(
validateDeclarativeJobset
createJobsetInputsRowAndData
updateDeclarativeJobset
handleDeclarativeJobsetBuild
handleDeclarativeJobsetJson
);
sub updateDeclarativeJobset {
my ($db, $project, $jobsetName, $declSpec) = @_;
sub validateDeclarativeJobset {
my ($config, $project, $jobsetName, $declSpec) = @_;
my @allowed_keys = qw(
enabled
@ -39,6 +41,7 @@ sub updateDeclarativeJobset {
checkinterval
schedulingshares
enableemail
enable_dynamic_run_command
emailoverride
keepnr
);
@ -61,16 +64,39 @@ sub updateDeclarativeJobset {
}
}
my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($config->{dynamicruncommand}->{enable}
&& $project->{enable_dynamic_run_command}))
{
die "Dynamic RunCommand is not enabled by the server or the parent project.";
}
return %update;
}
sub createJobsetInputsRowAndData {
my ($name, $declSpec) = @_;
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
return ($row, $data);
}
sub updateDeclarativeJobset {
my ($config, $db, $project, $jobsetName, $declSpec) = @_;
my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
$db->txn_do(sub {
my $jobset = $project->jobsets->update_or_create(\%update);
$jobset->jobsetinputs->delete;
foreach my $name (keys %{$declSpec->{"inputs"}}) {
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
my $input = $jobset->jobsetinputs->create($row);
$input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
}
@ -81,6 +107,7 @@ sub updateDeclarativeJobset {
sub handleDeclarativeJobsetJson {
my ($db, $project, $declSpec) = @_;
my $config = getHydraConfig();
$db->txn_do(sub {
my @kept = keys %$declSpec;
push @kept, ".jobsets";
@ -88,7 +115,7 @@ sub handleDeclarativeJobsetJson {
foreach my $jobsetName (keys %$declSpec) {
my $spec = $declSpec->{$jobsetName};
eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
1;
} or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";

95
src/lib/Hydra/Plugin/RunCommand.pm
View File

@ -12,7 +12,74 @@ use Try::Tiny;
sub isEnabled {
my ($self) = @_;
return defined $self->{config}->{runcommand};
return areStaticCommandsEnabled($self->{config}) || areDynamicCommandsEnabled($self->{config});
}
sub areStaticCommandsEnabled {
my ($config) = @_;
if (defined $config->{runcommand}) {
return 1;
}
return 0;
}
sub areDynamicCommandsEnabled {
my ($config) = @_;
if ((defined $config->{dynamicruncommand})
&& $config->{dynamicruncommand}->{enable}) {
return 1;
}
return 0;
}
sub isBuildEligibleForDynamicRunCommand {
my ($build) = @_;
if ($build->get_column("buildstatus") != 0) {
return 0;
}
if ($build->get_column("job") =~ "^runCommandHook\..+") {
my $out = $build->buildoutputs->find({name => "out"});
if (!defined $out) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: no output named 'out'.";
return 0;
}
my $path = $out->path;
if (-l $path) {
$path = readlink($path);
}
if (! -e $path) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output doesn't exist locally. This is a bug.";
return 0;
}
if (! -x $path) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not executable.";
return 0;
}
if (! -f $path) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not a regular file or symlink.";
return 0;
}
if (! $build->jobset->supportsDynamicRunCommand()) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The project or jobset don't have dynamic runcommand enabled.";
return 0;
}
return 1;
}
return 0;
}
sub configSectionMatches {
@ -43,10 +110,11 @@ sub eventMatches {
}
sub fanoutToCommands {
my ($config, $event, $project, $jobset, $job) = @_;
my ($config, $event, $build) = @_;
my @commands;
# Calculate all the statically defined commands to execute
my $cfg = $config->{runcommand};
my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
@ -55,9 +123,10 @@ sub fanoutToCommands {
next unless eventMatches($conf, $event);
next unless configSectionMatches(
$matcher,
$project,
$jobset,
$job);
$build->jobset->get_column('project'),
$build->jobset->get_column('name'),
$build->get_column('job')
);
if (!defined($conf->{command})) {
warn "<runcommand> section for '$matcher' lacks a 'command' option";
@ -70,6 +139,18 @@ sub fanoutToCommands {
})
}
# Calculate all dynamically defined commands to execute
if (areDynamicCommandsEnabled($config)) {
if (isBuildEligibleForDynamicRunCommand($build)) {
my $job = $build->get_column('job');
my $out = $build->buildoutputs->find({name => "out"});
push(@commands, {
matcher => "DynamicRunCommand($job)",
command => $out->path
})
}
}
return \@commands;
}
@ -138,9 +219,7 @@ sub buildFinished {
my $commandsToRun = fanoutToCommands(
$self->{config},
$event,
$build->project->get_column('name'),
$build->jobset->get_column('name'),
$build->get_column('job')
$build
);
if (@$commandsToRun == 0) {

20
src/lib/Hydra/Schema/Result/Jobsets.pm
View File

@ -155,6 +155,12 @@ __PACKAGE__->table("jobsets");
data_type: 'text'
is_nullable: 1
=head2 enable_dynamic_run_command
data_type: 'boolean'
default_value: false
is_nullable: 0
=cut
__PACKAGE__->add_columns(
@ -207,6 +213,8 @@ __PACKAGE__->add_columns(
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"flake",
{ data_type => "text", is_nullable => 1 },
"enable_dynamic_run_command",
{ data_type => "boolean", default_value => \"false", is_nullable => 0 },
);
=head1 PRIMARY KEY
@ -354,8 +362,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cQOnMitrWGMoJX6kZGNW+w
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:17:33
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7wPE5ebeVTkenMCWG9Sgcg
use JSON::MaybeXS;
@ -378,6 +386,13 @@ __PACKAGE__->add_column(
"+id" => { retrieve_on_insert => 1 }
);
sub supportsDynamicRunCommand {
my ($self) = @_;
return $self->get_column('enable_dynamic_run_command') == 1
&& $self->project->supportsDynamicRunCommand();
}
sub as_json {
my $self = shift;
@ -406,6 +421,7 @@ sub as_json {
# boolean_columns
"enableemail" => $self->get_column("enableemail") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"visible" => $self->get_column("hidden") ? JSON::MaybeXS::false : JSON::MaybeXS::true,
"inputs" => { map { $_->name => $_ } $self->jobsetinputs }

19
src/lib/Hydra/Schema/Result/Projects.pm
View File

@ -88,6 +88,12 @@ __PACKAGE__->table("projects");
data_type: 'text'
is_nullable: 1
=head2 enable_dynamic_run_command
data_type: 'boolean'
default_value: false
is_nullable: 0
=cut
__PACKAGE__->add_columns(
@ -111,6 +117,8 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 1 },
"declvalue",
{ data_type => "text", is_nullable => 1 },
"enable_dynamic_run_command",
{ data_type => "boolean", default_value => \"false", is_nullable => 0 },
);
=head1 PRIMARY KEY
@ -228,8 +236,8 @@ Composing rels: L</projectmembers> -> username
__PACKAGE__->many_to_many("usernames", "projectmembers", "username");
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:r/wbX3FAm5/OFrrwOQL5fA
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ
use JSON::MaybeXS;
@ -238,6 +246,12 @@ sub builds {
return $self->jobsets->related_resultset('builds');
};
sub supportsDynamicRunCommand {
my ($self) = @_;
return $self->get_column('enable_dynamic_run_command') == 1;
}
sub as_json {
my $self = shift;
@ -251,6 +265,7 @@ sub as_json {
# boolean_columns
"enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"jobsets" => [ map { $_->name } $self->jobsets ]