ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #1103 from DeterminateSystems/runcommand/dynamic

Browse Source 
Dynamic RunCommand
This commit is contained in:
Graham Christensen
2022-04-19 10:09:47 -04:00
committed by GitHub
parent c44d9d9e91 01aedc92ed
commit 5c90edd19f
29 changed files with 1233 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/lib/Hydra/Controller/Build.pm
View File

@ -38,6 +38,17 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
$c->stash->{jobset} = $c->stash->{build}->jobset;
$c->stash->{job} = $c->stash->{build}->job;
$c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})];
$c->stash->{runcommandlogProblem} = undef;
if ($c->stash->{job} =~ qr/^runCommandHook\..*/) {
if (!$c->config->{dynamicruncommand}->{enable}) {
$c->stash->{runcommandlogProblem} = "disabled-server";
} elsif (!$c->stash->{project}->enable_dynamic_run_command) {
$c->stash->{runcommandlogProblem} = "disabled-project";
} elsif (!$c->stash->{jobset}->enable_dynamic_run_command) {
$c->stash->{runcommandlogProblem} = "disabled-jobset";
}
}
}

9
src/lib/Hydra/Controller/Jobset.pm
View File

@ -261,6 +261,14 @@ sub updateJobset {
my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($c->config->{dynamicruncommand}->{enable}
&& $jobset->project->enable_dynamic_run_command))
{
badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
}
$jobset->update(
{ name => $jobsetName
, description => trim($c->stash->{params}->{"description"})
@ -268,6 +276,7 @@ sub updateJobset {
, nixexprinput => $nixExprInput
, enabled => $enabled
, enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
, enable_dynamic_run_command => $enable_dynamic_run_command
, emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))

6
src/lib/Hydra/Controller/Project.pm
View File

@ -149,6 +149,11 @@ sub updateProject {
my $displayName = trim $c->stash->{params}->{displayname};
error($c, "You must specify a display name.") if $displayName eq "";
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
badRequest($c, "Dynamic RunCommand is not enabled by the server.");
}
$project->update(
{ name => $projectName
, displayname => $displayName
@ -157,6 +162,7 @@ sub updateProject {
, enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, owner => $owner
, enable_dynamic_run_command => $enable_dynamic_run_command
, declfile => trim($c->stash->{params}->{declarative}->{file})
, decltype => trim($c->stash->{params}->{declarative}->{type})
, declvalue => trim($c->stash->{params}->{declarative}->{value})

45
src/lib/Hydra/Helper/AddBuilds.pm
View File

@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
our @ISA = qw(Exporter);
our @EXPORT = qw(
validateDeclarativeJobset
createJobsetInputsRowAndData
updateDeclarativeJobset
handleDeclarativeJobsetBuild
handleDeclarativeJobsetJson
);
sub updateDeclarativeJobset {
my ($db, $project, $jobsetName, $declSpec) = @_;
sub validateDeclarativeJobset {
my ($config, $project, $jobsetName, $declSpec) = @_;
my @allowed_keys = qw(
enabled
@ -39,6 +41,7 @@ sub updateDeclarativeJobset {
checkinterval
schedulingshares
enableemail
enable_dynamic_run_command
emailoverride
keepnr
);
@ -61,16 +64,39 @@ sub updateDeclarativeJobset {
}
}
my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($config->{dynamicruncommand}->{enable}
&& $project->{enable_dynamic_run_command}))
{
die "Dynamic RunCommand is not enabled by the server or the parent project.";
}
return %update;
}
sub createJobsetInputsRowAndData {
my ($name, $declSpec) = @_;
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
return ($row, $data);
}
sub updateDeclarativeJobset {
my ($config, $db, $project, $jobsetName, $declSpec) = @_;
my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
$db->txn_do(sub {
my $jobset = $project->jobsets->update_or_create(\%update);
$jobset->jobsetinputs->delete;
foreach my $name (keys %{$declSpec->{"inputs"}}) {
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
my $input = $jobset->jobsetinputs->create($row);
$input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
}
@ -81,6 +107,7 @@ sub updateDeclarativeJobset {
sub handleDeclarativeJobsetJson {
my ($db, $project, $declSpec) = @_;
my $config = getHydraConfig();
$db->txn_do(sub {
my @kept = keys %$declSpec;
push @kept, ".jobsets";
@ -88,7 +115,7 @@ sub handleDeclarativeJobsetJson {
foreach my $jobsetName (keys %$declSpec) {
my $spec = $declSpec->{$jobsetName};
eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
1;
} or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";

95
src/lib/Hydra/Plugin/RunCommand.pm
View File

@ -12,7 +12,74 @@ use Try::Tiny;
sub isEnabled {
my ($self) = @_;
return defined $self->{config}->{runcommand};
return areStaticCommandsEnabled($self->{config}) || areDynamicCommandsEnabled($self->{config});
}
sub areStaticCommandsEnabled {
my ($config) = @_;
if (defined $config->{runcommand}) {
return 1;
}
return 0;
}
sub areDynamicCommandsEnabled {
my ($config) = @_;
if ((defined $config->{dynamicruncommand})
&& $config->{dynamicruncommand}->{enable}) {
return 1;
}
return 0;
}
sub isBuildEligibleForDynamicRunCommand {
my ($build) = @_;
if ($build->get_column("buildstatus") != 0) {
return 0;
}
if ($build->get_column("job") =~ "^runCommandHook\..+") {
my $out = $build->buildoutputs->find({name => "out"});
if (!defined $out) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: no output named 'out'.";
return 0;
}
my $path = $out->path;
if (-l $path) {
$path = readlink($path);
}
if (! -e $path) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output doesn't exist locally. This is a bug.";
return 0;
}
if (! -x $path) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not executable.";
return 0;
}
if (! -f $path) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not a regular file or symlink.";
return 0;
}
if (! $build->jobset->supportsDynamicRunCommand()) {
warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The project or jobset don't have dynamic runcommand enabled.";
return 0;
}
return 1;
}
return 0;
}
sub configSectionMatches {
@ -43,10 +110,11 @@ sub eventMatches {
}
sub fanoutToCommands {
my ($config, $event, $project, $jobset, $job) = @_;
my ($config, $event, $build) = @_;
my @commands;
# Calculate all the statically defined commands to execute
my $cfg = $config->{runcommand};
my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
@ -55,9 +123,10 @@ sub fanoutToCommands {
next unless eventMatches($conf, $event);
next unless configSectionMatches(
$matcher,
$project,
$jobset,
$job);
$build->jobset->get_column('project'),
$build->jobset->get_column('name'),
$build->get_column('job')
);
if (!defined($conf->{command})) {
warn "<runcommand> section for '$matcher' lacks a 'command' option";
@ -70,6 +139,18 @@ sub fanoutToCommands {
})
}
# Calculate all dynamically defined commands to execute
if (areDynamicCommandsEnabled($config)) {
if (isBuildEligibleForDynamicRunCommand($build)) {
my $job = $build->get_column('job');
my $out = $build->buildoutputs->find({name => "out"});
push(@commands, {
matcher => "DynamicRunCommand($job)",
command => $out->path
})
}
}
return \@commands;
}
@ -138,9 +219,7 @@ sub buildFinished {
my $commandsToRun = fanoutToCommands(
$self->{config},
$event,
$build->project->get_column('name'),
$build->jobset->get_column('name'),
$build->get_column('job')
$build
);
if (@$commandsToRun == 0) {

20
src/lib/Hydra/Schema/Result/Jobsets.pm
View File

@ -155,6 +155,12 @@ __PACKAGE__->table("jobsets");
data_type: 'text'
is_nullable: 1
=head2 enable_dynamic_run_command
data_type: 'boolean'
default_value: false
is_nullable: 0
=cut
__PACKAGE__->add_columns(
@ -207,6 +213,8 @@ __PACKAGE__->add_columns(
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"flake",
{ data_type => "text", is_nullable => 1 },
"enable_dynamic_run_command",
{ data_type => "boolean", default_value => \"false", is_nullable => 0 },
);
=head1 PRIMARY KEY
@ -354,8 +362,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cQOnMitrWGMoJX6kZGNW+w
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:17:33
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7wPE5ebeVTkenMCWG9Sgcg
use JSON::MaybeXS;
@ -378,6 +386,13 @@ __PACKAGE__->add_column(
"+id" => { retrieve_on_insert => 1 }
);
sub supportsDynamicRunCommand {
my ($self) = @_;
return $self->get_column('enable_dynamic_run_command') == 1
&& $self->project->supportsDynamicRunCommand();
}
sub as_json {
my $self = shift;
@ -406,6 +421,7 @@ sub as_json {
# boolean_columns
"enableemail" => $self->get_column("enableemail") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"visible" => $self->get_column("hidden") ? JSON::MaybeXS::false : JSON::MaybeXS::true,
"inputs" => { map { $_->name => $_ } $self->jobsetinputs }

19
src/lib/Hydra/Schema/Result/Projects.pm
View File

@ -88,6 +88,12 @@ __PACKAGE__->table("projects");
data_type: 'text'
is_nullable: 1
=head2 enable_dynamic_run_command
data_type: 'boolean'
default_value: false
is_nullable: 0
=cut
__PACKAGE__->add_columns(
@ -111,6 +117,8 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 1 },
"declvalue",
{ data_type => "text", is_nullable => 1 },
"enable_dynamic_run_command",
{ data_type => "boolean", default_value => \"false", is_nullable => 0 },
);
=head1 PRIMARY KEY
@ -228,8 +236,8 @@ Composing rels: L</projectmembers> -> username
__PACKAGE__->many_to_many("usernames", "projectmembers", "username");
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:r/wbX3FAm5/OFrrwOQL5fA
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ
use JSON::MaybeXS;
@ -238,6 +246,12 @@ sub builds {
return $self->jobsets->related_resultset('builds');
};
sub supportsDynamicRunCommand {
my ($self) = @_;
return $self->get_column('enable_dynamic_run_command') == 1;
}
sub as_json {
my $self = shift;
@ -251,6 +265,7 @@ sub as_json {
# boolean_columns
"enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
"jobsets" => [ map { $_->name } $self->jobsets ]

15
src/root/build.tt
View File

@ -149,7 +149,7 @@ END;
[% IF build.dependents %]<li class="nav-item"><a class="nav-link" href="#tabs-usedby" data-toggle="tab">Used By</a></li>[% END%]
[% IF drvAvailable %]<li class="nav-item"><a class="nav-link" href="#tabs-build-deps" data-toggle="tab">Build Dependencies</a></li>[% END %]
[% IF localStore && available %]<li class="nav-item"><a class="nav-link" href="#tabs-runtime-deps" data-toggle="tab">Runtime Dependencies</a></li>[% END %]
[% IF runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs</a></li>[% END %]
[% IF runcommandlogProblem || runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs[% IF runcommandlogProblem %] <span class="badge badge-warning">Disabled</span>[% END %]</a></li>[% END %]
</ul>
<div id="generic-tabs" class="tab-content">
@ -489,6 +489,19 @@ END;
[% END %]
<div id="tabs-runcommandlogs" class="tab-pane">
[% IF runcommandlogProblem %]
<div class="alert alert-warning" role="alert">
[% IF runcommandlogProblem == "disabled-server" %]
This server does not enable Dynamic RunCommand support.
[% ELSIF runcommandlogProblem == "disabled-project" %]
This project does not enable Dynamic RunCommand support.
[% ELSIF runcommandlogProblem == "disabled-jobset" %]
This jobset does not enable Dynamic RunCommand support.
[% ELSE %]
Dynamic RunCommand is not enabled: [% runcommandlogProblem %].
[% END %]
</div>
[% END %]
<div class="d-flex flex-column">
[% FOREACH runcommandlog IN runcommandlogs %]
<div class="p-2 border-bottom">

15
src/root/edit-jobset.tt
View File

@ -157,6 +157,21 @@
</div>
</div>
<div class="form-group row">
<label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
<div class="col-sm-9">
<input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF !project.enable_dynamic_run_command %]
title="The parent project has not enabled dynamic RunCommands" disabled
[% ELSIF jobset.enable_dynamic_run_command %]
checked
[% END %]
/>
</div>
</div>
<div class="form-group row">
<label class="col-sm-3" for="editjobsetenableemail">Email notification</label>
<div class="col-sm-9">

14
src/root/edit-project.tt
View File

@ -52,6 +52,20 @@
</div>
</div>
<div class="form-group row">
<label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
<div class="col-sm-9">
<input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF project.enable_dynamic_run_command %]
checked
[% END %]
/>
</div>
</div>
<div class="form-group row">
<label class="col-sm-3" for="editprojectdeclfile">
Declarative spec file

4
src/root/jobset.tt
View File

@ -160,6 +160,10 @@
<th>Scheduling shares:</th>
<td>[% jobset.schedulingshares %] [% IF totalShares %] ([% f = format("%.2f"); f(jobset.schedulingshares / totalShares * 100) %]% out of [% totalShares %] shares)[% END %]</td>
</tr>
<tr>
<th>Enable Dynamic RunCommand Hooks:</th>
<td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr>
[% IF emailNotification %]
<tr>
<th>Enable email notification:</th>

4
src/root/project.tt
View File

@ -92,6 +92,10 @@
<th>Enabled:</th>
<td>[% project.enabled ? "Yes" : "No" %]</td>
</tr>
<tr>
<th>Enable Dynamic RunCommand Hooks:</th>
<td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr>
</table>
</div>

2
src/script/hydra-eval-jobset
View File

@ -619,7 +619,7 @@ sub checkJobsetWrapped {
} else {
# Update the jobset with the spec's inputs, and the continue
# evaluating the .jobsets jobset.
updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
$jobset->discard_changes;
$inputInfo->{"declInput"} = [ $declInput ];
$inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];

2
src/sql/hydra.sql
View File

@ -49,6 +49,7 @@ create table Projects (
declfile text, -- File containing declarative jobset specification
decltype text, -- Type of the input containing declarative jobset specification
declvalue text, -- Value of the input containing declarative jobset specification
enable_dynamic_run_command boolean not null default false,
foreign key (owner) references Users(userName) on update cascade
);
@ -88,6 +89,7 @@ create table Jobsets (
startTime integer, -- if jobset is currently running
type integer not null default 0, -- 0 == legacy, 1 == flake
flake text,
enable_dynamic_run_command boolean not null default false,
constraint jobsets_schedulingshares_nonzero_check check (schedulingShares > 0),
constraint jobsets_type_known_check check (type = 0 or type = 1),
-- If the type is 0, then nixExprInput and nixExprPath should be non-null and other type-specific fields should be null

4
src/sql/upgrade-82.sql Normal file
View File

@ -0,0 +1,4 @@
ALTER TABLE Jobsets
ADD COLUMN enable_dynamic_run_command boolean not null default false;
ALTER TABLE Projects
ADD COLUMN enable_dynamic_run_command boolean not null default false;