2009-10-26 15:39:14 +00:00
|
|
|
package Hydra::Helper::AddBuilds;
|
|
|
|
|
|
|
|
use strict;
|
2013-09-25 15:51:03 +02:00
|
|
|
use utf8;
|
2015-04-14 15:20:56 +02:00
|
|
|
use Encode;
|
2014-09-30 00:20:54 +02:00
|
|
|
use JSON;
|
2011-11-30 15:25:28 +01:00
|
|
|
use Nix::Store;
|
2013-02-13 18:26:00 +01:00
|
|
|
use Nix::Config;
|
2012-03-13 12:10:19 +01:00
|
|
|
use Hydra::Model::DB;
|
2009-10-26 15:39:14 +00:00
|
|
|
use Hydra::Helper::Nix;
|
2010-07-27 11:14:24 +00:00
|
|
|
use Digest::SHA qw(sha256_hex);
|
2010-09-02 09:00:06 +00:00
|
|
|
use File::Basename;
|
|
|
|
use File::stat;
|
2010-07-27 11:14:24 +00:00
|
|
|
use File::Path;
|
2011-12-04 22:01:53 +01:00
|
|
|
use File::Temp;
|
2013-02-13 18:26:00 +01:00
|
|
|
use File::Spec;
|
2013-02-13 16:49:28 +00:00
|
|
|
use File::Slurp;
|
2013-11-11 21:17:22 +00:00
|
|
|
use Hydra::Helper::CatalystUtils;
|
2009-10-26 15:39:14 +00:00
|
|
|
|
|
|
|
our @ISA = qw(Exporter);
|
2011-12-01 13:25:54 +01:00
|
|
|
our @EXPORT = qw(
|
2017-02-21 16:17:31 +01:00
|
|
|
updateDeclarativeJobset
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
handleDeclarativeJobsetBuild
|
2011-12-01 13:25:54 +01:00
|
|
|
);
|
|
|
|
|
2009-10-26 15:39:14 +00:00
|
|
|
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
sub updateDeclarativeJobset {
|
|
|
|
my ($db, $project, $jobsetName, $declSpec) = @_;
|
|
|
|
|
|
|
|
my @allowed_keys = qw(
|
|
|
|
enabled
|
|
|
|
hidden
|
2019-10-27 14:57:53 +01:00
|
|
|
type
|
|
|
|
flake
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
description
|
|
|
|
nixexprinput
|
|
|
|
nixexprpath
|
|
|
|
checkinterval
|
|
|
|
schedulingshares
|
|
|
|
enableemail
|
|
|
|
emailoverride
|
|
|
|
keepnr
|
|
|
|
);
|
|
|
|
my %update = ( name => $jobsetName );
|
|
|
|
foreach my $key (@allowed_keys) {
|
|
|
|
$update{$key} = $declSpec->{$key};
|
|
|
|
delete $declSpec->{$key};
|
|
|
|
}
|
|
|
|
txn_do($db, sub {
|
|
|
|
my $jobset = $project->jobsets->update_or_create(\%update);
|
|
|
|
$jobset->jobsetinputs->delete;
|
|
|
|
while ((my $name, my $data) = each %{$declSpec->{"inputs"}}) {
|
|
|
|
my $input = $jobset->jobsetinputs->create(
|
|
|
|
{ name => $name,
|
|
|
|
type => $data->{type},
|
|
|
|
emailresponsible => $data->{emailresponsible}
|
|
|
|
});
|
|
|
|
$input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
|
|
|
|
}
|
|
|
|
delete $declSpec->{"inputs"};
|
2018-06-29 17:41:23 -07:00
|
|
|
die "invalid keys ($declSpec) in declarative specification file\n" if (%{$declSpec});
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
});
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
sub handleDeclarativeJobsetBuild {
|
|
|
|
my ($db, $project, $build) = @_;
|
|
|
|
|
|
|
|
eval {
|
|
|
|
my $id = $build->id;
|
|
|
|
die "Declarative jobset build $id failed" unless $build->buildstatus == 0;
|
|
|
|
my $declPath = ($build->buildoutputs)[0]->path;
|
2020-03-03 22:32:13 -05:00
|
|
|
my $declText = eval {
|
|
|
|
readNixFile($declPath)
|
|
|
|
};
|
|
|
|
if ($@) {
|
|
|
|
print STDERR "ERROR: failed to readNixFile $declPath: ", $@, "\n";
|
|
|
|
die;
|
|
|
|
}
|
|
|
|
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
my $declSpec = decode_json($declText);
|
|
|
|
txn_do($db, sub {
|
|
|
|
my @kept = keys %$declSpec;
|
|
|
|
push @kept, ".jobsets";
|
|
|
|
$project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
|
|
|
|
while ((my $jobsetName, my $spec) = each %$declSpec) {
|
2018-09-11 08:03:58 -03:00
|
|
|
eval {
|
|
|
|
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
|
|
|
|
};
|
|
|
|
if ($@) {
|
|
|
|
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
|
|
|
|
}
|
Enable declarative projects.
This allows fully declarative project specifications. This is best
illustrated by example:
* I create a new project, setting the declarative spec file to
"spec.json" and the declarative input to a git repo pointing
at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
declarative-hydra-example.git, reads spec.json as a jobset spec,
and updates the jobset's configuration accordingly:
{
"enabled": 1,
"hidden": false,
"description": "Jobsets",
"nixexprinput": "src",
"nixexprpath": "default.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
reads its output as a JSON representation of a dictionary of
jobset specs and creates a jobset named "master" configured
accordingly (In this example, this is the same configuration as
.jobsets itself, except using release.nix instead of default.nix):
{
"enabled": 1,
"hidden": false,
"description": "js",
"nixexprinput": "src",
"nixexprpath": "release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
"nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
}
}
2016-03-11 18:14:58 -05:00
|
|
|
}
|
|
|
|
});
|
|
|
|
};
|
|
|
|
$project->jobsets->find({ name => ".jobsets" })->update({ errormsg => $@, errortime => time, fetcherrormsg => undef })
|
|
|
|
if defined $@;
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
2013-10-04 17:01:47 +02:00
|
|
|
1;
|