ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into persona

Browse Source 
Conflicts:
	src/lib/Hydra/Helper/CatalystUtils.pm
	src/root/layout.tt
	src/root/topbar.tt
	src/root/user.tt
This commit is contained in:
Eelco Dolstra
2013-11-05 11:11:48 +01:00
parent ccd2292458 1f711403be
commit b54cfbf032
114 changed files with 3593 additions and 1883 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
src/lib/Hydra/Helper/AddBuilds.pm
View File

@ -2,6 +2,7 @@ package Hydra::Helper::AddBuilds;
use strict;
use feature 'switch';
use utf8;
use XML::Simple;
use IPC::Run;
use Nix::Store;
@ -15,6 +16,7 @@ use File::Path;
use File::Temp;
use File::Spec;
use File::Slurp;
use Hydra::Helper::PluginHooks;
our @ISA = qw(Exporter);
our @EXPORT = qw(
@ -86,10 +88,7 @@ sub fetchInputBuild {
{ order_by => "me.id DESC", rows => 1
, where => \ attrsToSQL($attrs, "me.id") });
if (!defined $prevBuild || !isValidPath(getMainOutput($prevBuild)->path)) {
print STDERR "input `", $name, "': no previous build available\n";
return undef;
}
return () if !defined $prevBuild || !isValidPath(getMainOutput($prevBuild)->path);
#print STDERR "input `", $name, "': using build ", $prevBuild->id, "\n";
@ -148,9 +147,8 @@ sub fetchInputSystemBuild {
return @inputs;
}
sub fetchInput {
my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_;
my ($plugins, $db, $project, $jobset, $name, $type, $value, $emailresponsible) = @_;
my @inputs;
if ($type eq "build") {
@ -159,7 +157,7 @@ sub fetchInput {
elsif ($type eq "sysbuild") {
@inputs = fetchInputSystemBuild($db, $project, $jobset, $name, $value);
}
elsif ($type eq "string") {
elsif ($type eq "string" || $type eq "nix") {
die unless defined $value;
@inputs = { value => $value };
}
@ -170,7 +168,7 @@ sub fetchInput {
else {
my $found = 0;
foreach my $plugin (@{$plugins}) {
@inputs = $plugin->fetchInput($type, $name, $value);
@inputs = $plugin->fetchInput($type, $name, $value, $project, $jobset);
if (defined $inputs[0]) {
$found = 1;
last;
@ -179,7 +177,10 @@ sub fetchInput {
die "input `$name' has unknown type `$type'." unless $found;
}
$_->{type} = $type foreach @inputs;
foreach my $input (@inputs) {
$input->{type} = $type;
$input->{emailresponsible} = $emailresponsible;
}
return @inputs;
}
@ -243,6 +244,9 @@ sub inputsToArgs {
when ("boolean") {
push @res, "--arg", $input, booleanToString($exprType, $alt->{value});
}
when ("nix") {
push @res, "--arg", $input, $alt->{value};
}
default {
push @res, "--arg", $input, buildInputToString($exprType, $alt);
}
@ -287,17 +291,25 @@ sub evalJobs {
my $validJob = 1;
foreach my $arg (@{$job->{arg}}) {
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}];
if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) {
$validJob = 0;
}
}
if ($validJob) {
push(@filteredJobs, $job);
$validJob = 0 if $input->{type} eq "sysbuild" && $input->{system} ne $job->{system};
}
push(@filteredJobs, $job) if $validJob;
}
$jobs->{job} = \@filteredJobs;
return ($jobs, $nixExprInput);
my %jobNames;
my $errors;
foreach my $job (@{$jobs->{job}}) {
$jobNames{$job->{jobName}}++;
if ($jobNames{$job->{jobName}} == 2) {
$errors .= "warning: there are multiple jobs named ‘$job->{jobName}’; support for this will go away soon!\n\n";
}
}
# Handle utf-8 characters in error messages. No idea why this works.
utf8::decode($_->{msg}) foreach @{$jobs->{error}};
return ($jobs, $nixExprInput, $errors);
}
@ -389,7 +401,7 @@ sub getPrevJobsetEval {
# Check whether to add the build described by $buildInfo.
sub checkBuild {
my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap) = @_;
my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
my @outputNames = sort keys %{$buildInfo->{output}};
die unless scalar @outputNames;
@ -410,9 +422,7 @@ sub checkBuild {
my $build;
txn_do($db, sub {
my $job = $jobset->jobs->update_or_create(
{ name => $jobName
});
my $job = $jobset->jobs->update_or_create({ name => $jobName });
# Don't add a build that has already been scheduled for this
# job, or has been built but is still a "current" build for
@ -433,19 +443,19 @@ sub checkBuild {
# semantically unnecessary (because they're implied by
# the eval), but they give a factor 1000 speedup on
# the Nixpkgs jobset with PostgreSQL.
{ project => $project->name, jobset => $jobset->name, job => $job->name,
{ project => $jobset->project->name, jobset => $jobset->name, job => $jobName,
name => $firstOutputName, path => $firstOutputPath },
{ rows => 1, columns => ['id'], join => ['buildoutputs'] });
if (defined $prevBuild) {
print STDERR " already scheduled/built as build ", $prevBuild->id, "\n";
$buildIds->{$prevBuild->id} = 0;
$buildMap->{$prevBuild->id} = { id => $prevBuild->id, jobName => $jobName, new => 0, drvPath => $drvPath };
return;
}
}
# Prevent multiple builds with the same (job, outPath) from
# being added.
my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath};
my $prev = $$jobOutPathMap{$jobName . "\t" . $firstOutputPath};
if (defined $prev) {
print STDERR " already scheduled as build ", $prev, "\n";
return;
@ -511,12 +521,13 @@ sub checkBuild {
$build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} })
foreach @outputNames;
$buildIds->{$build->id} = 1;
$$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id;
$buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath };
$$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id;
if ($build->iscachedbuild) {
print STDERR " marked as cached build ", $build->id, "\n";
addBuildProducts($db, $build);
notifyBuildFinished($plugins, $build, []);
} else {
print STDERR " added to queue as build ", $build->id, "\n";
}
@ -545,6 +556,7 @@ sub checkBuild {
, uri => $input->{uri}
, revision => $input->{revision}
, value => $input->{value}
, emailresponsible => $input->{emailresponsible}
, dependency => $input->{id}
, path => $input->{storePath} || "" # !!! temporary hack
, sha256hash => $input->{sha256hash}
@ -556,29 +568,4 @@ sub checkBuild {
};
sub restartBuild {
my ($db, $build) = @_;
txn_do($db, sub {
my @paths;
push @paths, $build->drvpath;
push @paths, $_->drvpath foreach $build->buildsteps;
my $r = `nix-store --clear-failed-paths @paths`;
$build->update(
{ finished => 0
, busy => 0
, locker => ""
, iscachedbuild => 0
});
$build->buildproducts->delete_all;
# Reset the stats for the evals to which this build belongs.
# !!! Should do this in a trigger.
foreach my $m ($build->jobsetevalmembers->all) {
$m->eval->update({nrsucceeded => undef});
}
});
}
1;

80
src/lib/Hydra/Helper/CatalystUtils.pm
View File

@ -15,8 +15,8 @@ use feature qw/switch/;
our @ISA = qw(Exporter);
our @EXPORT = qw(
getBuild getPreviousBuild getNextBuild getPreviousSuccessfulBuild
error notFound
requireLogin requireProjectOwner requireAdmin requirePost isAdmin isProjectOwner
error notFound accessDenied
forceLogin requireUser requireProjectOwner requireAdmin requirePost isAdmin isProjectOwner
trim
getLatestFinishedEval
sendEmail
@ -27,6 +27,7 @@ our @EXPORT = qw(
parseJobsetName
showJobName
showStatus
getResponsibleAuthors
);
@ -102,6 +103,12 @@ sub notFound {
}
sub accessDenied {
my ($c, $msg) = @_;
error($c, $msg, 403);
}
sub backToReferer {
my ($c) = @_;
$c->response->redirect($c->session->{referer} || $c->uri_for('/'));
@ -110,26 +117,33 @@ sub backToReferer {
}
sub requireLogin {
sub forceLogin {
my ($c) = @_;
$c->session->{referer} = $c->request->uri;
error($c, "This page requires you to sign in.", 403);
accessDenied($c, "This page requires you to sign in.");
}
sub requireUser {
my ($c) = @_;
forceLogin($c) if !$c->user_exists;
}
sub isProjectOwner {
my ($c, $project) = @_;
return $c->user_exists && ($c->check_user_roles('admin') || $c->user->username eq $project->owner->username || defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
return
$c->user_exists &&
(isAdmin($c) ||
$c->user->username eq $project->owner->username ||
defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
}
sub requireProjectOwner {
my ($c, $project) = @_;
requireLogin($c) if !$c->user_exists;
error($c, "Only the project members or administrators can perform this operation.", 403)
requireUser($c);
accessDenied($c, "Only the project members or administrators can perform this operation.")
unless isProjectOwner($c, $project);
}
@ -142,8 +156,8 @@ sub isAdmin {
sub requireAdmin {
my ($c) = @_;
requireLogin($c) if !$c->user_exists;
error($c, "Only administrators can perform this operation.", 403)
requireUser($c);
accessDenied($c, "Only administrators can perform this operation.")
unless isAdmin($c);
}
@ -206,12 +220,12 @@ sub paramToList {
# Security checking of filenames.
Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$]*)";
Readonly our $pathCompRE => "(?:[A-Za-z0-9-\+\._\$][A-Za-z0-9-\+\._\$:]*)";
Readonly our $relPathRE => "(?:$pathCompRE(?:/$pathCompRE)*)";
Readonly our $relNameRE => "(?:[A-Za-z0-9-_][A-Za-z0-9-\._]*)";
Readonly our $attrNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)";
Readonly our $projectNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)";
Readonly our $jobsetNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)";
Readonly our $jobsetNameRE => "(?:[A-Za-z_][A-Za-z0-9-_\.]*)";
Readonly our $jobNameRE => "(?:$attrNameRE(?:\\.$attrNameRE)*)";
Readonly our $systemRE => "(?:[a-z0-9_]+-[a-z0-9_]+)";
Readonly our $userNameRE => "(?:[a-z][a-z0-9_\.]*)";
@ -246,4 +260,42 @@ sub showStatus {
}
# Determine who broke/fixed the build.
sub getResponsibleAuthors {
my ($build, $plugins) = @_;
my $prevBuild = getPreviousBuild($build);
my $nrCommits = 0;
my %authors;
my @emailable_authors;
if ($prevBuild) {
foreach my $curInput ($build->buildinputs_builds) {
next unless ($curInput->type eq "git" || $curInput->type eq "hg");
my $prevInput = $prevBuild->buildinputs_builds->find({ name => $curInput->name });
next unless defined $prevInput;
next if $curInput->type ne $prevInput->type;
next if $curInput->uri ne $prevInput->uri;
next if $curInput->revision eq $prevInput->revision;
my @commits;
foreach my $plugin (@{$plugins}) {
push @commits, @{$plugin->getCommits($curInput->type, $curInput->uri, $prevInput->revision, $curInput->revision)};
}
foreach my $commit (@commits) {
#print STDERR "$commit->{revision} by $commit->{author}\n";
$authors{$commit->{author}} = $commit->{email};
push @emailable_authors, $commit->{email} if $curInput->emailresponsible;
$nrCommits++;
}
}
}
return (\%authors, $nrCommits, \@emailable_authors);
}
1;

157
src/lib/Hydra/Helper/Nix.pm
View File

@ -7,6 +7,7 @@ use File::Basename;
use Config::General;
use Hydra::Helper::CatalystUtils;
use Hydra::Model::DB;
use Nix::Store;
our @ISA = qw(Exporter);
our @EXPORT = qw(
@ -16,11 +17,13 @@ our @EXPORT = qw(
getPrimaryBuildsForView
getPrimaryBuildTotal
getViewResult getLatestSuccessfulViewResult
jobsetOverview removeAsciiEscapes getDrvLogPath logContents
jobsetOverview removeAsciiEscapes getDrvLogPath findLog logContents
getMainOutput
getEvals getMachines
pathIsInsidePrefix
captureStdoutStderr);
captureStdoutStderr run grab
getTotalShares
cancelBuilds restartBuilds);
sub getHydraHome {
@ -42,11 +45,12 @@ sub getHydraConfig {
# doesn't work.
sub txn_do {
my ($db, $coderef) = @_;
my $res;
while (1) {
eval {
$db->txn_do($coderef);
$res = $db->txn_do($coderef);
};
last if !$@;
return $res if !$@;
die $@ unless $@ =~ "database is locked";
}
}
@ -253,21 +257,46 @@ sub getLatestSuccessfulViewResult {
sub getDrvLogPath {
my ($drvPath) = @_;
my $base = basename $drvPath;
my $fn =
($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/"
. substr($base, 0, 2) . "/"
. substr($base, 2);
return $fn if -f $fn;
$fn .= ".bz2";
return $fn if -f $fn;
my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2);
my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
return $_ if (-f $_);
}
return undef;
}
# Find the log of the derivation denoted by $drvPath. It it doesn't
# exist, try other derivations that produced its outputs (@outPaths).
sub findLog {
my ($c, $drvPath, @outPaths) = @_;
if (defined $drvPath) {
my $logPath = getDrvLogPath($drvPath);
return $logPath if defined $logPath;
}
return undef if scalar @outPaths == 0;
my @steps = $c->model('DB::BuildSteps')->search(
{ path => { -in => [@outPaths] } },
{ select => ["drvpath"]
, distinct => 1
, join => "buildstepoutputs"
});
foreach my $step (@steps) {
next unless defined $step->drvpath;
my $logPath = getDrvLogPath($step->drvpath);
return $logPath if defined $logPath;
}
return undef;
}
sub logContents {
my ($drvPath, $tail) = @_;
my $logPath = getDrvLogPath($drvPath);
die unless defined $logPath;
my ($logPath, $tail) = @_;
my $cmd;
if ($logPath =~ /.bz2$/) {
$cmd = "bzip2 -d < $logPath";
@ -381,7 +410,7 @@ sub getEvals {
}
sub getMachines {
my $machinesConf = $ENV{"NIX_REMOTE_SYSTEMS"} || "/etc/nix.machines";
my $machinesConf = $ENV{"NIX_REMOTE_SYSTEMS"} || "/etc/nix/machines";
# Read the list of machines.
my %machines = ();
@ -472,4 +501,102 @@ sub captureStdoutStderr {
}
sub run {
my (%args) = @_;
my $res = { stdout => "", stderr => "" };
my $stdin = "";
eval {
local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required
alarm $args{timeout} if defined $args{timeout};
my @x = ($args{cmd}, \$stdin, \$res->{stdout});
push @x, \$res->{stderr} if $args{grabStderr} // 1;
IPC::Run::run(@x,
init => sub { chdir $args{dir} or die "changing to $args{dir}" if defined $args{dir}; });
alarm 0;
};
if ($@) {
die unless $@ eq "timeout\n"; # propagate unexpected errors
$res->{status} = -1;
$res->{stderr} = "timeout\n";
} else {
$res->{status} = $?;
chomp $res->{stdout} if $args{chomp} // 0;
}
return $res;
}
sub grab {
my (%args) = @_;
my $res = run(%args, grabStderr => 0);
die "command `@{$args{cmd}}' failed with exit status $res->{status}" if $res->{status};
return $res->{stdout};
}
sub getTotalShares {
my ($db) = @_;
return $db->resultset('Jobsets')->search(
{ 'project.enabled' => 1, 'me.enabled' => { '!=' => 0 } },
{ join => 'project', select => { sum => 'schedulingshares' }, as => 'sum' })->single->get_column('sum');
}
sub cancelBuilds($$) {
my ($db, $builds) = @_;
return txn_do($db, sub {
$builds = $builds->search({ finished => 0, busy => 0 });
my $n = $builds->count;
my $time = time();
$builds->update(
{ finished => 1,
, iscachedbuild => 0, buildstatus => 4 # = cancelled
, starttime => $time
, stoptime => $time
});
return $n;
});
}
sub restartBuilds($$) {
my ($db, $builds) = @_;
my $n = 0;
txn_do($db, sub {
my @paths;
$builds = $builds->search({ finished => 1 });
foreach my $build ($builds->all) {
next if !isValidPath($build->drvpath);
push @paths, $build->drvpath;
push @paths, $_->drvpath foreach $build->buildsteps;
registerRoot $build->drvpath;
$build->update(
{ finished => 0
, busy => 0
, locker => ""
, iscachedbuild => 0
});
$n++;
# Reset the stats for the evals to which this build belongs.
# !!! Should do this in a trigger.
$build->jobsetevals->update({nrsucceeded => undef});
}
# Clear Nix's negative failure cache.
# FIXME: Add this to the API.
system("nix-store", "--clear-failed-paths", @paths);
});
return $n;
}
1;

22
src/lib/Hydra/Helper/PluginHooks.pm Normal file
View File

@ -0,0 +1,22 @@
package Hydra::Helper::PluginHooks;
use strict;
use Exporter;
our @ISA = qw(Exporter);
our @EXPORT = qw(
notifyBuildFinished);
sub notifyBuildFinished {
my ($plugins, $build, $dependents) = @_;
foreach my $plugin (@{$plugins}) {
eval {
$plugin->buildFinished($build, $dependents);
};
if ($@) {
print STDERR "$plugin->buildFinished: $@\n";
}
}
}
1;