ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge branch 'build-ng'

Browse Source
This commit is contained in:
Eelco Dolstra
2015-08-12 20:32:48 +02:00
parent cf9516baa4 48d6f0de2a
commit b0c8eecd37
80 changed files with 4157 additions and 1037 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
src/lib/Hydra/Helper/AddBuilds.pm
View File

@ -22,20 +22,10 @@ use Hydra::Helper::CatalystUtils;
our @ISA = qw(Exporter);
our @EXPORT = qw(
fetchInput evalJobs checkBuild inputsToArgs
getReleaseName addBuildProducts restartBuild
getPrevJobsetEval
restartBuild getPrevJobsetEval
);
sub getReleaseName {
my ($outPath) = @_;
return undef unless -f "$outPath/nix-support/hydra-release-name";
my $releaseName = read_file("$outPath/nix-support/hydra-release-name");
chomp $releaseName;
return $releaseName;
}
sub parseJobName {
# Parse a job specification of the form `<project>:<jobset>:<job>
# [attrs]'. The project, jobset and attrs may be omitted. The
@ -299,7 +289,7 @@ sub inputsToArgs {
my ($inputInfo, $exprType) = @_;
my @res = ();
foreach my $input (keys %{$inputInfo}) {
foreach my $input (sort keys %{$inputInfo}) {
push @res, "-I", "$input=$inputInfo->{$input}->[0]->{storePath}"
if scalar @{$inputInfo->{$input}} == 1
&& defined $inputInfo->{$input}->[0]->{storePath};
@ -367,80 +357,6 @@ sub evalJobs {
}
sub addBuildProducts {
my ($db, $build) = @_;
my $productnr = 1;
my $explicitProducts = 0;
my $storeDir = $Nix::Config::storeDir . "/";
foreach my $output ($build->buildoutputs->all) {
my $outPath = $output->path;
if (-e "$outPath/nix-support/hydra-build-products") {
$explicitProducts = 1;
open LIST, "$outPath/nix-support/hydra-build-products" or die;
while (<LIST>) {
/^([\w\-]+)\s+([\w\-]+)\s+("[^"]*"|\S+)(\s+(\S+))?$/ or next;
my $type = $1;
my $subtype = $2 eq "none" ? "" : $2;
my $path = substr($3, 0, 1) eq "\"" ? substr($3, 1, -1) : $3;
my $defaultPath = $5;
# Ensure that the path exists and points into the Nix store.
next unless File::Spec->file_name_is_absolute($path);
$path = pathIsInsidePrefix($path, $Nix::Config::storeDir);
next unless defined $path;
next unless -e $path;
# FIXME: check that the path is in the input closure
# of the build?
my $fileSize, my $sha1, my $sha256;
if (-f $path) {
my $st = stat($path) or die "cannot stat $path: $!";
$fileSize = $st->size;
$sha1 = hashFile("sha1", 0, $path);
$sha256 = hashFile("sha256", 0, $path);
}
my $name = $path eq $outPath ? "" : basename $path;
$db->resultset('BuildProducts')->create(
{ build => $build->id
, productnr => $productnr++
, type => $type
, subtype => $subtype
, path => $path
, filesize => $fileSize
, sha1hash => $sha1
, sha256hash => $sha256
, name => $name
, defaultpath => $defaultPath
});
}
close LIST;
}
}
return if $explicitProducts;
foreach my $output ($build->buildoutputs->all) {
my $outPath = $output->path;
next unless -d $outPath;
$db->resultset('BuildProducts')->create(
{ build => $build->id
, productnr => $productnr++
, type => "nix-build"
, subtype => $output->name eq "out" ? "" : $output->name
, path => $outPath
, name => $build->nixname
});
}
}
# Return the most recent evaluation of the given jobset (that
# optionally had new builds), or undefined if no such evaluation
# exists.
@ -513,40 +429,6 @@ sub checkBuild {
my $time = time();
# Are the outputs already in the Nix store? Then add a cached
# build.
my %extraFlags;
my $allValid = 1;
my $buildStatus;
my $releaseName;
foreach my $name (@outputNames) {
my $path = $buildInfo->{outputs}->{$name};
if (isValidPath($path)) {
if (-f "$path/nix-support/failed") {
$buildStatus = 6;
} else {
$buildStatus //= 0;
}
$releaseName //= getReleaseName($path);
} else {
$allValid = 0;
last;
}
}
if ($allValid) {
%extraFlags =
( finished => 1
, iscachedbuild => 1
, buildstatus => $buildStatus
, starttime => $time
, stoptime => $time
, releasename => $releaseName
);
} else {
%extraFlags = ( finished => 0 );
}
# Add the build to the database.
$build = $job->builds->create(
{ timestamp => $time
@ -562,10 +444,10 @@ sub checkBuild {
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
, priority => $buildInfo->{schedulingPriority}
, finished => 0
, busy => 0
, locker => ""
, iscurrent => 1
, %extraFlags
});
$build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
@ -574,13 +456,7 @@ sub checkBuild {
$buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath };
$$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id;
if ($build->iscachedbuild) {
#print STDERR " marked as cached build ", $build->id, "\n";
addBuildProducts($db, $build);
notifyBuildFinished($plugins, $build, []);
} else {
print STDERR "added build ${\$build->id} (${\$jobset->project->name}:${\$jobset->name}:$jobName)\n";
}
print STDERR "added build ${\$build->id} (${\$jobset->project->name}:${\$jobset->name}:$jobName)\n";
});
return $build;

8
src/lib/Hydra/Helper/CatalystUtils.pm
View File

@ -23,6 +23,7 @@ our @EXPORT = qw(
showStatus
getResponsibleAuthors
setCacheHeaders
approxTableSize
);
@ -296,4 +297,11 @@ sub setCacheHeaders {
}
sub approxTableSize {
my ($c, $name) = @_;
return $c->model('DB')->schema->storage->dbh->selectrow_hashref(
"select reltuples::int from pg_class where relname = lower(?)", { }, $name)->{"reltuples"};
}
1;

2
src/lib/Hydra/Helper/Email.pm
View File

@ -13,7 +13,7 @@ sub sendEmail {
my ($config, $to, $subject, $body, $extraHeaders) = @_;
my $url = getBaseUrl($config);
my $sender = $config->{'notification_sender'} // (($ENV{'USER'} // "hydra") . "@" . $url);
my $sender = $config->{'notification_sender'} // (($ENV{'USER'} // "hydra") . "@" . hostname_long);
my @headers = (
To => $to,

14
src/lib/Hydra/Helper/Nix.pm
View File

@ -133,8 +133,9 @@ sub getDrvLogPath {
my $base = basename $drvPath;
my $bucketed = substr($base, 0, 2) . "/" . substr($base, 2);
my $fn = ($ENV{NIX_LOG_DIR} || "/nix/var/log/nix") . "/drvs/";
for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
return $_ if (-f $_);
my $fn2 = Hydra::Model::DB::getHydraPath . "/build-logs/";
for ($fn2 . $bucketed, $fn2 . $bucketed . ".bz2", $fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
return $_ if -f $_;
}
return undef;
}
@ -423,7 +424,7 @@ sub getTotalShares {
sub cancelBuilds($$) {
my ($db, $builds) = @_;
return txn_do($db, sub {
$builds = $builds->search({ finished => 0, busy => 0 });
$builds = $builds->search({ finished => 0 });
my $n = $builds->count;
my $time = time();
$builds->update(
@ -448,7 +449,7 @@ sub restartBuilds($$) {
foreach my $build ($builds->all) {
next if !isValidPath($build->drvpath);
push @paths, $build->drvpath;
push @paths, $_->path foreach $build->buildoutputs->all;
push @buildIds, $build->id;
registerRoot $build->drvpath;
}
@ -464,9 +465,10 @@ sub restartBuilds($$) {
# !!! Should do this in a trigger.
$db->resultset('JobsetEvals')->search({ build => \@buildIds }, { join => 'buildIds' })->update({ nrsucceeded => undef });
# Clear Nix's negative failure cache.
# Clear the failed paths cache.
# FIXME: Add this to the API.
system("nix-store", "--clear-failed-paths", @paths);
# FIXME: clear the dependencies?
$db->resultset('FailedPaths')->search({ path => [ @paths ]})->delete;
});
return scalar(@buildIds);