Merge the BuildSchedulingInfo table into the Builds table

This simplifies the code and improves performance since it reduces
the number of joins.
This commit is contained in:
Eelco Dolstra
2012-02-29 02:22:49 +01:00
parent 19fe4b9b4a
commit 25334715f8
21 changed files with 194 additions and 301 deletions

View File

@ -136,13 +136,11 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
my $nr = $c->request->params->{nr} ;
error($c, "Parameter not defined!") if !defined $nr;
my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, join => ['schedulingInfo'] , order_by => ["busy DESC", "priority DESC", "timestamp"], '+select' => ['schedulingInfo.priority', 'schedulingInfo.busy'], '+as' => ['priority', 'busy'] });
my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["busy DESC", "priority DESC", "timestamp"]});
my @list ;
foreach my $b (@builds) {
push @list, buildToHash($b) ;
}
my @list;
push @list, buildToHash($_) foreach @builds;
$c->stash->{'plain'} = {
data => scalar (JSON::Any->objToJson(\@list))
};
@ -151,18 +149,18 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) {
my ($self, $c) = @_;
my $nrQueuedBuilds = $c->model('DB::BuildSchedulingInfo')->count();
my $nrQueuedBuilds = $c->model('DB::Builds')->search({finished => 0})->count();
$c->stash->{'plain'} = {
data => " $nrQueuedBuilds"
data => "$nrQueuedBuilds"
};
$c->forward('Hydra::View::Plain');
}
sub nrrunning : Chained('api') PathPart('nrrunning') Args(0) {
my ($self, $c) = @_;
my $nrRunningBuilds = $c->model('DB::BuildSchedulingInfo')->search({ busy => 1 }, {})->count();
$c->stash->{'plain'} = {
data => " $nrRunningBuilds"
my $nrRunningBuilds = $c->model('DB::Builds')->search({finished => 0, busy => 1 })->count();
$c->stash->{'plain'} = {
data => "$nrRunningBuilds"
};
$c->forward('Hydra::View::Plain');
}

View File

@ -57,9 +57,9 @@ sub index : Chained('admin') PathPart('') Args(0) {
, '+as' => ['idle']
})];
$c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
{ 'me.busy' => 1, 'schedulingInfo.busy' => 1 },
{ join => [ 'schedulingInfo', 'build' ]
, order_by => [ 'machine' ]
{ finished => 0, 'me.busy' => 1, 'build.busy' => 1, },
{ join => [ 'build' ]
, order_by => [ 'machine', 'stepnr' ]
} ) ];
$c->stash->{template} = 'admin.tt';
}
@ -296,13 +296,15 @@ sub machine_disable : Chained('machine') PathPart('disable') Args(0) {
sub clear_queue_non_current : Chained('admin') Path('clear-queue-non-current') Args(0) {
my ($self, $c) = @_;
$c->model('DB::Builds')->search({iscurrent => 0, busy => 0}, { join => 'schedulingInfo' })->delete_all;
# !!! Mark the builds as cancelled instead.
$c->model('DB::Builds')->search({finished => 0, iscurrent => 0, busy => 0})->delete_all;
$c->res->redirect("/admin");
}
sub clear_queue : Chained('admin') Path('clear-queue') Args(0) {
my ($self, $c) = @_;
$c->model('DB::Builds')->search({busy => 0}, { join => 'schedulingInfo' })->delete_all;
# !!! Mark the builds as cancelled instead.
$c->model('DB::Builds')->search({finished => 0, busy => 0})->delete_all;
$c->res->redirect("/admin");
}

View File

@ -41,11 +41,10 @@ sub view_build : Chained('build') PathPart('') Args(0) {
$c->stash->{drvAvailable} = isValidPath $build->drvpath;
$c->stash->{flashMsg} = $c->flash->{buildMsg};
my $pathHash = $c->stash->{available} ? queryPathHash($build->outpath) : "Not available";
$c->stash->{pathHash} = $pathHash;
$c->stash->{pathHash} = $c->stash->{available} ? queryPathHash($build->outpath) : undef;
if (!$build->finished && $build->schedulingInfo->busy) {
my $logfile = $build->schedulingInfo->logfile;
if (!$build->finished && $build->busy) {
my $logfile = $build->logfile;
$c->stash->{logtext} = `cat $logfile` if defined $logfile && -e $logfile;
}
@ -81,13 +80,13 @@ sub view_build : Chained('build') PathPart('') Args(0) {
];
}
my $r = joinWithResultInfo( $c, $c->model('DB::Builds'))->search(
{ eval => { -in => $build->jobsetevalmembers->get_column('eval')->as_query } }
, { join => 'jobsetevalmembers', order_by => [ 'project', 'jobset', 'job'], distinct => 1 }
);
if ($r->count <= 100) {
$c->stash->{relatedbuilds} = [$r->all];
}
#my $r = joinWithResultInfo( $c, $c->model('DB::Builds'))->search(
# { eval => { -in => $build->jobsetevalmembers->all->get_column('eval')->as_query } }
# , { join => 'jobsetevalmembers', order_by => [ 'project', 'jobset', 'job'], distinct => 1 }
# );
#if ($r->count <= 100) {
# $c->stash->{relatedbuilds} = [$r->all];
#}
}
@ -141,7 +140,7 @@ sub showLog {
my $url = $c->request->uri->as_string;
$url =~ s/tail-reload/tail/g;
$c->stash->{url} = $url;
$c->stash->{reload} = defined $c->stash->{build}->schedulingInfo && $c->stash->{build}->schedulingInfo->busy;
$c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy;
$c->stash->{title} = "";
$c->stash->{contents} = (scalar `$pipestart | tail -n 50`) || " ";
$c->stash->{template} = 'plain-reload.tt';
@ -406,21 +405,19 @@ sub cancel : Chained('build') PathPart Args(0) {
txn_do($c->model('DB')->schema, sub {
error($c, "This build cannot be cancelled.")
if $build->finished || $build->schedulingInfo->busy;
if $build->finished || $build->busy;
# !!! Actually, it would be nice to be able to cancel busy
# builds as well, but we would have to send a signal or
# something to the build process.
$build->update({finished => 1, timestamp => time});
$build->update({finished => 1, busy => 0, timestamp => time});
$c->model('DB::BuildResultInfo')->create(
{ id => $build->id
, iscachedbuild => 0
, buildstatus => 4 # = cancelled
});
$build->schedulingInfo->delete;
});
$c->flash->{buildMsg} = "Build has been cancelled.";

View File

@ -36,9 +36,15 @@ sub overview : Chained('job') PathPart('') Args(0) {
, order_by => 'timestamp DESC', rows => 10
}) ];
$c->stash->{runningBuilds} = [$c->stash->{job}->builds->search({busy => 1}, { join => ['schedulingInfo', 'project'] , order_by => ["priority DESC", "timestamp"]
, '+select' => ['project.enabled', 'schedulingInfo.priority', 'schedulingInfo.disabled', 'schedulingInfo.busy']
, '+as' => ['enabled', 'priority', 'disabled', 'busy'] })];
$c->stash->{runningBuilds} = [
$c->stash->{job}->builds->search(
{ busy => 1 },
{ join => ['project']
, order_by => ["priority DESC", "timestamp"]
, '+select' => ['project.enabled']
, '+as' => ['enabled']
}
) ];
$c->stash->{systems} = [$c->stash->{job}->builds->search({iscurrent => 1}, {select => ["system"], distinct => 1})];
}

View File

@ -65,10 +65,10 @@ sub jobsetIndex {
my @as = ();
push(@select, "job"); push(@as, "job");
foreach my $system (@systems) {
push(@select, "(SELECT buildstatus FROM BuildResultInfo bri NATURAL JOIN Builds b WHERE b.id = (SELECT MAX(id) FROM Builds t WHERE t.project = me.project AND t.jobset = me.jobset AND t.job = me.job AND t.system = '$system' AND t.iscurrent = 1 ))");
push(@select, "(select buildstatus from BuildResultInfo bri join Builds b using (id) where b.id = (select max(id) from Builds t where t.project = me.project and t.jobset = me.jobset and t.job = me.job and t.system = '$system' and t.iscurrent = 1 ))");
push(@as, $system);
push(@select, "(SELECT b.id FROM BuildResultInfo bri NATURAL JOIN Builds b WHERE b.id = (SELECT MAX(id) FROM Builds t WHERE t.project = me.project AND t.jobset = me.jobset AND t.job = me.job AND t.system = '$system' AND t.iscurrent = 1 ))");
push(@as, $system."-build");
push(@select, "(select b.id from BuildResultInfo bri join Builds b using (id) where b.id = (select max(id) from Builds t where t.project = me.project and t.jobset = me.jobset and t.job = me.job and t.system = '$system' and t.iscurrent = 1 ))");
push(@as, "$system-build");
}
$c->stash->{activeJobsStatus} =
[ $c->model('DB')->resultset('ActiveJobsForJobset')->search(

View File

@ -22,8 +22,8 @@ sub begin :Private {
$c->stash->{tracker} = $ENV{"HYDRA_TRACKER"} ;
if (scalar(@args) == 0 || $args[0] ne "static") {
$c->stash->{nrRunningBuilds} = $c->model('DB::BuildSchedulingInfo')->search({ busy => 1 }, {})->count();
$c->stash->{nrQueuedBuilds} = $c->model('DB::BuildSchedulingInfo')->count();
$c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count();
$c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count();
}
}
@ -74,7 +74,7 @@ sub queue :Local {
my ($self, $c) = @_;
$c->stash->{template} = 'queue.tt';
$c->stash->{queue} = [$c->model('DB::Builds')->search(
{finished => 0}, {join => ['schedulingInfo', 'project'] , order_by => ["priority DESC", "timestamp"], '+select' => ['project.enabled', 'schedulingInfo.priority', 'schedulingInfo.disabled', 'schedulingInfo.busy'], '+as' => ['enabled', 'priority', 'disabled', 'busy'] })];
{finished => 0}, {join => ['project'] , order_by => ["priority DESC", "timestamp"], '+select' => ['project.enabled'], '+as' => ['enabled'] })];
$c->stash->{flashMsg} = $c->flash->{buildMsg};
}
@ -99,8 +99,8 @@ sub timeline :Local {
sub status :Local {
my ($self, $c) = @_;
$c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
{ 'me.busy' => 1, 'schedulingInfo.busy' => 1 },
{ join => [ 'schedulingInfo', 'build' ]
{ 'me.busy' => 1, 'build.busy' => 1 },
{ join => [ 'build' ]
, order_by => [ 'machine' ]
} ) ];
}

View File

@ -851,12 +851,11 @@ sub checkBuild {
, nixexprpath => $jobset->nixexprpath
});
$currentBuilds->{$build->id} = 1;
if (isValidPath($outPath)) {
print STDERR "marked as cached build ", $build->id, "\n";
$build->update({ finished => 1 });
$build->update({ finished => 1 });
$build->create_related('buildresultinfo',
{ iscachedbuild => 1
, buildstatus => 0
@ -869,8 +868,8 @@ sub checkBuild {
addBuildProducts($db, $build);
} else {
print STDERR "added to queue as build ", $build->id, "\n";
$build->create_related('buildschedulinginfo',
{ priority => $priority
$build->update(
{ priority => $priority
, busy => 0
, locker => ""
});
@ -906,24 +905,23 @@ sub restartBuild {
my ($db, $build) = @_;
txn_do($db, sub {
my $drvpath = $build->drvpath ;
my $outpath = $build->outpath ;
my $drvpath = $build->drvpath;
my $outpath = $build->outpath;
my $paths = "";
foreach my $bs ($build->buildsteps) {
$paths = $paths . " " . $bs->outpath;
$paths = $paths . " " . $bs->outpath;
}
my $r = `nix-store --clear-failed-paths $paths $outpath`;
$build->update({finished => 0, timestamp => time});
$build->resultInfo->delete;
$db->resultset('BuildSchedulingInfo')->create(
{ id => $build->id
, priority => 0 # don't know the original priority anymore...
$build->update(
{ finished => 0
, timestamp => time
, busy => 0
, locker => ""
});
});
$build->resultInfo->delete;
});
}

View File

@ -82,9 +82,7 @@ sub getBuildStats {
$c->stash->{scheduledBuilds} = $builds->search({finished => 0}) || 0;
$c->stash->{busyBuilds} = $builds->search(
{finished => 0, busy => 1},
{join => 'schedulingInfo'}) || 0;
$c->stash->{busyBuilds} = $builds->search({finished => 0, busy => 1}) || 0;
my $res;
$res = $builds->search({},

View File

@ -195,13 +195,13 @@ sub jobsetOverview {
return $project->jobsets->search( isProjectOwner($c, $project) ? {} : { hidden => 0 },
{ order_by => "name"
, "+select" =>
[ "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildSchedulingInfo WHERE me.project = a.project AND me.name = a.jobset AND a.isCurrent = 1)"
, "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildResultInfo WHERE me.project = a.project AND me.name = a.jobset AND buildstatus <> 0 AND a.isCurrent = 1)"
, "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildResultInfo WHERE me.project = a.project AND me.name = a.jobset AND buildstatus = 0 AND a.isCurrent = 1)"
, "(SELECT COUNT(*) FROM Builds AS a WHERE me.project = a.project AND me.name = a.jobset AND a.isCurrent = 1)"
[ "(select count(*) from Builds as a where a.finished = 0 and me.project = a.project and me.name = a.jobset and a.isCurrent = 1)"
, "(select count(*) from Builds as a join BuildResultInfo r using (id) where me.project = a.project and me.name = a.jobset and buildstatus <> 0 and a.isCurrent = 1)"
, "(select count(*) from Builds as a join BuildResultInfo r using (id) where me.project = a.project and me.name = a.jobset and buildstatus = 0 and a.isCurrent = 1)"
, "(select count(*) from Builds as a where me.project = a.project and me.name = a.jobset and a.isCurrent = 1)"
]
, "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"]
});
, "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"]
});
}
sub getViewResult {

View File

@ -46,7 +46,7 @@ __PACKAGE__->table("BuildProducts");
=head2 filesize
data_type: 'integer'
data_type: 'bigint'
is_nullable: 1
=head2 sha1hash
@ -91,7 +91,7 @@ __PACKAGE__->add_columns(
"subtype",
{ data_type => "text", is_nullable => 0 },
"filesize",
{ data_type => "integer", is_nullable => 1 },
{ data_type => "bigint", is_nullable => 1 },
"sha1hash",
{ data_type => "text", is_nullable => 1 },
"sha256hash",
@ -133,8 +133,8 @@ Related object: L<Hydra::Schema::Builds>
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {});
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:suSgQkBLXzu0yD4YicRS1A
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:18
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dzTKwZ7bby7kplnSgta3Gw
# You can replace this text with custom content, and it will be preserved on regeneration
1;

View File

@ -1,120 +0,0 @@
use utf8;
package Hydra::Schema::BuildSchedulingInfo;
# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE
=head1 NAME
Hydra::Schema::BuildSchedulingInfo
=cut
use strict;
use warnings;
use base 'DBIx::Class::Core';
=head1 TABLE: C<BuildSchedulingInfo>
=cut
__PACKAGE__->table("BuildSchedulingInfo");
=head1 ACCESSORS
=head2 id
data_type: 'integer'
is_auto_increment: 1
is_foreign_key: 1
is_nullable: 0
=head2 priority
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 busy
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 locker
data_type: 'text'
default_value: (empty string)
is_nullable: 0
=head2 logfile
data_type: 'text'
is_nullable: 1
=head2 disabled
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 starttime
data_type: 'integer'
is_nullable: 1
=cut
__PACKAGE__->add_columns(
"id",
{
data_type => "integer",
is_auto_increment => 1,
is_foreign_key => 1,
is_nullable => 0,
},
"priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"busy",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"locker",
{ data_type => "text", default_value => "", is_nullable => 0 },
"logfile",
{ data_type => "text", is_nullable => 1 },
"disabled",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"starttime",
{ data_type => "integer", is_nullable => 1 },
);
=head1 PRIMARY KEY
=over 4
=item * L</id>
=back
=cut
__PACKAGE__->set_primary_key("id");
=head1 RELATIONS
=head2 id
Type: belongs_to
Related object: L<Hydra::Schema::Builds>
=cut
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }, {});
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uz7y9Ly+ADRrtrPfEk9lGA
# You can replace this text with custom content, and it will be preserved on regeneration
1;

View File

@ -151,10 +151,4 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {})
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5H+OkGT0zQEWkAjU+OlBdg
__PACKAGE__->belongs_to(
"schedulingInfo",
"Hydra::Schema::BuildSchedulingInfo",
{ id => "build" },
);
1;

View File

@ -130,6 +130,39 @@ __PACKAGE__->table("Builds");
data_type: 'text'
is_nullable: 1
=head2 priority
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 busy
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 locker
data_type: 'text'
is_nullable: 1
=head2 logfile
data_type: 'text'
is_nullable: 1
=head2 disabled
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 starttime
data_type: 'integer'
is_nullable: 1
=cut
__PACKAGE__->add_columns(
@ -173,6 +206,18 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 1 },
"nixexprpath",
{ data_type => "text", is_nullable => 1 },
"priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"busy",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"locker",
{ data_type => "text", is_nullable => 1 },
"logfile",
{ data_type => "text", is_nullable => 1 },
"disabled",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"starttime",
{ data_type => "integer", is_nullable => 1 },
);
=head1 PRIMARY KEY
@ -249,21 +294,6 @@ __PACKAGE__->might_have(
{},
);
=head2 buildschedulinginfo
Type: might_have
Related object: L<Hydra::Schema::BuildSchedulingInfo>
=cut
__PACKAGE__->might_have(
"buildschedulinginfo",
"Hydra::Schema::BuildSchedulingInfo",
{ "foreign.id" => "self.id" },
{},
);
=head2 buildsteps
Type: has_many
@ -350,8 +380,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RRtBPTdD946kA5133+c4kw
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:54
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:VnnyFTwnLncGb2Dj2/giiA
use Hydra::Helper::Nix;
@ -369,12 +399,6 @@ __PACKAGE__->has_many(
{ "foreign.build" => "self.id" },
);
__PACKAGE__->belongs_to(
"schedulingInfo",
"Hydra::Schema::BuildSchedulingInfo",
{ id => "id" },
);
__PACKAGE__->belongs_to(
"resultInfo",
"Hydra::Schema::BuildResultInfo",
@ -410,11 +434,11 @@ sub makeQueries {
my $joinWithStatusChange =
<<QUERY;
natural join BuildResultInfo r
join BuildResultInfo r using (id)
left join Builds b on
b.id =
(select max(id)
from builds c natural join buildresultinfo r2
(select max(c.id)
from builds c join buildresultinfo r2 on c.id = r2.id
where
x.project = c.project and x.jobset = c.jobset and x.job = c.job and x.system = c.system and
x.id > c.id and
@ -432,11 +456,12 @@ QUERY
x.id, x.finished, x.timestamp, x.project, x.jobset, x.job, x.nixname,
x.description, x.drvpath, x.outpath, x.system, x.longdescription,
x.license, x.homepage, x.maintainers, x.isCurrent, x.nixExprInput,
x.nixExprPath, x.maxsilent, x.timeout,
x.nixExprPath, x.maxsilent, x.timeout, x.priority, x.busy, x.locker,
x.logfile, x.disabled, x.startTime,
b.id as statusChangeId, b.timestamp as statusChangeTime
from
(select
(select max(id) from builds b
(select max(b.id) from builds b
where
project = activeJobs.project and jobset = activeJobs.jobset
and job = activeJobs.job and system = activeJobs.system
@ -457,7 +482,7 @@ QUERY
select *
from
(select
(select max(id) from builds b
(select max(b.id) from builds b
where
project = activeJobs.project and jobset = activeJobs.jobset
and job = activeJobs.job and system = activeJobs.system

View File

@ -26,31 +26,15 @@ __PACKAGE__->table("SchemaVersion");
=head2 version
data_type: 'integer'
is_auto_increment: 1
is_nullable: 0
=cut
__PACKAGE__->add_columns(
"version",
{ data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
);
=head1 PRIMARY KEY
=over 4
=item * L</version>
=back
=cut
__PACKAGE__->set_primary_key("version");
__PACKAGE__->add_columns("version", { data_type => "integer", is_nullable => 0 });
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:F/jsSRq8pxR4mWq/N4qYGw
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:18
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LFD28W0GvvrOOylCM98SEQ
# You can replace this text with custom code or comments, and it will be preserved on regeneration