hydra-queue-runner: Improved scheduling

Each jobset now has a "scheduling share" that determines how much of
the build farm's time it is entitled to.  For instance, if a jobset
has 100 shares and the total number of shares of all jobsets is 1000,
it's entitled to 10% of the build farm's time.  When there is a free
build slot for a given system type, the queue runner will select the
jobset that is furthest below its scheduling share over a certain time
window (currently, the last day).  Withing that jobset, it will pick
the build with the highest priority.

So meta.schedulingPriority now only determines the order of builds
within a jobset, not between jobsets.  This makes it much easier to
prioritise one jobset over another (e.g. nixpkgs:trunk over
nixpkgs:stdenv).

src/lib/Hydra/Controller/Jobset.pm

@@ -50,7 +50,9 @@ sub jobset_GET {
 
     $c->stash->{evals} = getEvals($self, $c, scalar $c->stash->{jobset}->jobsetevals, 0, 10);
 
-    ($c->stash->{latestEval}) = $c->stash->{jobset}->jobsetevals->search({}, { rows => 1, order_by => ["id desc"] });
+    $c->stash->{latestEval} = $c->stash->{jobset}->jobsetevals->search({}, { rows => 1, order_by => ["id desc"] })->single;
+
+    $c->stash->{totalShares} = getTotalShares($c->model('DB')->schema);
 
     $self->status_ok(
         $c,
@@ -161,22 +163,22 @@ sub jobs_tab : Chained('jobsetChain') PathPart('jobs-tab') Args(0) {
         my @builds = $eval->builds->search(
             { job => { ilike => $filter } },
             { columns => ['id', 'job', 'finished', 'buildstatus'] });
-	foreach my $b (@builds) {
-	    my $jobName = $b->get_column('job');
-	    $evals->{$eval->id}->{$jobName} = 
-	        { id => $b->id, finished => $b->finished, buildstatus => $b->buildstatus };
-	    $jobs{$jobName} = 1;
-	    $nrBuilds++;
-	}
-	last if $nrBuilds >= 10000;
+        foreach my $b (@builds) {
+            my $jobName = $b->get_column('job');
+            $evals->{$eval->id}->{$jobName} = 
+                { id => $b->id, finished => $b->finished, buildstatus => $b->buildstatus };
+            $jobs{$jobName} = 1;
+            $nrBuilds++;
+        }
+        last if $nrBuilds >= 10000;
     }
 
     if ($c->request->params->{showInactive}) {
-	$c->stash->{showInactive} = 1;
-	foreach my $job ($c->stash->{jobset}->jobs->search({ name => { ilike => $filter } })) {
-	    next if defined $jobs{$job->name};
-	    $c->stash->{inactiveJobs}->{$job->name} = $jobs{$job->name} = 1;
-	}
+        $c->stash->{showInactive} = 1;
+        foreach my $job ($c->stash->{jobset}->jobs->search({ name => { ilike => $filter } })) {
+            next if defined $jobs{$job->name};
+            $c->stash->{inactiveJobs}->{$job->name} = $jobs{$job->name} = 1;
+        }
     }
 
     $c->stash->{evals} = $evals;
@@ -209,6 +211,7 @@ sub edit : Chained('jobsetChain') PathPart Args(0) {
 
     $c->stash->{template} = 'edit-jobset.tt';
     $c->stash->{edit} = 1;
+    $c->stash->{totalShares} = getTotalShares($c->model('DB')->schema);
 }
 
 
@@ -287,6 +290,7 @@ sub updateJobset {
         , keepnr => int(trim($c->stash->{params}->{keepnr}))
         , checkinterval => int(trim($c->stash->{params}->{checkinterval}))
         , triggertime => $enabled ? $jobset->triggertime // time() : undef
+        , schedulingshares => int($c->stash->{params}->{schedulingshares})
         });
 
     # Process the inputs of this jobset.

src/lib/Hydra/Controller/Project.pm

@@ -201,6 +201,7 @@ sub create_jobset : Chained('projectChain') PathPart('create-jobset') Args(0) {
     $c->stash->{template} = 'edit-jobset.tt';
     $c->stash->{create} = 1;
     $c->stash->{edit} = 1;
+    $c->stash->{totalShares} = getTotalShares($c->model('DB')->schema);
 }
 
 

src/lib/Hydra/Helper/Nix.pm

@@ -20,7 +20,8 @@ our @EXPORT = qw(
     getMainOutput
     getEvals getMachines
     pathIsInsidePrefix
-    captureStdoutStderr run grab);
+    captureStdoutStderr run grab
+    getTotalShares);
 
 
 sub getHydraHome {
@@ -533,4 +534,12 @@ sub grab {
 }
 
 
+sub getTotalShares {
+    my ($db) = @_;
+    return $db->resultset('Jobsets')->search(
+        { 'project.enabled' => 1, 'me.enabled' => 1 },
+        { join => 'project', select => { sum => 'schedulingshares' }, as => 'sum' })->single->get_column('sum');
+}
+
+
 1;

src/lib/Hydra/Schema/CachedDarcsInputs.pm

@@ -15,6 +15,18 @@ use warnings;
 
 use base 'DBIx::Class::Core';
 
+=head1 COMPONENTS LOADED
+
+=over 4
+
+=item * L<Hydra::Component::ToJSON>
+
+=back
+
+=cut
+
+__PACKAGE__->load_components("+Hydra::Component::ToJSON");
+
 =head1 TABLE: C<CachedDarcsInputs>
 
 =cut
@@ -28,11 +40,6 @@ __PACKAGE__->table("CachedDarcsInputs");
   data_type: 'text'
   is_nullable: 0
 
-=head2 branch
-
-  data_type: 'text'
-  is_nullable: 0
-
 =head2 revision
 
   data_type: 'text'
@@ -48,6 +55,11 @@ __PACKAGE__->table("CachedDarcsInputs");
   data_type: 'text'
   is_nullable: 0
 
+=head2 revcount
+
+  data_type: 'integer'
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -55,12 +67,12 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_nullable => 0 },
   "revision",
   { data_type => "text", is_nullable => 0 },
-  "revcount",
-  { data_type => "integer", is_nullable => 0 },
   "sha256hash",
   { data_type => "text", is_nullable => 0 },
   "storepath",
   { data_type => "text", is_nullable => 0 },
+  "revcount",
+  { data_type => "integer", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -69,8 +81,6 @@ __PACKAGE__->add_columns(
 
 =item * L</uri>
 
-=item * L</branch>
-
 =item * L</revision>
 
 =back
@@ -80,7 +90,9 @@ __PACKAGE__->add_columns(
 __PACKAGE__->set_primary_key("uri", "revision");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fx3yosWMmJ+MnvL/dSWtFA
+# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-09-20 11:08:50
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Yl1slt3SAizijgu0KUTn0A
 
+
+# You can replace this text with custom code or comments, and it will be preserved on regeneration
 1;

src/lib/Hydra/Schema/Jobsets.pm

@@ -118,6 +118,12 @@ __PACKAGE__->table("Jobsets");
   default_value: 300
   is_nullable: 0
 
+=head2 schedulingshares
+
+  data_type: 'integer'
+  default_value: 100
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -151,6 +157,8 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 3, is_nullable => 0 },
   "checkinterval",
   { data_type => "integer", default_value => 300, is_nullable => 0 },
+  "schedulingshares",
+  { data_type => "integer", default_value => 100, is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -272,7 +280,7 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tsGR8MhZRIUeNwpcVczMUw
+# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-09-20 12:15:23
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pD6tGW0Ob3fuA1p0uQnBWw
 
 1;