ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into flake

Browse Source 
Also update flake.lock
This commit is contained in:
Eelco Dolstra
2020-02-10 17:58:10 +01:00
parent ba8814a245 add4f610ce
commit 100e09a5b3
49 changed files with 648 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/script/Makefile.am
View File

@ -3,6 +3,7 @@ EXTRA_DIST = \
hydra-eval-guile-jobs.in
distributable_scripts = \
hydra-backfill-ids \
hydra-init \
hydra-eval-jobset \
hydra-server \

164
src/script/hydra-backfill-ids Executable file
View File

@ -0,0 +1,164 @@
#! /usr/bin/env perl
use strict;
use utf8;
use Hydra::Model::DB;
STDOUT->autoflush();
STDERR->autoflush(1);
binmode STDERR, ":encoding(utf8)";
my $db = Hydra::Model::DB->new();
my $vacuum = $db->storage->dbh->prepare("VACUUM;");
my $dryRun = defined $ENV{'HYDRA_DRY_RUN'};
my $batchSize = 10000;
my $iterationsPerVacuum = 500;
sub backfillJobsJobsetId {
my ($skipLocked) = @_;
my $logPrefix;
if ($skipLocked) {
$logPrefix = "(pass 1/2)";
} else {
$logPrefix = "(pass 2/2)";
}
print STDERR "$logPrefix Backfilling Jobs records where jobset_id is NULL...\n";
my $totalToGoSth = $db->storage->dbh->prepare(<<QUERY);
SELECT COUNT(*) FROM jobs WHERE jobset_id IS NULL
QUERY
$totalToGoSth->execute();
my ($totalToGo) = $totalToGoSth->fetchrow_array;
my $skipLockedStmt = $skipLocked ? "FOR UPDATE SKIP LOCKED" : "";
my $update10kJobs = $db->storage->dbh->prepare(<<QUERY);
UPDATE jobs
SET jobset_id = (
SELECT jobsets.id
FROM jobsets
WHERE jobsets.name = jobs.jobset
AND jobsets.project = jobs.project
)
WHERE (jobs.project, jobs.jobset, jobs.name) in (
SELECT jobsprime.project, jobsprime.jobset, jobsprime.name
FROM jobs jobsprime
WHERE jobsprime.jobset_id IS NULL
$skipLockedStmt
LIMIT ?
);
QUERY
print STDERR "$logPrefix Total Jobs records without a jobset_id: $totalToGo\n";
my $iteration = 0;
my $affected;
do {
$iteration++;
$affected = $update10kJobs->execute($batchSize);
print STDERR "$logPrefix (batch #$iteration; $totalToGo remaining) Jobs.jobset_id: affected $affected rows...\n";
$totalToGo -= $affected;
if ($iteration % $iterationsPerVacuum == 0) {
print STDERR "$logPrefix (batch #$iteration) Vacuuming...\n";
$vacuum->execute();
}
} while ($affected > 0);
if ($skipLocked) {
backfillJobsJobsetId(0);
}
}
sub backfillBuildsJobsetId {
my ($skipLocked) = @_;
my $logPrefix;
if ($skipLocked) {
$logPrefix = "(pass 1/2)";
print STDERR "$logPrefix Backfilling unlocked Builds records where jobset_id is NULL...\n";
} else {
$logPrefix = "(pass 2/2)";
print STDERR "$logPrefix Backfilling all Builds records where jobset_id is NULL...\n";
}
my $skipLockedStmt = $skipLocked ? "FOR UPDATE SKIP LOCKED" : "";
my $update10kBuilds = $db->storage->dbh->prepare(<<"QUERY");
WITH updateprogress AS (
UPDATE builds
SET jobset_id = (
SELECT jobsets.id
FROM jobsets
WHERE jobsets.name = builds.jobset
AND jobsets.project = builds.project
)
WHERE builds.id in (
SELECT buildprime.id
FROM builds buildprime
WHERE buildprime.jobset_id IS NULL
AND buildprime.id >= ?
ORDER BY buildprime.id
$skipLockedStmt
LIMIT ?
)
RETURNING id
)
SELECT
count(*) AS affected,
max(updateprogress.id) AS highest_id
FROM updateprogress;
QUERY
my $lowestNullIdSth = $db->storage->dbh->prepare(<<QUERY);
SELECT id FROM builds WHERE jobset_id IS NULL ORDER BY id LIMIT 1
QUERY
$lowestNullIdSth->execute();
my ($highestId) = $lowestNullIdSth->fetchrow_array;
my $totalToGoSth = $db->storage->dbh->prepare(<<QUERY);
SELECT COUNT(*) FROM builds WHERE jobset_id IS NULL AND id >= ?
QUERY
$totalToGoSth->execute($highestId);
my ($totalToGo) = $totalToGoSth->fetchrow_array;
print STDERR "$logPrefix Total Builds records without a jobset_id: $totalToGo, starting at $highestId\n";
my $iteration = 0;
my $affected;
do {
my $previousHighId = $highestId;
$iteration++;
$update10kBuilds->execute($highestId, $batchSize);
($affected, $highestId) = $update10kBuilds->fetchrow_array;
print STDERR "$logPrefix (batch #$iteration; $totalToGo remaining) Builds.jobset_id: affected $affected rows; max ID: $previousHighId -> $highestId\n";
$totalToGo -= $affected;
if ($iteration % $iterationsPerVacuum == 0) {
print STDERR "$logPrefix (batch #$iteration) Vacuuming...\n";
$vacuum->execute();
}
} while ($affected > 0);
if ($skipLocked) {
backfillBuildsJobsetId(0);
}
}
die "syntax: $0\n" unless @ARGV == 0;
print STDERR "Beginning with a VACUUM\n";
$vacuum->execute();
backfillJobsJobsetId(1);
backfillBuildsJobsetId(1);
print STDERR "Ending with a VACUUM\n";
$vacuum->execute();

10
src/script/hydra-eval-jobset
View File

@ -426,7 +426,12 @@ sub checkBuild {
my $build;
txn_do($db, sub {
my $job = $jobset->jobs->update_or_create({ name => $jobName });
my $job = $jobset->jobs->update_or_create({
name => $jobName,
jobset_id => $jobset->id,
project => $jobset->project,
jobset => $jobset->name,
});
# Don't add a build that has already been scheduled for this
# job, or has been built but is still a "current" build for
@ -473,6 +478,9 @@ sub checkBuild {
# Add the build to the database.
$build = $job->builds->create(
{ timestamp => $time
, project => $jobset->project
, jobset => $jobset->name
, jobset_id => $jobset->id
, description => null($buildInfo->{description})
, license => null($buildInfo->{license})
, homepage => null($buildInfo->{homepage})

11
src/script/hydra-init
View File

@ -44,6 +44,17 @@ my @versions = $db->resultset('SchemaVersion')->all;
die "couldn't get Hydra schema version!" if scalar @versions != 1;
my $schemaVersion = $versions[0]->version;
if ($schemaVersion <= 60) {
print STDERR <<QUOTE;
WARNING: Schema version 62 and 63 make nullable jobset_id fields on
Builds and Jobs non-nullable. On big Hydra servers, this
migration will take many hours. Because of that, the
migration is not automatic, and must be performed manually.
To backfill these IDs, run: hydra-fill-ids
QUOTE
}
for (my $n = $schemaVersion; $n < $maxSchemaVersion; $n++) {
my $m = $n + 1;
print STDERR "upgrading Hydra schema from version $n to $m\n";