Normalize nixexpr{input,path} from builds to jobsetevals.

Duplicating this data on every record of the builds table cost
approximately 4G of duplication.

Note that the database migration included took about 4h45m on an
untuned server which uses very slow rotational disks in a RAID5 setup,
with not a lot of RAM. I imagine in production it might take an hour
or two, but not 4. If this should become a chunked migration, I can do
that.

Note: Because of the question about chunked migrations, I have NOT
YET tested this migration thoroughly enough for merge.
This commit is contained in:
Graham Christensen
2021-01-22 07:14:24 -05:00
parent 53c2fc2216
commit 9516b256f1
9 changed files with 62 additions and 36 deletions

View File

@ -162,13 +162,6 @@ create table Builds (
isChannel integer not null default 0, -- meta.isHydraChannel
isCurrent integer default 0,
-- Copy of the nixExprInput/nixExprPath fields of the jobset that
-- instantiated this build. Needed if we want to reproduce this
-- build. FIXME: this should be stored in JobsetEvals, storing it
-- here is denormal.
nixExprInput text,
nixExprPath text,
-- Priority within a jobset, set via meta.schedulingPriority.
priority integer not null default 0,
@ -466,6 +459,8 @@ create table JobsetEvals (
nrSucceeded integer, -- set lazily when all builds are finished
flake text, -- immutable flake reference
nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression
nixExprPath text, -- relative path of the Nix or Guix expression
foreign key (project) references Projects(name) on delete cascade on update cascade,
foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade

31
src/sql/upgrade-71.sql Normal file
View File

@ -0,0 +1,31 @@
ALTER TABLE JobsetEvals
ADD COLUMN nixExprInput text,
ADD COLUMN nixExprPath text;
-- This migration took 4.5 hours on a server
-- with 5400RPM drives, against a copy of hydra's
-- production dataset. It might take a significantly
-- less amount of time there, and not justify a
-- batched migration.
UPDATE jobsetevals
SET (nixexprinput, nixexprpath) = (
SELECT builds.nixexprinput, builds.nixexprpath
FROM builds
LEFT JOIN jobsetevalmembers
ON jobsetevalmembers.build = builds.id
WHERE jobsetevalmembers.eval = jobsetevals.id
LIMIT 1
)
WHERE jobsetevals.id in (
SELECT jobsetevalsprime.id
FROM jobsetevals as jobsetevalsprime
WHERE jobsetevalsprime.nixexprinput IS NULL
-- AND jobsetevalsprime.id > ? --------- These are in case of a batched migration
ORDER BY jobsetevalsprime.id ASC -- /
-- LIMIT ? -- ----------------------
);
ALTER TABLE builds
DROP COLUMN nixexprinput,
DROP COLUMN nixexprpath;