From 7517c134c57c7ce967a7b603482a23f4952331a0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Dec 2023 00:29:23 -0500 Subject: [PATCH 1/3] flake.lock: Update Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newer 2.19-maintenance has some `restricted-eval` fixes that benefit Hydra. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-12-11) → 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 2871e70a..66e30323 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1701122567, - "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "lastModified": 1702510710, + "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=", "owner": "NixOS", "repo": "nix", - "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.19.2", + "ref": "2.19-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 5a5aef55..8280e076 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.nix.url = "github:NixOS/nix/2.19.2"; + inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: From abd858d3dcd8dbcb2ee2d17ff5b03dbce46821e7 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 19 Dec 2023 07:54:40 +1000 Subject: [PATCH 2/3] Document the `store_uri` parameter by way of example --- doc/manual/src/configuration.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md index 02210449..4954040c 100644 --- a/doc/manual/src/configuration.md +++ b/doc/manual/src/configuration.md @@ -74,6 +74,30 @@ following: } } +Populating a Cache +------------------ + +A common use for Hydra is to pre-build and cache derivations which +take a long time to build. While it is possible to direcly access the +Hydra server's store over SSH, a more scalable option is to upload +built derivations to a remote store like an [S3-compatible object +store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting +the `store_uri` parameter will cause Hydra to sign and upload +derivations as they are built: + +``` +store_uri = s3://cache-bucket-name?compression=zstd¶llel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key +``` + +This example uses [Zstandard](https://github.com/facebook/zstd) +compression on derivations to reduce CPU usage on the server, but +[Brotli](https://brotli.org/) compression for derivation listings and +build logs because it has better browser support. + +See [`nix help +stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html) +for a description of the store URI format. + Statsd Configuration -------------------- From 75f26f1fc4f114b7f930c57ba37212ea67990de1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Dec 2023 19:10:58 -0500 Subject: [PATCH 3/3] Clean up `std::optional` dereferencing in the queue runner Instead of doing this partial operation a number of times, assert (with a comment, get a reference to the thing inside, and use that just once. (This refactor was done twice, "just once" for each time.) --- src/hydra-queue-runner/queue-monitor.cc | 26 ++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 6c339af6..09a2871e 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -495,12 +495,14 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); - if (/* localStore != destStore && */ localStore->isValidPath(*path)) + auto pathOpt = i.second.path(*localStore, step->drv->name, i.first); + assert(pathOpt); // CA derivations not yet supported + auto & path = *pathOpt; + if (/* localStore != destStore && */ localStore->isValidPath(path)) avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{*path, {}}}, infos); + localStore->querySubstitutablePathInfos({{path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -509,25 +511,27 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name, i.first); + auto pathOpt = i.second.path(*localStore, step->drv->name, i.first); + assert(pathOpt); // CA derivations not yet supported + auto & path = *pathOpt; try { time_t startTime = time(0); - if (localStore->isValidPath(*path)) + if (localStore->isValidPath(path)) printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); else { printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath)); - localStore->ensurePath(*path); + localStore->ensurePath(path); // FIXME: should copy directly from substituter to destStore. } copyClosure(*localStore, *destStore, - StorePathSet { *path }, + StorePathSet { path }, NoRepair, CheckSigs, NoSubstitute); time_t stopTime = time(0); @@ -535,13 +539,13 @@ Step::ptr State::createStep(ref destStore, { auto mc = startDbUpdate(); pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); txn.commit(); } } catch (Error & e) { printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(*path), + localStore->printStorePath(path), localStore->printStorePath(drvPath), e.what()); valid = false;