ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into nix-next

Browse Source
This commit is contained in:
John Ericson 2024-01-22 13:01:40 -05:00
commit 89cfe26533
2 changed files with 39 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
doc/manual/src/configuration.md
View File

@ -74,6 +74,30 @@ following:
}
}
Populating a Cache
------------------
A common use for Hydra is to pre-build and cache derivations which
take a long time to build. While it is possible to direcly access the
Hydra server's store over SSH, a more scalable option is to upload
built derivations to a remote store like an [S3-compatible object
store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting
the `store_uri` parameter will cause Hydra to sign and upload
derivations as they are built:
```
store_uri = s3://cache-bucket-name?compression=zstd&parallel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key
```
This example uses [Zstandard](https://github.com/facebook/zstd)
compression on derivations to reduce CPU usage on the server, but
[Brotli](https://brotli.org/) compression for derivation listings and
build logs because it has better browser support.
See [`nix help
stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html)
for a description of the store URI format.
Statsd Configuration
--------------------

26
src/hydra-queue-runner/queue-monitor.cc
View File

@ -495,12 +495,14 @@ Step::ptr State::createStep(ref<Store> destStore,
size_t avail = 0;
for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name, i.first);
if (/* localStore != destStore && */ localStore->isValidPath(*path))
auto pathOpt = i.second.path(*localStore, step->drv->name, i.first);
assert(pathOpt); // CA derivations not yet supported
auto & path = *pathOpt;
if (/* localStore != destStore && */ localStore->isValidPath(path))
avail++;
else if (useSubstitutes) {
SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
localStore->querySubstitutablePathInfos({{path, {}}}, infos);
if (infos.size() == 1)
avail++;
}
@ -509,25 +511,27 @@ Step::ptr State::createStep(ref<Store> destStore,
if (missing.size() == avail) {
valid = true;
for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name, i.first);
auto pathOpt = i.second.path(*localStore, step->drv->name, i.first);
assert(pathOpt); // CA derivations not yet supported
auto & path = *pathOpt;
try {
time_t startTime = time(0);
if (localStore->isValidPath(*path))
if (localStore->isValidPath(path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(*path),
localStore->printStorePath(path),
localStore->printStorePath(drvPath));
else {
printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(*path),
localStore->printStorePath(path),
localStore->printStorePath(drvPath));
localStore->ensurePath(*path);
localStore->ensurePath(path);
// FIXME: should copy directly from substituter to destStore.
}
copyClosure(*localStore, *destStore,
StorePathSet { *path },
StorePathSet { path },
NoRepair, CheckSigs, NoSubstitute);
time_t stopTime = time(0);
@ -535,13 +539,13 @@ Step::ptr State::createStep(ref<Store> destStore,
{
auto mc = startDbUpdate();
pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path);
txn.commit();
}
} catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(*path),
localStore->printStorePath(path),
localStore->printStorePath(drvPath),
e.what());
valid = false;