From 39a4e4791ea9ae4c29400f322bc1d1b71453cf1d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Apr 2024 17:30:43 -0400 Subject: [PATCH 01/27] Switch (back) to Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-creating `nix-next` after using it in #1375. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11) → 'github:NixOS/nix/aa438b8fbaebbbdb922655127053c4e8ea3e55bb' (2024-04-12) --- flake.lock | 7 +++---- flake.nix | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 966431f9..0b5c3bbb 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,15 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1712849398, - "narHash": "sha256-10z/SoidVl9/lh56cMLj7ntJZHtVrumFvmn1YEqXmaM=", + "lastModified": 1712957033, + "narHash": "sha256-/P298re0Qga0eJV3Q3BppvYbVEN0CKXMnxMQX192hcE=", "owner": "NixOS", "repo": "nix", - "rev": "60824fa97c588a0faf68ea61260a47e388b0a4e5", + "rev": "aa438b8fbaebbbdb922655127053c4e8ea3e55bb", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.21-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index c6646200..9d8c7142 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small"; - inputs.nix.url = "github:NixOS/nix/2.21-maintenance"; + inputs.nix.url = "github:NixOS/nix"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: From 5e910fa2cec113642ded1294c0c47100670ef587 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 May 2024 11:25:14 -0400 Subject: [PATCH 02/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/00ca2b05b8fbbef09be5d1e4820857605d4c31b6' (2024-05-03) → 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index aa29e826..8b3c146d 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1714701944, - "narHash": "sha256-trTxWfGElp0rkjquqG5I5RYVoxo8foCflxJFUtHwnOQ=", + "lastModified": 1715180119, + "narHash": "sha256-6tdv6d0lzpS+EeUTRZ15vS0YdcSpkmRBrJ5ABHC0QiU=", "owner": "NixOS", "repo": "nix", - "rev": "00ca2b05b8fbbef09be5d1e4820857605d4c31b6", + "rev": "0930058189f350a3729cd5aef2ffc8dae2ad436e", "type": "github" }, "original": { From e4f2c84f8dfd2863e40c74a35fb90b35749fbc94 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 May 2024 20:02:54 -0400 Subject: [PATCH 03/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08) → 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8b3c146d..4155cb06 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1715180119, - "narHash": "sha256-6tdv6d0lzpS+EeUTRZ15vS0YdcSpkmRBrJ5ABHC0QiU=", + "lastModified": 1715986148, + "narHash": "sha256-mR8FVWkQxl7jmO14WRRYNJdTG0yno9x5VQX/bQp7Mx4=", "owner": "NixOS", "repo": "nix", - "rev": "0930058189f350a3729cd5aef2ffc8dae2ad436e", + "rev": "beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b", "type": "github" }, "original": { From e4552ddf918aa25a89f594413b626dbb64a9214b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 May 2024 18:11:37 -0400 Subject: [PATCH 04/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17) → 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4155cb06..cf51e51f 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1715986148, - "narHash": "sha256-mR8FVWkQxl7jmO14WRRYNJdTG0yno9x5VQX/bQp7Mx4=", + "lastModified": 1716243047, + "narHash": "sha256-SfQwsgwgvvGixkRIwZbULsKGKoRUPf9yH+0hdBKkgRo=", "owner": "NixOS", "repo": "nix", - "rev": "beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b", + "rev": "a57abbd143f8ed44e823c3244e93507f64020878", "type": "github" }, "original": { From 71c4e2dc5b834a38e199f7a59b8928cc60c115fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 May 2024 18:00:16 -0400 Subject: [PATCH 05/27] Dedup more protocol code Use https://github.com/NixOS/nix/pull/10749 --- src/hydra-queue-runner/build-remote.cc | 36 +++----------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index ad510e1b..b4275ecc 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -267,32 +267,6 @@ static BuildResult performBuild( return result; } -static std::map queryPathInfos( - ::Machine::Connection & conn, - Store & localStore, - StorePathSet & outputs, - size_t & totalNarSize -) -{ - - /* Get info about each output path. */ - std::map infos; - conn.to << ServeProto::Command::QueryPathInfos; - ServeProto::write(localStore, conn, outputs); - conn.to.flush(); - while (true) { - auto storePathS = readString(conn.from); - if (storePathS == "") break; - - auto storePath = localStore.parseStorePath(storePathS); - auto info = ServeProto::Serialise::read(localStore, conn); - totalNarSize += info.narSize; - infos.insert_or_assign(std::move(storePath), std::move(info)); - } - - return infos; -} - static void copyPathFromRemote( ::Machine::Connection & conn, NarMemberDatas & narMembers, @@ -478,12 +452,6 @@ void State::buildRemote(ref destStore, throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } - // Do not attempt to speak a newer version of the protocol. - // - // Per https://github.com/NixOS/nix/issues/9584 should be handled as - // part of `handshake` in upstream nix. - conn.remoteVersion = std::min(conn.remoteVersion, our_version); - { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; @@ -552,8 +520,10 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); + auto infos = conn.queryPathInfos(*localStore, outputs); + size_t totalNarSize = 0; - auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize); + for (auto & [_, info] : infos) totalNarSize += info.narSize; if (totalNarSize > maxOutputSize) { result.stepStatus = bsNarSizeLimitExceeded; From 346badc66ff68bccb20ccf9f48964da7bf60f2cf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 22 May 2024 22:00:38 -0400 Subject: [PATCH 06/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20) → 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index cf51e51f..6a6721b7 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1716243047, - "narHash": "sha256-SfQwsgwgvvGixkRIwZbULsKGKoRUPf9yH+0hdBKkgRo=", + "lastModified": 1716420255, + "narHash": "sha256-F8CpJTkbVl805LO5nvU5ivBdIHjcFUg8uk1EePGPu8s=", "owner": "NixOS", "repo": "nix", - "rev": "a57abbd143f8ed44e823c3244e93507f64020878", + "rev": "5845fd59c34198ad52a7f7bcb6d3ea7176ca437b", "type": "github" }, "original": { From d55bea2a1edf67645ace7221c3be4af192d198f3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 May 2024 13:34:30 -0400 Subject: [PATCH 07/27] Utilize `nix::Machine` more fully With https://github.com/NixOS/nix/pull/9839, the `storeUri` field is much better structured, so we can use it while still opening the SSH connection ourselves. --- src/hydra-queue-runner/build-remote.cc | 60 +++++++++++--------- src/hydra-queue-runner/builder.cc | 12 ++-- src/hydra-queue-runner/dispatcher.cc | 2 +- src/hydra-queue-runner/hydra-queue-runner.cc | 18 +++--- src/hydra-queue-runner/state.hh | 13 +---- 5 files changed, 49 insertions(+), 56 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index b4275ecc..617cce44 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -21,28 +21,23 @@ using namespace nix; -namespace nix::build_remote { - -static Strings extraStoreArgs(std::string & machine) +bool ::Machine::isLocalhost() const { - Strings result; - try { - auto parsed = parseURL(machine); - if (parsed.scheme != "ssh") { - throw SysError("Currently, only (legacy-)ssh stores are supported!"); - } - machine = parsed.authority.value_or(""); - auto remoteStore = parsed.query.find("remote-store"); - if (remoteStore != parsed.query.end()) { - result = {"--store", shellEscape(remoteStore->second)}; - } - } catch (BadURL &) { - // We just try to continue with `machine->sshName` here for backwards compat. - } - - return result; + return storeUri.params.empty() && std::visit(overloaded { + [](const StoreReference::Auto &) { + return true; + }, + [](const StoreReference::Specified & s) { + return + (s.scheme == "local" || s.scheme == "unix") || + ((s.scheme == "ssh" || s.scheme == "ssh-ng") && + s.authority == "localhost"); + }, + }, storeUri.variant); } +namespace nix::build_remote { + static std::unique_ptr openConnection( ::Machine::ptr machine, SSHMaster & master) { @@ -51,7 +46,11 @@ static std::unique_ptr openConnection( command.push_back("--builders"); command.push_back(""); } else { - command.splice(command.end(), extraStoreArgs(machine->sshName)); + auto remoteStore = machine->storeUri.params.find("remote-store"); + if (remoteStore != machine->storeUri.params.end()) { + command.push_back("--store"); + command.push_back(shellEscape(remoteStore->second)); + } } return master.startCommand(std::move(command), { @@ -187,7 +186,7 @@ static BasicDerivation sendInputs( MaintainCount mc2(nrStepsCopyingTo); printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", - localStore.printStorePath(step.drvPath), conn.machine->sshName); + localStore.printStorePath(step.drvPath), conn.machine->storeUri.render()); auto now1 = std::chrono::steady_clock::now(); @@ -393,8 +392,13 @@ void State::buildRemote(ref destStore, updateStep(ssConnecting); + auto * pSpecified = std::get_if(&machine->storeUri.variant); + if (!pSpecified || pSpecified->scheme != "ssh") { + throw Error("Currently, only (legacy-)ssh stores are supported!"); + } + SSHMaster master { - machine->sshName, + pSpecified->authority, machine->sshKey, machine->sshPublicHostKey, false, // no SSH master yet @@ -445,11 +449,11 @@ void State::buildRemote(ref destStore, conn.to, conn.from, our_version, - machine->sshName); + machine->storeUri.render()); } catch (EndOfFile & e) { child->sshPid.wait(); std::string s = chomp(readFile(result.logFile)); - throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); + throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s); } { @@ -480,7 +484,7 @@ void State::buildRemote(ref destStore, /* Do the build. */ printMsg(lvlDebug, "building ‘%s’ on ‘%s’", localStore->printStorePath(step->drvPath), - machine->sshName); + machine->storeUri.render()); updateStep(ssBuilding); @@ -503,7 +507,7 @@ void State::buildRemote(ref destStore, get a build log. */ if (result.isCached) { printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’", - localStore->printStorePath(step->drvPath), machine->sshName); + localStore->printStorePath(step->drvPath), machine->storeUri.render()); unlink(result.logFile.c_str()); result.logFile = ""; } @@ -532,7 +536,7 @@ void State::buildRemote(ref destStore, /* Copy each path. */ printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", - localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); + localStore->printStorePath(step->drvPath), machine->storeUri.render(), totalNarSize); build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); auto now2 = std::chrono::steady_clock::now(); @@ -571,7 +575,7 @@ void State::buildRemote(ref destStore, info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4); info->lastFailure = now; int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30); - printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta); + printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta); info->disabledUntil = now + std::chrono::seconds(delta); } throw; diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 5269febd..a0773511 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -41,7 +41,7 @@ void State::builder(MachineReservation::ptr reservation) } catch (std::exception & e) { printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s", localStore->printStorePath(reservation->step->drvPath), - reservation->machine->sshName, + reservation->machine->storeUri.render(), e.what()); } } @@ -150,7 +150,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, buildOptions.buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1)); } if (!buildOneDone) @@ -196,7 +196,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, { auto mc = startDbUpdate(); pqxx::work txn(*conn); - stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy); + stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy); txn.commit(); } @@ -253,7 +253,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* Finish the step in the database. */ if (stepNr) { pqxx::work txn(*conn); - finishBuildStep(txn, result, buildId, stepNr, machine->sshName); + finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render()); txn.commit(); } @@ -261,7 +261,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, issue). Retry a number of times. */ if (result.canRetry) { printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s", - localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg); + localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg); assert(stepNr); bool retry; { @@ -452,7 +452,7 @@ void State::failStep( build->finishedInDB) continue; createBuildStep(txn, - 0, build->id, step, machine ? machine->sshName : "", + 0, build->id, step, machine ? machine->storeUri.render() : "", result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId); } diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 6d738ded..0f5a65e7 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -255,7 +255,7 @@ system_time State::doDispatch() /* Can this machine do this step? */ if (!mi.machine->supportsStep(step)) { debug("machine '%s' does not support step '%s' (system type '%s')", - mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform); + mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform); continue; } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5ffa7fe6..d7a88a0d 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -153,8 +153,8 @@ void State::parseMachines(const std::string & contents) using MaxJobs = std::remove_const::type; auto machine = std::make_shared<::Machine>(nix::Machine { - // `storeUri`, not yet used - "", + // `storeUri` + tokens[0], // `systemTypes` tokenizeString(tokens[1], ","), // `sshKey` @@ -175,25 +175,23 @@ void State::parseMachines(const std::string & contents) : "", }); - machine->sshName = tokens[0]; - /* Re-use the State object of the previous machine with the same name. */ - auto i = oldMachines.find(machine->sshName); + auto i = oldMachines.find(machine->storeUri.variant); if (i == oldMachines.end()) - printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName); + printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render()); else - printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName); + printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render()); machine->state = i == oldMachines.end() ? std::make_shared<::Machine::State>() : i->second->state; - newMachines[machine->sshName] = machine; + newMachines[machine->storeUri.variant] = machine; } for (auto & m : oldMachines) if (newMachines.find(m.first) == newMachines.end()) { if (m.second->enabled) - printInfo("removing machine ‘%1%’", m.first); + printInfo("removing machine ‘%1%’", m.second->storeUri.render()); /* Add a disabled ::Machine object to make sure stats are maintained. */ auto machine = std::make_shared<::Machine>(*(m.second)); @@ -657,7 +655,7 @@ void State::dumpStatus(Connection & conn) machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone; machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone; } - statusJson["machines"][m->sshName] = machine; + statusJson["machines"][m->storeUri.render()] = machine; } } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 5d242cdf..e2fb0c9c 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -240,10 +239,6 @@ struct Machine : nix::Machine { typedef std::shared_ptr ptr; - /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way - we are not yet used to, but once we are, we don't need this. */ - std::string sshName; - struct State { typedef std::shared_ptr ptr; counter currentJobs{0}; @@ -293,11 +288,7 @@ struct Machine : nix::Machine return true; } - bool isLocalhost() - { - std::regex r("^(ssh://|ssh-ng://)?localhost$"); - return std::regex_search(sshName, r); - } + bool isLocalhost() const; // A connection to a machine struct Connection : nix::ServeProto::BasicClientConnection { @@ -357,7 +348,7 @@ private: /* The build machines. */ std::mutex machinesReadyLock; - typedef std::map Machines; + typedef std::map Machines; nix::Sync machines; // FIXME: use atomic_shared_ptr /* Various stats. */ From bede2a141a3c1073d89b7852b08ac58c71d6778b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 May 2024 09:59:32 -0400 Subject: [PATCH 08/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22) → 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6a6721b7..327d44ad 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1716420255, - "narHash": "sha256-F8CpJTkbVl805LO5nvU5ivBdIHjcFUg8uk1EePGPu8s=", + "lastModified": 1716472735, + "narHash": "sha256-506pEWmMufBGD9hM99WZynH9Vxkv3o3IJJsPiVAs4T4=", "owner": "NixOS", "repo": "nix", - "rev": "5845fd59c34198ad52a7f7bcb6d3ea7176ca437b", + "rev": "2c42e7b8d9ea32e59c01334852599b548b214d31", "type": "github" }, "original": { From 09a1e64ed2d16ffbbcbc0c98458c621f5d939cf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 May 2024 00:32:11 -0400 Subject: [PATCH 09/27] Dedup with nix: use `nix::Machine::parseConfig` Companion to https://github.com/NixOS/nix/pull/10763 --- src/hydra-queue-runner/hydra-queue-runner.cc | 41 +------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index d7a88a0d..99411f9f 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -135,45 +135,8 @@ void State::parseMachines(const std::string & contents) oldMachines = *machines_; } - for (auto line : tokenizeString(contents, "\n")) { - line = trim(std::string(line, 0, line.find('#'))); - auto tokens = tokenizeString>(line); - if (tokens.size() < 3) continue; - tokens.resize(8); - - if (tokens[5] == "-") tokens[5] = ""; - auto supportedFeatures = tokenizeString(tokens[5], ","); - - if (tokens[6] == "-") tokens[6] = ""; - auto mandatoryFeatures = tokenizeString(tokens[6], ","); - - for (auto & f : mandatoryFeatures) - supportedFeatures.insert(f); - - using MaxJobs = std::remove_const::type; - - auto machine = std::make_shared<::Machine>(nix::Machine { - // `storeUri` - tokens[0], - // `systemTypes` - tokenizeString(tokens[1], ","), - // `sshKey` - tokens[2] == "-" ? "" : tokens[2], - // `maxJobs` - tokens[3] != "" - ? string2Int(tokens[3]).value() - : 1, - // `speedFactor` - atof(tokens[4].c_str()), - // `supportedFeatures` - std::move(supportedFeatures), - // `mandatoryFeatures` - std::move(mandatoryFeatures), - // `sshPublicHostKey` - tokens[7] != "" && tokens[7] != "-" - ? base64Decode(tokens[7]) - : "", - }); + for (auto && machine_ : nix::Machine::parseConfig({}, contents)) { + auto machine = std::make_shared<::Machine>(std::move(machine_)); /* Re-use the State object of the previous machine with the same name. */ From 2feddd8511cfa7a06a2c2b31da2a8a6707d81511 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 May 2024 17:05:41 -0400 Subject: [PATCH 10/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23) → 'github:NixOS/nix/ef5c846e257e1e284ad47ed6be4308d190fe6531' (2024-05-29) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 327d44ad..38b9aded 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1716472735, - "narHash": "sha256-506pEWmMufBGD9hM99WZynH9Vxkv3o3IJJsPiVAs4T4=", + "lastModified": 1717016009, + "narHash": "sha256-RJ0k7fvKjuUGt439F9uFtUw44SM87A+XKBf6sH6qPy8=", "owner": "NixOS", "repo": "nix", - "rev": "2c42e7b8d9ea32e59c01334852599b548b214d31", + "rev": "ef5c846e257e1e284ad47ed6be4308d190fe6531", "type": "github" }, "original": { From 8a8ac148772d19188e8e99900cf50ae7b1fc085d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Feb 2025 21:30:49 -0500 Subject: [PATCH 11/27] Test using Hydra with flakes It seemed there was no self-contained end-to-end test actually doing this?! Among other things, this will help ensure that the switch-over to `nix-eval-jobs` is correct. --- t/evaluator/evaluate-flake.t | 67 ++++++++++++++++++++++++++++++++ t/jobs/flake-checks/flake.nix | 6 +++ t/jobs/flake-hydraJobs/flake.nix | 6 +++ t/lib/HydraTestContext.pm | 42 +++++++++++++++----- 4 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 t/evaluator/evaluate-flake.t create mode 100644 t/jobs/flake-checks/flake.nix create mode 100644 t/jobs/flake-hydraJobs/flake.nix diff --git a/t/evaluator/evaluate-flake.t b/t/evaluator/evaluate-flake.t new file mode 100644 index 00000000..d884f25a --- /dev/null +++ b/t/evaluator/evaluate-flake.t @@ -0,0 +1,67 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; +use Test2::V0; +use File::Copy qw(cp); + +my $ctx = test_context( + nix_config => qq| + experimental-features = nix-command flakes + |, + hydra_config => q| + + evaluator_pure_eval = false + + | +); + +sub checkFlake { + my ($flake) = @_; + + cp($ctx->jobsdir . "/basic.nix", $ctx->jobsdir . "/" . $flake); + cp($ctx->jobsdir . "/config.nix", $ctx->jobsdir . "/" . $flake); + cp($ctx->jobsdir . "/empty-dir-builder.sh", $ctx->jobsdir . "/" . $flake); + cp($ctx->jobsdir . "/fail.sh", $ctx->jobsdir . "/" . $flake); + cp($ctx->jobsdir . "/succeed-with-failed.sh", $ctx->jobsdir . "/" . $flake); + + chmod 0755, $ctx->jobsdir . "/" . $flake . "/empty-dir-builder.sh"; + chmod 0755, $ctx->jobsdir . "/" . $flake . "/fail.sh"; + chmod 0755, $ctx->jobsdir . "/" . $flake . "/succeed-with-failed.sh"; + + my $builds = $ctx->makeAndEvaluateJobset( + flake => 'path:' . $ctx->jobsdir . "/" . $flake, + build => 1 + ); + + subtest "Build: succeed_with_failed" => sub { + my $build = $builds->{"succeed_with_failed"}; + + is($build->finished, 1, "Build should be finished."); + is($build->buildstatus, 6, "succeeeded-but-failed should have buildstatus 6."); + }; + + subtest "Build: empty_dir" => sub { + my $build = $builds->{"empty_dir"}; + + is($build->finished, 1, "Build should be finished."); + is($build->buildstatus, 0, "Should have succeeded."); + }; + + subtest "Build: fails" => sub { + my $build = $builds->{"fails"}; + + is($build->finished, 1, "Build should be finished."); + is($build->buildstatus, 1, "Should have failed."); + }; +} + +subtest "Flake using `checks`" => sub { + checkFlake 'flake-checks' +}; + +subtest "Flake using `hydraJobs`" => sub { + checkFlake 'flake-hydraJobs' +}; + +done_testing; diff --git a/t/jobs/flake-checks/flake.nix b/t/jobs/flake-checks/flake.nix new file mode 100644 index 00000000..489fa9ec --- /dev/null +++ b/t/jobs/flake-checks/flake.nix @@ -0,0 +1,6 @@ +{ + outputs = { ... }: { + checks = + import ./basic.nix; + }; +} diff --git a/t/jobs/flake-hydraJobs/flake.nix b/t/jobs/flake-hydraJobs/flake.nix new file mode 100644 index 00000000..c02ccddd --- /dev/null +++ b/t/jobs/flake-hydraJobs/flake.nix @@ -0,0 +1,6 @@ +{ + outputs = { ... }: { + hydraJobs = + import ./basic.nix; + }; +} diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index d1de2212..34d41eb2 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -165,14 +165,25 @@ sub nix_state_dir { sub makeAndEvaluateJobset { my ($self, %opts) = @_; - my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeAndEvaluateJobset.\n"; + my $expression = $opts{'expression'}; + my $flake = $opts{'flake'}; + if (not $expression and not $flake) { + die "One of 'expression' or 'flake' must be passed to makeEvaluateJobset.\n"; + } + my $jobsdir = $opts{'jobsdir'} // $self->jobsdir; my $should_build = $opts{'build'} // 0; - my $jobsetCtx = $self->makeJobset( - expression => $expression, + my %args = ( jobsdir => $jobsdir, ); + if ($expression) { + $args{expression} = $expression; + } + if ($flake) { + $args{flake} = $flake; + } + my $jobsetCtx = $self->makeJobset(%args); my $jobset = $jobsetCtx->{"jobset"}; evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0.\n"; @@ -195,7 +206,7 @@ sub makeAndEvaluateJobset { # # In return, you get a hash of the user, project, and jobset records. # -# This always uses an `expression` from the `jobsdir` directory. +# This always uses an `expression` or `flake` from the `jobsdir` directory. # # Hash Parameters: # @@ -204,7 +215,12 @@ sub makeAndEvaluateJobset { sub makeJobset { my ($self, %opts) = @_; - my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeJobset.\n"; + my $expression = $opts{'expression'}; + my $flake = $opts{'flake'}; + if (not $expression and not $flake) { + die "One of 'expression' or 'flake' must be passed to makeJobset.\n"; + } + my $jobsdir = $opts{'jobsdir'} // $self->jobsdir; # Create a new user for this test @@ -222,12 +238,20 @@ sub makeJobset { }); # Create a new jobset for this test and set up the inputs - my $jobset = $project->jobsets->create({ + my %args = ( name => rand_chars(), - nixexprinput => "jobs", - nixexprpath => $expression, emailoverride => "" - }); + ); + if ($expression) { + $args{type} = 0; + $args{nixexprinput} = "jobs"; + $args{nixexprpath} = $expression; + } + if ($flake) { + $args{type} = 1; + $args{flake} = $flake; + } + my $jobset = $project->jobsets->create(\%args); my $jobsetinput = $jobset->jobsetinputs->create({name => "jobs", type => "path"}); $jobsetinput->jobsetinputalts->create({altnr => 0, value => $jobsdir}); From 141b5fd0b5053230e1823f43257fa0ab58b39373 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 7 Feb 2025 16:30:14 -0500 Subject: [PATCH 12/27] Improve tests around constituents - Test how shorter names are preferred when multiple jobs resolve to the same derivation. - Test the exact aggregate map we get, by looking in the DB. --- t/evaluator/evaluate-constituents-gc.t | 49 ++++++++++++++++++++++++-- t/jobs/constituents.nix | 4 ++- t/lib/HydraTestContext.pm | 19 ++++++++-- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/t/evaluator/evaluate-constituents-gc.t b/t/evaluator/evaluate-constituents-gc.t index a9b23e6c..2a5337b2 100644 --- a/t/evaluator/evaluate-constituents-gc.t +++ b/t/evaluator/evaluate-constituents-gc.t @@ -5,13 +5,58 @@ use Test2::V0; my $ctx = test_context(); -my $builds = $ctx->makeAndEvaluateJobset( - expression => 'constituents.nix', +my $expression = 'constituents.nix'; +my $jobsetCtx = $ctx->makeJobset( + expression => $expression, +); +my $builds = $ctx->evaluateJobset( + jobset => $jobsetCtx->{"jobset"}, + expression => $expression, + build => 0, ); my $constituentA = $builds->{"constituentA"}; my $directAggregate = $builds->{"direct_aggregate"}; my $indirectAggregate = $builds->{"indirect_aggregate"}; +my $mixedAggregate = $builds->{"mixed_aggregate"}; + +# Ensure that we get exactly the aggregates we expect +my %expected_constituents = ( + 'direct_aggregate' => { + 'constituentA' => 1, + }, + 'indirect_aggregate' => { + 'constituentA' => 1, + }, + 'mixed_aggregate' => { + # Note that `constituentA_alias` becomes `constituentA`, because + # the shorter name is preferred + 'constituentA' => 1, + 'constituentB' => 1, + }, +); + +my $rs = $ctx->db->resultset('AggregateConstituents')->search( + {}, + { + join => [ 'aggregate', 'constituent' ], # Use correct relationship names + columns => [], + '+select' => [ 'aggregate.job', 'constituent.job' ], + '+as' => [ 'aggregate_job', 'constituent_job' ], + } +); + +my %actual_constituents; +while (my $row = $rs->next) { + my $aggregate_job = $row->get_column('aggregate_job'); + my $constituent_job = $row->get_column('constituent_job'); + $actual_constituents{$aggregate_job} //= {}; + $actual_constituents{$aggregate_job}{$constituent_job} = 1; +} + +is(\%actual_constituents, \%expected_constituents, "Exact aggregate constituents as expected"); + +# Check that deletion also doesn't work accordingly is(system('nix-store', '--delete', $constituentA->drvpath), 256, "Deleting a constituent derivation fails"); is(system('nix-store', '--delete', $directAggregate->drvpath), 256, "Deleting the direct aggregate derivation fails"); diff --git a/t/jobs/constituents.nix b/t/jobs/constituents.nix index 5b7106b9..b8b88702 100644 --- a/t/jobs/constituents.nix +++ b/t/jobs/constituents.nix @@ -5,6 +5,8 @@ rec { builder = ./empty-dir-builder.sh; }; + constituentA_alias = constituentA; + constituentB = mkDerivation { name = "empty-dir-B"; builder = ./empty-dir-builder.sh; @@ -32,7 +34,7 @@ rec { name = "mixed_aggregate"; _hydraAggregate = true; constituents = [ - "constituentA" + "constituentA_alias" constituentB ]; builder = ./empty-dir-builder.sh; diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index 34d41eb2..27b0be0a 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -172,7 +172,6 @@ sub makeAndEvaluateJobset { } my $jobsdir = $opts{'jobsdir'} // $self->jobsdir; - my $should_build = $opts{'build'} // 0; my %args = ( jobsdir => $jobsdir, @@ -184,12 +183,28 @@ sub makeAndEvaluateJobset { $args{flake} = $flake; } my $jobsetCtx = $self->makeJobset(%args); - my $jobset = $jobsetCtx->{"jobset"}; + + return $self->evaluateJobset( + jobset => $jobsetCtx->{"jobset"}, + expression => $expression, + flake => $flake, + build => $opts{"build"} // 0, + ) +} + +sub evaluateJobset { + my ($self, %opts) = @_; + + my $jobset = $opts{'jobset'}; + + my $expression = $opts{'expression'} // $opts{'flake'}; evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0.\n"; my $builds = {}; + my $should_build = $opts{'build'}; + for my $build ($jobset->builds) { if ($should_build) { runBuild($build) or die "Build '".$build->job."' from jobs/$expression should exit with return code 0.\n"; From 0c9726af59f5a19b5fe2928a5542ed3fa8ec2849 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Tue, 16 Jul 2024 04:01:14 +0200 Subject: [PATCH 13/27] flake: add nix-eval-jobs as input (cherry picked from commit 684cc50d86608cccf7500ce00af89ea34c488473) --- flake.lock | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 6 +++++ 2 files changed, 73 insertions(+) diff --git a/flake.lock b/flake.lock index 897785ed..8055275b 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,26 @@ { "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { @@ -46,6 +67,30 @@ "type": "github" } }, + "nix-eval-jobs": { + "inputs": { + "flake-parts": "flake-parts", + "nix-github-actions": [], + "nixpkgs": [ + "nixpkgs" + ], + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1733814344, + "narHash": "sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU=", + "owner": "nix-community", + "repo": "nix-eval-jobs", + "rev": "889ea1406736b53cf165b6c28398aae3969418d1", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "release-2.24", + "repo": "nix-eval-jobs", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1726688310, @@ -66,8 +111,30 @@ "inputs": { "libgit2": "libgit2", "nix": "nix", + "nix-eval-jobs": "nix-eval-jobs", "nixpkgs": "nixpkgs" } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1723303070, + "narHash": "sha256-krGNVA30yptyRonohQ+i9cnK+CfCpedg6z3qzqVJcTs=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "14c092e0326de759e16b37535161b3cb9770cea3", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index fccd45b9..d25f451c 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,9 @@ inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; inputs.nix.inputs.libgit2.follows = "libgit2"; + inputs.nix-eval-jobs.url = "github:nix-community/nix-eval-jobs/release-2.24"; + inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs"; + # hide nix dev tooling from our lock file inputs.nix.inputs.flake-parts.follows = ""; inputs.nix.inputs.git-hooks-nix.follows = ""; @@ -15,6 +18,9 @@ inputs.nix.inputs.nixpkgs-23-11.follows = ""; inputs.nix.inputs.flake-compat.follows = ""; + # hide nix-eval-jobs dev tooling from our lock file + inputs.nix-eval-jobs.inputs.nix-github-actions.follows = ""; + outputs = { self, nixpkgs, nix, ... }: let systems = [ "x86_64-linux" "aarch64-linux" ]; From d84ff32ce600204c6473889a3ff16cd6053533c9 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Tue, 16 Jul 2024 04:22:41 +0200 Subject: [PATCH 14/27] hydra-eval-jobset: Use `nix-eval-jobs` instead of `hydra-eval-jobs` incrementally ingest eval results nix-eval-jobs streams output, unlike hydra-eval-jobs. Now that we've migrated, we can use this to: 1. Use less RAM by avoiding buffering a whole eval's worth of metadata into a Perl string and an array of JSON objects. 2. Make evals latency a bit lower by allowing the queue runner to start ingesting builds faster. Also use the newly-restored constituents support in `nix-eval-jobs` Note, we pass --workers and --max-memory-size to n-e-j Lost in the h-e-j -> n-e-j migration, causing evaluation to always be single threaded and limited to 4GiB RAM. Follow the config settings like h-e-j used to do (via C++ code). `nix-eval-jobs` should check `hydraJobs` and then `checks` with flakes (cherry picked from commit 6d4ccff43c41adaf6e4b2b9bced7243bc2f6e97b) (cherry picked from commit b0e9b4b2f99f9d8f5c4e780e89f955c394b5ced4) (cherry picked from commit cdfc5c81e8037d3e4818a3e459d0804b2c157ea9) (cherry picked from commit 4b107e6ff36bd89958fba36e0fe0340903e7cd13) Co-Authored-By: Maximilian Bosch --- flake.nix | 4 +- package.nix | 3 + src/script/hydra-eval-jobset | 190 +++++++++++++-------- t/evaluator/evaluate-constituents-broken.t | 6 +- t/evaluator/evaluate-meta.t | 22 +++ t/jobs/meta.nix | 17 ++ t/queue-runner/constituents.t | 4 +- 7 files changed, 167 insertions(+), 79 deletions(-) create mode 100644 t/evaluator/evaluate-meta.t create mode 100644 t/jobs/meta.nix diff --git a/flake.nix b/flake.nix index d25f451c..f12b8c2f 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,7 @@ # hide nix-eval-jobs dev tooling from our lock file inputs.nix-eval-jobs.inputs.nix-github-actions.follows = ""; - outputs = { self, nixpkgs, nix, ... }: + outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }: let systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -32,6 +32,7 @@ overlays.default = final: prev: { hydra = final.callPackage ./package.nix { inherit (nixpkgs.lib) fileset; + nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default; rawSrc = self; nix-perl-bindings = final.nixComponents.nix-perl-bindings; }; @@ -75,6 +76,7 @@ packages = forEachSystem (system: { hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix { inherit (nixpkgs.lib) fileset; + nix-eval-jobs = nix-eval-jobs.packages.${system}.default; rawSrc = self; nix = nix.packages.${system}.nix; nix-perl-bindings = nix.hydraJobs.perlBindings.${system}; diff --git a/package.nix b/package.nix index f944fe2b..ceb2c706 100644 --- a/package.nix +++ b/package.nix @@ -50,6 +50,7 @@ , xz , gnutar , gnused +, nix-eval-jobs , rpm , dpkg @@ -190,6 +191,7 @@ stdenv.mkDerivation (finalAttrs: { openldap postgresql_13 pixz + nix-eval-jobs ]; checkInputs = [ @@ -218,6 +220,7 @@ stdenv.mkDerivation (finalAttrs: { darcs gnused breezy + nix-eval-jobs ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 72a386f5..9d500a13 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -17,6 +17,7 @@ use Hydra::Helper::Nix; use Hydra::Model::DB; use Hydra::Plugin; use Hydra::Schema; +use IPC::Run; use JSON::MaybeXS; use Net::Statsd; use Nix::Store; @@ -357,22 +358,32 @@ sub evalJobs { my @cmd; if (defined $flakeRef) { - @cmd = ("hydra-eval-jobs", - "--flake", $flakeRef, - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1); + my $nix_expr = + "let " . + "flake = builtins.getFlake (toString \"$flakeRef\"); " . + "in " . + "flake.hydraJobs " . + "or flake.checks " . + "or (throw \"flake '$flakeRef' does not provide any Hydra jobs or checks\")"; + + @cmd = ("nix-eval-jobs", "--expr", $nix_expr); } else { my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; - @cmd = ("hydra-eval-jobs", + @cmd = ("nix-eval-jobs", "<" . $nixExprInputName . "/" . $nixExprPath . ">", - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1, inputsToArgs($inputInfo)); } - push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true"; + push @cmd, ("--gc-roots-dir", getGCRootsDir); + push @cmd, ("--max-jobs", 1); + push @cmd, "--meta"; + push @cmd, "--constituents"; + push @cmd, "--force-recurse"; + push @cmd, ("--option", "allow-import-from-derivation", "false") if $config->{allow_import_from_derivation} // "true" ne "true"; + push @cmd, ("--workers", $config->{evaluator_workers} // 1); + push @cmd, ("--max-memory-size", $config->{evaluator_max_memory_size} // 4096); if (defined $ENV{'HYDRA_DEBUG'}) { sub escape { @@ -384,14 +395,33 @@ sub evalJobs { print STDERR "evaluator: @escaped\n"; } - (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd); - die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) - . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") - if $res; + my $evalProc = IPC::Run::start \@cmd, + '>', IPC::Run::new_chunker, \my $out, + '2>', \my $err; - print STDERR "$stderr"; + return sub { + while (1) { + $evalProc->pump; + if (!defined $out && !defined $err) { + $evalProc->finish; + if ($?) { + die "nix-eval-jobs returned " . ($? & 127 ? "signal $?" : "exit code " . ($? >> 8)) . "\n"; + } + return; + } - return decode_json($jobsJSON); + if (defined $err) { + print STDERR "$err"; + undef $err; + } + + if (defined $out && $out ne '') { + my $job = decode_json($out); + undef $out; + return $job; + } + } + }; } @@ -420,7 +450,7 @@ sub checkBuild { my $firstOutputName = $outputNames[0]; my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName}; - my $jobName = $buildInfo->{jobName} or die; + my $jobName = $buildInfo->{attr} or die; my $drvPath = $buildInfo->{drvPath} or die; my $build; @@ -474,9 +504,30 @@ sub checkBuild { my $time = time(); - sub null { - my ($s) = @_; - return $s eq "" ? undef : $s; + sub getMeta { + my ($s, $def) = @_; + return ($s || "") eq "" ? $def : $s; + } + + sub getMetaStrings { + my ($v, $k, $acc) = @_; + my $t = ref $v; + + if ($t eq 'HASH') { + push @$acc, $v->{$k} if exists $v->{$k}; + } elsif ($t eq 'ARRAY') { + getMetaStrings($_, $k, $acc) foreach @$v; + } elsif (defined $v) { + push @$acc, $v; + } + } + + sub getMetaConcatStrings { + my ($v, $k) = @_; + + my @strings; + getMetaStrings($v, $k, \@strings); + return join(", ", @strings) || undef; } # Add the build to the database. @@ -484,19 +535,19 @@ sub checkBuild { { timestamp => $time , jobset_id => $jobset->id , job => $jobName - , description => null($buildInfo->{description}) - , license => null($buildInfo->{license}) - , homepage => null($buildInfo->{homepage}) - , maintainers => null($buildInfo->{maintainers}) - , maxsilent => $buildInfo->{maxSilent} - , timeout => $buildInfo->{timeout} - , nixname => $buildInfo->{nixName} + , description => getMeta($buildInfo->{meta}->{description}, undef) + , license => getMetaConcatStrings($buildInfo->{meta}->{license}, "shortName") + , homepage => getMeta($buildInfo->{meta}->{homepage}, undef) + , maintainers => getMetaConcatStrings($buildInfo->{meta}->{maintainers}, "email") + , maxsilent => getMeta($buildInfo->{meta}->{maxSilent}, 7200) + , timeout => getMeta($buildInfo->{meta}->{timeout}, 36000) + , nixname => $buildInfo->{name} , drvpath => $drvPath , system => $buildInfo->{system} - , priority => $buildInfo->{schedulingPriority} + , priority => getMeta($buildInfo->{meta}->{schedulingPriority}, 100) , finished => 0 , iscurrent => 1 - , ischannel => $buildInfo->{isChannel} + , ischannel => getMeta($buildInfo->{meta}->{isChannel}, 0) }); $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} }) @@ -665,7 +716,7 @@ sub checkJobsetWrapped { return; } - # Hash the arguments to hydra-eval-jobs and check the + # Hash the arguments to nix-eval-jobs and check the # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo)); @@ -687,19 +738,12 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = clock_gettime(CLOCK_MONOTONIC); - my $jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); - my $evalStop = clock_gettime(CLOCK_MONOTONIC); - - if ($jobsetsJobset) { - my @keys = keys %$jobs; - die "The .jobsets jobset must only have a single job named 'jobsets'" - unless (scalar @keys) == 1 && $keys[0] eq "jobsets"; - } - Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000)); + my $evalStop; + my $jobsIter = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); if ($dryRun) { - foreach my $name (keys %{$jobs}) { - my $job = $jobs->{$name}; + while (defined(my $job = $jobsIter->())) { + my $name = $job->{attr}; if (defined $job->{drvPath}) { print STDERR "good job $name: $job->{drvPath}\n"; } else { @@ -709,36 +753,20 @@ sub checkJobsetWrapped { return; } - die "Jobset contains a job with an empty name. Make sure the jobset evaluates to an attrset of jobs.\n" - if defined $jobs->{""}; - - $jobs->{$_}->{jobName} = $_ for keys %{$jobs}; - - my $jobOutPathMap = {}; - my $jobsetChanged = 0; - my $dbStart = clock_gettime(CLOCK_MONOTONIC); - - # Store the error messages for jobs that failed to evaluate. my $evaluationErrorTime = time; my $evaluationErrorMsg = ""; - foreach my $job (values %{$jobs}) { - next unless defined $job->{error}; - $evaluationErrorMsg .= - ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . - ":\n" . $job->{error} . "\n\n"; - } - setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); - my $evaluationErrorRecord = $db->resultset('EvaluationErrors')->create( { errormsg => $evaluationErrorMsg , errortime => $evaluationErrorTime } ); + my $jobOutPathMap = {}; + my $jobsetChanged = 0; my %buildMap; - $db->txn_do(sub { + $db->txn_do(sub { my $prevEval = getPrevJobsetEval($db, $jobset, 1); # Clear the "current" flag on all builds. Since we're in a @@ -751,7 +779,7 @@ sub checkJobsetWrapped { , evaluationerror => $evaluationErrorRecord , timestamp => time , checkouttime => abs(int($checkoutStop - $checkoutStart)) - , evaltime => abs(int($evalStop - $evalStart)) + , evaltime => 0 , hasnewbuilds => 0 , nrbuilds => 0 , flake => $flakeRef @@ -759,11 +787,24 @@ sub checkJobsetWrapped { , nixexprpath => $jobset->nixexprpath }); - # Schedule each successfully evaluated job. - foreach my $job (permute(values %{$jobs})) { - next if defined $job->{error}; - #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; - checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); + my @jobsWithConstituents; + + while (defined(my $job = $jobsIter->())) { + if ($jobsetsJobset) { + die "The .jobsets jobset must only have a single job named 'jobsets'" + unless $job->{attr} eq "jobsets"; + } + + $evaluationErrorMsg .= + ($job->{attr} ne "" ? "in job ‘$job->{attr}’" : "at top-level") . + ":\n" . $job->{error} . "\n\n" if defined $job->{error}; + + checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins) + unless defined $job->{error}; + + if (defined $job->{constituents}) { + push @jobsWithConstituents, $job; + } } # Have any builds been added or removed since last time? @@ -801,21 +842,20 @@ sub checkJobsetWrapped { $drvPathToId{$x->{drvPath}} = $x; } - foreach my $job (values %{$jobs}) { - next unless $job->{constituents}; - + foreach my $job (values @jobsWithConstituents) { + next unless defined $job->{constituents}; if (defined $job->{error}) { - die "aggregate job ‘$job->{jobName}’ failed with the error: $job->{error}\n"; + die "aggregate job ‘$job->{attr}’ failed with the error: $job->{error}\n"; } my $x = $drvPathToId{$job->{drvPath}} or - die "aggregate job ‘$job->{jobName}’ has no corresponding build record.\n"; + die "aggregate job ‘$job->{attr}’ has no corresponding build record.\n"; foreach my $drvPath (@{$job->{constituents}}) { my $constituent = $drvPathToId{$drvPath}; if (defined $constituent) { $db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}}); } else { - warn "aggregate job ‘$job->{jobName}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n"; + warn "aggregate job ‘$job->{attr}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n"; } } } @@ -857,11 +897,15 @@ sub checkJobsetWrapped { $jobset->update({ enabled => 0 }) if $jobset->enabled == 2; $jobset->update({ lastcheckedtime => time, forceeval => undef }); + + $evaluationErrorRecord->update({ errormsg => $evaluationErrorMsg }); + setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); + + $evalStop = clock_gettime(CLOCK_MONOTONIC); + $ev->update({ evaltime => abs(int($evalStop - $evalStart)) }); }); - my $dbStop = clock_gettime(CLOCK_MONOTONIC); - - Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000)); + Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000)); Net::Statsd::increment("hydra.evaluator.evals"); Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged; } diff --git a/t/evaluator/evaluate-constituents-broken.t b/t/evaluator/evaluate-constituents-broken.t index ed25d192..0e5960bf 100644 --- a/t/evaluator/evaluate-constituents-broken.t +++ b/t/evaluator/evaluate-constituents-broken.t @@ -18,14 +18,14 @@ isnt($res, 0, "hydra-eval-jobset exits non-zero"); ok(utf8::decode($stderr), "Stderr output is UTF8-clean"); like( $stderr, - qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/, + qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/, "The stderr record includes a relevant error message" ); -$jobset->discard_changes; # refresh from DB +$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB like( $jobset->errormsg, - qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/, + qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/, "The jobset records a relevant error message" ); diff --git a/t/evaluator/evaluate-meta.t b/t/evaluator/evaluate-meta.t new file mode 100644 index 00000000..9f546a7f --- /dev/null +++ b/t/evaluator/evaluate-meta.t @@ -0,0 +1,22 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; +use Test2::V0; + +my $ctx = test_context(); + +my $builds = $ctx->makeAndEvaluateJobset( + expression => "meta.nix", + build => 1 +); + +my $build = $builds->{"full-of-meta"}; + +is($build->finished, 1, "Build should be finished."); +is($build->description, "This is the description of the job.", "Wrong description extracted from the build."); +is($build->license, "MIT, BSD", "Wrong licenses extracted from the build."); +is($build->homepage, "https://example.com/", "Wrong homepage extracted from the build."); +is($build->maintainers, 'alice@example.com, bob@not.found', "Wrong maintainers extracted from the build."); + +done_testing; diff --git a/t/jobs/meta.nix b/t/jobs/meta.nix new file mode 100644 index 00000000..9204e384 --- /dev/null +++ b/t/jobs/meta.nix @@ -0,0 +1,17 @@ +with import ./config.nix; +{ + full-of-meta = + mkDerivation { + name = "full-of-meta"; + builder = ./empty-dir-builder.sh; + + meta = { + description = "This is the description of the job."; + license = [ { shortName = "MIT"; } "BSD" ]; + homepage = "https://example.com/"; + maintainers = [ "alice@example.com" { email = "bob@not.found"; } ]; + + outPath = "${placeholder "out"}"; + }; + }; +} diff --git a/t/queue-runner/constituents.t b/t/queue-runner/constituents.t index c6333642..e1b8d733 100644 --- a/t/queue-runner/constituents.t +++ b/t/queue-runner/constituents.t @@ -22,11 +22,11 @@ is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix like( $jobset->errormsg, - qr/^does-not-exist: does not exist$/m, + qr/^"does-not-exist": does not exist$/m, "Evaluating jobs/broken-constituent.nix should log an error for does-not-exist"); like( $jobset->errormsg, - qr/^does-not-evaluate: error: assertion 'false' failed$/m, + qr/^"does-not-evaluate": "error: assertion 'false' failed/m, "Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate"); done_testing; From 2f92846e5a37b284ffe3df0f6082911dbfc6f0e2 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Tue, 16 Jul 2024 04:04:08 +0200 Subject: [PATCH 15/27] hydra-eval-jobs: remove, replaced by nix-eval-jobs (cherry picked from commit ed7c58708cd3affd62a598a22a500ed2adf318bf) --- package.nix | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 587 ------------------------- src/hydra-eval-jobs/meson.build | 8 - src/meson.build | 1 - t/meson.build | 1 - 5 files changed, 1 insertion(+), 598 deletions(-) delete mode 100644 src/hydra-eval-jobs/hydra-eval-jobs.cc delete mode 100644 src/hydra-eval-jobs/meson.build diff --git a/package.nix b/package.nix index ceb2c706..a9ec12c8 100644 --- a/package.nix +++ b/package.nix @@ -235,7 +235,7 @@ stdenv.mkDerivation (finalAttrs: { shellHook = '' pushd $(git rev-parse --show-toplevel) >/dev/null - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH PERL5LIB=$(pwd)/src/lib:$PERL5LIB export HYDRA_HOME="$(pwd)/src/" mkdir -p .hydra-data diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc deleted file mode 100644 index b83cae91..00000000 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ /dev/null @@ -1,587 +0,0 @@ -#include -#include -#include -#include - -#include "shared.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-gc.hh" -#include "eval-inline.hh" -#include "eval-settings.hh" -#include "signals.hh" -#include "terminal.hh" -#include "util.hh" -#include "get-drvs.hh" -#include "globals.hh" -#include "common-eval-args.hh" -#include "flake/flakeref.hh" -#include "flake/flake.hh" -#include "attr-path.hh" -#include "derivations.hh" -#include "local-fs-store.hh" - -#include "hydra-config.hh" - -#include -#include -#include - -#include - -void check_pid_status_nonblocking(pid_t check_pid) -{ - // Only check 'initialized' and known PID's - if (check_pid <= 0) { return; } - - int wstatus = 0; - pid_t pid = waitpid(check_pid, &wstatus, WNOHANG); - // -1 = failure, WNOHANG: 0 = no change - if (pid <= 0) { return; } - - std::cerr << "child process (" << pid << ") "; - - if (WIFEXITED(wstatus)) { - std::cerr << "exited with status=" << WEXITSTATUS(wstatus) << std::endl; - } else if (WIFSIGNALED(wstatus)) { - std::cerr << "killed by signal=" << WTERMSIG(wstatus) << std::endl; - } else if (WIFSTOPPED(wstatus)) { - std::cerr << "stopped by signal=" << WSTOPSIG(wstatus) << std::endl; - } else if (WIFCONTINUED(wstatus)) { - std::cerr << "continued" << std::endl; - } -} - -using namespace nix; - -static Path gcRootsDir; -static size_t maxMemorySize; - -struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs -{ - Path releaseExpr; - bool flake = false; - bool dryRun = false; - - MyArgs() : MixCommonArgs("hydra-eval-jobs") - { - addFlag({ - .longName = "gc-roots-dir", - .description = "garbage collector roots directory", - .labels = {"path"}, - .handler = {&gcRootsDir} - }); - - addFlag({ - .longName = "dry-run", - .description = "don't create store derivations", - .handler = {&dryRun, true} - }); - - addFlag({ - .longName = "flake", - .description = "build a flake", - .handler = {&flake, true} - }); - - expectArg("expr", &releaseExpr); - } -}; - -static MyArgs myArgs; - -static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute) -{ - Strings res; - std::function rec; - - rec = [&](Value & v) { - state.forceValue(v, noPos); - if (v.type() == nString) - res.emplace_back(v.string_view()); - else if (v.isList()) - for (unsigned int n = 0; n < v.listSize(); ++n) - rec(*v.listElems()[n]); - else if (v.type() == nAttrs) { - auto a = v.attrs()->find(state.symbols.create(subAttribute)); - if (a != v.attrs()->end()) - res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes"))); - } - }; - - Value * v = drv.queryMeta(name); - if (v) rec(*v); - - return concatStringsSep(", ", res); -} - -static void worker( - EvalState & state, - Bindings & autoArgs, - AutoCloseFD & to, - AutoCloseFD & from) -{ - Value vTop; - - if (myArgs.flake) { - using namespace flake; - - auto [flakeRef, fragment, outputSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(fetchSettings, myArgs.releaseExpr, absPath(".")); - - auto vFlake = state.allocValue(); - - auto lockedFlake = lockFlake( - flakeSettings, - state, - flakeRef, - LockFlags { - .updateLockFile = false, - .useRegistries = false, - .allowUnlocked = false, - }); - - callFlake(state, lockedFlake, *vFlake); - - auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value; - state.forceValue(*vOutputs, noPos); - - auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs")); - if (!aHydraJobs) - aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks")); - if (!aHydraJobs) - throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); - - vTop = *aHydraJobs->value; - - } else { - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); - } - - auto vRoot = state.allocValue(); - state.autoCallFunction(autoArgs, vTop, *vRoot); - - while (true) { - /* Wait for the master to send us a job name. */ - writeLine(to.get(), "next"); - - auto s = readLine(from.get()); - if (s == "exit") break; - if (!hasPrefix(s, "do ")) abort(); - std::string attrPath(s, 3); - - debug("worker process %d at '%s'", getpid(), attrPath); - - /* Evaluate it and send info back to the master. */ - nlohmann::json reply; - - try { - auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; - - auto v = state.allocValue(); - state.autoCallFunction(autoArgs, *vTmp, *v); - - if (auto drv = getDerivation(state, *v, false)) { - - // CA derivations do not have static output paths, so we - // have to defensively not query output paths in case we - // encounter one. - PackageInfo::Outputs outputs = drv->queryOutputs( - !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); - - if (drv->querySystem() == "unknown") - state.error("derivation must have a 'system' attribute").debugThrow(); - - auto drvPath = state.store->printStorePath(drv->requireDrvPath()); - - nlohmann::json job; - - job["nixName"] = drv->queryName(); - job["system"] =drv->querySystem(); - job["drvPath"] = drvPath; - job["description"] = drv->queryMetaString("description"); - job["license"] = queryMetaStrings(state, *drv, "license", "shortName"); - job["homepage"] = drv->queryMetaString("homepage"); - job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email"); - job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100); - job["timeout"] = drv->queryMetaInt("timeout", 36000); - job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200); - job["isChannel"] = drv->queryMetaBool("isHydraChannel", false); - - /* If this is an aggregate, then get its constituents. */ - auto a = v->attrs()->get(state.symbols.create("_hydraAggregate")); - if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) { - auto a = v->attrs()->get(state.symbols.create("constituents")); - if (!a) - state.error("derivation must have a ‘constituents’ attribute").debugThrow(); - - NixStringContext context; - state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false); - for (auto & c : context) - std::visit(overloaded { - [&](const NixStringContextElem::Built & b) { - job["constituents"].push_back(b.drvPath->to_string(*state.store)); - }, - [&](const NixStringContextElem::Opaque & o) { - }, - [&](const NixStringContextElem::DrvDeep & d) { - }, - }, c.raw); - - state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute"); - for (unsigned int n = 0; n < a->value->listSize(); ++n) { - auto v = a->value->listElems()[n]; - state.forceValue(*v, noPos); - if (v->type() == nString) - job["namedConstituents"].push_back(v->string_view()); - } - } - - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - auto localStore = state.store.dynamic_pointer_cast(); - if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); - if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root); - } - - nlohmann::json out; - for (auto & [outputName, optOutputPath] : outputs) { - if (optOutputPath) { - out[outputName] = state.store->printStorePath(*optOutputPath); - } else { - // See the `queryOutputs` call above; we should - // not encounter missing output paths otherwise. - assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); - out[outputName] = nullptr; - } - } - job["outputs"] = std::move(out); - reply["job"] = std::move(job); - } - - else if (v->type() == nAttrs) { - auto attrs = nlohmann::json::array(); - StringSet ss; - for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) { - std::string name(state.symbols[i->name]); - if (name.find(' ') != std::string::npos) { - printError("skipping job with illegal name '%s'", name); - continue; - } - attrs.push_back(name); - } - reply["attrs"] = std::move(attrs); - } - - else if (v->type() == nNull) - ; - - else state.error("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow(); - - } catch (EvalError & e) { - auto msg = e.msg(); - // Transmits the error we got from the previous evaluation - // in the JSON output. - reply["error"] = filterANSIEscapes(msg, true); - // Don't forget to print it into the STDERR log, this is - // what's shown in the Hydra UI. - printError(msg); - } - - writeLine(to.get(), reply.dump()); - - /* If our RSS exceeds the maximum, exit. The master will - start a new process. */ - struct rusage r; - getrusage(RUSAGE_SELF, &r); - if ((size_t) r.ru_maxrss > maxMemorySize * 1024) break; - } - - writeLine(to.get(), "restart"); -} - -int main(int argc, char * * argv) -{ - /* Prevent undeclared dependencies in the evaluation via - $NIX_PATH. */ - unsetenv("NIX_PATH"); - - return handleExceptions(argv[0], [&]() { - - auto config = std::make_unique(); - - auto nrWorkers = config->getIntOption("evaluator_workers", 1); - maxMemorySize = config->getIntOption("evaluator_max_memory_size", 4096); - - initNix(); - initGC(); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - auto pureEval = config->getBoolOption("evaluator_pure_eval", myArgs.flake); - - /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ - settings.builders = ""; - - /* Prevent access to paths outside of the Nix search path and - to the environment. */ - evalSettings.restrictEval = true; - - /* When building a flake, use pure evaluation (no access to - 'getEnv', 'currentSystem' etc. */ - evalSettings.pureEval = pureEval; - - if (myArgs.dryRun) settings.readOnlyMode = true; - - if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); - - if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); - - struct State - { - std::set todo{""}; - std::set active; - nlohmann::json jobs; - std::exception_ptr exc; - }; - - std::condition_variable wakeup; - - Sync state_; - - /* Start a handler thread per worker process. */ - auto handler = [&]() - { - pid_t pid = -1; - try { - AutoCloseFD from, to; - - while (true) { - - /* Start a new worker process if necessary. */ - if (pid == -1) { - Pipe toPipe, fromPipe; - toPipe.create(); - fromPipe.create(); - pid = startProcess( - [&, - to{std::make_shared(std::move(fromPipe.writeSide))}, - from{std::make_shared(std::move(toPipe.readSide))} - ]() - { - try { - auto evalStore = myArgs.evalStoreUrl - ? openStore(*myArgs.evalStoreUrl) - : openStore(); - EvalState state(myArgs.lookupPath, - evalStore, fetchSettings, evalSettings); - Bindings & autoArgs = *myArgs.getAutoArgs(state); - worker(state, autoArgs, *to, *from); - } catch (Error & e) { - nlohmann::json err; - auto msg = e.msg(); - err["error"] = filterANSIEscapes(msg, true); - printError(msg); - writeLine(to->get(), err.dump()); - // Don't forget to print it into the STDERR log, this is - // what's shown in the Hydra UI. - writeLine(to->get(), "restart"); - } - }, - ProcessOptions { .allowVfork = false }); - from = std::move(fromPipe.readSide); - to = std::move(toPipe.writeSide); - debug("created worker process %d", pid); - } - - /* Check whether the existing worker process is still there. */ - auto s = readLine(from.get()); - if (s == "restart") { - pid = -1; - continue; - } else if (s != "next") { - auto json = nlohmann::json::parse(s); - throw Error("worker error: %s", (std::string) json["error"]); - } - - /* Wait for a job name to become available. */ - std::string attrPath; - - while (true) { - checkInterrupt(); - auto state(state_.lock()); - if ((state->todo.empty() && state->active.empty()) || state->exc) { - writeLine(to.get(), "exit"); - return; - } - if (!state->todo.empty()) { - attrPath = *state->todo.begin(); - state->todo.erase(state->todo.begin()); - state->active.insert(attrPath); - break; - } else - state.wait(wakeup); - } - - /* Tell the worker to evaluate it. */ - writeLine(to.get(), "do " + attrPath); - - /* Wait for the response. */ - auto response = nlohmann::json::parse(readLine(from.get())); - - /* Handle the response. */ - StringSet newAttrs; - - if (response.find("job") != response.end()) { - auto state(state_.lock()); - state->jobs[attrPath] = response["job"]; - } - - if (response.find("attrs") != response.end()) { - for (auto & i : response["attrs"]) { - std::string path = i; - if (path.find(".") != std::string::npos){ - path = "\"" + path + "\""; - } - auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) path; - newAttrs.insert(s); - } - } - - if (response.find("error") != response.end()) { - auto state(state_.lock()); - state->jobs[attrPath]["error"] = response["error"]; - } - - /* Add newly discovered job names to the queue. */ - { - auto state(state_.lock()); - state->active.erase(attrPath); - for (auto & s : newAttrs) - state->todo.insert(s); - wakeup.notify_all(); - } - } - } catch (...) { - check_pid_status_nonblocking(pid); - auto state(state_.lock()); - state->exc = std::current_exception(); - wakeup.notify_all(); - } - }; - - std::vector threads; - for (size_t i = 0; i < nrWorkers; i++) - threads.emplace_back(std::thread(handler)); - - for (auto & thread : threads) - thread.join(); - - auto state(state_.lock()); - - if (state->exc) - std::rethrow_exception(state->exc); - - /* For aggregate jobs that have named consistuents - (i.e. constituents that are a job name rather than a - derivation), look up the referenced job and add it to the - dependencies of the aggregate derivation. */ - auto store = openStore(); - - for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { - auto jobName = i.key(); - auto & job = i.value(); - - auto named = job.find("namedConstituents"); - if (named == job.end()) continue; - - std::unordered_map brokenJobs; - auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state]( - const std::string & childJobName) -> std::optional { - auto childJob = state->jobs.find(childJobName); - if (childJob == state->jobs.end()) { - printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName); - brokenJobs[childJobName] = "does not exist"; - return std::nullopt; - } - if (childJob->find("error") != childJob->end()) { - std::string error = (*childJob)["error"]; - printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error); - brokenJobs[childJobName] = error; - return std::nullopt; - } - return *childJob; - }; - - if (myArgs.dryRun) { - for (std::string jobName2 : *named) { - auto job2 = getNonBrokenJobOrRecordError(jobName2); - if (!job2) { - continue; - } - std::string drvPath2 = (*job2)["drvPath"]; - job["constituents"].push_back(drvPath2); - } - } else { - auto drvPath = store->parseStorePath((std::string) job["drvPath"]); - auto drv = store->readDerivation(drvPath); - - for (std::string jobName2 : *named) { - auto job2 = getNonBrokenJobOrRecordError(jobName2); - if (!job2) { - continue; - } - auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); - auto drv2 = store->readDerivation(drvPath2); - job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first}; - } - - if (brokenJobs.empty()) { - std::string drvName(drvPath.name()); - assert(hasSuffix(drvName, drvExtension)); - drvName.resize(drvName.size() - drvExtension.size()); - - auto hashModulo = hashDerivationModulo(*store, drv, true); - if (hashModulo.kind != DrvHash::Kind::Regular) continue; - auto h = hashModulo.hashes.find("out"); - if (h == hashModulo.hashes.end()) continue; - auto outPath = store->makeOutputPath("out", h->second, drvName); - drv.env["out"] = store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput::InputAddressed { .path = outPath }); - auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); - - debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); - - job["drvPath"] = newDrvPath; - job["outputs"]["out"] = store->printStorePath(outPath); - } - } - - job.erase("namedConstituents"); - - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - auto localStore = store.dynamic_pointer_cast(); - if (gcRootsDir != "" && localStore) { - auto drvPath = job["drvPath"].get(); - Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); - if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root); - } - - if (!brokenJobs.empty()) { - std::stringstream ss; - for (const auto& [jobName, error] : brokenJobs) { - ss << jobName << ": " << error << "\n"; - } - job["error"] = ss.str(); - } - } - - std::cout << state->jobs.dump(2) << "\n"; - }); -} diff --git a/src/hydra-eval-jobs/meson.build b/src/hydra-eval-jobs/meson.build deleted file mode 100644 index 916212e1..00000000 --- a/src/hydra-eval-jobs/meson.build +++ /dev/null @@ -1,8 +0,0 @@ -hydra_eval_jobs = executable('hydra-eval-jobs', - 'hydra-eval-jobs.cc', - dependencies: [ - libhydra_dep, - nix_dep, - ], - install: true, -) diff --git a/src/meson.build b/src/meson.build index 8c7562ed..52b821bc 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,6 +1,5 @@ # Native code subdir('libhydra') -subdir('hydra-eval-jobs') subdir('hydra-evaluator') subdir('hydra-queue-runner') diff --git a/t/meson.build b/t/meson.build index 11044a03..c3c58458 100644 --- a/t/meson.build +++ b/t/meson.build @@ -27,7 +27,6 @@ testenv.prepend('PERL5LIB', separator: ':' ) testenv.prepend('PATH', - fs.parent(hydra_eval_jobs.full_path()), fs.parent(hydra_evaluator.full_path()), fs.parent(hydra_queue_runner.full_path()), meson.project_source_root() / 'src/script', From 85383b952222e62baeda050b282682c68a8d2581 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 10 Dec 2024 11:31:40 -0500 Subject: [PATCH 16/27] Render the `nix-eval-jobs` version too --- package.nix | 3 ++- src/lib/Hydra/Controller/Root.pm | 1 + src/root/layout.tt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package.nix b/package.nix index a9ec12c8..e0046024 100644 --- a/package.nix +++ b/package.nix @@ -266,7 +266,8 @@ stdenv.mkDerivation (finalAttrs: { --prefix PATH ':' $out/bin:$hydraPath \ --set HYDRA_RELEASE ${version} \ --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${nix.name or "unknown"} + --set NIX_RELEASE ${nix.name or "unknown"} \ + --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"} done ''; diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index aa1ad5ab..a231d7c0 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -51,6 +51,7 @@ sub begin :Private { $c->stash->{curUri} = $c->request->uri; $c->stash->{version} = $ENV{"HYDRA_RELEASE"} || ""; $c->stash->{nixVersion} = $ENV{"NIX_RELEASE"} || ""; + $c->stash->{nixEvalJobsVersion} = $ENV{"NIX_EVAL_JOBS_RELEASE"} || ""; $c->stash->{curTime} = time; $c->stash->{logo} = defined $c->config->{hydra_logo} ? "/logo" : ""; $c->stash->{tracker} = defined $c->config->{tracker} ? $c->config->{tracker} : ""; diff --git a/src/root/layout.tt b/src/root/layout.tt index d67ff1b8..399962b4 100644 --- a/src/root/layout.tt +++ b/src/root/layout.tt @@ -93,7 +93,7 @@