From 39a4e4791ea9ae4c29400f322bc1d1b71453cf1d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Apr 2024 17:30:43 -0400 Subject: [PATCH 01/10] Switch (back) to Nix master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-creating `nix-next` after using it in #1375. Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11) → 'github:NixOS/nix/aa438b8fbaebbbdb922655127053c4e8ea3e55bb' (2024-04-12) --- flake.lock | 7 +++---- flake.nix | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 966431f9..0b5c3bbb 100644 --- a/flake.lock +++ b/flake.lock @@ -42,16 +42,15 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1712849398, - "narHash": "sha256-10z/SoidVl9/lh56cMLj7ntJZHtVrumFvmn1YEqXmaM=", + "lastModified": 1712957033, + "narHash": "sha256-/P298re0Qga0eJV3Q3BppvYbVEN0CKXMnxMQX192hcE=", "owner": "NixOS", "repo": "nix", - "rev": "60824fa97c588a0faf68ea61260a47e388b0a4e5", + "rev": "aa438b8fbaebbbdb922655127053c4e8ea3e55bb", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.21-maintenance", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index c6646200..9d8c7142 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small"; - inputs.nix.url = "github:NixOS/nix/2.21-maintenance"; + inputs.nix.url = "github:NixOS/nix"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; outputs = { self, nixpkgs, nix }: From 5e910fa2cec113642ded1294c0c47100670ef587 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 May 2024 11:25:14 -0400 Subject: [PATCH 02/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/00ca2b05b8fbbef09be5d1e4820857605d4c31b6' (2024-05-03) → 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index aa29e826..8b3c146d 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1714701944, - "narHash": "sha256-trTxWfGElp0rkjquqG5I5RYVoxo8foCflxJFUtHwnOQ=", + "lastModified": 1715180119, + "narHash": "sha256-6tdv6d0lzpS+EeUTRZ15vS0YdcSpkmRBrJ5ABHC0QiU=", "owner": "NixOS", "repo": "nix", - "rev": "00ca2b05b8fbbef09be5d1e4820857605d4c31b6", + "rev": "0930058189f350a3729cd5aef2ffc8dae2ad436e", "type": "github" }, "original": { From e4f2c84f8dfd2863e40c74a35fb90b35749fbc94 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 May 2024 20:02:54 -0400 Subject: [PATCH 03/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08) → 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8b3c146d..4155cb06 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1715180119, - "narHash": "sha256-6tdv6d0lzpS+EeUTRZ15vS0YdcSpkmRBrJ5ABHC0QiU=", + "lastModified": 1715986148, + "narHash": "sha256-mR8FVWkQxl7jmO14WRRYNJdTG0yno9x5VQX/bQp7Mx4=", "owner": "NixOS", "repo": "nix", - "rev": "0930058189f350a3729cd5aef2ffc8dae2ad436e", + "rev": "beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b", "type": "github" }, "original": { From e4552ddf918aa25a89f594413b626dbb64a9214b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 May 2024 18:11:37 -0400 Subject: [PATCH 04/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17) → 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4155cb06..cf51e51f 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1715986148, - "narHash": "sha256-mR8FVWkQxl7jmO14WRRYNJdTG0yno9x5VQX/bQp7Mx4=", + "lastModified": 1716243047, + "narHash": "sha256-SfQwsgwgvvGixkRIwZbULsKGKoRUPf9yH+0hdBKkgRo=", "owner": "NixOS", "repo": "nix", - "rev": "beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b", + "rev": "a57abbd143f8ed44e823c3244e93507f64020878", "type": "github" }, "original": { From 71c4e2dc5b834a38e199f7a59b8928cc60c115fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 May 2024 18:00:16 -0400 Subject: [PATCH 05/10] Dedup more protocol code Use https://github.com/NixOS/nix/pull/10749 --- src/hydra-queue-runner/build-remote.cc | 36 +++----------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index ad510e1b..b4275ecc 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -267,32 +267,6 @@ static BuildResult performBuild( return result; } -static std::map queryPathInfos( - ::Machine::Connection & conn, - Store & localStore, - StorePathSet & outputs, - size_t & totalNarSize -) -{ - - /* Get info about each output path. */ - std::map infos; - conn.to << ServeProto::Command::QueryPathInfos; - ServeProto::write(localStore, conn, outputs); - conn.to.flush(); - while (true) { - auto storePathS = readString(conn.from); - if (storePathS == "") break; - - auto storePath = localStore.parseStorePath(storePathS); - auto info = ServeProto::Serialise::read(localStore, conn); - totalNarSize += info.narSize; - infos.insert_or_assign(std::move(storePath), std::move(info)); - } - - return infos; -} - static void copyPathFromRemote( ::Machine::Connection & conn, NarMemberDatas & narMembers, @@ -478,12 +452,6 @@ void State::buildRemote(ref destStore, throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); } - // Do not attempt to speak a newer version of the protocol. - // - // Per https://github.com/NixOS/nix/issues/9584 should be handled as - // part of `handshake` in upstream nix. - conn.remoteVersion = std::min(conn.remoteVersion, our_version); - { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; @@ -552,8 +520,10 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); + auto infos = conn.queryPathInfos(*localStore, outputs); + size_t totalNarSize = 0; - auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize); + for (auto & [_, info] : infos) totalNarSize += info.narSize; if (totalNarSize > maxOutputSize) { result.stepStatus = bsNarSizeLimitExceeded; From 346badc66ff68bccb20ccf9f48964da7bf60f2cf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 22 May 2024 22:00:38 -0400 Subject: [PATCH 06/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20) → 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index cf51e51f..6a6721b7 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1716243047, - "narHash": "sha256-SfQwsgwgvvGixkRIwZbULsKGKoRUPf9yH+0hdBKkgRo=", + "lastModified": 1716420255, + "narHash": "sha256-F8CpJTkbVl805LO5nvU5ivBdIHjcFUg8uk1EePGPu8s=", "owner": "NixOS", "repo": "nix", - "rev": "a57abbd143f8ed44e823c3244e93507f64020878", + "rev": "5845fd59c34198ad52a7f7bcb6d3ea7176ca437b", "type": "github" }, "original": { From d55bea2a1edf67645ace7221c3be4af192d198f3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 May 2024 13:34:30 -0400 Subject: [PATCH 07/10] Utilize `nix::Machine` more fully With https://github.com/NixOS/nix/pull/9839, the `storeUri` field is much better structured, so we can use it while still opening the SSH connection ourselves. --- src/hydra-queue-runner/build-remote.cc | 60 +++++++++++--------- src/hydra-queue-runner/builder.cc | 12 ++-- src/hydra-queue-runner/dispatcher.cc | 2 +- src/hydra-queue-runner/hydra-queue-runner.cc | 18 +++--- src/hydra-queue-runner/state.hh | 13 +---- 5 files changed, 49 insertions(+), 56 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index b4275ecc..617cce44 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -21,28 +21,23 @@ using namespace nix; -namespace nix::build_remote { - -static Strings extraStoreArgs(std::string & machine) +bool ::Machine::isLocalhost() const { - Strings result; - try { - auto parsed = parseURL(machine); - if (parsed.scheme != "ssh") { - throw SysError("Currently, only (legacy-)ssh stores are supported!"); - } - machine = parsed.authority.value_or(""); - auto remoteStore = parsed.query.find("remote-store"); - if (remoteStore != parsed.query.end()) { - result = {"--store", shellEscape(remoteStore->second)}; - } - } catch (BadURL &) { - // We just try to continue with `machine->sshName` here for backwards compat. - } - - return result; + return storeUri.params.empty() && std::visit(overloaded { + [](const StoreReference::Auto &) { + return true; + }, + [](const StoreReference::Specified & s) { + return + (s.scheme == "local" || s.scheme == "unix") || + ((s.scheme == "ssh" || s.scheme == "ssh-ng") && + s.authority == "localhost"); + }, + }, storeUri.variant); } +namespace nix::build_remote { + static std::unique_ptr openConnection( ::Machine::ptr machine, SSHMaster & master) { @@ -51,7 +46,11 @@ static std::unique_ptr openConnection( command.push_back("--builders"); command.push_back(""); } else { - command.splice(command.end(), extraStoreArgs(machine->sshName)); + auto remoteStore = machine->storeUri.params.find("remote-store"); + if (remoteStore != machine->storeUri.params.end()) { + command.push_back("--store"); + command.push_back(shellEscape(remoteStore->second)); + } } return master.startCommand(std::move(command), { @@ -187,7 +186,7 @@ static BasicDerivation sendInputs( MaintainCount mc2(nrStepsCopyingTo); printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", - localStore.printStorePath(step.drvPath), conn.machine->sshName); + localStore.printStorePath(step.drvPath), conn.machine->storeUri.render()); auto now1 = std::chrono::steady_clock::now(); @@ -393,8 +392,13 @@ void State::buildRemote(ref destStore, updateStep(ssConnecting); + auto * pSpecified = std::get_if(&machine->storeUri.variant); + if (!pSpecified || pSpecified->scheme != "ssh") { + throw Error("Currently, only (legacy-)ssh stores are supported!"); + } + SSHMaster master { - machine->sshName, + pSpecified->authority, machine->sshKey, machine->sshPublicHostKey, false, // no SSH master yet @@ -445,11 +449,11 @@ void State::buildRemote(ref destStore, conn.to, conn.from, our_version, - machine->sshName); + machine->storeUri.render()); } catch (EndOfFile & e) { child->sshPid.wait(); std::string s = chomp(readFile(result.logFile)); - throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s); + throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s); } { @@ -480,7 +484,7 @@ void State::buildRemote(ref destStore, /* Do the build. */ printMsg(lvlDebug, "building ‘%s’ on ‘%s’", localStore->printStorePath(step->drvPath), - machine->sshName); + machine->storeUri.render()); updateStep(ssBuilding); @@ -503,7 +507,7 @@ void State::buildRemote(ref destStore, get a build log. */ if (result.isCached) { printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’", - localStore->printStorePath(step->drvPath), machine->sshName); + localStore->printStorePath(step->drvPath), machine->storeUri.render()); unlink(result.logFile.c_str()); result.logFile = ""; } @@ -532,7 +536,7 @@ void State::buildRemote(ref destStore, /* Copy each path. */ printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", - localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); + localStore->printStorePath(step->drvPath), machine->storeUri.render(), totalNarSize); build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); auto now2 = std::chrono::steady_clock::now(); @@ -571,7 +575,7 @@ void State::buildRemote(ref destStore, info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4); info->lastFailure = now; int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30); - printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta); + printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta); info->disabledUntil = now + std::chrono::seconds(delta); } throw; diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 5269febd..a0773511 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -41,7 +41,7 @@ void State::builder(MachineReservation::ptr reservation) } catch (std::exception & e) { printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s", localStore->printStorePath(reservation->step->drvPath), - reservation->machine->sshName, + reservation->machine->storeUri.render(), e.what()); } } @@ -150,7 +150,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, buildOptions.buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1)); } if (!buildOneDone) @@ -196,7 +196,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, { auto mc = startDbUpdate(); pqxx::work txn(*conn); - stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy); + stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy); txn.commit(); } @@ -253,7 +253,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* Finish the step in the database. */ if (stepNr) { pqxx::work txn(*conn); - finishBuildStep(txn, result, buildId, stepNr, machine->sshName); + finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render()); txn.commit(); } @@ -261,7 +261,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, issue). Retry a number of times. */ if (result.canRetry) { printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s", - localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg); + localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg); assert(stepNr); bool retry; { @@ -452,7 +452,7 @@ void State::failStep( build->finishedInDB) continue; createBuildStep(txn, - 0, build->id, step, machine ? machine->sshName : "", + 0, build->id, step, machine ? machine->storeUri.render() : "", result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId); } diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 6d738ded..0f5a65e7 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -255,7 +255,7 @@ system_time State::doDispatch() /* Can this machine do this step? */ if (!mi.machine->supportsStep(step)) { debug("machine '%s' does not support step '%s' (system type '%s')", - mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform); + mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform); continue; } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5ffa7fe6..d7a88a0d 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -153,8 +153,8 @@ void State::parseMachines(const std::string & contents) using MaxJobs = std::remove_const::type; auto machine = std::make_shared<::Machine>(nix::Machine { - // `storeUri`, not yet used - "", + // `storeUri` + tokens[0], // `systemTypes` tokenizeString(tokens[1], ","), // `sshKey` @@ -175,25 +175,23 @@ void State::parseMachines(const std::string & contents) : "", }); - machine->sshName = tokens[0]; - /* Re-use the State object of the previous machine with the same name. */ - auto i = oldMachines.find(machine->sshName); + auto i = oldMachines.find(machine->storeUri.variant); if (i == oldMachines.end()) - printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName); + printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render()); else - printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName); + printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render()); machine->state = i == oldMachines.end() ? std::make_shared<::Machine::State>() : i->second->state; - newMachines[machine->sshName] = machine; + newMachines[machine->storeUri.variant] = machine; } for (auto & m : oldMachines) if (newMachines.find(m.first) == newMachines.end()) { if (m.second->enabled) - printInfo("removing machine ‘%1%’", m.first); + printInfo("removing machine ‘%1%’", m.second->storeUri.render()); /* Add a disabled ::Machine object to make sure stats are maintained. */ auto machine = std::make_shared<::Machine>(*(m.second)); @@ -657,7 +655,7 @@ void State::dumpStatus(Connection & conn) machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone; machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone; } - statusJson["machines"][m->sshName] = machine; + statusJson["machines"][m->storeUri.render()] = machine; } } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 5d242cdf..e2fb0c9c 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -240,10 +239,6 @@ struct Machine : nix::Machine { typedef std::shared_ptr ptr; - /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way - we are not yet used to, but once we are, we don't need this. */ - std::string sshName; - struct State { typedef std::shared_ptr ptr; counter currentJobs{0}; @@ -293,11 +288,7 @@ struct Machine : nix::Machine return true; } - bool isLocalhost() - { - std::regex r("^(ssh://|ssh-ng://)?localhost$"); - return std::regex_search(sshName, r); - } + bool isLocalhost() const; // A connection to a machine struct Connection : nix::ServeProto::BasicClientConnection { @@ -357,7 +348,7 @@ private: /* The build machines. */ std::mutex machinesReadyLock; - typedef std::map Machines; + typedef std::map Machines; nix::Sync machines; // FIXME: use atomic_shared_ptr /* Various stats. */ From bede2a141a3c1073d89b7852b08ac58c71d6778b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 May 2024 09:59:32 -0400 Subject: [PATCH 08/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22) → 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6a6721b7..327d44ad 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1716420255, - "narHash": "sha256-F8CpJTkbVl805LO5nvU5ivBdIHjcFUg8uk1EePGPu8s=", + "lastModified": 1716472735, + "narHash": "sha256-506pEWmMufBGD9hM99WZynH9Vxkv3o3IJJsPiVAs4T4=", "owner": "NixOS", "repo": "nix", - "rev": "5845fd59c34198ad52a7f7bcb6d3ea7176ca437b", + "rev": "2c42e7b8d9ea32e59c01334852599b548b214d31", "type": "github" }, "original": { From 09a1e64ed2d16ffbbcbc0c98458c621f5d939cf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 May 2024 00:32:11 -0400 Subject: [PATCH 09/10] Dedup with nix: use `nix::Machine::parseConfig` Companion to https://github.com/NixOS/nix/pull/10763 --- src/hydra-queue-runner/hydra-queue-runner.cc | 41 +------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index d7a88a0d..99411f9f 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -135,45 +135,8 @@ void State::parseMachines(const std::string & contents) oldMachines = *machines_; } - for (auto line : tokenizeString(contents, "\n")) { - line = trim(std::string(line, 0, line.find('#'))); - auto tokens = tokenizeString>(line); - if (tokens.size() < 3) continue; - tokens.resize(8); - - if (tokens[5] == "-") tokens[5] = ""; - auto supportedFeatures = tokenizeString(tokens[5], ","); - - if (tokens[6] == "-") tokens[6] = ""; - auto mandatoryFeatures = tokenizeString(tokens[6], ","); - - for (auto & f : mandatoryFeatures) - supportedFeatures.insert(f); - - using MaxJobs = std::remove_const::type; - - auto machine = std::make_shared<::Machine>(nix::Machine { - // `storeUri` - tokens[0], - // `systemTypes` - tokenizeString(tokens[1], ","), - // `sshKey` - tokens[2] == "-" ? "" : tokens[2], - // `maxJobs` - tokens[3] != "" - ? string2Int(tokens[3]).value() - : 1, - // `speedFactor` - atof(tokens[4].c_str()), - // `supportedFeatures` - std::move(supportedFeatures), - // `mandatoryFeatures` - std::move(mandatoryFeatures), - // `sshPublicHostKey` - tokens[7] != "" && tokens[7] != "-" - ? base64Decode(tokens[7]) - : "", - }); + for (auto && machine_ : nix::Machine::parseConfig({}, contents)) { + auto machine = std::make_shared<::Machine>(std::move(machine_)); /* Re-use the State object of the previous machine with the same name. */ From 2feddd8511cfa7a06a2c2b31da2a8a6707d81511 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 May 2024 17:05:41 -0400 Subject: [PATCH 10/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23) → 'github:NixOS/nix/ef5c846e257e1e284ad47ed6be4308d190fe6531' (2024-05-29) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 327d44ad..38b9aded 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1716472735, - "narHash": "sha256-506pEWmMufBGD9hM99WZynH9Vxkv3o3IJJsPiVAs4T4=", + "lastModified": 1717016009, + "narHash": "sha256-RJ0k7fvKjuUGt439F9uFtUw44SM87A+XKBf6sH6qPy8=", "owner": "NixOS", "repo": "nix", - "rev": "2c42e7b8d9ea32e59c01334852599b548b214d31", + "rev": "ef5c846e257e1e284ad47ed6be4308d190fe6531", "type": "github" }, "original": {