From 54675a0d94adae6c6164e93ba088edcc1f527647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Sun, 13 Feb 2022 14:24:36 +0100
Subject: [PATCH 001/285] Fix local store detection and related issues

- Add localStore into the stash because it's used in templates
- Hide the Channels button for non-local stores because the link 404s
  anyway
- Fix a style issue when having popovers in dark mode
---
 src/lib/Hydra/Controller/Root.pm | 1 +
 src/root/static/css/hydra.css    | 2 +-
 src/root/topbar.tt               | 6 +++---
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index c6843d29..641002e7 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -52,6 +52,7 @@ sub begin :Private {
     $c->stash->{tracker} = defined $c->config->{tracker} ? $c->config->{tracker} : "";
     $c->stash->{flashMsg} = $c->flash->{flashMsg};
     $c->stash->{successMsg} = $c->flash->{successMsg};
+    $c->stash->{localStore} = isLocalStore;
 
     $c->stash->{isPrivateHydra} = $c->config->{private} // "0" ne "0";
 
diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 475d61c2..252ba815 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -151,7 +151,7 @@ td.step-status span.warn {
     html {
         background-color: #1f1f1f;
     }
-    body, div.popover {
+    body, div.popover, div.popover-body {
         background-color: #1f1f1f;
         color: #fafafa !important;
     }
diff --git a/src/root/topbar.tt b/src/root/topbar.tt
index fdfbf431..7165aa15 100644
--- a/src/root/topbar.tt
+++ b/src/root/topbar.tt
@@ -42,7 +42,7 @@
       <div class="dropdown-divider"></div>
       [% INCLUDE menuItem uri = c.uri_for(c.controller('Project').action_for('project'), [project.name]) title = "Overview" %]
       [% INCLUDE menuItem uri = c.uri_for(c.controller('Project').action_for('all'), [project.name]) title = "Latest builds" %]
-      [% INCLUDE menuItem uri = c.uri_for('/project' project.name 'channel' 'latest') title = "Channel" %]
+      [% IF localStore %][% INCLUDE menuItem uri = c.uri_for('/project' project.name 'channel' 'latest') title = "Channel" %][% END %]
     [% END %]
   [% END %]
 
@@ -59,7 +59,7 @@
       [% INCLUDE menuItem
         uri = c.uri_for(c.controller('Jobset').action_for('all'), [project.name, jobset.name])
         title = "Latest builds" %]
-      [% INCLUDE menuItem uri = c.uri_for('/jobset' project.name jobset.name 'channel' 'latest') title = "Channel" %]
+      [% IF localStore %][% INCLUDE menuItem uri = c.uri_for('/jobset' project.name jobset.name 'channel' 'latest') title = "Channel" %][% END %]
     [% END %]
   [% END %]
 
@@ -73,7 +73,7 @@
       [% INCLUDE menuItem
         uri = c.uri_for(c.controller('Job').action_for('all'), [project.name, jobset.name, job])
         title = "Latest builds" %]
-      [% INCLUDE menuItem uri = c.uri_for('/job' project.name jobset.name job 'channel' 'latest') title = "Channel" %]
+      [% IF localStore %][% INCLUDE menuItem uri = c.uri_for('/job' project.name jobset.name job 'channel' 'latest') title = "Channel" %][% END %]
     [% END %]
   [% END %]
 

From 1c846765273018ff9b97edbfcaf9e5884fd9e140 Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Mon, 10 May 2021 14:32:24 +0200
Subject: [PATCH 002/285] Fit more content on screen

---
 src/root/static/css/hydra.css | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 475d61c2..53304b19 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -1,5 +1,5 @@
 div.skip-topbar {
-    padding-top: 40px;
+    padding-top: 20px;
     margin-bottom: 1.5em;
 }
 
@@ -146,6 +146,26 @@ td.step-status span.warn {
     padding-top: 1.5rem;
 }
 
+.container {
+    max-width: 80%;
+}
+
+.tab-content {
+    margin-right: 0 !important;
+}
+
+body {
+    line-height: 1;
+}
+
+.navbar-nav {
+    line-height: 1.5;
+}
+
+.dropdown-item {
+    line-height: 1.5;
+}
+
 @media (prefers-color-scheme: dark) {
     /* Prevent some flickering */
     html {

From f1f2fc742700c6f80f87758687afedefd9d08ac6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 7 Mar 2022 19:07:06 +0000
Subject: [PATCH 003/285] build(deps): bump actions/checkout from 2 to 3

Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7d98e3ff..8d32f581 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,7 +6,7 @@ jobs:
   tests:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v12

From 5db8642224e996d0ebe3492cd55dfef6276d5a15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 16:02:51 +0100
Subject: [PATCH 004/285] Factor out a struct representing a connection to a
 machine

---
 src/hydra-queue-runner/state.hh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 8f303d28..059b03a1 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -290,6 +290,16 @@ struct Machine
     {
         return sshName == "localhost";
     }
+
+    // A connection to a machine
+    struct Connection {
+        nix::FdSink to;
+        nix::FdSource from;
+        unsigned int remoteVersion;
+
+        // Backpointer to the machine
+        ptr machine;
+    };
 };
 
 

From 2f494b783425d6703f23bc4f2cdbc70592a31990 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 10:42:44 +0100
Subject: [PATCH 005/285] Factor out the creation of the log file

---
 src/hydra-queue-runner/build-remote.cc | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 464a35c8..2e258484 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -175,6 +175,18 @@ StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths
     return sorted;
 }
 
+std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
+{
+    string base(drvPath.to_string());
+    auto logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
+
+    createDirs(dirOf(logFile));
+
+    AutoCloseFD logFD = open(logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666);
+    if (!logFD) throw SysError("creating log file ‘%s’", logFile);
+
+    return {std::move(logFile), std::move(logFD)};
+}
 
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
@@ -185,14 +197,9 @@ void State::buildRemote(ref<Store> destStore,
 {
     assert(BuildResult::TimedOut == 8);
 
-    string base(step->drvPath.to_string());
-    result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
-    AutoDelete autoDelete(result.logFile, false);
-
-    createDirs(dirOf(result.logFile));
-
-    AutoCloseFD logFD = open(result.logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666);
-    if (!logFD) throw SysError("creating log file ‘%s’", result.logFile);
+    auto [logFile, logFD] = openLogFile(logDir, step->drvPath);
+    AutoDelete logFileDel(logFile, false);
+    result.logFile = logFile;
 
     nix::Path tmpDir = createTempDir();
     AutoDelete tmpDirDel(tmpDir, true);
@@ -316,7 +323,7 @@ void State::buildRemote(ref<Store> destStore,
             result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
         }
 
-        autoDelete.cancel();
+        logFileDel.cancel();
 
         /* Truncate the log to get rid of messages about substitutions
            etc. on the remote system. */

From 9f1b911625cbc92023b6aec936a505a46af3172f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 11:35:38 +0100
Subject: [PATCH 006/285] Factor more stuff out

---
 src/hydra-queue-runner/build-remote.cc | 224 ++++++++++++++-----------
 1 file changed, 122 insertions(+), 102 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 2e258484..360a8ef7 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -188,6 +188,87 @@ std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const Store
     return {std::move(logFile), std::move(logFD)};
 }
 
+void handshake(Machine::Connection & conn, unsigned int repeats)
+{
+    conn.to << SERVE_MAGIC_1 << 0x204;
+    conn.to.flush();
+
+    unsigned int magic = readInt(conn.from);
+    if (magic != SERVE_MAGIC_2)
+        throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName);
+    conn.remoteVersion = readInt(conn.from);
+    if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200)
+        throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName);
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0)
+        throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName);
+}
+
+StorePathSet sendInputs(
+    State & state,
+    Step & step,
+    Store & localStore,
+    Store & destStore,
+    Machine::Connection & conn,
+    unsigned int & overhead,
+    counter & nrStepsWaiting,
+    counter & nrStepsCopyingTo
+)
+{
+
+    StorePathSet inputs;
+    BasicDerivation basicDrv(*step.drv);
+
+    for (auto & p : step.drv->inputSrcs)
+        inputs.insert(p);
+
+    for (auto & input : step.drv->inputDrvs) {
+        auto drv2 = localStore.readDerivation(input.first);
+        for (auto & name : input.second) {
+            if (auto i = get(drv2.outputs, name)) {
+                auto outPath = i->path(localStore, drv2.name, name);
+                inputs.insert(*outPath);
+                basicDrv.inputSrcs.insert(*outPath);
+            }
+        }
+    }
+
+    /* Ensure that the inputs exist in the destination store. This is
+        a no-op for regular stores, but for the binary cache store,
+        this will copy the inputs to the binary cache from the local
+        store. */
+    if (localStore.getUri() != destStore.getUri()) {
+        StorePathSet closure;
+        localStore.computeFSClosure(step.drv->inputSrcs, closure);
+        copyPaths(localStore, destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
+    }
+
+    {
+        auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
+        mc1.reset();
+        MaintainCount<counter> mc2(nrStepsCopyingTo);
+
+        printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
+            localStore.printStorePath(step.drvPath), conn.machine->sshName);
+
+        auto now1 = std::chrono::steady_clock::now();
+
+        /* Copy the input closure. */
+        if (conn.machine->isLocalhost()) {
+            StorePathSet closure;
+            destStore.computeFSClosure(inputs, closure);
+            copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
+        } else {
+            copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, inputs, true);
+        }
+
+        auto now2 = std::chrono::steady_clock::now();
+
+        overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    }
+
+    return inputs;
+}
+
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
@@ -230,33 +311,21 @@ void State::buildRemote(ref<Store> destStore,
                process. Meh. */
         });
 
-        FdSource from(child.from.get());
-        FdSink to(child.to.get());
+        Machine::Connection conn;
+        conn.from = child.from.get();
+        conn.to = child.to.get();
+        conn.machine = machine;
 
         Finally updateStats([&]() {
-            bytesReceived += from.read;
-            bytesSent += to.written;
+            bytesReceived += conn.from.read;
+            bytesSent += conn.to.written;
         });
 
-        /* Handshake. */
-        unsigned int remoteVersion;
-
         try {
-            to << SERVE_MAGIC_1 << 0x204;
-            to.flush();
-
-            unsigned int magic = readInt(from);
-            if (magic != SERVE_MAGIC_2)
-                throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", machine->sshName);
-            remoteVersion = readInt(from);
-            if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
-                throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", machine->sshName);
-            if (GET_PROTOCOL_MINOR(remoteVersion) < 3 && repeats > 0)
-                throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", machine->sshName);
-
+          handshake(conn, repeats);
         } catch (EndOfFile & e) {
             child.pid.wait();
-            string s = chomp(readFile(result.logFile));
+            std::string s = chomp(readFile(result.logFile));
             throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
         }
 
@@ -272,61 +341,12 @@ void State::buildRemote(ref<Store> destStore,
            outputs of the input derivations. */
         updateStep(ssSendingInputs);
 
-        StorePathSet inputs;
-        BasicDerivation basicDrv(*step->drv);
-
-        for (auto & p : step->drv->inputSrcs)
-            inputs.insert(p);
-
-        for (auto & input : step->drv->inputDrvs) {
-            auto drv2 = localStore->readDerivation(input.first);
-            for (auto & name : input.second) {
-                if (auto i = get(drv2.outputs, name)) {
-                    auto outPath = i->path(*localStore, drv2.name, name);
-                    inputs.insert(*outPath);
-                    basicDrv.inputSrcs.insert(*outPath);
-                }
-            }
-        }
-
-        /* Ensure that the inputs exist in the destination store. This is
-           a no-op for regular stores, but for the binary cache store,
-           this will copy the inputs to the binary cache from the local
-           store. */
-        if (localStore != std::shared_ptr<Store>(destStore)) {
-            StorePathSet closure;
-            localStore->computeFSClosure(step->drv->inputSrcs, closure);
-            copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
-        }
-
-        {
-            auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
-            mc1.reset();
-            MaintainCount<counter> mc2(nrStepsCopyingTo);
-
-            printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
-                localStore->printStorePath(step->drvPath), machine->sshName);
-
-            auto now1 = std::chrono::steady_clock::now();
-
-            /* Copy the input closure. */
-            if (machine->isLocalhost()) {
-                StorePathSet closure;
-                destStore->computeFSClosure(inputs, closure);
-                copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
-            } else {
-                copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true);
-            }
-
-            auto now2 = std::chrono::steady_clock::now();
-
-            result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-        }
+        StorePathSet inputs = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo);
 
         logFileDel.cancel();
 
         /* Truncate the log to get rid of messages about substitutions
-           etc. on the remote system. */
+            etc. on the remote system. */
         if (lseek(logFD.get(), SEEK_SET, 0) != 0)
             throw SysError("seeking to the start of log file ‘%s’", result.logFile);
 
@@ -342,31 +362,31 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssBuilding);
 
-        to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
-        writeDerivation(to, *localStore, basicDrv);
-        to << maxSilentTime << buildTimeout;
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
-            to << maxLogSize;
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
-            to << repeats // == build-repeat
+        conn.to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
+        writeDerivation(conn.to, *localStore, BasicDerivation(*step->drv));
+        conn.to << maxSilentTime << buildTimeout;
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
+            conn.to << maxLogSize;
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
+            conn.to << repeats // == build-repeat
                << step->isDeterministic; // == enforce-determinism
         }
-        to.flush();
+        conn.to.flush();
 
         result.startTime = time(0);
         int res;
         {
             MaintainCount<counter> mc(nrStepsBuilding);
-            res = readInt(from);
+            res = readInt(conn.from);
         }
         result.stopTime = time(0);
 
-        result.errorMsg = readString(from);
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
-            result.timesBuilt = readInt(from);
-            result.isNonDeterministic = readInt(from);
-            auto start = readInt(from);
-            auto stop = readInt(from);
+        result.errorMsg = readString(conn.from);
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
+            result.timesBuilt = readInt(conn.from);
+            result.isNonDeterministic = readInt(conn.from);
+            auto start = readInt(conn.from);
+            auto stop = readInt(conn.from);
             if (start && start) {
                 /* Note: this represents the duration of a single
                     round, rather than all rounds. */
@@ -374,8 +394,8 @@ void State::buildRemote(ref<Store> destStore,
                 result.stopTime = stop;
             }
         }
-        if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
-            worker_proto::read(*localStore, from, Phantom<DrvOutputs> {});
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
+            worker_proto::read(*localStore, conn.from, Phantom<DrvOutputs> {});
         }
         switch ((BuildResult::Status) res) {
             case BuildResult::Built:
@@ -451,19 +471,19 @@ void State::buildRemote(ref<Store> destStore,
             /* Get info about each output path. */
             std::map<StorePath, ValidPathInfo> infos;
             size_t totalNarSize = 0;
-            to << cmdQueryPathInfos;
-            worker_proto::write(*localStore, to, outputs);
-            to.flush();
+            conn.to << cmdQueryPathInfos;
+            worker_proto::write(*localStore, conn.to, outputs);
+            conn.to.flush();
             while (true) {
-                auto storePathS = readString(from);
+                auto storePathS = readString(conn.from);
                 if (storePathS == "") break;
-                auto deriver = readString(from); // deriver
-                auto references = worker_proto::read(*localStore, from, Phantom<StorePathSet> {});
-                readLongLong(from); // download size
-                auto narSize = readLongLong(from);
-                auto narHash = Hash::parseAny(readString(from), htSHA256);
-                auto ca = parseContentAddressOpt(readString(from));
-                readStrings<StringSet>(from); // sigs
+                auto deriver = readString(conn.from); // deriver
+                auto references = worker_proto::read(*localStore, conn.from, Phantom<StorePathSet> {});
+                readLongLong(conn.from); // download size
+                auto narSize = readLongLong(conn.from);
+                auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
+                auto ca = parseContentAddressOpt(readString(conn.from));
+                readStrings<StringSet>(conn.from); // sigs
                 ValidPathInfo info(localStore->parseStorePath(storePathS), narHash);
                 assert(outputs.count(info.path));
                 info.references = references;
@@ -502,10 +522,10 @@ void State::buildRemote(ref<Store> destStore,
                        lambda function only gets executed if someone tries to read
                        from source2, we will send the command from here rather
                        than outside the lambda. */
-                    to << cmdDumpStorePath << localStore->printStorePath(path);
-                    to.flush();
+                    conn.to << cmdDumpStorePath << localStore->printStorePath(path);
+                    conn.to.flush();
 
-                    TeeSource tee(from, sink);
+                    TeeSource tee(conn.from, sink);
                     extractNarData(tee, localStore->printStorePath(path), narMembers);
                 });
 

From 365776f5d77112842d4a094a596cb725123b535d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 12:14:37 +0100
Subject: [PATCH 007/285] Factor out the building part

---
 src/hydra-queue-runner/build-remote.cc | 207 ++++++++++++++++---------
 src/hydra-queue-runner/state.hh        |   2 +
 2 files changed, 132 insertions(+), 77 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 360a8ef7..7934d401 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -269,6 +269,121 @@ StorePathSet sendInputs(
     return inputs;
 }
 
+struct BuildOptions {
+    unsigned int maxSilentTime, buildTimeout, repeats;
+    size_t maxLogSize;
+    bool enforceDeterminism;
+};
+
+void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
+{
+    RemoteResult thisArrow;
+
+    startTime = buildResult.startTime;
+    stopTime = buildResult.stopTime;
+    timesBuilt = buildResult.timesBuilt;
+    errorMsg = buildResult.errorMsg;
+    isNonDeterministic = buildResult.isNonDeterministic;
+
+    switch ((BuildResult::Status) buildResult.status) {
+        case BuildResult::Built:
+            stepStatus = bsSuccess;
+            break;
+        case BuildResult::Substituted:
+        case BuildResult::AlreadyValid:
+            stepStatus = bsSuccess;
+            isCached = true;
+            break;
+        case BuildResult::PermanentFailure:
+            stepStatus = bsFailed;
+            canCache = true;
+            errorMsg = "";
+            break;
+        case BuildResult::InputRejected:
+        case BuildResult::OutputRejected:
+            stepStatus = bsFailed;
+            canCache = true;
+            break;
+        case BuildResult::TransientFailure:
+            stepStatus = bsFailed;
+            canRetry = true;
+            errorMsg = "";
+            break;
+        case BuildResult::TimedOut:
+            stepStatus = bsTimedOut;
+            errorMsg = "";
+            break;
+        case BuildResult::MiscFailure:
+            stepStatus = bsAborted;
+            canRetry = true;
+            break;
+        case BuildResult::LogLimitExceeded:
+            stepStatus = bsLogLimitExceeded;
+            break;
+        case BuildResult::NotDeterministic:
+            stepStatus = bsNotDeterministic;
+            canRetry = false;
+            canCache = true;
+            break;
+        default:
+            stepStatus = bsAborted;
+            break;
+    }
+
+}
+
+BuildResult performBuild(
+    Machine::Connection & conn,
+    Store & localStore,
+    StorePath drvPath,
+    const BasicDerivation & drv,
+    const BuildOptions & options,
+    counter & nrStepsBuilding
+)
+{
+
+    BuildResult result;
+
+    conn.to << cmdBuildDerivation << localStore.printStorePath(drvPath);
+    writeDerivation(conn.to, localStore, drv);
+    conn.to << options.maxSilentTime << options.buildTimeout;
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
+        conn.to << options.maxLogSize;
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
+        conn.to << options.repeats // == build-repeat
+          << options.enforceDeterminism;
+    }
+    conn.to.flush();
+
+    result.startTime = time(0);
+
+    {
+        MaintainCount<counter> mc(nrStepsBuilding);
+        result.status = (BuildResult::Status)readInt(conn.from);
+    }
+    result.stopTime = time(0);
+
+
+    result.errorMsg = readString(conn.from);
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
+        result.timesBuilt = readInt(conn.from);
+        result.isNonDeterministic = readInt(conn.from);
+        auto start = readInt(conn.from);
+        auto stop = readInt(conn.from);
+        if (start && start) {
+            /* Note: this represents the duration of a single
+                round, rather than all rounds. */
+            result.startTime = start;
+            result.stopTime = stop;
+        }
+    }
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
+        result.builtOutputs = worker_proto::read(localStore, conn.from, Phantom<DrvOutputs> {});
+    }
+
+    return result;
+}
+
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
@@ -362,85 +477,23 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssBuilding);
 
-        conn.to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
-        writeDerivation(conn.to, *localStore, BasicDerivation(*step->drv));
-        conn.to << maxSilentTime << buildTimeout;
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
-            conn.to << maxLogSize;
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
-            conn.to << repeats // == build-repeat
-               << step->isDeterministic; // == enforce-determinism
-        }
-        conn.to.flush();
+        BuildResult buildResult = performBuild(
+            conn,
+            *localStore,
+            step->drvPath,
+            BasicDerivation(*step->drv),
+            {
+              .maxSilentTime = maxSilentTime,
+              .buildTimeout = buildTimeout,
+              .repeats = repeats,
+              .maxLogSize = maxLogSize,
+              .enforceDeterminism = step->isDeterministic,
+            },
+            nrStepsBuilding
+        );
 
-        result.startTime = time(0);
-        int res;
-        {
-            MaintainCount<counter> mc(nrStepsBuilding);
-            res = readInt(conn.from);
-        }
-        result.stopTime = time(0);
+        result.updateWithBuildResult(buildResult);
 
-        result.errorMsg = readString(conn.from);
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
-            result.timesBuilt = readInt(conn.from);
-            result.isNonDeterministic = readInt(conn.from);
-            auto start = readInt(conn.from);
-            auto stop = readInt(conn.from);
-            if (start && start) {
-                /* Note: this represents the duration of a single
-                    round, rather than all rounds. */
-                result.startTime = start;
-                result.stopTime = stop;
-            }
-        }
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
-            worker_proto::read(*localStore, conn.from, Phantom<DrvOutputs> {});
-        }
-        switch ((BuildResult::Status) res) {
-            case BuildResult::Built:
-                result.stepStatus = bsSuccess;
-                break;
-            case BuildResult::Substituted:
-            case BuildResult::AlreadyValid:
-                result.stepStatus = bsSuccess;
-                result.isCached = true;
-                break;
-            case BuildResult::PermanentFailure:
-                result.stepStatus = bsFailed;
-                result.canCache = true;
-                result.errorMsg = "";
-                break;
-            case BuildResult::InputRejected:
-            case BuildResult::OutputRejected:
-                result.stepStatus = bsFailed;
-                result.canCache = true;
-                break;
-            case BuildResult::TransientFailure:
-                result.stepStatus = bsFailed;
-                result.canRetry = true;
-                result.errorMsg = "";
-                break;
-            case BuildResult::TimedOut:
-                result.stepStatus = bsTimedOut;
-                result.errorMsg = "";
-                break;
-            case BuildResult::MiscFailure:
-                result.stepStatus = bsAborted;
-                result.canRetry = true;
-                break;
-            case BuildResult::LogLimitExceeded:
-                result.stepStatus = bsLogLimitExceeded;
-                break;
-            case BuildResult::NotDeterministic:
-                result.stepStatus = bsNotDeterministic;
-                result.canRetry = false;
-                result.canCache = true;
-                break;
-            default:
-                result.stepStatus = bsAborted;
-                break;
-        }
         if (result.stepStatus != bsSuccess) return;
 
         result.errorMsg = "";
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 059b03a1..6292a2db 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -72,6 +72,8 @@ struct RemoteResult
     {
         return stepStatus == bsCachedFailure ? bsFailed : stepStatus;
     }
+
+    void updateWithBuildResult(const nix::BuildResult &);
 };
 
 

From a778a89f0424b5bffa66818cf0b46ffa945403c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 15:16:32 +0100
Subject: [PATCH 008/285] Factor out the `queryPathInfos` part of the build

---
 src/hydra-queue-runner/build-remote.cc | 65 +++++++++++++++-----------
 1 file changed, 39 insertions(+), 26 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 7934d401..0451ccb7 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -384,6 +384,44 @@ BuildResult performBuild(
     return result;
 }
 
+std::map<StorePath, ValidPathInfo> queryPathInfos(
+    Machine::Connection & conn,
+    Store & localStore,
+    StorePathSet & outputs,
+    size_t & totalNarSize
+)
+{
+
+    /* Get info about each output path. */
+    std::map<StorePath, ValidPathInfo> infos;
+    conn.to << cmdQueryPathInfos;
+    worker_proto::write(localStore, conn.to, outputs);
+    conn.to.flush();
+    while (true) {
+        auto storePathS = readString(conn.from);
+        if (storePathS == "") break;
+        auto deriver = readString(conn.from); // deriver
+        auto references = worker_proto::read(localStore, conn.from, Phantom<StorePathSet> {});
+        readLongLong(conn.from); // download size
+        auto narSize = readLongLong(conn.from);
+        auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
+        auto ca = parseContentAddressOpt(readString(conn.from));
+        readStrings<StringSet>(conn.from); // sigs
+        ValidPathInfo info(localStore.parseStorePath(storePathS), narHash);
+        assert(outputs.count(info.path));
+        info.references = references;
+        info.narSize = narSize;
+        totalNarSize += info.narSize;
+        info.narHash = narHash;
+        info.ca = ca;
+        if (deriver != "")
+            info.deriver = localStore.parseStorePath(deriver);
+        infos.insert_or_assign(info.path, info);
+    }
+
+    return infos;
+}
+
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
@@ -521,33 +559,8 @@ void State::buildRemote(ref<Store> destStore,
                    outputs.insert(*i.second.second);
             }
 
-            /* Get info about each output path. */
-            std::map<StorePath, ValidPathInfo> infos;
             size_t totalNarSize = 0;
-            conn.to << cmdQueryPathInfos;
-            worker_proto::write(*localStore, conn.to, outputs);
-            conn.to.flush();
-            while (true) {
-                auto storePathS = readString(conn.from);
-                if (storePathS == "") break;
-                auto deriver = readString(conn.from); // deriver
-                auto references = worker_proto::read(*localStore, conn.from, Phantom<StorePathSet> {});
-                readLongLong(conn.from); // download size
-                auto narSize = readLongLong(conn.from);
-                auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
-                auto ca = parseContentAddressOpt(readString(conn.from));
-                readStrings<StringSet>(conn.from); // sigs
-                ValidPathInfo info(localStore->parseStorePath(storePathS), narHash);
-                assert(outputs.count(info.path));
-                info.references = references;
-                info.narSize = narSize;
-                totalNarSize += info.narSize;
-                info.narHash = narHash;
-                info.ca = ca;
-                if (deriver != "")
-                    info.deriver = localStore->parseStorePath(deriver);
-                infos.insert_or_assign(info.path, info);
-            }
+            auto infos = queryPathInfos(conn, *localStore, outputs, totalNarSize);
 
             if (totalNarSize > maxOutputSize) {
                 result.stepStatus = bsNarSizeLimitExceeded;

From fd0ae78eba058ab456590da698b45082fcafb519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 15:26:31 +0100
Subject: [PATCH 009/285] Factor out the copying from the build store

---
 src/hydra-queue-runner/build-remote.cc | 76 +++++++++++++++++---------
 1 file changed, 49 insertions(+), 27 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 0451ccb7..79e5a231 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -422,6 +422,54 @@ std::map<StorePath, ValidPathInfo> queryPathInfos(
     return infos;
 }
 
+void copyPathFromRemote(
+    Machine::Connection & conn,
+    NarMemberDatas & narMembers,
+    Store & localStore,
+    Store & destStore,
+    const ValidPathInfo & info
+)
+{
+      /* Receive the NAR from the remote and add it to the
+          destination store. Meanwhile, extract all the info from the
+          NAR that getBuildOutput() needs. */
+      auto source2 = sinkToSource([&](Sink & sink)
+      {
+          /* Note: we should only send the command to dump the store
+              path to the remote if the NAR is actually going to get read
+              by the destination store, which won't happen if this path
+              is already valid on the destination store. Since this
+              lambda function only gets executed if someone tries to read
+              from source2, we will send the command from here rather
+              than outside the lambda. */
+          conn.to << cmdDumpStorePath << localStore.printStorePath(info.path);
+          conn.to.flush();
+
+          TeeSource tee(conn.from, sink);
+          extractNarData(tee, localStore.printStorePath(info.path), narMembers);
+      });
+
+      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
+}
+
+void copyPathsFromRemote(
+    Machine::Connection & conn,
+    NarMemberDatas & narMembers,
+    Store & localStore,
+    Store & destStore,
+    const std::map<StorePath, ValidPathInfo> & infos
+)
+{
+      auto pathsSorted = reverseTopoSortPaths(infos);
+
+      for (auto & path : pathsSorted) {
+          auto & info = infos.find(path)->second;
+          copyPathFromRemote(conn, narMembers, localStore, destStore, info);
+      }
+
+}
+
+
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
@@ -571,33 +619,7 @@ void State::buildRemote(ref<Store> destStore,
             printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
                 localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
 
-            auto pathsSorted = reverseTopoSortPaths(infos);
-
-            for (auto & path : pathsSorted) {
-                auto & info = infos.find(path)->second;
-
-                /* Receive the NAR from the remote and add it to the
-                   destination store. Meanwhile, extract all the info from the
-                   NAR that getBuildOutput() needs. */
-                auto source2 = sinkToSource([&](Sink & sink)
-                {
-                    /* Note: we should only send the command to dump the store
-                       path to the remote if the NAR is actually going to get read
-                       by the destination store, which won't happen if this path
-                       is already valid on the destination store. Since this
-                       lambda function only gets executed if someone tries to read
-                       from source2, we will send the command from here rather
-                       than outside the lambda. */
-                    conn.to << cmdDumpStorePath << localStore->printStorePath(path);
-                    conn.to.flush();
-
-                    TeeSource tee(conn.from, sink);
-                    extractNarData(tee, localStore->printStorePath(path), narMembers);
-                });
-
-                destStore->addToStore(info, *source2, NoRepair, NoCheckSigs);
-            }
-
+            copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos);
             auto now2 = std::chrono::steady_clock::now();
 
             result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

From b430d41afd6a4ca0e38343ae2eee4aa6307cf980 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Mon, 21 Mar 2022 16:33:25 +0100
Subject: [PATCH 010/285] Use the `BuildOptions` more eagerly

---
 src/hydra-queue-runner/build-remote.cc | 20 ++++----------------
 src/hydra-queue-runner/builder.cc      | 16 +++++++++-------
 src/hydra-queue-runner/state.hh        |  9 +++++++--
 3 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 79e5a231..1e06e501 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -269,12 +269,6 @@ StorePathSet sendInputs(
     return inputs;
 }
 
-struct BuildOptions {
-    unsigned int maxSilentTime, buildTimeout, repeats;
-    size_t maxLogSize;
-    bool enforceDeterminism;
-};
-
 void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 {
     RemoteResult thisArrow;
@@ -337,7 +331,7 @@ BuildResult performBuild(
     Store & localStore,
     StorePath drvPath,
     const BasicDerivation & drv,
-    const BuildOptions & options,
+    const State::BuildOptions & options,
     counter & nrStepsBuilding
 )
 {
@@ -472,7 +466,7 @@ void copyPathsFromRemote(
 
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
-    unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
+    const BuildOptions & buildOptions,
     RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
     std::function<void(StepState)> updateStep,
     NarMemberDatas & narMembers)
@@ -523,7 +517,7 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         try {
-          handshake(conn, repeats);
+          handshake(conn, buildOptions.repeats);
         } catch (EndOfFile & e) {
             child.pid.wait();
             std::string s = chomp(readFile(result.logFile));
@@ -568,13 +562,7 @@ void State::buildRemote(ref<Store> destStore,
             *localStore,
             step->drvPath,
             BasicDerivation(*step->drv),
-            {
-              .maxSilentTime = maxSilentTime,
-              .buildTimeout = buildTimeout,
-              .repeats = repeats,
-              .maxLogSize = maxLogSize,
-              .enforceDeterminism = step->isDeterministic,
-            },
+            buildOptions,
             nrStepsBuilding
         );
 
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 89aa7d15..b25b4e63 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -98,8 +98,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        it). */
     BuildID buildId;
     std::optional<StorePath> buildDrvPath;
-    unsigned int maxSilentTime, buildTimeout;
-    unsigned int repeats = step->isDeterministic ? 1 : 0;
+    BuildOptions buildOptions;
+    buildOptions.repeats = step->isDeterministic ? 1 : 0;
+    buildOptions.maxLogSize = maxLogSize;
+    buildOptions.enforceDeterminism = step->isDeterministic;
 
     auto conn(dbPool.get());
 
@@ -134,18 +136,18 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
             {
                 auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
                 if (i != jobsetRepeats.end())
-                    repeats = std::max(repeats, i->second);
+                    buildOptions.repeats = std::max(buildOptions.repeats, i->second);
             }
         }
         if (!build) build = *dependents.begin();
 
         buildId = build->id;
         buildDrvPath = build->drvPath;
-        maxSilentTime = build->maxSilentTime;
-        buildTimeout = build->buildTimeout;
+        buildOptions.maxSilentTime = build->maxSilentTime;
+        buildOptions.buildTimeout = build->buildTimeout;
 
         printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
-            localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
+            localStore->printStorePath(step->drvPath), buildOptions.repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
     }
 
     if (!buildOneDone)
@@ -206,7 +208,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         try {
             /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
         } catch (Error & e) {
             if (activeStep->state_.lock()->cancelled) {
                 printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 6292a2db..f4d8ccce 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -447,6 +447,12 @@ private:
 public:
     State();
 
+    struct BuildOptions {
+        unsigned int maxSilentTime, buildTimeout, repeats;
+        size_t maxLogSize;
+        bool enforceDeterminism;
+    };
+
 private:
 
     nix::MaintainCount<counter> startDbUpdate();
@@ -531,8 +537,7 @@ private:
 
     void buildRemote(nix::ref<nix::Store> destStore,
         Machine::ptr machine, Step::ptr step,
-        unsigned int maxSilentTime, unsigned int buildTimeout,
-        unsigned int repeats,
+        const BuildOptions & buildOptions,
         RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
         std::function<void(StepState)> updateStep,
         NarMemberDatas & narMembers);

From 92b627ac1b3e0f53f6b35fc5940406c12fa977da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Thu, 24 Mar 2022 09:39:24 +0100
Subject: [PATCH 011/285] Remove an accidental re-indenting of a comment

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/hydra-queue-runner/build-remote.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 1e06e501..901bbc89 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -233,9 +233,9 @@ StorePathSet sendInputs(
     }
 
     /* Ensure that the inputs exist in the destination store. This is
-        a no-op for regular stores, but for the binary cache store,
-        this will copy the inputs to the binary cache from the local
-        store. */
+       a no-op for regular stores, but for the binary cache store,
+       this will copy the inputs to the binary cache from the local
+       store. */
     if (localStore.getUri() != destStore.getUri()) {
         StorePathSet closure;
         localStore.computeFSClosure(step.drv->inputSrcs, closure);

From 6e571e26ff068386bc949eaab6646e90613b23c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <theophane@hufschmitt.net>
Date: Thu, 24 Mar 2022 14:27:45 +0100
Subject: [PATCH 012/285] Build the resolved derivation and not the original
 one

---
 src/hydra-queue-runner/build-remote.cc | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 901bbc89..62461a65 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -203,7 +203,7 @@ void handshake(Machine::Connection & conn, unsigned int repeats)
         throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName);
 }
 
-StorePathSet sendInputs(
+BasicDerivation sendInputs(
     State & state,
     Step & step,
     Store & localStore,
@@ -214,19 +214,13 @@ StorePathSet sendInputs(
     counter & nrStepsCopyingTo
 )
 {
-
-    StorePathSet inputs;
     BasicDerivation basicDrv(*step.drv);
 
-    for (auto & p : step.drv->inputSrcs)
-        inputs.insert(p);
-
     for (auto & input : step.drv->inputDrvs) {
         auto drv2 = localStore.readDerivation(input.first);
         for (auto & name : input.second) {
             if (auto i = get(drv2.outputs, name)) {
                 auto outPath = i->path(localStore, drv2.name, name);
-                inputs.insert(*outPath);
                 basicDrv.inputSrcs.insert(*outPath);
             }
         }
@@ -255,10 +249,10 @@ StorePathSet sendInputs(
         /* Copy the input closure. */
         if (conn.machine->isLocalhost()) {
             StorePathSet closure;
-            destStore.computeFSClosure(inputs, closure);
+            destStore.computeFSClosure(basicDrv.inputSrcs, closure);
             copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
         } else {
-            copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, inputs, true);
+            copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, basicDrv.inputSrcs, true);
         }
 
         auto now2 = std::chrono::steady_clock::now();
@@ -266,7 +260,7 @@ StorePathSet sendInputs(
         overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     }
 
-    return inputs;
+    return basicDrv;
 }
 
 void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
@@ -535,8 +529,7 @@ void State::buildRemote(ref<Store> destStore,
            copy the immediate sources of the derivation and the required
            outputs of the input derivations. */
         updateStep(ssSendingInputs);
-
-        StorePathSet inputs = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo);
+        BasicDerivation resolvedDrv = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo);
 
         logFileDel.cancel();
 
@@ -561,7 +554,7 @@ void State::buildRemote(ref<Store> destStore,
             conn,
             *localStore,
             step->drvPath,
-            BasicDerivation(*step->drv),
+            resolvedDrv,
             buildOptions,
             nrStepsBuilding
         );

From a179f0be610f37635c981de441c73d246b88f1fb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Apr 2022 15:26:03 +0000
Subject: [PATCH 013/285] build(deps): bump cachix/install-nix-action from 16
 to 17

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 16 to 17.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v16...v17)

---
updated-dependencies:
- dependency-name: cachix/install-nix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3ba4aba6..0f5f43da 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,6 +9,6 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v16
+    - uses: cachix/install-nix-action@v17
     #- run: nix flake check
     - run: nix-build -A checks.x86_64-linux.build -A checks.x86_64-linux.validate-openapi

From 3c71be5b5b9b8fd4739c925ab46642f6121ec218 Mon Sep 17 00:00:00 2001
From: Ulrik Strid <ulrik.strid@outlook.com>
Date: Wed, 18 May 2022 08:14:00 +0200
Subject: [PATCH 014/285] GithubPulls: Don't fail on missing `Link`

---
 src/lib/Hydra/Plugin/GithubPulls.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/GithubPulls.pm b/src/lib/Hydra/Plugin/GithubPulls.pm
index db0e8d25..9d8412c3 100644
--- a/src/lib/Hydra/Plugin/GithubPulls.pm
+++ b/src/lib/Hydra/Plugin/GithubPulls.pm
@@ -30,7 +30,7 @@ sub _iterate {
         $pulls->{$pull->{number}} = $pull;
     }
     # TODO Make Link header parsing more robust!!!
-    my @links = split ',', $res->header("Link");
+    my @links = split ',', ($res->header("Link") // "");
     my $next = "";
     foreach my $link (@links) {
         my ($url, $rel) = split ";", $link;

From a8b590014b4b3f59aaf8eec3465f600b5899b2fd Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 22 May 2022 14:14:14 +0200
Subject: [PATCH 015/285] Fix email notifications for jobsets w/git-inputs

I started to wonder quite recently why Hydra doesn't send email
notifications anymore to me. I saw the following issue in the log of
`hydra-notify.service`:

    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1887289]: fatal: unsafe repository ('/var/lib/hydra/scm/git/3e70c16c266ef70dc4198705a688acccf71e932878f178277c9ac47d133cc663' is owned by someone else)
    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1887289]: To add an exception for this directory, call:
    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1887289]:         git config --global --add safe.directory /var/lib/hydra/scm/git/3e70c16c266ef70dc4198705a688acccf71e932878f178277c9ac47d133cc663
    May 22 11:57:29 hydra 9bik0bxyxbrklhx6lqwifd6af8kj84va-hydra-notify[1886654]: error running build_finished hooks: command `git log --pretty=format:%H%x09%an%x09%ae%x09%at b0c30a7557685d25a8ab3f34fdb775e66db0bc4c..eaf28389fcebc2beca13a802f79b2cca6e9ca309 --git-dir=.git' failed with e>

This is also a problem because of Git's fix for CVE-2022-24765[1], so I
applied the same fix as for Nix[2], by using `--git-dir` which skips the
code-path for the ownership-check[3].

[1] https://lore.kernel.org/git/xmqqv8veb5i6.fsf@gitster.g/
[2] https://github.com/NixOS/nix/pull/6440
[3] To quote `git(1)`:
    > Specifying the location of the ".git" directory using this option
    > (or GIT_DIR environment variable) turns off the repository
    > discovery that tries to find a directory with ".git" subdirectory
---
 src/lib/Hydra/Plugin/GitInput.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm
index aca35c30..e5fc7de9 100644
--- a/src/lib/Hydra/Plugin/GitInput.pm
+++ b/src/lib/Hydra/Plugin/GitInput.pm
@@ -261,7 +261,7 @@ sub getCommits {
 
     my $clonePath = getSCMCacheDir . "/git/" . sha256_hex($uri);
 
-    my $out = grab(cmd => ["git", "log", "--pretty=format:%H%x09%an%x09%ae%x09%at", "$rev1..$rev2"], dir => $clonePath);
+    my $out = grab(cmd => ["git", "--git-dir=.git", "log", "--pretty=format:%H%x09%an%x09%ae%x09%at", "$rev1..$rev2"], dir => $clonePath);
 
     my $res = [];
     foreach my $line (split /\n/, $out) {

From b6ea85a601ddac9cb0716d8cb4d446439fa0778f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= <josef.kemetmueller@ims.co.at>
Date: Fri, 27 May 2022 11:40:49 +0200
Subject: [PATCH 016/285] scmdiff: Hardcode `--git-dir`

The newest version of git refuses to work on repositories not owned by
the current user. This leads to issues with the /api/scmdiff endpoint:

May 27 11:16:05 myhydra hydra-server[923698]: fatal: unsafe repository ('/var/lib/hydra/scm/git/57ea036ec7ecd85c8dd085e02ecc6f12dd5c079a6203d16aea49f586cadfb2be' is owned by someone else)
May 27 11:16:05 myhydra hydra-server[923698]: To add an exception for this directory, call:
May 27 11:16:05 myhydra hydra-server[923698]:         git config --global --add safe.directory /var/lib/hydra/scm/git/57ea036ec7ecd85c8dd085e02ecc6f12dd5c079a6203d16aea49f586cadfb2be
May 27 11:16:05 myhydra hydra-server[923701]: warning: Not a git repository. Use --no-index to compare two paths outside a working tree
May 27 11:16:05 myhydra hydra-server[923701]: usage: git diff --no-index [<options>] <path> <path>

I used the same solution that was used in NixOS/nix#6440.

Fixes #1214
---
 src/lib/Hydra/Controller/API.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
index 6f10ef57..8ebed599 100644
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -216,8 +216,8 @@ sub scmdiff : Path('/api/scmdiff') Args(0) {
     } elsif ($type eq "git") {
         my $clonePath = getSCMCacheDir . "/git/" . sha256_hex($uri);
         die if ! -d $clonePath;
-        $diff .= `(cd $clonePath; git log $rev1..$rev2)`;
-        $diff .= `(cd $clonePath; git diff $rev1..$rev2)`;
+        $diff .= `(cd $clonePath; git --git-dir .git log $rev1..$rev2)`;
+        $diff .= `(cd $clonePath; git --git-dir .git diff $rev1..$rev2)`;
     }
 
     $c->stash->{'plain'} = { data => (scalar $diff) || " " };

From 5c01800fbe49939c53566457047283eb5c93f51b Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 16 Jun 2022 14:54:57 +0200
Subject: [PATCH 017/285] flake: Update Nix to 2.9.1

NOTE: I'm well-aware that we have to be careful with this to avoid new
regressions on hydra.nixos.org, so this should only be merged after
extensive testing from more people.

Motivation: I updated Nix in my deployment to 2.9.1 and decided to also
update Hydra in one go (and compile it against the newer Nix). Given
that this also updates the C++ code in `hydra-{queue-runner,eval-jobs}`
this patch might become useful in the future though.
---
 flake.lock                              | 12 +++++++-----
 flake.nix                               |  2 +-
 src/hydra-eval-jobs/Makefile.am         |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc  | 10 +++++-----
 src/hydra-queue-runner/queue-monitor.cc |  2 +-
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/flake.lock b/flake.lock
index 5c726a24..1310c53d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -39,16 +39,18 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1649172203,
-        "narHash": "sha256-Q3nYaXqbseDOvZrlePKeIrx0/KzqyrtNpxHIUbtFHuI=",
+        "lastModified": 1654014617,
+        "narHash": "sha256-qNL3lQPBsnStkru3j1ajN/H+knXI+X3dku8/dBfSw3g=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "5fe4fe823c193cbb7bfa05a468de91eeab09058d",
+        "rev": "624e38aa43f304fbb78b4779172809add042b513",
         "type": "github"
       },
       "original": {
-        "id": "nix",
-        "type": "indirect"
+        "owner": "NixOS",
+        "ref": "2.9.1",
+        "repo": "nix",
+        "type": "github"
       }
     },
     "nixpkgs": {
diff --git a/flake.nix b/flake.nix
index 01b0c988..794555ec 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,7 +5,7 @@
   # even 2.7.0's Nixpkgs pin).
   inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  #inputs.nix.url = github:NixOS/nix/2.7.0;
+  inputs.nix.url = github:NixOS/nix/2.9.1;
 
   outputs = { self, newNixpkgs, nixpkgs, nix }:
     let
diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am
index 7a4e9c91..90742a30 100644
--- a/src/hydra-eval-jobs/Makefile.am
+++ b/src/hydra-eval-jobs/Makefile.am
@@ -1,5 +1,5 @@
 bin_PROGRAMS = hydra-eval-jobs
 
 hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
-hydra_eval_jobs_LDADD = $(NIX_LIBS)
+hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd
 hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 918bd451..18d39620 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -197,21 +197,21 @@ static void worker(
 
                 /* If this is an aggregate, then get its constituents. */
                 auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
-                if (a && state.forceBool(*a->value, *a->pos)) {
+                if (a && state.forceBool(*a->value, a->pos)) {
                     auto a = v->attrs->get(state.symbols.create("constituents"));
                     if (!a)
                         throw EvalError("derivation must have a ‘constituents’ attribute");
 
 
                     PathSet context;
-                    state.coerceToString(*a->pos, *a->value, context, true, false);
+                    state.coerceToString(a->pos, *a->value, context, true, false);
                     for (auto & i : context)
                         if (i.at(0) == '!') {
                             size_t index = i.find("!", 1);
                             job["constituents"].push_back(std::string(i, index + 1));
                         }
 
-                    state.forceList(*a->value, *a->pos);
+                    state.forceList(*a->value, a->pos);
                     for (unsigned int n = 0; n < a->value->listSize(); ++n) {
                         auto v = a->value->listElems()[n];
                         state.forceValue(*v, noPos);
@@ -243,8 +243,8 @@ static void worker(
             else if (v->type() == nAttrs) {
                 auto attrs = nlohmann::json::array();
                 StringSet ss;
-                for (auto & i : v->attrs->lexicographicOrder()) {
-                    std::string name(i->name);
+                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
+                    std::string name(state.symbols[i->name]);
                     if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) {
                         printError("skipping job with illegal name '%s'", name);
                         continue;
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 3bde0d99..12d55b79 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -460,7 +460,7 @@ Step::ptr State::createStep(ref<Store> destStore,
     step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
 
     step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore);
-    step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1";
+    step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1";
 
     step->systemType = step->drv->platform;
     {

From 183f58ca9f10a0a6a18e42c75168145a1de864df Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Thu, 16 Jun 2022 22:48:39 +0200
Subject: [PATCH 018/285] Remove url literal

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 794555ec..2e891364 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,7 +5,7 @@
   # even 2.7.0's Nixpkgs pin).
   inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = github:NixOS/nix/2.9.1;
+  inputs.nix.url = "github:NixOS/nix/2.9.1";
 
   outputs = { self, newNixpkgs, nixpkgs, nix }:
     let

From 750978a19232583e17620a1bd80435e957e7213a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Sat, 18 Jun 2022 13:22:42 +0200
Subject: [PATCH 019/285] Add gitea push hook

---
 doc/manual/src/webhooks.md       | 20 +++++++++++++++++---
 src/lib/Hydra/Controller/API.pm  | 16 ++++++++++++++++
 src/lib/Hydra/Controller/Root.pm |  3 ++-
 3 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/webhooks.md b/doc/manual/src/webhooks.md
index 2b26cd61..674e1064 100644
--- a/doc/manual/src/webhooks.md
+++ b/doc/manual/src/webhooks.md
@@ -1,9 +1,12 @@
 # Webhooks
 
-Hydra can be notified by github's webhook to trigger a new evaluation when a
+Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a
 jobset has a github repo in its input.
-To set up a github webhook go to `https://github.com/<yourhandle>/<yourrepo>/settings` and in the `Webhooks` tab
-click on `Add webhook`.
+
+## GitHub
+
+To set up a webhook for a GitHub repository go to `https://github.com/<yourhandle>/<yourrepo>/settings`
+and in the `Webhooks` tab click on `Add webhook`.
 
 - In `Payload URL` fill in `https://<your-hydra-domain>/api/push-github`.
 - In `Content type` switch to `application/json`.
@@ -11,3 +14,14 @@ click on `Add webhook`.
 - For `Which events would you like to trigger this webhook?` keep the default option for events on `Just the push event.`.
 
 Then add the hook with `Add webhook`.
+
+## Gitea
+
+To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance
+and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down.
+
+- In `Target URL` fill in `https://<your-hydra-domain>/api/push-gitea`.
+- Keep HTTP method `POST`, POST Content Type `application/json` and Trigger On `Push Events`.
+- Change the branch filter to match the git branch hydra builds.
+
+Then add the hook with `Add webhook`.
diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
index 6f10ef57..12073595 100644
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -285,6 +285,22 @@ sub push_github : Chained('api') PathPart('push-github') Args(0) {
     $c->response->body("");
 }
 
+sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) {
+    my ($self, $c) = @_;
+
+    $c->{stash}->{json}->{jobsetsTriggered} = [];
+
+    my $in = $c->request->{data};
+    my $url = $in->{repository}->{clone_url} or die;
+    print STDERR "got push from Gitea repository $url\n";
+
+    triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search(
+        { 'project.enabled' => 1, 'me.enabled' => 1 },
+        { join => 'project'
+        , where => \ [ 'me.flake like ? or exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value like ?)', [ 'flake', "%$url%"], [ 'value', "%$url%" ] ]
+        });
+    $c->response->body("");
+}
 
 
 1;
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index c6843d29..1b33db2a 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -32,6 +32,7 @@ sub noLoginNeeded {
 
   return $whitelisted ||
          $c->request->path eq "api/push-github" ||
+         $c->request->path eq "api/push-gitea" ||
          $c->request->path eq "google-login" ||
          $c->request->path eq "github-redirect" ||
          $c->request->path eq "github-login" ||
@@ -77,7 +78,7 @@ sub begin :Private {
     $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};
 
     # XSRF protection: require POST requests to have the same origin.
-    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github") {
+    if ($c->req->method eq "POST" && $c->req->path ne "api/push-github" && $c->req->path ne "api/push-gitea") {
         my $referer = $c->req->header('Referer');
         $referer //= $c->req->header('Origin');
         my $base = $c->req->base;

From bab671124df0bc43a521d708bf86eece908d33af Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Thu, 30 Jun 2022 00:24:09 +0200
Subject: [PATCH 020/285] replace nix cat-store with nix store cat

the former was deprecated in favor of the latter
---
 src/lib/Hydra/Controller/Build.pm | 4 ++--
 src/lib/Hydra/Helper/Nix.pm       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index 552f31af..c7811c62 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -234,7 +234,7 @@ sub serveFile {
     elsif ($ls->{type} eq "regular") {
 
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
-                                                      "cat-store", "--store", getStoreUri(), "$path"]) };
+                                                      "store", "cat", "--store", getStoreUri(), "$path"]) };
 
         # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple.
         my $type = "text/plain";
@@ -366,7 +366,7 @@ sub contents : Chained('buildChain') PathPart Args(1) {
 
     # FIXME: don't use shell invocations below.
 
-    # FIXME: use nix cat-store
+    # FIXME: use nix store cat
 
     my $res;
 
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 514fb439..71a8a7d7 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -537,7 +537,7 @@ sub getStoreUri {
 sub readNixFile {
     my ($path) = @_;
     return grab(cmd => ["nix", "--experimental-features", "nix-command",
-                        "cat-store", "--store", getStoreUri(), "$path"]);
+                        "store", "cat", "--store", getStoreUri(), "$path"]);
 }
 
 

From bb1f04ed8669cc012fc552a40a376f3b93228833 Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Thu, 30 Jun 2022 00:32:31 +0200
Subject: [PATCH 021/285] AddBuilds: fix declarative jobsets with dynamic
 runcommand enabled

$project->{enable_dynamic_run_command} is undefined
---
 src/lib/Hydra/Helper/AddBuilds.pm     |  2 +-
 t/Helper/AddBuilds/dynamic-disabled.t | 36 +++++++++++++++++++++------
 t/Helper/AddBuilds/dynamic-enabled.t  | 36 +++++++++++++++++++++------
 3 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index 9e3ddfd2..a6373be5 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -67,7 +67,7 @@ sub validateDeclarativeJobset {
     my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
     if ($enable_dynamic_run_command
         && !($config->{dynamicruncommand}->{enable}
-            && $project->{enable_dynamic_run_command}))
+            && $project->enable_dynamic_run_command))
     {
         die "Dynamic RunCommand is not enabled by the server or the parent project.";
     }
diff --git a/t/Helper/AddBuilds/dynamic-disabled.t b/t/Helper/AddBuilds/dynamic-disabled.t
index 0507b03e..0c91f382 100644
--- a/t/Helper/AddBuilds/dynamic-disabled.t
+++ b/t/Helper/AddBuilds/dynamic-disabled.t
@@ -6,11 +6,31 @@ use Test2::V0;
 require Catalyst::Test;
 use HTTP::Request::Common qw(POST PUT GET DELETE);
 use JSON::MaybeXS qw(decode_json encode_json);
-use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
-use Hydra::Helper::Nix qw(getHydraConfig);
 
 my $ctx = test_context();
 
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+my $project_with_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_with_dynamic_runcommand',
+    displayname => 'Tests with dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 1,
+});
+my $project_without_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_without_dynamic_runcommand',
+    displayname => 'Tests without dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 0,
+});
+
 sub makeJobsetSpec {
     my ($dynamic) = @_;
 
@@ -29,14 +49,16 @@ sub makeJobsetSpec {
 };
 
 subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
-    my $config = getHydraConfig();
+    my $config = Hydra::Helper::Nix->getHydraConfig();
+    require Hydra::Helper::AddBuilds;
+    Hydra::Helper::AddBuilds->import( qw(validateDeclarativeJobset) );
 
     subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
         like(
             dies {
               validateDeclarativeJobset(
                   $config,
-                  { enable_dynamic_run_command => 1 },
+                  $project_with_dynamic_run_command,
                   "test-jobset",
                   makeJobsetSpec(JSON::MaybeXS::true),
               ),
@@ -49,7 +71,7 @@ subtest "validate declarative jobset with dynamic RunCommand disabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 1 },
+                $project_with_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),
@@ -61,7 +83,7 @@ subtest "validate declarative jobset with dynamic RunCommand disabled by server"
             dies {
               validateDeclarativeJobset(
                   $config,
-                  { enable_dynamic_run_command => 0 },
+                  $project_without_dynamic_run_command,
                   "test-jobset",
                   makeJobsetSpec(JSON::MaybeXS::true),
               ),
@@ -74,7 +96,7 @@ subtest "validate declarative jobset with dynamic RunCommand disabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 0 },
+                $project_without_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),
diff --git a/t/Helper/AddBuilds/dynamic-enabled.t b/t/Helper/AddBuilds/dynamic-enabled.t
index d2f5a386..46497bed 100644
--- a/t/Helper/AddBuilds/dynamic-enabled.t
+++ b/t/Helper/AddBuilds/dynamic-enabled.t
@@ -6,8 +6,6 @@ use Test2::V0;
 require Catalyst::Test;
 use HTTP::Request::Common qw(POST PUT GET DELETE);
 use JSON::MaybeXS qw(decode_json encode_json);
-use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
-use Hydra::Helper::Nix qw(getHydraConfig);
 
 my $ctx = test_context(
     hydra_config => q|
@@ -17,6 +15,28 @@ my $ctx = test_context(
     |
 );
 
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+my $project_with_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_with_dynamic_runcommand',
+    displayname => 'Tests with dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 1,
+});
+my $project_without_dynamic_run_command = $db->resultset('Projects')->create({
+    name => 'tests_without_dynamic_runcommand',
+    displayname => 'Tests without dynamic runcommand',
+    owner => 'alice',
+    enable_dynamic_run_command => 0,
+});
+
 sub makeJobsetSpec {
     my ($dynamic) = @_;
 
@@ -35,13 +55,15 @@ sub makeJobsetSpec {
 };
 
 subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
-    my $config = getHydraConfig();
+    my $config = Hydra::Helper::Nix->getHydraConfig();
+    require Hydra::Helper::AddBuilds;
+    Hydra::Helper::AddBuilds->import( qw(validateDeclarativeJobset) );
 
     subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 1 },
+                $project_with_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::true)
             ),
@@ -52,7 +74,7 @@ subtest "validate declarative jobset with dynamic RunCommand enabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 1 },
+                $project_with_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),
@@ -64,7 +86,7 @@ subtest "validate declarative jobset with dynamic RunCommand enabled by server"
             dies {
               validateDeclarativeJobset(
                   $config,
-                  { enable_dynamic_run_command => 0 },
+                  $project_without_dynamic_run_command,
                   "test-jobset",
                   makeJobsetSpec(JSON::MaybeXS::true),
               ),
@@ -77,7 +99,7 @@ subtest "validate declarative jobset with dynamic RunCommand enabled by server"
         ok(
             validateDeclarativeJobset(
                 $config,
-                { enable_dynamic_run_command => 0 },
+                $project_without_dynamic_run_command,
                 "test-jobset",
                 makeJobsetSpec(JSON::MaybeXS::false)
             ),

From a81c6a3a80d1055aa80934ab229e2dc49594edd2 Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Fri, 1 Jul 2022 22:21:32 +0200
Subject: [PATCH 022/285] Match URIs that don't end in .git

Co-authored-by: Charlotte <lotte@chir.rs>
---
 src/lib/Hydra/Controller/API.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
index 12073595..5eeb0c04 100644
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -292,6 +292,7 @@ sub push_gitea : Chained('api') PathPart('push-gitea') Args(0) {
 
     my $in = $c->request->{data};
     my $url = $in->{repository}->{clone_url} or die;
+    $url =~ s/.git$//;
     print STDERR "got push from Gitea repository $url\n";
 
     triggerJobset($self, $c, $_, 0) foreach $c->model('DB::Jobsets')->search(

From 38e033e7ceb8fecea9fe3d8c621e77d1af884948 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Sun, 10 Jul 2022 13:31:21 +0200
Subject: [PATCH 023/285] Remove yet another URL literal

---
 shell.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shell.nix b/shell.nix
index 9e967032..1ad58f49 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 # The `default.nix` in flake-compat reads `flake.nix` and `flake.lock` from `src` and
 # returns an attribute set of the shape `{ defaultNix, shellNix }`
 
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
   src = ./.;
 }).shellNix

From 8dd1daac8a9bd1a5b4d8f37969c344d5ef593c6e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 12:14:50 +0200
Subject: [PATCH 024/285] Update to Nix 2.10

---
 flake.nix        | 28 +++++++++++++---------------
 hydra-module.nix |  2 +-
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2e891364..36863913 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,7 +5,7 @@
   # even 2.7.0's Nixpkgs pin).
   inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:NixOS/nix/2.9.1";
+  inputs.nix.url = "github:NixOS/nix/2.10.0";
 
   outputs = { self, newNixpkgs, nixpkgs, nix }:
     let
@@ -14,7 +14,7 @@
 
       pkgs = import nixpkgs {
         system = "x86_64-linux";
-        overlays = [ self.overlay nix.overlay ];
+        overlays = [ self.overlay nix.overlays.default ];
       };
 
       # NixOS configuration used for VM tests.
@@ -332,7 +332,7 @@
               url = "mirror://cpan/authors/id/A/AA/AAR/Net-LDAP-Server-0.43.tar.gz";
               sha256 = "0qmh3cri3fpccmwz6bhwp78yskrb3qmalzvqn0a23hqbsfs4qv6x";
             };
-            propagatedBuildInputs = with final.perlPackages; [ NetLDAP ConvertASN1 ];
+            propagatedBuildInputs = with final.perlPackages; [ perlldap ConvertASN1 ];
             meta = {
               description = "LDAP server side protocol handling";
               license = with final.lib.licenses; [ artistic1 ];
@@ -359,7 +359,7 @@
               url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-Server-Test-0.22.tar.gz";
               sha256 = "13idip7jky92v4adw60jn2gcc3zf339gsdqlnc9nnvqzbxxp285i";
             };
-            propagatedBuildInputs = with final.perlPackages; [ NetLDAP NetLDAPServer TestMore DataDump NetLDAPSID ];
+            propagatedBuildInputs = with final.perlPackages; [ perlldap NetLDAPServer DataDump NetLDAPSID ];
             meta = {
               description = "test Net::LDAP code";
               license = with final.lib.licenses; [ artistic1 ];
@@ -373,8 +373,8 @@
               url = "mirror://cpan/authors/id/I/IL/ILMARI/Catalyst-Authentication-Store-LDAP-1.016.tar.gz";
               sha256 = "0cm399vxqqf05cjgs1j5v3sk4qc6nmws5nfhf52qvpbwc4m82mq8";
             };
-            propagatedBuildInputs = with final.perlPackages; [ NetLDAP CatalystPluginAuthentication ClassAccessorFast ];
-            buildInputs = with final.perlPackages; [ TestMore TestMockObject TestException NetLDAPServerTest ];
+            propagatedBuildInputs = with final.perlPackages; [ perlldap CatalystPluginAuthentication ClassAccessor ];
+            buildInputs = with final.perlPackages; [ TestMockObject TestException NetLDAPServerTest ];
             meta = {
               description = "Authentication from an LDAP Directory";
               license = with final.lib.licenses; [ artistic1 ];
@@ -486,7 +486,6 @@
                 CatalystPluginSessionStateCookie
                 CatalystPluginSessionStoreFastMmap
                 CatalystPluginStackTrace
-                CatalystPluginUnicodeEncoding
                 CatalystTraitForRequestProxyBase
                 CatalystViewDownload
                 CatalystViewJSON
@@ -533,7 +532,6 @@
                 TermSizeAny
                 TermReadKey
                 Test2Harness
-                TestMore
                 TestPostgreSQL
                 TextDiff
                 TextTable
@@ -558,9 +556,9 @@
               libtool
               unzip
               nukeReferences
-              pkgconfig
+              pkg-config
               libpqxx
-              gitAndTools.topGit
+              top-git
               mercurial
               darcs
               subversion
@@ -585,7 +583,7 @@
             cacert
             # FIXME: foreman is broken on all nix/nixpkgs pin, up to and
             # including 2.7.0
-            newNixpkgs.legacyPackages.${final.system}.foreman
+            newNixpkgs.legacyPackages.${final.stdenv.system}.foreman
             glibcLocales
             libressl.nc
             openldap
@@ -602,11 +600,11 @@
               pixz
               gzip
               bzip2
-              lzma
+              xz
               gnutar
               unzip
               git
-              gitAndTools.topGit
+              top-git
               mercurial
               darcs
               gnused
@@ -661,7 +659,7 @@
 
           dontStrip = true;
 
-          meta.description = "Build of Hydra on ${system}";
+          meta.description = "Build of Hydra on ${final.stdenv.system}";
           passthru = { inherit perlDeps; inherit (final) nix; };
         };
       };
@@ -966,7 +964,7 @@
 
       nixosModules.hydra = {
         imports = [ ./hydra-module.nix ];
-        nixpkgs.overlays = [ self.overlay nix.overlay ];
+        nixpkgs.overlays = [ self.overlay nix.overlays.default ];
       };
 
       nixosModules.hydraTest = {
diff --git a/hydra-module.nix b/hydra-module.nix
index 0df5e690..f826ef36 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -268,7 +268,7 @@ in
         environment = env // {
           HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
         };
-        path = [ pkgs.utillinux ];
+        path = [ pkgs.util-linux ];
         preStart = ''
           ln -sf ${hydraConf} ${baseDir}/hydra.conf
 

From e06c480fd67d71fe1ab175c3f14e694f08a0cae0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 11:58:38 +0200
Subject: [PATCH 025/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'newNixpkgs':
    'github:NixOS/nixpkgs/6e3ee8957637a60f5072e33d78e05c0f65c54366' (2022-03-15)
  → 'github:NixOS/nixpkgs/de5b3dd17034e6106e75746e81618e5bd408de8a' (2022-07-10)
• Updated input 'nix':
    'github:NixOS/nix/624e38aa43f304fbb78b4779172809add042b513' (2022-05-31)
  → 'github:NixOS/nix/b9cf655150b52d071c85a337cb5db96e735fa64a' (2022-07-11)
• Updated input 'nix/nixpkgs':
    'github:NixOS/nixpkgs/530a53dcbc9437363471167a5e4762c5fcfa34a1' (2022-02-19)
  → 'github:NixOS/nixpkgs/2fa57ed190fd6c7c746319444f34b5917666e5c1' (2022-05-31)
---
 flake.lock | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1310c53d..befc8e5b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -18,11 +18,11 @@
     },
     "newNixpkgs": {
       "locked": {
-        "lastModified": 1647380550,
-        "narHash": "sha256-909TI9poX7CIUiFx203WL29YON6m/I6k0ExbZvR7bLM=",
+        "lastModified": 1657425264,
+        "narHash": "sha256-3aHvoI2e8vJKw3hvnHECaBpSsL5mxVsVtaLCnTdNcH8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6e3ee8957637a60f5072e33d78e05c0f65c54366",
+        "rev": "de5b3dd17034e6106e75746e81618e5bd408de8a",
         "type": "github"
       },
       "original": {
@@ -39,33 +39,34 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1654014617,
-        "narHash": "sha256-qNL3lQPBsnStkru3j1ajN/H+knXI+X3dku8/dBfSw3g=",
+        "lastModified": 1657569404,
+        "narHash": "sha256-zJONRtGALmYifrWKzcH6MMfSKLxeuW2iqG13500OrY4=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "624e38aa43f304fbb78b4779172809add042b513",
+        "rev": "b9cf655150b52d071c85a337cb5db96e735fa64a",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.9.1",
+        "ref": "2.10.0",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1645296114,
-        "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
+        "lastModified": 1653988320,
+        "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
+        "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1",
         "type": "github"
       },
       "original": {
-        "id": "nixpkgs",
-        "ref": "nixos-21.05-small",
-        "type": "indirect"
+        "owner": "NixOS",
+        "ref": "nixos-22.05-small",
+        "repo": "nixpkgs",
+        "type": "github"
       }
     },
     "nixpkgs-regression": {
@@ -78,9 +79,10 @@
         "type": "github"
       },
       "original": {
-        "id": "nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
         "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
-        "type": "indirect"
+        "type": "github"
       }
     },
     "root": {

From c72bed5cb4f24853ebb7daaf26bb67d3cb4397f7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 14:28:55 +0200
Subject: [PATCH 026/285] Fix tests

Use $NIX_REMOTE instead of the legacy environment variables.
---
 t/lib/HydraTestContext.pm      | 16 ++++++++++------
 t/queue-runner/notifications.t |  4 +---
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index badb3728..53eaa0f7 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -53,16 +53,19 @@ sub new {
     my $hydra_config = $opts{'hydra_config'} || "";
     $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config;
     if ($opts{'use_external_destination_store'} // 1) {
-        $hydra_config = "store_uri = file:$dir/nix/dest-store\n" . $hydra_config;
+        $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config;
     }
 
     write_file($ENV{'HYDRA_CONFIG'}, $hydra_config);
 
-    $ENV{'NIX_LOG_DIR'} = "$dir/nix/var/log/nix";
+    my $nix_store_dir = "$dir/nix/store";
+    my $nix_state_dir = "$dir/nix/var/nix";
+    my $nix_log_dir = "$dir/nix/var/log/nix";
+
     $ENV{'NIX_REMOTE_SYSTEMS'} = '';
-    $ENV{'NIX_REMOTE'} = '';
-    $ENV{'NIX_STATE_DIR'} = "$dir/nix/var/nix";
-    $ENV{'NIX_STORE_DIR'} = "$dir/nix/store";
+    $ENV{'NIX_REMOTE'} = "local?store=$nix_store_dir&state=$nix_state_dir&log=$nix_log_dir";
+    $ENV{'NIX_STATE_DIR'} = $nix_state_dir; # FIXME: remove
+    $ENV{'NIX_STORE_DIR'} = $nix_store_dir; # FIXME: remove
 
     my $pgsql = Test::PostgreSQL->new(
         extra_initdb_args => "--locale C.UTF-8"
@@ -73,7 +76,8 @@ sub new {
         _db => undef,
         db_handle => $pgsql,
         tmpdir => $dir,
-        nix_state_dir => "$dir/nix/var/nix",
+        nix_state_dir => $nix_state_dir,
+        nix_log_dir => $nix_log_dir,
         testdir => abs_path(dirname(__FILE__) . "/.."),
         jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
     }, $class;
diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t
index b35b2b2f..1966cde1 100644
--- a/t/queue-runner/notifications.t
+++ b/t/queue-runner/notifications.t
@@ -33,9 +33,6 @@ my $ctx = test_context(
 # the build locally.
 
 subtest "Pre-build the job, upload to the cache, and then delete locally" => sub {
-    my $scratchlogdir = File::Temp->newdir();
-    $ENV{'NIX_LOG_DIR'} = "$scratchlogdir";
-
     my $outlink = $ctx->tmpdir . "/basic-canbesubstituted";
     is(system('nix-build', $ctx->jobsdir . '/notifications.nix', '-A', 'canbesubstituted', '--out-link', $outlink), 0, "Building notifications.nix succeeded");
     is(system('nix', 'copy', '--to', "file://${binarycachedir}", $outlink), 0, "Copying the closure to the binary cache succeeded");
@@ -46,6 +43,7 @@ subtest "Pre-build the job, upload to the cache, and then delete locally" => sub
     is(system('nix', 'log', $outpath), 0, "Reading the output's log succeeds");
     is(system('nix-store', '--delete', $outpath), 0, "Deleting the notifications.nix output succeeded");
     is(system("nix-collect-garbage"), 0, "Delete all the system's garbage");
+    File::Path::rmtree($ctx->{nix_log_dir});
 };
 
 subtest "Ensure substituting the job works, but reading the log fails" => sub {

From d5ba1bba50a7fa2f343d9ad284e293e13bb13f85 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 14:46:48 +0200
Subject: [PATCH 027/285] Fix deprecation warning

---
 flake.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index 36863913..db4eb854 100644
--- a/flake.nix
+++ b/flake.nix
@@ -681,7 +681,7 @@
         tests.install.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           simpleTest {
-            machine = hydraServer;
+            nodes.machine = hydraServer;
             testScript =
               ''
                 machine.wait_for_job("hydra-init")
@@ -696,7 +696,7 @@
         tests.notifications.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           simpleTest {
-            machine = { pkgs, ... }: {
+            nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
               services.hydra-dev.extraConfig = ''
                 <influxdb>
@@ -753,7 +753,7 @@
         tests.gitea.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           makeTest {
-            machine = { pkgs, ... }: {
+            nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
               services.hydra-dev.extraConfig = ''
                 <gitea_authorization>

From 3e001a8f05cbd0f7f5a8c0fc2d8da8f839d7b3f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 14:58:25 +0200
Subject: [PATCH 028/285] Remove newNixpkgs and a lot of packages that are in
 Nixpkgs 22.05

---
 flake.lock |  17 ---
 flake.nix  | 419 +----------------------------------------------------
 2 files changed, 3 insertions(+), 433 deletions(-)

diff --git a/flake.lock b/flake.lock
index befc8e5b..012d2407 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,22 +16,6 @@
         "type": "github"
       }
     },
-    "newNixpkgs": {
-      "locked": {
-        "lastModified": 1657425264,
-        "narHash": "sha256-3aHvoI2e8vJKw3hvnHECaBpSsL5mxVsVtaLCnTdNcH8=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "de5b3dd17034e6106e75746e81618e5bd408de8a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "nix": {
       "inputs": {
         "lowdown-src": "lowdown-src",
@@ -87,7 +71,6 @@
     },
     "root": {
       "inputs": {
-        "newNixpkgs": "newNixpkgs",
         "nix": "nix",
         "nixpkgs": [
           "nix",
diff --git a/flake.nix b/flake.nix
index db4eb854..cb1d9aea 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,13 +1,10 @@
 {
   description = "A Nix-based continuous build system";
 
-  # FIXME: All the pinned versions of nix/nixpkgs have a broken foreman (yes,
-  # even 2.7.0's Nixpkgs pin).
-  inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
   inputs.nixpkgs.follows = "nix/nixpkgs";
   inputs.nix.url = "github:NixOS/nix/2.10.0";
 
-  outputs = { self, newNixpkgs, nixpkgs, nix }:
+  outputs = { self, nixpkgs, nix }:
     let
 
       version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
@@ -41,153 +38,8 @@
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlay = final: prev: {
 
-        # Overlay these packages to use dependencies from the Nixpkgs everything
-        # else uses, to side-step the version difference: glibc is 2.32 in the
-        # nix-pinned Nixpkgs, but 2.33 in the newNixpkgs commit.
-        civetweb = (final.callPackage "${newNixpkgs}/pkgs/development/libraries/civetweb" { }).overrideAttrs
-          # Can be dropped once newNixpkgs points to a revision containing
-          # https://github.com/NixOS/nixpkgs/pull/167751
-          ({ cmakeFlags ? [ ], ... }: {
-            cmakeFlags = cmakeFlags ++ [
-              "-DCIVETWEB_ENABLE_IPV6=1"
-            ];
-          });
-        prometheus-cpp = final.callPackage "${newNixpkgs}/pkgs/development/libraries/prometheus-cpp" { };
-
         # Add LDAP dependencies that aren't currently found within nixpkgs.
         perlPackages = prev.perlPackages // {
-          TestPostgreSQL = final.perlPackages.buildPerlModule {
-            pname = "Test-PostgreSQL";
-            version = "1.28-1";
-            src = final.fetchFromGitHub {
-              owner = "grahamc";
-              repo = "Test-postgresql";
-              rev = "release-1.28-1";
-              hash = "sha256-SFC1C3q3dbcBos18CYd/s0TIcfJW4g04ld0+XQXVToQ=";
-            };
-            buildInputs = with final.perlPackages; [ ModuleBuildTiny TestSharedFork pkgs.postgresql ];
-            propagatedBuildInputs = with final.perlPackages; [ DBDPg DBI FileWhich FunctionParameters Moo TieHashMethod TryTiny TypeTiny ];
-
-            makeMakerFlags = "POSTGRES_HOME=${final.postgresql}";
-
-            meta = {
-              homepage = "https://github.com/grahamc/Test-postgresql/releases/tag/release-1.28-1";
-              description = "PostgreSQL runner for tests";
-              license = with final.lib.licenses; [ artistic2 ];
-            };
-          };
-
-          FunctionParameters = final.perlPackages.buildPerlPackage {
-            pname = "Function-Parameters";
-            version = "2.001003";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/M/MA/MAUKE/Function-Parameters-2.001003.tar.gz";
-              sha256 = "eaa22c6b43c02499ec7db0758c2dd218a3b2ab47a714b2bdf8010b5ee113c242";
-            };
-            buildInputs = with final.perlPackages; [ DirSelf TestFatal ];
-            meta = {
-              description = "Define functions and methods with parameter lists (\"subroutine signatures\")";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          CatalystPluginPrometheusTiny = final.perlPackages.buildPerlPackage {
-            pname = "Catalyst-Plugin-PrometheusTiny";
-            version = "0.005";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/S/SY/SYSPETE/Catalyst-Plugin-PrometheusTiny-0.005.tar.gz";
-              sha256 = "a42ef09efdc3053899ae007c41220d3ed7207582cc86e491b4f534539c992c5a";
-            };
-            buildInputs = with final.perlPackages; [ HTTPMessage Plack SubOverride TestDeep ];
-            propagatedBuildInputs = with final.perlPackages; [ CatalystRuntime Moose PrometheusTiny PrometheusTinyShared ];
-            meta = {
-              description = "Prometheus metrics for Catalyst";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          CryptArgon2 = final.perlPackages.buildPerlModule {
-            pname = "Crypt-Argon2";
-            version = "0.010";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Argon2-0.010.tar.gz";
-              sha256 = "3ea1c006f10ef66fd417e502a569df15c4cc1c776b084e35639751c41ce6671a";
-            };
-            nativeBuildInputs = [ pkgs.ld-is-cc-hook ];
-            meta = {
-              description = "Perl interface to the Argon2 key derivation functions";
-              license = final.lib.licenses.cc0;
-            };
-          };
-
-          CryptPassphrase = final.perlPackages.buildPerlPackage {
-            pname = "Crypt-Passphrase";
-            version = "0.003";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Passphrase-0.003.tar.gz";
-              sha256 = "685aa090f8179a86d6896212ccf8ccfde7a79cce857199bb14e2277a10d240ad";
-            };
-            meta = {
-              description = "A module for managing passwords in a cryptographically agile manner";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          CryptPassphraseArgon2 = final.perlPackages.buildPerlPackage {
-            pname = "Crypt-Passphrase-Argon2";
-            version = "0.002";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/L/LE/LEONT/Crypt-Passphrase-Argon2-0.002.tar.gz";
-              sha256 = "3906ff81697d13804ee21bd5ab78ffb1c4408b4822ce020e92ecf4737ba1f3a8";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ CryptArgon2 CryptPassphrase ];
-            meta = {
-              description = "An Argon2 encoder for Crypt::Passphrase";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          DataRandom = final.perlPackages.buildPerlPackage {
-            pname = "Data-Random";
-            version = "0.13";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/B/BA/BAREFOOT/Data-Random-0.13.tar.gz";
-              sha256 = "eb590184a8db28a7e49eab09e25f8650c33f1f668b6a472829de74a53256bfc0";
-            };
-            buildInputs = with final.perlPackages; [ FileShareDirInstall TestMockTime ];
-            meta = {
-              description = "Perl module to generate random data";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          DirSelf = final.perlPackages.buildPerlPackage {
-            pname = "Dir-Self";
-            version = "0.11";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/M/MA/MAUKE/Dir-Self-0.11.tar.gz";
-              sha256 = "e251a51abc7d9ba3e708f73c2aa208e09d47a0c528d6254710fa78cc8d6885b5";
-            };
-            meta = {
-              homepage = "https://github.com/mauke/Dir-Self";
-              description = "A __DIR__ constant for the directory your source file is in";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          HashSharedMem = final.perlPackages.buildPerlModule {
-            pname = "Hash-SharedMem";
-            version = "0.005";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/Z/ZE/ZEFRAM/Hash-SharedMem-0.005.tar.gz";
-              sha256 = "324776808602f7bdc44adaa937895365454029a926fa611f321c9bf6b940bb5e";
-            };
-            buildInputs = with final.perlPackages; [ ScalarString ];
-            meta = {
-              description = "Efficient shared mutable hash";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
 
           PrometheusTiny = final.perlPackages.buildPerlPackage {
             pname = "Prometheus-Tiny";
@@ -204,269 +56,6 @@
             };
           };
 
-          PrometheusTinyShared = final.perlPackages.buildPerlPackage {
-            pname = "Prometheus-Tiny-Shared";
-            version = "0.023";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-Shared-0.023.tar.gz";
-              sha256 = "7c2c72397be5d8e4839d1bf4033c1800f467f2509689673c6419df48794f2abe";
-            };
-            buildInputs = with final.perlPackages; [ DataRandom HTTPMessage Plack TestDifferences TestException ];
-            propagatedBuildInputs = with final.perlPackages; [ HashSharedMem JSONXS PrometheusTiny ];
-            meta = {
-              homepage = "https://github.com/robn/Prometheus-Tiny-Shared";
-              description = "A tiny Prometheus client with a shared database behind it";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          ReadonlyX = final.perlPackages.buildPerlModule {
-            pname = "ReadonlyX";
-            version = "1.04";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/S/SA/SANKO/ReadonlyX-1.04.tar.gz";
-              sha256 = "81bb97dba93ac6b5ccbce04a42c3590eb04557d75018773ee18d5a30fcf48188";
-            };
-            buildInputs = with final.perlPackages; [ ModuleBuildTiny TestFatal ];
-            meta = {
-              homepage = "https://github.com/sanko/readonly";
-              description = "Faster facility for creating read-only scalars, arrays, hashes";
-              license = final.lib.licenses.artistic2;
-            };
-          };
-
-          TieHashMethod = final.perlPackages.buildPerlPackage {
-            pname = "Tie-Hash-Method";
-            version = "0.02";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/Y/YV/YVES/Tie-Hash-Method-0.02.tar.gz";
-              sha256 = "d513fbb51413f7ca1e64a1bdce6194df7ec6076dea55066d67b950191eec32a9";
-            };
-            meta = {
-              description = "Tied hash with specific methods overriden by callbacks";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          Test2Harness = final.perlPackages.buildPerlPackage {
-            pname = "Test2-Harness";
-            version = "1.000042";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Harness-1.000042.tar.gz";
-              sha256 = "aaf231a68af1a6ffd6a11188875fcf572e373e43c8285945227b9d687b43db2d";
-            };
-
-            checkPhase = ''
-              patchShebangs ./t ./scripts/yath
-              ./scripts/yath test -j $NIX_BUILD_CORES
-            '';
-
-            propagatedBuildInputs = with final.perlPackages; [ DataUUID Importer LongJump ScopeGuard TermTable Test2PluginMemUsage Test2PluginUUID Test2Suite gotofile ];
-            meta = {
-              description = "A new and improved test harness with better Test2 integration";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          Test2PluginMemUsage = prev.perlPackages.buildPerlPackage {
-            pname = "Test2-Plugin-MemUsage";
-            version = "0.002003";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Plugin-MemUsage-0.002003.tar.gz";
-              sha256 = "5e0662d5a823ae081641f5ce82843111eec1831cd31f883a6c6de54afdf87c25";
-            };
-            buildInputs = with final.perlPackages; [ Test2Suite ];
-            meta = {
-              description = "Collect and display memory usage information";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          Test2PluginUUID = prev.perlPackages.buildPerlPackage {
-            pname = "Test2-Plugin-UUID";
-            version = "0.002001";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Test2-Plugin-UUID-0.002001.tar.gz";
-              sha256 = "4c6c8d484d7153d8779dc155a992b203095b5c5aa1cfb1ee8bcedcd0601878c9";
-            };
-            buildInputs = with final.perlPackages;[ Test2Suite ];
-            propagatedBuildInputs = with final.perlPackages; [ DataUUID ];
-            meta = {
-              description = "Use REAL UUIDs in Test2";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          LongJump = final.perlPackages.buildPerlPackage {
-            pname = "Long-Jump";
-            version = "0.000001";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/Long-Jump-0.000001.tar.gz";
-              sha256 = "d5d6456d86992b559d8f66fc90960f919292cd3803c13403faac575762c77af4";
-            };
-            buildInputs = with final.perlPackages; [ Test2Suite ];
-            meta = {
-              description = "Mechanism for returning to a specific point from a deeply nested stack";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          gotofile = final.perlPackages.buildPerlPackage {
-            pname = "goto-file";
-            version = "0.005";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/E/EX/EXODIST/goto-file-0.005.tar.gz";
-              sha256 = "c6cdd5ee4a6cdcbdbf314d92a4f9985dbcdf9e4258048cae76125c052aa31f77";
-            };
-            buildInputs = with final.perlPackages; [ Test2Suite ];
-            meta = {
-              description = "Stop parsing the current file and move on to a different one";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          NetLDAPServer = prev.perlPackages.buildPerlPackage {
-            pname = "Net-LDAP-Server";
-            version = "0.43";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/A/AA/AAR/Net-LDAP-Server-0.43.tar.gz";
-              sha256 = "0qmh3cri3fpccmwz6bhwp78yskrb3qmalzvqn0a23hqbsfs4qv6x";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ perlldap ConvertASN1 ];
-            meta = {
-              description = "LDAP server side protocol handling";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          NetLDAPSID = prev.perlPackages.buildPerlPackage {
-            pname = "Net-LDAP-SID";
-            version = "0.0001";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-SID-0.001.tar.gz";
-              sha256 = "1mnnpkmj8kpb7qw50sm8h4sd8py37ssy2xi5hhxzr5whcx0cvhm8";
-            };
-            meta = {
-              description = "Active Directory Security Identifier manipulation";
-              license = with final.lib.licenses; [ artistic2 ];
-            };
-          };
-
-          NetLDAPServerTest = prev.perlPackages.buildPerlPackage {
-            pname = "Net-LDAP-Server-Test";
-            version = "0.22";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KA/KARMAN/Net-LDAP-Server-Test-0.22.tar.gz";
-              sha256 = "13idip7jky92v4adw60jn2gcc3zf339gsdqlnc9nnvqzbxxp285i";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ perlldap NetLDAPServer DataDump NetLDAPSID ];
-            meta = {
-              description = "test Net::LDAP code";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          CatalystAuthenticationStoreLDAP = prev.perlPackages.buildPerlPackage {
-            pname = "Catalyst-Authentication-Store-LDAP";
-            version = "1.016";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/I/IL/ILMARI/Catalyst-Authentication-Store-LDAP-1.016.tar.gz";
-              sha256 = "0cm399vxqqf05cjgs1j5v3sk4qc6nmws5nfhf52qvpbwc4m82mq8";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ perlldap CatalystPluginAuthentication ClassAccessor ];
-            buildInputs = with final.perlPackages; [ TestMockObject TestException NetLDAPServerTest ];
-            meta = {
-              description = "Authentication from an LDAP Directory";
-              license = with final.lib.licenses; [ artistic1 ];
-            };
-          };
-
-          PerlCriticCommunity = prev.perlPackages.buildPerlModule {
-            pname = "Perl-Critic-Community";
-            version = "1.0.0";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/D/DB/DBOOK/Perl-Critic-Community-v1.0.0.tar.gz";
-              sha256 = "311b775da4193e9de94cf5225e993cc54dd096ae1e7ef60738cdae1d9b8854e7";
-            };
-            buildInputs = with final.perlPackages; [ ModuleBuildTiny ];
-            propagatedBuildInputs = with final.perlPackages; [ PPI PathTiny PerlCritic PerlCriticPolicyVariablesProhibitLoopOnHash PerlCriticPulp ];
-            meta = {
-              homepage = "https://github.com/Grinnz/Perl-Critic-Freenode";
-              description = "Community-inspired Perl::Critic policies";
-              license = final.lib.licenses.artistic2;
-            };
-          };
-
-          PerlCriticPolicyVariablesProhibitLoopOnHash = prev.perlPackages.buildPerlPackage {
-            pname = "Perl-Critic-Policy-Variables-ProhibitLoopOnHash";
-            version = "0.008";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/X/XS/XSAWYERX/Perl-Critic-Policy-Variables-ProhibitLoopOnHash-0.008.tar.gz";
-              sha256 = "12f5f0be96ea1bdc7828058577bd1c5c63ca23c17fac9c3709452b3dff5b84e0";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ PerlCritic ];
-            meta = {
-              description = "Don't write loops on hashes, only on keys and values of hashes";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          PerlCriticPulp = prev.perlPackages.buildPerlPackage {
-            pname = "Perl-Critic-Pulp";
-            version = "99";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KR/KRYDE/Perl-Critic-Pulp-99.tar.gz";
-              sha256 = "b8fda842fcbed74d210257c0a284b6dc7b1d0554a47a3de5d97e7d542e23e7fe";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ IOString ListMoreUtils PPI PerlCritic PodMinimumVersion ];
-            meta = {
-              homepage = "http://user42.tuxfamily.org/perl-critic-pulp/index.html";
-              description = "Some add-on policies for Perl::Critic";
-              license = final.lib.licenses.gpl3Plus;
-            };
-          };
-
-          PodMinimumVersion = prev.perlPackages.buildPerlPackage {
-            pname = "Pod-MinimumVersion";
-            version = "50";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/K/KR/KRYDE/Pod-MinimumVersion-50.tar.gz";
-              sha256 = "0bd2812d9aacbd99bb71fa103a4bb129e955c138ba7598734207dc9fb67b5a6f";
-            };
-            propagatedBuildInputs = with final.perlPackages; [ IOString PodParser ];
-            meta = {
-              homepage = "http://user42.tuxfamily.org/pod-minimumversion/index.html";
-              description = "Determine minimum Perl version of POD directives";
-              license = final.lib.licenses.free;
-            };
-          };
-
-          StringCompareConstantTime = final.perlPackages.buildPerlPackage {
-            pname = "String-Compare-ConstantTime";
-            version = "0.321";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/F/FR/FRACTAL/String-Compare-ConstantTime-0.321.tar.gz";
-              sha256 = "0b26ba2b121d8004425d4485d1d46f59001c83763aa26624dff6220d7735d7f7";
-            };
-            meta = {
-              description = "Timing side-channel protected string compare";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-          UUID4Tiny = final.perlPackages.buildPerlPackage {
-            pname = "UUID4-Tiny";
-            version = "0.002";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/C/CV/CVLIBRARY/UUID4-Tiny-0.002.tar.gz";
-              sha256 = "e7535b31e386d432dec7adde214348389e1d5cf753e7ed07f1ae04c4360840cf";
-            };
-            meta = {
-              description = "Cryptographically secure v4 UUIDs for Linux x64";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
         };
 
         hydra = with final; let
@@ -581,9 +170,7 @@
 
           checkInputs = [
             cacert
-            # FIXME: foreman is broken on all nix/nixpkgs pin, up to and
-            # including 2.7.0
-            newNixpkgs.legacyPackages.${final.stdenv.system}.foreman
+            foreman
             glibcLocales
             libressl.nc
             openldap
@@ -1015,7 +602,7 @@
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {
-              system.configurationRevision = self.rev;
+              system.configurationRevision = self.lastModifiedDate;
 
               boot.isContainer = true;
               networking.useDHCP = false;

From 481ca71d6e7a391b7c93da7149f50ada02e9303f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jul 2022 15:03:27 +0200
Subject: [PATCH 029/285] Use new flake output naming convention

---
 flake.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.nix b/flake.nix
index cb1d9aea..c5d4b547 100644
--- a/flake.nix
+++ b/flake.nix
@@ -11,7 +11,7 @@
 
       pkgs = import nixpkgs {
         system = "x86_64-linux";
-        overlays = [ self.overlay nix.overlays.default ];
+        overlays = [ self.overlays.default nix.overlays.default ];
       };
 
       # NixOS configuration used for VM tests.
@@ -36,7 +36,7 @@
     rec {
 
       # A Nixpkgs overlay that provides a 'hydra' package.
-      overlay = final: prev: {
+      overlays.default = final: prev: {
 
         # Add LDAP dependencies that aren't currently found within nixpkgs.
         perlPackages = prev.perlPackages // {
@@ -547,11 +547,11 @@
       checks.x86_64-linux.validate-openapi = hydraJobs.tests.validate-openapi;
 
       packages.x86_64-linux.hydra = pkgs.hydra;
-      defaultPackage.x86_64-linux = pkgs.hydra;
+      packages.x86_64-linux.default = pkgs.hydra;
 
       nixosModules.hydra = {
         imports = [ ./hydra-module.nix ];
-        nixpkgs.overlays = [ self.overlay nix.overlays.default ];
+        nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
       };
 
       nixosModules.hydraTest = {

From 9656f16509665a2d49ff7de2905d7115b5fe4440 Mon Sep 17 00:00:00 2001
From: Amanda Cameron <git@amanda.darkdna.net>
Date: Fri, 22 Jul 2022 10:35:38 -0400
Subject: [PATCH 030/285] Update hydra-module.nix to use newer nixos options

nix.trustedUsers is deprecated as of 22.05, and since the nix.extraOptions config is just doing something similar, I moved that to the new nix.settings as well
---
 hydra-module.nix | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 0df5e690..1f46a3ca 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -228,8 +228,12 @@ in
         useDefaultShell = true;
       };
 
-    nix.trustedUsers = [ "hydra-queue-runner" ];
-
+    nix.settings = {
+      trusted-users = [ "hydra-queue-runner" ];
+      gc-keep-outputs = true;
+      gc-keep-derivations = true;
+    };
+    
     services.hydra-dev.extraConfig =
       ''
         using_frontend_proxy = 1
@@ -256,11 +260,6 @@ in
 
     environment.variables = hydraEnv;
 
-    nix.extraOptions = ''
-      gc-keep-outputs = true
-      gc-keep-derivations = true
-    '';
-
     systemd.services.hydra-init =
       { wantedBy = [ "multi-user.target" ];
         requires = optional haveLocalDB "postgresql.service";

From a58e2f1a642233672f9941a77c0f4bd5c3597878 Mon Sep 17 00:00:00 2001
From: Marco Rebhan <me@dblsaiko.net>
Date: Thu, 4 Aug 2022 21:18:55 +0200
Subject: [PATCH 031/285] Use libmagic for better output MIME detection

---
 flake.nix                         |  1 +
 src/lib/Hydra/Controller/Build.pm | 39 ++++++-------------------------
 2 files changed, 8 insertions(+), 32 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2e891364..93186ee0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -503,6 +503,7 @@
                 DigestSHA1
                 EmailMIME
                 EmailSender
+                FileLibMagic
                 FileSlurper
                 FileWhich
                 final.nix.perl-bindings
diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index c7811c62..18a0eba3 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -7,15 +7,16 @@ use base 'Hydra::Base::Controller::NixChannel';
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 use File::Basename;
+use File::LibMagic;
 use File::stat;
 use Data::Dump qw(dump);
 use Nix::Store;
 use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
-use MIME::Types;
 use JSON::PP;
 
+use feature 'state';
 
 sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
     my ($self, $c, $id) = @_;
@@ -236,14 +237,10 @@ sub serveFile {
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };
 
-        # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple.
-        my $type = "text/plain";
-        if ($path =~ /.*\.(\S{1,})$/xms) {
-            my $ext = $1;
-            my $mimeTypes = MIME::Types->new(only_complete => 1);
-            my $t = $mimeTypes->mimeTypeOf($ext);
-            $type = ref $t ? $t->type : $t if $t;
-        }
+        # Detect MIME type.
+        state $magic = File::LibMagic->new(follow_symlinks => 1);
+        my $info = $magic->info_from_filename($path);
+        my $type = $info->{mime_with_encoding};
         $c->response->content_type($type);
         $c->forward('Hydra::View::Plain');
     }
@@ -288,29 +285,7 @@ sub download : Chained('buildChain') PathPart {
     my $path = $product->path;
     $path .= "/" . join("/", @path) if scalar @path > 0;
 
-    if (isLocalStore) {
-
-        notFound($c, "File '" . $product->path . "' does not exist.") unless -e $product->path;
-
-        # Make sure the file is in the Nix store.
-        $path = checkPath($self, $c, $path);
-
-        # If this is a directory but no "/" is attached, then redirect.
-        if (-d $path && substr($c->request->uri, -1) ne "/") {
-            return $c->res->redirect($c->request->uri . "/");
-        }
-
-        $path = "$path/index.html" if -d $path && -e "$path/index.html";
-
-        notFound($c, "File '$path' does not exist.") if !-e $path;
-
-        notFound($c, "Path '$path' is a directory.") if -d $path;
-
-        $c->serve_static_file($path);
-
-    } else {
-        serveFile($c, $path);
-    }
+    serveFile($c, $path);
 
     $c->response->headers->last_modified($c->stash->{build}->stoptime);
 }

From f6d45b0f0cdedf940df67a935f4eb337b43300e7 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 8 Aug 2022 13:35:56 +0200
Subject: [PATCH 032/285] doc/configuration: fix ldap role mapping example

To group is called `cancel-build`, not `cancel-builds` (note the
trailing `s`).
---
 doc/manual/src/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index 2700625d..ab68df43 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -185,7 +185,7 @@ Example configuration:
     hydra_admin = admin
     # Allow all users in the dev group to restart jobs and cancel builds
     dev = restart-jobs
-    dev = cancel-builds
+    dev = cancel-build
   </role_mapping>
 </ldap>
 ```

From 74caaa696e5732313a621ada9a0d9283a57fa764 Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Thu, 11 Aug 2022 13:30:19 +0300
Subject: [PATCH 033/285] Run the JS code to make build trees collapsible at
 the right time

---
 src/root/build.tt            |  4 ++--
 src/root/common.tt           |  6 +++++-
 src/root/static/js/common.js | 27 ++++++++++++++++-----------
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/root/build.tt b/src/root/build.tt
index 027ce3e4..93a02e0f 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -481,11 +481,11 @@ END;
 [% END %]
 
   [% IF drvAvailable %]
-    [% INCLUDE makeLazyTab tabName="tabs-build-deps" uri=c.uri_for('/build' build.id 'build-deps') %]
+    [% INCLUDE makeLazyTab tabName="tabs-build-deps" uri=c.uri_for('/build' build.id 'build-deps') callback="makeTreeCollapsible" %]
   [% END %]
 
   [% IF available %]
-    [% INCLUDE makeLazyTab tabName="tabs-runtime-deps" uri=c.uri_for('/build' build.id 'runtime-deps') %]
+    [% INCLUDE makeLazyTab tabName="tabs-runtime-deps" uri=c.uri_for('/build' build.id 'runtime-deps') callback="makeTreeCollapsible" %]
   [% END %]
 
   <div id="tabs-runcommandlogs" class="tab-pane">
diff --git a/src/root/common.tt b/src/root/common.tt
index 6b4e406e..32d6c8cc 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -520,7 +520,11 @@ BLOCK makeLazyTab %]
     <center><span class="spinner-border spinner-border-sm"/></center>
   </div>
   <script>
-    $(function() { makeLazyTab("[% tabName %]", "[% uri %]"); });
+    [% IF callback.defined %]
+    $(function() { makeLazyTab("[% tabName %]", "[% uri %]", [% callback %] ); });
+    [% ELSE %]
+    $(function() { makeLazyTab("[% tabName %]", "[% uri %]", null ); });
+    [% END %]
   </script>
 [% END;
 
diff --git a/src/root/static/js/common.js b/src/root/static/js/common.js
index 433c06d7..c51f769a 100644
--- a/src/root/static/js/common.js
+++ b/src/root/static/js/common.js
@@ -1,10 +1,9 @@
-$(document).ready(function() {
-
+function makeTreeCollapsible(tab) {
     /*** Tree toggles in logfiles. ***/
 
     /* Set the appearance of the toggle depending on whether the
        corresponding subtree is initially shown or hidden. */
-    $(".tree-toggle").map(function() {
+    tab.find(".tree-toggle").map(function() {
         if ($(this).siblings("ul:hidden").length == 0) {
             $(this).text("-");
         } else {
@@ -13,7 +12,7 @@ $(document).ready(function() {
     });
 
     /* When a toggle is clicked, show or hide the subtree. */
-    $(".tree-toggle").click(function() {
+    tab.find(".tree-toggle").click(function() {
         if ($(this).siblings("ul:hidden").length != 0) {
             $(this).siblings("ul").show();
             $(this).text("-");
@@ -24,21 +23,23 @@ $(document).ready(function() {
     });
 
     /* Implementation of the expand all link. */
-    $(".tree-expand-all").click(function() {
-        $(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
+    tab.find(".tree-expand-all").click(function() {
+        tab.find(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
             $(this).siblings("ul").show();
             $(this).text("-");
         });
     });
 
     /* Implementation of the collapse all link. */
-    $(".tree-collapse-all").click(function() {
-        $(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
+    tab.find(".tree-collapse-all").click(function() {
+        tab.find(".tree-toggle", $(this).parent().siblings(".tree")).map(function() {
             $(this).siblings("ul").hide();
             $(this).text("+");
         });
     });
+}
 
+$(document).ready(function() {
     $("table.clickable-rows").click(function(event) {
         if ($(event.target).closest("a").length) return;
         link = $(event.target).parents("tr").find("a.row-link");
@@ -132,7 +133,7 @@ $(document).ready(function() {
 
 var tabsLoaded = {};
 
-function makeLazyTab(tabName, uri) {
+function makeLazyTab(tabName, uri, callback) {
     $('.nav-tabs').bind('show.bs.tab', function(e) {
         var pattern = /#.+/gi;
         var id = e.target.toString().match(pattern)[0];
@@ -140,11 +141,15 @@ function makeLazyTab(tabName, uri) {
           tabsLoaded[id] = 1;
           $('#' + tabName).load(uri, function(response, status, xhr) {
             var lazy = xhr.getResponseHeader("X-Hydra-Lazy") === "Yes";
+            var tab = $('#' + tabName);
             if (status == "error" && !lazy) {
-              $('#' + tabName).html("<div class='alert alert-error'>Error loading tab: " + xhr.status + " " + xhr.statusText + "</div>");
+              tab.html("<div class='alert alert-error'>Error loading tab: " + xhr.status + " " + xhr.statusText + "</div>");
             }
             else {
-              $('#' + tabName).html(response);
+              tab.html(response);
+              if (callback) {
+                callback(tab);
+              }
             }
           });
         }

From 93bbd6925b8c08d8080d0a317b5f30694da1e67c Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Fri, 12 Aug 2022 09:46:17 +0300
Subject: [PATCH 034/285] Also restore the "expand all" and "collapse all"
 buttons

---
 src/root/build-deps.tt        | 2 +-
 src/root/deps.tt              | 9 ++++++++-
 src/root/static/css/hydra.css | 5 +++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/root/build-deps.tt b/src/root/build-deps.tt
index 157b113e..f15342fa 100644
--- a/src/root/build-deps.tt
+++ b/src/root/build-deps.tt
@@ -4,6 +4,6 @@
 
 <div class="dep-tree">
   <ul class="tree">
-    [% INCLUDE renderNode node=buildTimeGraph %]
+    [% INCLUDE renderNode node=buildTimeGraph isRoot=1 %]
   </ul>
 </div>
diff --git a/src/root/deps.tt b/src/root/deps.tt
index a2c1fbba..6daa9725 100644
--- a/src/root/deps.tt
+++ b/src/root/deps.tt
@@ -19,9 +19,16 @@
           <tt>[% node.name %]</tt> (<em>no info</em>)
         [% END %]
       </span></span>
+      [% IF isRoot %]
+      <span class="dep-tree-buttons">
+        (<a href="#" class="tree-collapse-all">collapse all</a>
+        &ndash;
+        <a href="#" class="tree-expand-all">expand all</a>)
+      </span>
+      [% END %]
       [% IF node.refs.size > 0 %]
         <ul class="subtree">
-          [% FOREACH ref IN node.refs; INCLUDE renderNode node=ref; END %]
+          [% FOREACH ref IN node.refs; INCLUDE renderNode node=ref isRoot=0; END %]
         </ul>
       [% END %]
     [% END %]
diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 3b50d246..aac3f1ea 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -33,6 +33,11 @@ span:target > span.dep-tree-line {
     font-weight: bold;
 }
 
+span.dep-tree-buttons {
+    font-style: italic;
+    padding-left: 10px;
+}
+
 span.disabled-project, span.disabled-jobset, span.disabled-job {
     text-decoration: line-through;
 }

From 2b8a8fdd9cd7c6ea7d2d892fa9c93acdfef0c25d Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Fri, 12 Aug 2022 09:46:32 +0300
Subject: [PATCH 035/285] Make the tree a little less dense

---
 src/root/static/css/tree.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/root/static/css/tree.css b/src/root/static/css/tree.css
index 71df2f12..e80f6871 100644
--- a/src/root/static/css/tree.css
+++ b/src/root/static/css/tree.css
@@ -9,6 +9,7 @@ ul.tree, ul.subtree {
 ul.subtree > li {
     position: relative;
     padding-left: 2.0em;
+    line-height: 140%;
     border-left: 0.1em solid #6185a0;
 }
 

From 9addaeb17fc9663f706b58b33e0de8ec57f4bf2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Fri, 8 Jul 2022 22:23:35 +0200
Subject: [PATCH 036/285] Add a squiggly line to the Hydra link on hover

The effect is the same as the one on links in mail bodys on
https://lists.apache.org/
---
 src/root/layout.tt            |  2 +-
 src/root/static/css/hydra.css | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/root/layout.tt b/src/root/layout.tt
index 8eb1f119..d67ff1b8 100644
--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@@ -93,7 +93,7 @@
       <footer class="navbar">
         <hr />
         <small>
-          <em><a href="http://nixos.org/hydra" target="_blank">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
+          <em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
           [% IF c.user_exists %]
           You are signed in as <tt>[% HTML.escape(c.user.username) %]</tt>
             [%- IF c.user.type == 'google' %] via Google[% END %].
diff --git a/src/root/static/css/hydra.css b/src/root/static/css/hydra.css
index 3b50d246..251e0e75 100644
--- a/src/root/static/css/hydra.css
+++ b/src/root/static/css/hydra.css
@@ -166,6 +166,16 @@ body {
     line-height: 1.5;
 }
 
+a.squiggle:hover {
+    background-image: url("data:image/svg+xml;charset=utf8,%3Csvg id='squiggle-link' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' xmlns:ev='http://www.w3.org/2001/xml-events' viewBox='0 0 10 18'%3E%3Cstyle type='text/css'%3E.squiggle{animation:shift .5s linear infinite;}@keyframes shift {from {transform:translateX(-10px);}to {transform:translateX(0);}}%3C/style%3E%3Cpath fill='none' stroke='%230056b3' stroke-width='0.65' class='squiggle' d='M0,17.5 c 2.5,0,2.5,-1.5,5,-1.5 s 2.5,1.5,5,1.5 c 2.5,0,2.5,-1.5,5,-1.5 s 2.5,1.5,5,1.5' /%3E%3C/svg%3E");
+    background-position: 0 100%;
+    background-size: auto 24px;
+    background-repeat: repeat;
+    text-decoration: none;
+    border-bottom: none;
+    padding-bottom: 1px;
+}
+
 @media (prefers-color-scheme: dark) {
     /* Prevent some flickering */
     html {

From 371402c3c1d76a7d4e2e1691fda3d90397c8929f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Wed, 17 Aug 2022 11:41:37 +0200
Subject: [PATCH 037/285] Drop the HipChat plugin

https://en.wikipedia.org/wiki/HipChat says:
> Following this, HipChat and Stride customers were migrated to the
> Slack group collaboration platform in a transition that was completed by
> February 2019.
---
 doc/manual/src/plugins/README.md            | 11 ---
 src/lib/Hydra/Plugin/HipChatNotification.pm | 89 ---------------------
 2 files changed, 100 deletions(-)
 delete mode 100644 src/lib/Hydra/Plugin/HipChatNotification.pm

diff --git a/doc/manual/src/plugins/README.md b/doc/manual/src/plugins/README.md
index 26ee2649..6e46224f 100644
--- a/doc/manual/src/plugins/README.md
+++ b/doc/manual/src/plugins/README.md
@@ -172,17 +172,6 @@ Sets Gitlab CI status.
 
 - `gitlab_authorization.<projectId>`
 
-## HipChat notification
-
-Sends hipchat chat notifications when a build finish.
-
-### Configuration options
-
-- `hipchat.[].jobs`
-- `hipchat.[].builds`
-- `hipchat.[].token`
-- `hipchat.[].notify`
-
 ## InfluxDB notification
 
 Writes InfluxDB events when a builds finished.
diff --git a/src/lib/Hydra/Plugin/HipChatNotification.pm b/src/lib/Hydra/Plugin/HipChatNotification.pm
deleted file mode 100644
index 61cea57a..00000000
--- a/src/lib/Hydra/Plugin/HipChatNotification.pm
+++ /dev/null
@@ -1,89 +0,0 @@
-package Hydra::Plugin::HipChatNotification;
-
-use strict;
-use warnings;
-use parent 'Hydra::Plugin';
-use LWP::UserAgent;
-use Hydra::Helper::CatalystUtils;
-
-sub isEnabled {
-    my ($self) = @_;
-    return defined $self->{config}->{hipchat};
-}
-
-sub buildFinished {
-    my ($self, $topbuild, $dependents) = @_;
-
-    my $cfg = $self->{config}->{hipchat};
-    my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
-
-    my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000";
-
-    # Figure out to which rooms to send notification.  For each email
-    # room, we send one aggregate message.
-    my %rooms;
-    foreach my $build ($topbuild, @{$dependents}) {
-        my $prevBuild = getPreviousBuild($build);
-        my $jobName = showJobName $build;
-
-        foreach my $room (@config) {
-            my $force = $room->{force};
-            next unless $jobName =~ /^$room->{jobs}$/;
-
-            # If build is cancelled or aborted, do not send email.
-            next if ! $force && ($build->buildstatus == 4 || $build->buildstatus == 3);
-
-            # If there is a previous (that is not cancelled or aborted) build
-            # with same buildstatus, do not send email.
-            next if ! $force && defined $prevBuild && ($build->buildstatus == $prevBuild->buildstatus);
-
-            $rooms{$room->{room}} //= { room => $room, builds => [] };
-            push @{$rooms{$room->{room}}->{builds}}, $build;
-        }
-    }
-
-    return if scalar keys %rooms == 0;
-
-    my ($authors, $nrCommits) = getResponsibleAuthors($topbuild, $self->{plugins});
-
-    # Send a message to each room.
-    foreach my $roomId (keys %rooms) {
-        my $room = $rooms{$roomId};
-        my @deps = grep { $_->id != $topbuild->id } @{$room->{builds}};
-
-        my $img =
-            $topbuild->buildstatus == 0 ? "$baseurl/static/images/checkmark_16.png" :
-            $topbuild->buildstatus == 2 ? "$baseurl/static/images/dependency_16.png" :
-            $topbuild->buildstatus == 4 ? "$baseurl/static/images/cancelled_16.png" :
-            "$baseurl/static/images/error_16.png";
-
-        my $msg = "";
-        $msg .= "<img src='$img'/> ";
-        $msg .= "Job <a href='$baseurl/job/${\$topbuild->jobset->get_column('project')}/${\$topbuild->jobset->get_column('name')}/${\$topbuild->get_column('job')}'>${\showJobName($topbuild)}</a>";
-        $msg .= " (and ${\scalar @deps} others)" if scalar @deps > 0;
-        $msg .= ": <a href='$baseurl/build/${\$topbuild->id}'>" . showStatus($topbuild) . "</a>";
-
-        if (scalar keys %{$authors} > 0) {
-            # FIXME: HTML escaping
-            my @x = map { "<a href='mailto:$authors->{$_}'>$_</a>" } (sort keys %{$authors});
-            $msg .= ", likely due to ";
-            $msg .= "$nrCommits commits by " if $nrCommits > 1;
-            $msg .= join(" or ", scalar @x > 1 ? join(", ", @x[0..scalar @x - 2]) : (), $x[-1]);
-        }
-
-        print STDERR "sending hipchat notification to room $roomId: $msg\n";
-
-        my $ua = LWP::UserAgent->new();
-        my $resp = $ua->post('https://api.hipchat.com/v1/rooms/message?format=json&auth_token=' . $room->{room}->{token}, {
-            room_id => $roomId,
-            from => 'Hydra',
-            message => $msg,
-            message_format => 'html',
-            notify => $room->{room}->{notify} || 0,
-            color => $topbuild->buildstatus == 0 ? 'green' : 'red' });
-
-        print STDERR $resp->status_line, ": ", $resp->decoded_content,"\n" if !$resp->is_success;
-    }
-}
-
-1;

From 1dbc100837dcc705c6d2451603da17f14e72b606 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Sep 2022 16:18:52 +0200
Subject: [PATCH 038/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/b9cf655150b52d071c85a337cb5db96e735fa64a' (2022-07-11)
  → 'github:NixOS/nix/11e45768b34fdafdcf019ddbd337afa16127ff0f' (2022-08-27)
• Updated input 'nix/nixpkgs':
    'github:NixOS/nixpkgs/2fa57ed190fd6c7c746319444f34b5917666e5c1' (2022-05-31)
  → 'github:NixOS/nixpkgs/365e1b3a859281cf11b94f87231adeabbdd878a2' (2022-07-13)
---
 flake.lock | 14 +++++++-------
 flake.nix  |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/flake.lock b/flake.lock
index 012d2407..b41b843a 100644
--- a/flake.lock
+++ b/flake.lock
@@ -23,27 +23,27 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1657569404,
-        "narHash": "sha256-zJONRtGALmYifrWKzcH6MMfSKLxeuW2iqG13500OrY4=",
+        "lastModified": 1661606874,
+        "narHash": "sha256-9+rpYzI+SmxJn+EbYxjGv68Ucp22bdFUSy/4LkHkkDQ=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "b9cf655150b52d071c85a337cb5db96e735fa64a",
+        "rev": "11e45768b34fdafdcf019ddbd337afa16127ff0f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.10.0",
+        "ref": "2.11.0",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1653988320,
-        "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=",
+        "lastModified": 1657693803,
+        "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1",
+        "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index b731e684..2e413277 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:NixOS/nix/2.10.0";
+  inputs.nix.url = "github:NixOS/nix/2.11.0";
 
   outputs = { self, nixpkgs, nix }:
     let

From b16470c54434966b227de93c5229811c9c2afab4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Sep 2022 19:22:20 +0200
Subject: [PATCH 039/285] nix flake info -> nix flake metadata

This gets rid of a deprecation warning.
---
 src/script/hydra-eval-jobset | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 8804b2bb..c6f6c275 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -640,8 +640,8 @@ sub checkJobsetWrapped {
     my $flakeRef = $jobset->flake;
     if (defined $flakeRef) {
         (my $res, my $json, my $stderr) = captureStdoutStderr(
-            600, "nix", "flake", "info", "--tarball-ttl", 0, "--json", "--", $flakeRef);
-        die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
+            600, "nix", "flake", "metadata", "--refresh", "--json", "--", $flakeRef);
+        die "'nix flake metadata' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
             . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
             if $res;
         $flakeRef = decode_json($json)->{'url'};

From 44e1efff7f51cd884f3924fdb086f3268b753972 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Sep 2022 11:51:13 +0200
Subject: [PATCH 040/285] Send the right nix-serve client version

We were using protocol version 6 but requesting version 4. The only
reason that this worked was because of a broken version check in
'nix-store --serve'. That was fixed in
https://github.com/NixOS/nix/commit/c2d74569269d9642448a701aa6efe08ec0fe6f00,
which had the side-effect of breaking hydra-queue-runner.
---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 57a5f0df..21a6c331 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -234,7 +234,7 @@ void State::buildRemote(ref<Store> destStore,
         unsigned int remoteVersion;
 
         try {
-            to << SERVE_MAGIC_1 << 0x204;
+            to << SERVE_MAGIC_1 << 0x206;
             to.flush();
 
             unsigned int magic = readInt(from);

From d3fe4ffbf645b8fbc1110a30614a6871a23f0f23 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 9 Sep 2022 00:38:46 +0200
Subject: [PATCH 041/285] Job: expose `closuresize` and `size` (output size in
 the UI) as prometheus metrics

---
 src/lib/Hydra/Controller/Job.pm | 22 +++++++++++++++++++++-
 t/Hydra/Controller/Job/builds.t |  6 ++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm
index 378887ea..b392e8e1 100644
--- a/src/lib/Hydra/Controller/Job.pm
+++ b/src/lib/Hydra/Controller/Job.pm
@@ -69,7 +69,7 @@ sub prometheus : Chained('job') PathPart('prometheus') Args(0) {
 
     my $lastBuild = $c->stash->{jobset}->builds->find(
         { job => $c->stash->{job}, finished => 1 },
-        { order_by => 'id DESC', rows => 1, columns => [@buildListColumns] }
+        { order_by => 'id DESC', rows => 1, columns => ["stoptime", "buildstatus", "closuresize", "size"] }
     );
 
     $prometheus->new_counter(
@@ -92,6 +92,26 @@ sub prometheus : Chained('job') PathPart('prometheus') Args(0) {
         $c->stash->{job},
     )->inc($lastBuild->buildstatus > 0);
 
+    $prometheus->new_gauge(
+        name => "hydra_build_closure_size",
+        help => "Closure size of the last job's build in bytes",
+        labels => [ "project", "jobset", "job" ]
+    )->labels(
+        $c->stash->{project}->name,
+        $c->stash->{jobset}->name,
+        $c->stash->{job},
+    )->inc($lastBuild->closuresize);
+
+    $prometheus->new_gauge(
+        name => "hydra_build_output_size",
+        help => "Output size of the last job's build in bytes",
+        labels => [ "project", "jobset", "job" ]
+    )->labels(
+        $c->stash->{project}->name,
+        $c->stash->{jobset}->name,
+        $c->stash->{job},
+    )->inc($lastBuild->size);
+
     $c->stash->{'plain'} = { data => $prometheus->render };
     $c->forward('Hydra::View::Plain');
 }
diff --git a/t/Hydra/Controller/Job/builds.t b/t/Hydra/Controller/Job/builds.t
index e1097c3e..d933fbb8 100644
--- a/t/Hydra/Controller/Job/builds.t
+++ b/t/Hydra/Controller/Job/builds.t
@@ -55,6 +55,12 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {
 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
     my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
     ok($response->is_success, "The page showing the job's prometheus data returns 200.");
+    my $metrics = $response->content;
+
+    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
+    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
+    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
+    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
 };
 
 done_testing;

From 2ff67cbaccb8dcad239c7a92e7dc3d80e208e752 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 27 Sep 2022 22:44:27 +0200
Subject: [PATCH 042/285] docs/hacking: update nixos module option for
 nix-daemon

---
 doc/manual/src/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 9d98b00c..72a74c82 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -92,7 +92,7 @@ On NixOS:
 
 ```nix
 {
-  nix.trustedUsers = [ "YOURUSER" ];
+  nix.settings.trusted-users = [ "YOURUSER" ];
 }
 ```
 

From 94d19e19725905c3d10acd28cce78268a53d5ff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 29 Sep 2022 20:42:57 +0200
Subject: [PATCH 043/285] hydra: fix localhost detection when protocol prefix
 are used

---
 src/hydra-queue-runner/state.hh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 47e74f55..55c99afc 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -6,6 +6,7 @@
 #include <map>
 #include <memory>
 #include <queue>
+#include <regex>
 
 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -293,7 +294,8 @@ struct Machine
 
     bool isLocalhost()
     {
-        return sshName == "localhost";
+        std::regex r("^(ssh://|ssh-ng://)?localhost$");
+        return std::regex_search(sshName, r);
     }
 };
 

From 940c0d7d978ce35c13d7d78c5fb0e9b55f17e397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 29 Sep 2022 21:22:01 +0200
Subject: [PATCH 044/285] fix deprecated chown syntax

---
 hydra-module.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index b58be922..8e02dcbb 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -272,12 +272,12 @@ in
           ln -sf ${hydraConf} ${baseDir}/hydra.conf
 
           mkdir -m 0700 -p ${baseDir}/www
-          chown hydra-www.hydra ${baseDir}/www
+          chown hydra-www:hydra ${baseDir}/www
 
           mkdir -m 0700 -p ${baseDir}/queue-runner
           mkdir -m 0750 -p ${baseDir}/build-logs
           mkdir -m 0750 -p ${baseDir}/runcommand-logs
-          chown hydra-queue-runner.hydra \
+          chown hydra-queue-runner:hydra \
             ${baseDir}/queue-runner \
             ${baseDir}/build-logs \
             ${baseDir}/runcommand-logs
@@ -308,7 +308,7 @@ in
             rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
           fi
 
-          chown hydra.hydra ${cfg.gcRootsDir}
+          chown hydra:hydra ${cfg.gcRootsDir}
           chmod 2775 ${cfg.gcRootsDir}
         '';
         serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init";

From c737bed42f73f3ee1be576f5f739d4fc3d713f40 Mon Sep 17 00:00:00 2001
From: MaxHearnden <MaxHearnden@users.noreply.github.com>
Date: Wed, 19 Oct 2022 16:24:01 +0100
Subject: [PATCH 045/285] Fix insecure advice in the manual

Bring the manual in line with the configuration documentation
---
 doc/manual/src/projects.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md
index 95174f1b..a399406d 100644
--- a/doc/manual/src/projects.md
+++ b/doc/manual/src/projects.md
@@ -378,13 +378,18 @@ This section describes how it can be implemented for `gitea`, but the approach f
 analogous:
 
 * [Obtain an API token for your user](https://docs.gitea.io/en-us/api-usage/#authentication)
-* Add it to your `hydra.conf` like this:
+* Add it to a file which only users in the hydra group can read like this: see [including files](configuration.md#including-files) for more information
+  ```
+  <gitea_authorization>
+    your_username=your_token
+  </gitea_authorization>
+  ```
+
+* Include the file in your `hydra.conf` like this:
   ``` nix
   {
     services.hydra-dev.extraConfig = ''
-      <gitea_authorization>
-      your_username=your_token
-      </gitea_authorization>
+      Include /path/to/secret/file
     '';
   }
   ```

From 143c31734fc19d72a3ccfb7c064d829af9bfca72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Tue, 25 Oct 2022 10:04:29 +0200
Subject: [PATCH 046/285] Move all the build remote utils to their namespace

Just don't pollute the global one
---
 src/hydra-queue-runner/build-remote.cc | 134 +++++++++++++------------
 1 file changed, 70 insertions(+), 64 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 62461a65..a429467b 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -27,6 +27,8 @@ static void append(Strings & dst, const Strings & src)
     dst.insert(dst.end(), src.begin(), src.end());
 }
 
+namespace nix::build_remote {
+
 static Strings extraStoreArgs(std::string & machine)
 {
     Strings result;
@@ -263,63 +265,6 @@ BasicDerivation sendInputs(
     return basicDrv;
 }
 
-void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
-{
-    RemoteResult thisArrow;
-
-    startTime = buildResult.startTime;
-    stopTime = buildResult.stopTime;
-    timesBuilt = buildResult.timesBuilt;
-    errorMsg = buildResult.errorMsg;
-    isNonDeterministic = buildResult.isNonDeterministic;
-
-    switch ((BuildResult::Status) buildResult.status) {
-        case BuildResult::Built:
-            stepStatus = bsSuccess;
-            break;
-        case BuildResult::Substituted:
-        case BuildResult::AlreadyValid:
-            stepStatus = bsSuccess;
-            isCached = true;
-            break;
-        case BuildResult::PermanentFailure:
-            stepStatus = bsFailed;
-            canCache = true;
-            errorMsg = "";
-            break;
-        case BuildResult::InputRejected:
-        case BuildResult::OutputRejected:
-            stepStatus = bsFailed;
-            canCache = true;
-            break;
-        case BuildResult::TransientFailure:
-            stepStatus = bsFailed;
-            canRetry = true;
-            errorMsg = "";
-            break;
-        case BuildResult::TimedOut:
-            stepStatus = bsTimedOut;
-            errorMsg = "";
-            break;
-        case BuildResult::MiscFailure:
-            stepStatus = bsAborted;
-            canRetry = true;
-            break;
-        case BuildResult::LogLimitExceeded:
-            stepStatus = bsLogLimitExceeded;
-            break;
-        case BuildResult::NotDeterministic:
-            stepStatus = bsNotDeterministic;
-            canRetry = false;
-            canCache = true;
-            break;
-        default:
-            stepStatus = bsAborted;
-            break;
-    }
-
-}
-
 BuildResult performBuild(
     Machine::Connection & conn,
     Store & localStore,
@@ -457,6 +402,67 @@ void copyPathsFromRemote(
 
 }
 
+}
+
+/* using namespace nix::build_remote; */
+
+void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
+{
+    RemoteResult thisArrow;
+
+    startTime = buildResult.startTime;
+    stopTime = buildResult.stopTime;
+    timesBuilt = buildResult.timesBuilt;
+    errorMsg = buildResult.errorMsg;
+    isNonDeterministic = buildResult.isNonDeterministic;
+
+    switch ((BuildResult::Status) buildResult.status) {
+        case BuildResult::Built:
+            stepStatus = bsSuccess;
+            break;
+        case BuildResult::Substituted:
+        case BuildResult::AlreadyValid:
+            stepStatus = bsSuccess;
+            isCached = true;
+            break;
+        case BuildResult::PermanentFailure:
+            stepStatus = bsFailed;
+            canCache = true;
+            errorMsg = "";
+            break;
+        case BuildResult::InputRejected:
+        case BuildResult::OutputRejected:
+            stepStatus = bsFailed;
+            canCache = true;
+            break;
+        case BuildResult::TransientFailure:
+            stepStatus = bsFailed;
+            canRetry = true;
+            errorMsg = "";
+            break;
+        case BuildResult::TimedOut:
+            stepStatus = bsTimedOut;
+            errorMsg = "";
+            break;
+        case BuildResult::MiscFailure:
+            stepStatus = bsAborted;
+            canRetry = true;
+            break;
+        case BuildResult::LogLimitExceeded:
+            stepStatus = bsLogLimitExceeded;
+            break;
+        case BuildResult::NotDeterministic:
+            stepStatus = bsNotDeterministic;
+            canRetry = false;
+            canCache = true;
+            break;
+        default:
+            stepStatus = bsAborted;
+            break;
+    }
+
+}
+
 
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
@@ -467,7 +473,7 @@ void State::buildRemote(ref<Store> destStore,
 {
     assert(BuildResult::TimedOut == 8);
 
-    auto [logFile, logFD] = openLogFile(logDir, step->drvPath);
+    auto [logFile, logFD] = build_remote::openLogFile(logDir, step->drvPath);
     AutoDelete logFileDel(logFile, false);
     result.logFile = logFile;
 
@@ -480,7 +486,7 @@ void State::buildRemote(ref<Store> destStore,
 
         // FIXME: rewrite to use Store.
         Child child;
-        openConnection(machine, tmpDir, logFD.get(), child);
+        build_remote::openConnection(machine, tmpDir, logFD.get(), child);
 
         {
             auto activeStepState(activeStep->state_.lock());
@@ -511,7 +517,7 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         try {
-          handshake(conn, buildOptions.repeats);
+          build_remote::handshake(conn, buildOptions.repeats);
         } catch (EndOfFile & e) {
             child.pid.wait();
             std::string s = chomp(readFile(result.logFile));
@@ -529,7 +535,7 @@ void State::buildRemote(ref<Store> destStore,
            copy the immediate sources of the derivation and the required
            outputs of the input derivations. */
         updateStep(ssSendingInputs);
-        BasicDerivation resolvedDrv = sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo);
+        BasicDerivation resolvedDrv = build_remote::sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo);
 
         logFileDel.cancel();
 
@@ -550,7 +556,7 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssBuilding);
 
-        BuildResult buildResult = performBuild(
+        BuildResult buildResult = build_remote::performBuild(
             conn,
             *localStore,
             step->drvPath,
@@ -589,7 +595,7 @@ void State::buildRemote(ref<Store> destStore,
             }
 
             size_t totalNarSize = 0;
-            auto infos = queryPathInfos(conn, *localStore, outputs, totalNarSize);
+            auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize);
 
             if (totalNarSize > maxOutputSize) {
                 result.stepStatus = bsNarSizeLimitExceeded;
@@ -600,7 +606,7 @@ void State::buildRemote(ref<Store> destStore,
             printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
                 localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
 
-            copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos);
+            build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos);
             auto now2 = std::chrono::steady_clock::now();
 
             result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

From 70ad3a924affd56d57cdced7b31c5b36a7181701 Mon Sep 17 00:00:00 2001
From: Casey Ransom <casey@dailykos.com>
Date: Wed, 26 Oct 2022 16:13:40 -0400
Subject: [PATCH 047/285] exit with error if database connectivity lost

There's currently no automatic recovery for disconnected databases in
the evaluator. This means if the database is ever temporarily
unavailable, hydra-evaluator will sit and spin with no work
accomplished.

If this condition is caught, the daemon will exit and systemd will be
responsible for resuming the service.
---
 src/hydra-evaluator/hydra-evaluator.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc
index 2d7e68d9..a1ccf047 100644
--- a/src/hydra-evaluator/hydra-evaluator.cc
+++ b/src/hydra-evaluator/hydra-evaluator.cc
@@ -366,6 +366,9 @@ struct Evaluator
                     printInfo("received jobset event");
                 }
 
+            } catch (pqxx::broken_connection & e) {
+                printError("Database connection broken: %s", e.what());
+                std::_Exit(1);
             } catch (std::exception & e) {
                 printError("exception in database monitor thread: %s", e.what());
                 sleep(30);
@@ -473,6 +476,9 @@ struct Evaluator
         while (true) {
             try {
                 loop();
+            } catch (pqxx::broken_connection & e) {
+                printError("Database connection broken: %s", e.what());
+                std::_Exit(1);
             } catch (std::exception & e) {
                 printError("exception in main loop: %s", e.what());
                 sleep(30);

From d1d171ee9010be514bb7b95866ac275213ed4286 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 2 Nov 2022 17:30:32 +0100
Subject: [PATCH 048/285] renderInputDiff: Increase git hash length 6 -> 8

Nixpkgs has so many commits that length 6 is often ambiguous,
making the use of the shown values with git difficult.
---
 src/root/common.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/common.tt b/src/root/common.tt
index 32d6c8cc..4487cbe3 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -374,7 +374,7 @@ BLOCK renderInputDiff; %]
               [% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %]
                 [% IF bi1.type == "git" %]
                   <tr><td>
-                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 6) _ ' to ' _ bi2.revision.substr(0, 6)) %]</tt>
+                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 8) _ ' to ' _ bi2.revision.substr(0, 8)) %]</tt>
                   </td></tr>
                 [% ELSE %]
                   <tr><td>

From a8b89420bc2004b21652ec4de7faecc843faeb73 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Wed, 2 Nov 2022 11:19:33 +0100
Subject: [PATCH 049/285] Enable aarch64 support

---
 flake.nix | 63 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 37 insertions(+), 26 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2e413277..cd9f094d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -6,13 +6,15 @@
 
   outputs = { self, nixpkgs, nix }:
     let
-
       version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
 
-      pkgs = import nixpkgs {
-        system = "x86_64-linux";
+      systems = [ "x86_64-linux" "aarch64-linux" ];
+      forEachSystem = nixpkgs.lib.genAttrs systems;
+
+      pkgsBySystem = forEachSystem (system: import nixpkgs {
+        inherit system;
         overlays = [ self.overlays.default nix.overlays.default ];
-      };
+      });
 
       # NixOS configuration used for VM tests.
       hydraServer =
@@ -254,9 +256,10 @@
 
       hydraJobs = {
 
-        build.x86_64-linux = packages.x86_64-linux.hydra;
+        build = forEachSystem (system: packages.${system}.hydra);
 
-        manual =
+        manual = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
           pkgs.runCommand "hydra-manual-${version}" { }
             ''
               mkdir -p $out/share
@@ -264,10 +267,10 @@
 
               mkdir $out/nix-support
               echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
-            '';
+            '');
 
-        tests.install.x86_64-linux =
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
+        tests.install = forEachSystem (system:
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
           simpleTest {
             nodes.machine = hydraServer;
             testScript =
@@ -279,10 +282,11 @@
                 machine.wait_for_open_port("3000")
                 machine.succeed("curl --fail http://localhost:3000/")
               '';
-          };
+          });
 
-        tests.notifications.x86_64-linux =
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
+        tests.notifications = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
           simpleTest {
             nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
@@ -336,10 +340,11 @@
                   + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
               )
             '';
-          };
+          });
 
-        tests.gitea.x86_64-linux =
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
+        tests.gitea = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
           makeTest {
             nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
@@ -352,7 +357,7 @@
                 distributedBuilds = true;
                 buildMachines = [{
                   hostName = "localhost";
-                  systems = [ "x86_64-linux" ];
+                  systems = [ system ];
                 }];
                 binaryCaches = [ ];
               };
@@ -467,7 +472,7 @@
                 smallDrv = pkgs.writeText "jobset.nix" ''
                   { trivial = builtins.derivation {
                       name = "trivial";
-                      system = "x86_64-linux";
+                      system = "${system}";
                       builder = "/bin/sh";
                       allowSubstitutes = false;
                       preferLocalBuild = true;
@@ -531,31 +536,37 @@
 
                 machine.shutdown()
               '';
-          };
+          });
 
-        tests.validate-openapi = pkgs.runCommand "validate-openapi"
+        tests.validate-openapi = forEachSystem (system:
+          let pkgs = pkgsBySystem.${system}; in
+          pkgs.runCommand "validate-openapi"
           { buildInputs = [ pkgs.openapi-generator-cli ]; }
           ''
             openapi-generator-cli validate -i ${./hydra-api.yaml}
             touch $out
-          '';
+          '');
 
         container = nixosConfigurations.container.config.system.build.toplevel;
       };
 
-      checks.x86_64-linux.build = hydraJobs.build.x86_64-linux;
-      checks.x86_64-linux.install = hydraJobs.tests.install.x86_64-linux;
-      checks.x86_64-linux.validate-openapi = hydraJobs.tests.validate-openapi;
+      checks = forEachSystem (system: {
+        build = hydraJobs.build.${system};
+        install = hydraJobs.tests.install.${system};
+        validate-openapi = hydraJobs.tests.validate-openapi.${system};
+      });
 
-      packages.x86_64-linux.hydra = pkgs.hydra;
-      packages.x86_64-linux.default = pkgs.hydra;
+      packages = forEachSystem (system: {
+        hydra = pkgsBySystem.${system}.hydra;
+        default = pkgsBySystem.${system}.hydra;
+      });
 
       nixosModules.hydra = {
         imports = [ ./hydra-module.nix ];
         nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
       };
 
-      nixosModules.hydraTest = {
+      nixosModules.hydraTest = { pkgs, ... }: {
         imports = [ self.nixosModules.hydra ];
 
         services.hydra-dev.enable = true;

From fd765bc97a321d45b56449ccabb1a7eca1553597 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 22 Nov 2022 20:50:39 +0100
Subject: [PATCH 050/285] Fix "My Jobs" tab in user dashboard

Nowadays `Builds` doesn't reference `Project` directly anymore. This
means that simply resolving both `jobset` and `project` with a single
JOIN from `Builds` doesn't work anymore. Instead we need to resolve the
relation to `jobset` first and then the relation to `project`.

For similar fixes see e.g. c7c47596009687d1652522c556333cefce28ec51.
---
 src/lib/Hydra/Controller/User.pm    |  2 +-
 t/Hydra/Controller/User/dashboard.t | 30 +++++++++++++++++++++++++++++
 t/jobs/basic.nix                    |  2 ++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 t/Hydra/Controller/User/dashboard.t

diff --git a/src/lib/Hydra/Controller/User.pm b/src/lib/Hydra/Controller/User.pm
index 2a8affae..9e7d96e5 100644
--- a/src/lib/Hydra/Controller/User.pm
+++ b/src/lib/Hydra/Controller/User.pm
@@ -463,7 +463,7 @@ sub my_jobs_tab :Chained('dashboard_base') :PathPart('my-jobs-tab') :Args(0) {
         , "jobset.enabled" => 1
         },
         { order_by => ["project", "jobset", "job"]
-        , join => ["project", "jobset"]
+        , join => {"jobset" => "project"}
         })];
 }
 
diff --git a/t/Hydra/Controller/User/dashboard.t b/t/Hydra/Controller/User/dashboard.t
new file mode 100644
index 00000000..8a24585d
--- /dev/null
+++ b/t/Hydra/Controller/User/dashboard.t
@@ -0,0 +1,30 @@
+use strict;
+use warnings;
+use Setup;
+my $ctx = test_context();
+use HTTP::Request::Common;
+use Test2::V0;
+use Catalyst::Test ();
+Catalyst::Test->import('Hydra');
+require Hydra::Schema;
+require Hydra::Model::DB;
+my $db = $ctx->db();
+my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'alice@invalid.org', password => '!' });
+$user->setPassword('foobar');
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "basic.nix",
+    build => 1
+);
+my $login = request(POST '/login', Referer => 'http://localhost', Content => {
+        username => 'alice',
+        password => 'foobar',
+    });
+is($login->code, 302);
+my $cookie = $login->header("set-cookie");
+my $my_jobs = request(GET '/dashboard/alice/my-jobs-tab', Accept => 'application/json', Cookie => $cookie);
+ok($my_jobs->is_success);
+my $content = $my_jobs->content();
+ok($content =~ /empty_dir/);
+ok(!($content =~ /fails/));
+ok(!($content =~ /succeed_with_failed/));
+done_testing;
diff --git a/t/jobs/basic.nix b/t/jobs/basic.nix
index 2b76ab5d..9e207f80 100644
--- a/t/jobs/basic.nix
+++ b/t/jobs/basic.nix
@@ -4,6 +4,8 @@ with import ./config.nix;
     mkDerivation {
       name = "empty-dir";
       builder = ./empty-dir-builder.sh;
+      meta.maintainers = [ "alice@invalid.org" ];
+      meta.outPath = "${placeholder "out"}";
     };
 
   fails =

From 213879484d2d671eeb417f9c4fe488b8b83e5ce8 Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Mon, 5 Dec 2022 00:22:35 +0100
Subject: [PATCH 051/285] Fix example config

---
 doc/manual/src/configuration.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index ab68df43..e06dd35f 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -179,6 +179,7 @@ Example configuration:
       <role_search_options>
         deref = always
       </role_search_options>
+    </store>
   </config>
   <role_mapping>
     # Make all users in the hydra_admin group Hydra admins

From 7f816e3237ef1446ea43b4e0f4cbe1e0d53a3656 Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Mon, 5 Dec 2022 00:35:05 +0100
Subject: [PATCH 052/285] Fix link

---
 doc/manual/src/configuration.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index e06dd35f..02210449 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -131,8 +131,8 @@ use LDAP to manage roles and users.
 This is configured by defining the `<ldap>` block in the configuration file.
 In this block it's possible to configure the authentication plugin in the
 `<config>` block. All options are directly passed to `Catalyst::Authentication::Store::LDAP`.
-The documentation for the available settings can be found [here]
-(https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS).
+The documentation for the available settings can be found
+[here](https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS).
 
 Note that the bind password (if needed) should be supplied as an included file to
 prevent it from leaking to the Nix store.

From ad99d3366f8a0088782904fc66958fdf614c12db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= <josef.kemetmueller@gmail.com>
Date: Thu, 29 Dec 2022 22:26:59 +0100
Subject: [PATCH 053/285] Fix MIME types when serving .js and .css

To correctly render HTML reports we make sure to return the following MIME
types instead of "text/plain"

- *.css: "text/css"
- *.js: "application/javascript"

Fixes: #1267
---
 src/lib/Hydra/Controller/Build.pm | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index 18a0eba3..2d74f86a 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -238,9 +238,17 @@ sub serveFile {
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };
 
         # Detect MIME type.
-        state $magic = File::LibMagic->new(follow_symlinks => 1);
-        my $info = $magic->info_from_filename($path);
-        my $type = $info->{mime_with_encoding};
+        my $type = "text/plain";
+        if ($path =~ /.*\.(\S{1,})$/xms) {
+            my $ext = $1;
+            my $mimeTypes = MIME::Types->new(only_complete => 1);
+            my $t = $mimeTypes->mimeTypeOf($ext);
+            $type = ref $t ? $t->type : $t if $t;
+        } else {
+            state $magic = File::LibMagic->new(follow_symlinks => 1);
+            my $info = $magic->info_from_filename($path);
+            $type = $info->{mime_with_encoding};
+        }
         $c->response->content_type($type);
         $c->forward('Hydra::View::Plain');
     }

From 96e36201ebb7748d64f895947d198b370968edd0 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Tue, 29 Nov 2022 18:13:15 +0100
Subject: [PATCH 054/285] hydra-queue-runner: adapt to nlohmann::json

---
 src/hydra-queue-runner/hydra-queue-runner.cc | 249 +++++++++----------
 1 file changed, 114 insertions(+), 135 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 723bf223..b16fd770 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -8,6 +8,8 @@
 
 #include <prometheus/exposer.h>
 
+#include <nlohmann/json.hpp>
+
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "store-api.hh"
@@ -15,20 +17,11 @@
 
 #include "globals.hh"
 #include "hydra-config.hh"
-#include "json.hh"
 #include "s3-binary-cache-store.hh"
 #include "shared.hh"
 
 using namespace nix;
-
-
-namespace nix {
-
-template<> void toJSON<std::atomic<long>>(std::ostream & str, const std::atomic<long> & n) { str << n; }
-template<> void toJSON<std::atomic<uint64_t>>(std::ostream & str, const std::atomic<uint64_t> & n) { str << n; }
-template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
-
-}
+using nlohmann::json;
 
 
 std::string getEnvOrDie(const std::string & key)
@@ -542,181 +535,167 @@ std::shared_ptr<PathLocks> State::acquireGlobalLock()
 
 void State::dumpStatus(Connection & conn)
 {
-    std::ostringstream out;
-
+    auto root = json::object();
     {
-        JSONObject root(out);
         time_t now = time(0);
-        root.attr("status", "up");
-        root.attr("time", time(0));
-        root.attr("uptime", now - startedAt);
-        root.attr("pid", getpid());
+        root["status"] = "up";
+        root["time"] = time(0);
+        root["uptime"] = now - startedAt;
+        root["pid"] = getpid();
         {
             auto builds_(builds.lock());
-            root.attr("nrQueuedBuilds", builds_->size());
+            root["nrQueuedBuilds"] = builds_->size();
         }
         {
             auto steps_(steps.lock());
             for (auto i = steps_->begin(); i != steps_->end(); )
                 if (i->second.lock()) ++i; else i = steps_->erase(i);
-            root.attr("nrUnfinishedSteps", steps_->size());
+            root["nrUnfinishedSteps"] = steps_->size();
         }
         {
             auto runnable_(runnable.lock());
             for (auto i = runnable_->begin(); i != runnable_->end(); )
                 if (i->lock()) ++i; else i = runnable_->erase(i);
-            root.attr("nrRunnableSteps", runnable_->size());
+            root["nrRunnableSteps"] = runnable_->size();
         }
-        root.attr("nrActiveSteps", activeSteps_.lock()->size());
-        root.attr("nrStepsBuilding", nrStepsBuilding);
-        root.attr("nrStepsCopyingTo", nrStepsCopyingTo);
-        root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom);
-        root.attr("nrStepsWaiting", nrStepsWaiting);
-        root.attr("nrUnsupportedSteps", nrUnsupportedSteps);
-        root.attr("bytesSent", bytesSent);
-        root.attr("bytesReceived", bytesReceived);
-        root.attr("nrBuildsRead", nrBuildsRead);
-        root.attr("buildReadTimeMs", buildReadTimeMs);
-        root.attr("buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead);
-        root.attr("nrBuildsDone", nrBuildsDone);
-        root.attr("nrStepsStarted", nrStepsStarted);
-        root.attr("nrStepsDone", nrStepsDone);
-        root.attr("nrRetries", nrRetries);
-        root.attr("maxNrRetries", maxNrRetries);
+        root["nrActiveSteps"] = activeSteps_.lock()->size();
+        root["nrStepsBuilding"] = nrStepsBuilding.load();
+        root["nrStepsCopyingTo"] = nrStepsCopyingTo.load();
+        root["nrStepsCopyingFrom"] = nrStepsCopyingFrom.load();
+        root["nrStepsWaiting"] = nrStepsWaiting.load();
+        root["nrUnsupportedSteps"] = nrUnsupportedSteps.load();
+        root["bytesSent"] = bytesSent.load();
+        root["bytesReceived"] = bytesReceived.load();
+        root["nrBuildsRead"] = nrBuildsRead.load();
+        root["buildReadTimeMs"] = buildReadTimeMs.load();
+        root["buildReadTimeAvgMs"] = nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead;
+        root["nrBuildsDone"] = nrBuildsDone.load();
+        root["nrStepsStarted"] = nrStepsStarted.load();
+        root["nrStepsDone"] = nrStepsDone.load();
+        root["nrRetries"] = nrRetries.load();
+        root["maxNrRetries"] = maxNrRetries.load();
         if (nrStepsDone) {
-            root.attr("totalStepTime", totalStepTime);
-            root.attr("totalStepBuildTime", totalStepBuildTime);
-            root.attr("avgStepTime", (float) totalStepTime / nrStepsDone);
-            root.attr("avgStepBuildTime", (float) totalStepBuildTime / nrStepsDone);
+            root["totalStepTime"] = totalStepTime.load();
+            root["totalStepBuildTime"] = totalStepBuildTime.load();
+            root["avgStepTime"] = (float) totalStepTime / nrStepsDone;
+            root["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone;
         }
-        root.attr("nrQueueWakeups", nrQueueWakeups);
-        root.attr("nrDispatcherWakeups", nrDispatcherWakeups);
-        root.attr("dispatchTimeMs", dispatchTimeMs);
-        root.attr("dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups);
-        root.attr("nrDbConnections", dbPool.count());
-        root.attr("nrActiveDbUpdates", nrActiveDbUpdates);
+        root["nrQueueWakeups"] = nrQueueWakeups.load();
+        root["nrDispatcherWakeups"] = nrDispatcherWakeups.load();
+        root["dispatchTimeMs"] = dispatchTimeMs.load();
+        root["dispatchTimeAvgMs"] = nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups;
+        root["nrDbConnections"] = dbPool.count();
+        root["nrActiveDbUpdates"] = nrActiveDbUpdates.load();
 
         {
-            auto nested = root.object("machines");
+            auto nested = root["machines"];
             auto machines_(machines.lock());
             for (auto & i : *machines_) {
                 auto & m(i.second);
                 auto & s(m->state);
-                auto nested2 = nested.object(m->sshName);
-                nested2.attr("enabled", m->enabled);
-
-                {
-                    auto list = nested2.list("systemTypes");
-                    for (auto & s : m->systemTypes)
-                        list.elem(s);
-                }
-
-                {
-                    auto list = nested2.list("supportedFeatures");
-                    for (auto & s : m->supportedFeatures)
-                        list.elem(s);
-                }
-
-                {
-                    auto list = nested2.list("mandatoryFeatures");
-                    for (auto & s : m->mandatoryFeatures)
-                        list.elem(s);
-                }
-
-                nested2.attr("currentJobs", s->currentJobs);
-                if (s->currentJobs == 0)
-                    nested2.attr("idleSince", s->idleSince);
-                nested2.attr("nrStepsDone", s->nrStepsDone);
-                if (m->state->nrStepsDone) {
-                    nested2.attr("totalStepTime", s->totalStepTime);
-                    nested2.attr("totalStepBuildTime", s->totalStepBuildTime);
-                    nested2.attr("avgStepTime", (float) s->totalStepTime / s->nrStepsDone);
-                    nested2.attr("avgStepBuildTime", (float) s->totalStepBuildTime / s->nrStepsDone);
-                }
-
                 auto info(m->state->connectInfo.lock());
-                nested2.attr("disabledUntil", std::chrono::system_clock::to_time_t(info->disabledUntil));
-                nested2.attr("lastFailure", std::chrono::system_clock::to_time_t(info->lastFailure));
-                nested2.attr("consecutiveFailures", info->consecutiveFailures);
 
+                auto machine = nested[m->sshName] = {
+                    {"enabled",  m->enabled},
+                    {"systemTypes", m->systemTypes},
+                    {"supportedFeatures", m->supportedFeatures},
+                    {"mandatoryFeatures", m->mandatoryFeatures},
+                    {"nrStepsDone", s->nrStepsDone.load()},
+                    {"currentJobs", s->currentJobs.load()},
+                    {"disabledUntil", std::chrono::system_clock::to_time_t(info->disabledUntil)},
+                    {"lastFailure", std::chrono::system_clock::to_time_t(info->lastFailure)},
+                    {"consecutiveFailures", info->consecutiveFailures},
+                };
+
+                if (s->currentJobs == 0)
+                    machine["idleSince"] = s->idleSince.load();
+                if (m->state->nrStepsDone) {
+                    machine["totalStepTime"] = s->totalStepTime.load();
+                    machine["totalStepBuildTime"] = s->totalStepBuildTime.load();
+                    machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
+                    machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
+                }
             }
         }
 
         {
-            auto nested = root.object("jobsets");
+            auto jobsets_json = root["jobsets"];
             auto jobsets_(jobsets.lock());
             for (auto & jobset : *jobsets_) {
-                auto nested2 = nested.object(jobset.first.first + ":" + jobset.first.second);
-                nested2.attr("shareUsed", jobset.second->shareUsed());
-                nested2.attr("seconds", jobset.second->getSeconds());
+                jobsets_json[jobset.first.first + ":" + jobset.first.second] = {
+                    {"shareUsed", jobset.second->shareUsed()},
+                    {"seconds", jobset.second->getSeconds()},
+                };
             }
         }
 
         {
-            auto nested = root.object("machineTypes");
+            auto machineTypesJson = root["machineTypes"];
             auto machineTypes_(machineTypes.lock());
             for (auto & i : *machineTypes_) {
-                auto nested2 = nested.object(i.first);
-                nested2.attr("runnable", i.second.runnable);
-                nested2.attr("running", i.second.running);
+                auto machineTypeJson = machineTypesJson[i.first] = {
+                    {"runnable", i.second.runnable},
+                    {"running", i.second.running},
+                };
                 if (i.second.runnable > 0)
-                    nested2.attr("waitTime", i.second.waitTime.count() +
-                        i.second.runnable * (time(0) - lastDispatcherCheck));
+                    machineTypeJson["waitTime"] = i.second.waitTime.count() +
+                        i.second.runnable * (time(0) - lastDispatcherCheck);
                 if (i.second.running == 0)
-                    nested2.attr("lastActive", std::chrono::system_clock::to_time_t(i.second.lastActive));
+                    machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive);
             }
         }
 
         auto store = getDestStore();
 
-        auto nested = root.object("store");
-
         auto & stats = store->getStats();
-        nested.attr("narInfoRead", stats.narInfoRead);
-        nested.attr("narInfoReadAverted", stats.narInfoReadAverted);
-        nested.attr("narInfoMissing", stats.narInfoMissing);
-        nested.attr("narInfoWrite", stats.narInfoWrite);
-        nested.attr("narInfoCacheSize", stats.pathInfoCacheSize);
-        nested.attr("narRead", stats.narRead);
-        nested.attr("narReadBytes", stats.narReadBytes);
-        nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);
-        nested.attr("narWrite", stats.narWrite);
-        nested.attr("narWriteAverted", stats.narWriteAverted);
-        nested.attr("narWriteBytes", stats.narWriteBytes);
-        nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);
-        nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);
-        nested.attr("narCompressionSavings",
-            stats.narWriteBytes
-            ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
-            : 0.0);
-        nested.attr("narCompressionSpeed", // MiB/s
+        root["store"] = {
+            {"narInfoRead", stats.narInfoRead.load()},
+            {"narInfoReadAverted", stats.narInfoReadAverted.load()},
+            {"narInfoMissing", stats.narInfoMissing.load()},
+            {"narInfoWrite", stats.narInfoWrite.load()},
+            {"narInfoCacheSize", stats.pathInfoCacheSize.load()},
+            {"narRead", stats.narRead.load()},
+            {"narReadBytes", stats.narReadBytes.load()},
+            {"narReadCompressedBytes", stats.narReadCompressedBytes.load()},
+            {"narWrite", stats.narWrite.load()},
+            {"narWriteAverted", stats.narWriteAverted.load()},
+            {"narWriteBytes", stats.narWriteBytes.load()},
+            {"narWriteCompressedBytes", stats.narWriteCompressedBytes.load()},
+            {"narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs.load()},
+            {"narCompressionSavings",
+             stats.narWriteBytes
+             ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
+             : 0.0},
+            {"narCompressionSpeed", // MiB/s
             stats.narWriteCompressionTimeMs
             ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
-            : 0.0);
+            : 0.0},
+        };
 
         auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
         if (s3Store) {
-            auto nested2 = nested.object("s3");
             auto & s3Stats = s3Store->getS3Stats();
-            nested2.attr("put", s3Stats.put);
-            nested2.attr("putBytes", s3Stats.putBytes);
-            nested2.attr("putTimeMs", s3Stats.putTimeMs);
-            nested2.attr("putSpeed",
-                s3Stats.putTimeMs
-                ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
-                : 0.0);
-            nested2.attr("get", s3Stats.get);
-            nested2.attr("getBytes", s3Stats.getBytes);
-            nested2.attr("getTimeMs", s3Stats.getTimeMs);
-            nested2.attr("getSpeed",
-                s3Stats.getTimeMs
-                ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
-                : 0.0);
-            nested2.attr("head", s3Stats.head);
-            nested2.attr("costDollarApprox",
-                (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
-                + s3Stats.put / 1000.0 * 0.005 +
-                + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);
+            auto jsonS3 = root["s3"] = {
+                {"put", s3Stats.put.load()},
+                {"putBytes", s3Stats.putBytes.load()},
+                {"putTimeMs", s3Stats.putTimeMs.load()},
+                {"putSpeed",
+                 s3Stats.putTimeMs
+                 ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
+                 : 0.0},
+                {"get", s3Stats.get.load()},
+                {"getBytes", s3Stats.getBytes.load()},
+                {"getTimeMs", s3Stats.getTimeMs.load()},
+                {"getSpeed",
+                 s3Stats.getTimeMs
+                 ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
+                 : 0.0},
+                {"head", s3Stats.head.load()},
+                {"costDollarApprox",
+                        (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
+                        + s3Stats.put / 1000.0 * 0.005 +
+                        + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09},
+            };
         }
     }
 
@@ -725,7 +704,7 @@ void State::dumpStatus(Connection & conn)
         pqxx::work txn(conn);
         // FIXME: use PostgreSQL 9.5 upsert.
         txn.exec("delete from SystemStatus where what = 'queue-runner'");
-        txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", out.str());
+        txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", root.dump());
         txn.exec("notify status_dumped");
         txn.commit();
     }

From 5b35e1389885cf4c5d1058dc60017564b58e7e4c Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Sat, 28 Jan 2023 12:58:41 +0100
Subject: [PATCH 055/285] hydra-queue-runner: use initializer lists for
 constructing JSON

And also fix the parts that were broken
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 89 ++++++++++----------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index b16fd770..b84681d5 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -535,67 +535,65 @@ std::shared_ptr<PathLocks> State::acquireGlobalLock()
 
 void State::dumpStatus(Connection & conn)
 {
-    auto root = json::object();
+    time_t now = time(0);
+    json statusJson = {
+        {"status", "up"},
+        {"time", time(0)},
+        {"uptime", now - startedAt},
+        {"pid", getpid()},
+
+        {"nrQueuedBuilds", builds.lock()->size()},
+        {"nrActiveSteps", activeSteps_.lock()->size()},
+        {"nrStepsBuilding", nrStepsBuilding.load()},
+        {"nrStepsCopyingTo", nrStepsCopyingTo.load()},
+        {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
+        {"nrStepsWaiting", nrStepsWaiting.load()},
+        {"nrUnsupportedSteps", nrUnsupportedSteps.load()},
+        {"bytesSent", bytesSent.load()},
+        {"bytesReceived", bytesReceived.load()},
+        {"nrBuildsRead", nrBuildsRead.load()},
+        {"buildReadTimeMs", buildReadTimeMs.load()},
+        {"buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead},
+        {"nrBuildsDone", nrBuildsDone.load()},
+        {"nrStepsStarted", nrStepsStarted.load()},
+        {"nrStepsDone", nrStepsDone.load()},
+        {"nrRetries", nrRetries.load()},
+        {"maxNrRetries", maxNrRetries.load()},
+        {"nrQueueWakeups", nrQueueWakeups.load()},
+        {"nrDispatcherWakeups", nrDispatcherWakeups.load()},
+        {"dispatchTimeMs", dispatchTimeMs.load()},
+        {"dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups},
+        {"nrDbConnections", dbPool.count()},
+        {"nrActiveDbUpdates", nrActiveDbUpdates.load()},
+    };
     {
-        time_t now = time(0);
-        root["status"] = "up";
-        root["time"] = time(0);
-        root["uptime"] = now - startedAt;
-        root["pid"] = getpid();
-        {
-            auto builds_(builds.lock());
-            root["nrQueuedBuilds"] = builds_->size();
-        }
         {
             auto steps_(steps.lock());
             for (auto i = steps_->begin(); i != steps_->end(); )
                 if (i->second.lock()) ++i; else i = steps_->erase(i);
-            root["nrUnfinishedSteps"] = steps_->size();
+            statusJson["nrUnfinishedSteps"] = steps_->size();
         }
         {
             auto runnable_(runnable.lock());
             for (auto i = runnable_->begin(); i != runnable_->end(); )
                 if (i->lock()) ++i; else i = runnable_->erase(i);
-            root["nrRunnableSteps"] = runnable_->size();
+            statusJson["nrRunnableSteps"] = runnable_->size();
         }
-        root["nrActiveSteps"] = activeSteps_.lock()->size();
-        root["nrStepsBuilding"] = nrStepsBuilding.load();
-        root["nrStepsCopyingTo"] = nrStepsCopyingTo.load();
-        root["nrStepsCopyingFrom"] = nrStepsCopyingFrom.load();
-        root["nrStepsWaiting"] = nrStepsWaiting.load();
-        root["nrUnsupportedSteps"] = nrUnsupportedSteps.load();
-        root["bytesSent"] = bytesSent.load();
-        root["bytesReceived"] = bytesReceived.load();
-        root["nrBuildsRead"] = nrBuildsRead.load();
-        root["buildReadTimeMs"] = buildReadTimeMs.load();
-        root["buildReadTimeAvgMs"] = nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead;
-        root["nrBuildsDone"] = nrBuildsDone.load();
-        root["nrStepsStarted"] = nrStepsStarted.load();
-        root["nrStepsDone"] = nrStepsDone.load();
-        root["nrRetries"] = nrRetries.load();
-        root["maxNrRetries"] = maxNrRetries.load();
         if (nrStepsDone) {
-            root["totalStepTime"] = totalStepTime.load();
-            root["totalStepBuildTime"] = totalStepBuildTime.load();
-            root["avgStepTime"] = (float) totalStepTime / nrStepsDone;
-            root["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone;
+            statusJson["totalStepTime"] = totalStepTime.load();
+            statusJson["totalStepBuildTime"] = totalStepBuildTime.load();
+            statusJson["avgStepTime"] = (float) totalStepTime / nrStepsDone;
+            statusJson["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone;
         }
-        root["nrQueueWakeups"] = nrQueueWakeups.load();
-        root["nrDispatcherWakeups"] = nrDispatcherWakeups.load();
-        root["dispatchTimeMs"] = dispatchTimeMs.load();
-        root["dispatchTimeAvgMs"] = nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups;
-        root["nrDbConnections"] = dbPool.count();
-        root["nrActiveDbUpdates"] = nrActiveDbUpdates.load();
 
         {
-            auto nested = root["machines"];
             auto machines_(machines.lock());
             for (auto & i : *machines_) {
                 auto & m(i.second);
                 auto & s(m->state);
                 auto info(m->state->connectInfo.lock());
 
-                auto machine = nested[m->sshName] = {
+                json machine = {
                     {"enabled",  m->enabled},
                     {"systemTypes", m->systemTypes},
                     {"supportedFeatures", m->supportedFeatures},
@@ -615,11 +613,12 @@ void State::dumpStatus(Connection & conn)
                     machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
                     machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
                 }
+                statusJson["machines"][m->sshName] = machine;
             }
         }
 
         {
-            auto jobsets_json = root["jobsets"];
+            auto jobsets_json = statusJson["jobsets"] = json::object();
             auto jobsets_(jobsets.lock());
             for (auto & jobset : *jobsets_) {
                 jobsets_json[jobset.first.first + ":" + jobset.first.second] = {
@@ -630,7 +629,7 @@ void State::dumpStatus(Connection & conn)
         }
 
         {
-            auto machineTypesJson = root["machineTypes"];
+            auto machineTypesJson = statusJson["machineTypes"] = json::object();
             auto machineTypes_(machineTypes.lock());
             for (auto & i : *machineTypes_) {
                 auto machineTypeJson = machineTypesJson[i.first] = {
@@ -648,7 +647,7 @@ void State::dumpStatus(Connection & conn)
         auto store = getDestStore();
 
         auto & stats = store->getStats();
-        root["store"] = {
+        statusJson["store"] = {
             {"narInfoRead", stats.narInfoRead.load()},
             {"narInfoReadAverted", stats.narInfoReadAverted.load()},
             {"narInfoMissing", stats.narInfoMissing.load()},
@@ -675,7 +674,7 @@ void State::dumpStatus(Connection & conn)
         auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
         if (s3Store) {
             auto & s3Stats = s3Store->getS3Stats();
-            auto jsonS3 = root["s3"] = {
+            auto jsonS3 = statusJson["s3"] = {
                 {"put", s3Stats.put.load()},
                 {"putBytes", s3Stats.putBytes.load()},
                 {"putTimeMs", s3Stats.putTimeMs.load()},
@@ -704,7 +703,7 @@ void State::dumpStatus(Connection & conn)
         pqxx::work txn(conn);
         // FIXME: use PostgreSQL 9.5 upsert.
         txn.exec("delete from SystemStatus where what = 'queue-runner'");
-        txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", root.dump());
+        txn.exec_params0("insert into SystemStatus values ('queue-runner', $1)", statusJson.dump());
         txn.exec("notify status_dumped");
         txn.commit();
     }

From c7716817a92031f6d94259a3f9d411dd1f062b1e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 28 Jan 2023 09:27:48 +0100
Subject: [PATCH 056/285] Update Nix to 2.13

---
 flake.lock                             | 20 ++++++++++----------
 flake.nix                              |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc |  2 +-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/flake.lock b/flake.lock
index b41b843a..75023b95 100644
--- a/flake.lock
+++ b/flake.lock
@@ -23,32 +23,32 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1661606874,
-        "narHash": "sha256-9+rpYzI+SmxJn+EbYxjGv68Ucp22bdFUSy/4LkHkkDQ=",
-        "owner": "NixOS",
+        "lastModified": 1675514340,
+        "narHash": "sha256-JjnneK+TkhkxFoh6EEVKAzEBdxz0iucZsJ6+PWTTReQ=",
+        "owner": "nixos",
         "repo": "nix",
-        "rev": "11e45768b34fdafdcf019ddbd337afa16127ff0f",
+        "rev": "9157f94e775936798c1f8783eab929e77904e5ed",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
-        "ref": "2.11.0",
+        "owner": "nixos",
+        "ref": "2.13-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1657693803,
-        "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=",
+        "lastModified": 1670461440,
+        "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2",
+        "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.05-small",
+        "ref": "nixos-22.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index cd9f094d..208d9017 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:NixOS/nix/2.11.0";
+  inputs.nix.url = "github:nixos/nix/2.13-maintenance";
 
   outputs = { self, nixpkgs, nix }:
     let
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 18d39620..de7ae7ba 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -129,7 +129,7 @@ static void worker(
             LockFlags {
                 .updateLockFile = false,
                 .useRegistries = false,
-                .allowMutable = false,
+                .allowUnlocked = false,
             });
 
         callFlake(state, lockedFlake, *vFlake);

From ddd3ac3a4d57549c88812a0d1bad8a196899309d Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Tue, 29 Nov 2022 15:48:42 +0100
Subject: [PATCH 057/285] name tests

---
 flake.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flake.nix b/flake.nix
index 208d9017..ec9a708e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -272,6 +272,7 @@
         tests.install = forEachSystem (system:
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
           simpleTest {
+            name = "hydra-install";
             nodes.machine = hydraServer;
             testScript =
               ''
@@ -288,6 +289,7 @@
           let pkgs = pkgsBySystem.${system}; in
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
           simpleTest {
+            name = "hydra-notifications";
             nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
               services.hydra-dev.extraConfig = ''
@@ -346,6 +348,7 @@
           let pkgs = pkgsBySystem.${system}; in
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
           makeTest {
+            name = "hydra-gitea";
             nodes.machine = { pkgs, ... }: {
               imports = [ hydraServer ];
               services.hydra-dev.extraConfig = ''

From 73dff150397bbc3de94e6c4bf4d7ac59b3a681ab Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Thu, 1 Dec 2022 23:32:47 +0100
Subject: [PATCH 058/285] tests: ports are numbers

---
 flake.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index ec9a708e..5f0a7d24 100644
--- a/flake.nix
+++ b/flake.nix
@@ -280,7 +280,7 @@
                 machine.wait_for_job("hydra-server")
                 machine.wait_for_job("hydra-evaluator")
                 machine.wait_for_job("hydra-queue-runner")
-                machine.wait_for_open_port("3000")
+                machine.wait_for_open_port(3000)
                 machine.succeed("curl --fail http://localhost:3000/")
               '';
           });
@@ -317,7 +317,7 @@
 
               # Wait until InfluxDB can receive web requests
               machine.wait_for_job("influxdb")
-              machine.wait_for_open_port("8086")
+              machine.wait_for_open_port(8086)
 
               # Create an InfluxDB database where hydra will write to
               machine.succeed(
@@ -327,7 +327,7 @@
 
               # Wait until hydra-server can receive HTTP requests
               machine.wait_for_job("hydra-server")
-              machine.wait_for_open_port("3000")
+              machine.wait_for_open_port(3000)
 
               # Setup the project and jobset
               machine.succeed(

From 65c1249227dd33fec8856dc1680dfdd5b3598d0a Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Thu, 16 Feb 2023 19:24:53 +0100
Subject: [PATCH 059/285] systemd: hydra-queue-runner: wait for network-online

This prevents eval errors when a machine is just started and the network isn't yet online.
I'm running hydra on a laptop and the network takes a bit of time to come online (WLAN),
so it's nice if the evaluator starts only when the network actually goes online.

Otherwise an error like this can happen on the first eval(s):

```
error fetching latest change from git repo at `https://github.com/nixos/nixpkgs.git':
fatal: unable to access 'https://github.com/nixos/nixpkgs.git/': Could not resolve host: github.com
```
---
 hydra-module.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 8e02dcbb..70e17284 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -340,7 +340,7 @@ in
     systemd.services.hydra-queue-runner =
       { wantedBy = [ "multi-user.target" ];
         requires = [ "hydra-init.service" ];
-        after = [ "hydra-init.service" "network.target" ];
+        after = [ "hydra-init.service" "network-online.target" ];
         path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
         restartTriggers = [ hydraConf ];
         environment = env // {

From f44d3d6ec9b3ca0546a82d0230a9afee4ac179c8 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 4 Mar 2023 12:07:34 +0100
Subject: [PATCH 060/285] Update Nix to 2.13.3

Includes the following required fixes:

* perl-bindings are correctly initialized: https://github.com/NixOS/nix/commit/77d8066e83ec6120c954ce34290ee1ffe00da133
* /etc/ must be unwritable in build sandbox: https://github.com/NixOS/nix/commit/4acc684ef7b3117c6d6ac12837398a0008a53d85
---
 flake.lock | 8 ++++----
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index 75023b95..08fd86ad 100644
--- a/flake.lock
+++ b/flake.lock
@@ -23,16 +23,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1675514340,
-        "narHash": "sha256-JjnneK+TkhkxFoh6EEVKAzEBdxz0iucZsJ6+PWTTReQ=",
+        "lastModified": 1677045134,
+        "narHash": "sha256-jUc2ccTR8f6MGY2pUKgujm+lxSPNGm/ZAP+toX+nMNc=",
         "owner": "nixos",
         "repo": "nix",
-        "rev": "9157f94e775936798c1f8783eab929e77904e5ed",
+        "rev": "4acc684ef7b3117c6d6ac12837398a0008a53d85",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "2.13-maintenance",
+        "ref": "2.13.3",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 5f0a7d24..beeb90c1 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:nixos/nix/2.13-maintenance";
+  inputs.nix.url = "github:nixos/nix/2.13.3";
 
   outputs = { self, nixpkgs, nix }:
     let

From 810d2e6b51af82a948a12eae505a66c2e0f6f09f Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 6 Mar 2023 07:45:03 -0800
Subject: [PATCH 061/285] Drop unused IndexBuildOutputsOnPath index

Also it's larger than the actual table it's indexing lol.

    -[ RECORD 30 ]----------+-----------------------------------------
    table_name              | buildoutputs
    index_name              | indexbuildoutputsonpath
    index_scans_count       | 0
    index_size              | 31 GB
    table_reads_index_count | 2128699937
    table_reads_seq_count   | 0
    table_reads_count       | 2128699937
    table_writes_count      | 22442976
    table_size              | 28 GB
---
 src/sql/upgrade-83.sql | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 src/sql/upgrade-83.sql

diff --git a/src/sql/upgrade-83.sql b/src/sql/upgrade-83.sql
new file mode 100644
index 00000000..01603e78
--- /dev/null
+++ b/src/sql/upgrade-83.sql
@@ -0,0 +1,3 @@
+-- This index was introduced in a migration but was never recorded in
+-- hydra.sql (the source of truth), which is why `if exists` is required.
+drop index if exists IndexBuildOutputsOnPath;

From 8d53c3ca11855234e32ca9f1da0f544491e6cf09 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 6 Mar 2023 07:47:35 -0800
Subject: [PATCH 062/285] test: use ubuntu-latest

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0f5f43da..42cb6843 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -4,7 +4,7 @@ on:
   push:
 jobs:
   tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:

From a084e204ae8b8ffcf3a4a1912582a00c15914136 Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <Mindavi@users.noreply.github.com>
Date: Tue, 7 Mar 2023 21:56:20 +0100
Subject: [PATCH 063/285] systemd: hydra-queue-runner: wait for network.target
 too

Co-authored-by: Sandro <sandro.jaeckel@gmail.com>
---
 hydra-module.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 70e17284..1f0792d7 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -340,7 +340,7 @@ in
     systemd.services.hydra-queue-runner =
       { wantedBy = [ "multi-user.target" ];
         requires = [ "hydra-init.service" ];
-        after = [ "hydra-init.service" "network-online.target" ];
+        after = [ "hydra-init.service" "network.target" "network-online.target" ];
         path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
         restartTriggers = [ hydraConf ];
         environment = env // {

From f88bef15ed57c36dc33d220c8cdf1d5021b8fdbb Mon Sep 17 00:00:00 2001
From: Rob Vermaas <rob.vermaas@gmail.com>
Date: Mon, 13 Mar 2023 16:44:09 +0100
Subject: [PATCH 064/285] Use new Google for Web signin, the old way will be
 deprecated Mar 31st 2023

---
 src/root/auth.tt   | 5 +----
 src/root/topbar.tt | 6 ++++--
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/root/auth.tt b/src/root/auth.tt
index 360904d9..d1539765 100644
--- a/src/root/auth.tt
+++ b/src/root/auth.tt
@@ -82,7 +82,7 @@
       function onGoogleSignIn(googleUser) {
         requestJSON({
           url: "[% c.uri_for('/google-login') %]",
-          data: "id_token=" + googleUser.getAuthResponse().id_token,
+          data: "id_token=" + googleUser.credential,
           type: 'POST',
           success: function(data) {
             window.location.reload();
@@ -91,9 +91,6 @@
         return false;
         };
 
-      $("#google-signin").click(function() {
-        $(".g-signin2:first-child > div").click();
-      });
     </script>
   [% END %]
 
diff --git a/src/root/topbar.tt b/src/root/topbar.tt
index fdfbf431..1771222d 100644
--- a/src/root/topbar.tt
+++ b/src/root/topbar.tt
@@ -133,8 +133,10 @@
   [% ELSE %]
     [% WRAPPER makeSubMenu title="Sign in" id="sign-in-menu" align="right" %]
       [% IF c.config.enable_google_login %]
-        <div style="display: none" class="g-signin2" data-onsuccess="onGoogleSignIn" data-theme="dark"></div>
-        <a class="dropdown-item" href="#" id="google-signin">Sign in with Google</a>
+        <script src="https://accounts.google.com/gsi/client" async defer></script>
+        <div id="g_id_onload" data-client_id="[% c.config.google_client_id %]" data-callback="onGoogleSignIn">
+        </div>
+        <div class="g_id_signin" data-type="standard"></div>
         <div class="dropdown-divider"></div>
       [% END %]
       [% IF c.config.github_client_id %]

From b4099df91ec542a2b2eed38d07ff1a64c265711f Mon Sep 17 00:00:00 2001
From: Julien Malka <julien@malka.sh>
Date: Mon, 24 Apr 2023 16:30:03 +0200
Subject: [PATCH 065/285] hydra-eval-jobs: fix jobs containing a dot being
 dropped

---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index de7ae7ba..af839bba 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -245,7 +245,7 @@ static void worker(
                 StringSet ss;
                 for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
                     std::string name(state.symbols[i->name]);
-                    if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) {
+                    if (name.find(' ') != std::string::npos) {
                         printError("skipping job with illegal name '%s'", name);
                         continue;
                     }
@@ -416,7 +416,11 @@ int main(int argc, char * * argv)
 
                     if (response.find("attrs") != response.end()) {
                         for (auto & i : response["attrs"]) {
-                            auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i;
+                            std::string path = i;
+                            if (path.find(".") != std::string::npos){
+                                path = "\"" + path  + "\"";
+                            }
+                            auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) path;
                             newAttrs.insert(s);
                         }
                     }

From a0c8440a5c6eee911479c220706f74fd17e0c55f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 23 Jun 2023 13:14:49 +0200
Subject: [PATCH 066/285] Update to Nix 2.16 and NixOS 23.05
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:nixos/nix/4acc684ef7b3117c6d6ac12837398a0008a53d85' (2023-02-22)
  → 'github:NixOS/nix/84050709ea18f3285a85d729f40c8f8eddf5008e' (2023-06-06)
• Added input 'nix/flake-compat':
    'github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9' (2023-01-17)
• Updated input 'nixpkgs':
    follows 'nix/nixpkgs'
  → 'github:NixOS/nixpkgs/ef0bc3976340dab9a4e087a0bcff661a8b2e87f3' (2023-06-21)
---
 flake.lock | 46 +++++++++++++++++++++++++++++++---------------
 flake.nix  |  5 +++--
 2 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/flake.lock b/flake.lock
index 08fd86ad..ee85f6fa 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "lowdown-src": {
       "flake": false,
       "locked": {
@@ -18,37 +34,40 @@
     },
     "nix": {
       "inputs": {
+        "flake-compat": "flake-compat",
         "lowdown-src": "lowdown-src",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1677045134,
-        "narHash": "sha256-jUc2ccTR8f6MGY2pUKgujm+lxSPNGm/ZAP+toX+nMNc=",
-        "owner": "nixos",
+        "lastModified": 1686048923,
+        "narHash": "sha256-/XCWa2osNFIpPC5MkxlX6qTZf/DaTLwS3LWN0SRFiuU=",
+        "owner": "NixOS",
         "repo": "nix",
-        "rev": "4acc684ef7b3117c6d6ac12837398a0008a53d85",
+        "rev": "84050709ea18f3285a85d729f40c8f8eddf5008e",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
-        "ref": "2.13.3",
+        "owner": "NixOS",
+        "ref": "2.16.1",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1670461440,
-        "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=",
+        "lastModified": 1687379288,
+        "narHash": "sha256-cSuwfiqYfeVyqzCRkU9AvLTysmEuSal8nh6CYr+xWog=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425",
+        "rev": "ef0bc3976340dab9a4e087a0bcff661a8b2e87f3",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.11-small",
+        "ref": "nixos-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -72,10 +91,7 @@
     "root": {
       "inputs": {
         "nix": "nix",
-        "nixpkgs": [
-          "nix",
-          "nixpkgs"
-        ]
+        "nixpkgs": "nixpkgs"
       }
     }
   },
diff --git a/flake.nix b/flake.nix
index beeb90c1..6bbec9b0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,8 +1,9 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.follows = "nix/nixpkgs";
-  inputs.nix.url = "github:nixos/nix/2.13.3";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
+  inputs.nix.url = "github:NixOS/nix/2.16.1";
+  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
     let

From 9f69bb5c2c132e9ac7b8155972096b425155c6e1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 23 Jun 2023 15:06:34 +0200
Subject: [PATCH 067/285] Fix compilation against Nix 2.16

---
 configure.ac                                 |  2 --
 src/hydra-eval-jobs/hydra-eval-jobs.cc       | 31 ++++++++++++--------
 src/hydra-queue-runner/build-remote.cc       | 12 ++++----
 src/hydra-queue-runner/builder.cc            |  4 +--
 src/hydra-queue-runner/dispatcher.cc         |  8 ++---
 src/hydra-queue-runner/hydra-queue-runner.cc |  7 ++---
 src/hydra-queue-runner/queue-monitor.cc      | 14 ++++-----
 7 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/configure.ac b/configure.ac
index 0c823696..eec647c3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -10,8 +10,6 @@ AC_PROG_LN_S
 AC_PROG_LIBTOOL
 AC_PROG_CXX
 
-CXXFLAGS+=" -std=c++17"
-
 AC_PATH_PROG([XSLTPROC], [xsltproc])
 
 AC_ARG_WITH([docbook-xsl],
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index af839bba..79523944 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -25,7 +25,8 @@
 
 #include <nlohmann/json.hpp>
 
-void check_pid_status_nonblocking(pid_t check_pid) {
+void check_pid_status_nonblocking(pid_t check_pid)
+{
     // Only check 'initialized' and known PID's
     if (check_pid <= 0) { return; }
 
@@ -100,7 +101,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std:
         else if (v.type() == nAttrs) {
             auto a = v.attrs->find(state.symbols.create(subAttribute));
             if (a != v.attrs->end())
-                res.push_back(std::string(state.forceString(*a->value)));
+                res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
         }
     };
 
@@ -197,26 +198,30 @@ static void worker(
 
                 /* If this is an aggregate, then get its constituents. */
                 auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
-                if (a && state.forceBool(*a->value, a->pos)) {
+                if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
                     auto a = v->attrs->get(state.symbols.create("constituents"));
                     if (!a)
                         throw EvalError("derivation must have a ‘constituents’ attribute");
 
+                    NixStringContext context;
+                    state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
+                    for (auto & c : context)
+                        std::visit(overloaded {
+                            [&](const NixStringContextElem::Built & b) {
+                                job["constituents"].push_back(state.store->printStorePath(b.drvPath));
+                            },
+                            [&](const NixStringContextElem::Opaque & o) {
+                            },
+                            [&](const NixStringContextElem::DrvDeep & d) {
+                            },
+                        }, c.raw());
 
-                    PathSet context;
-                    state.coerceToString(a->pos, *a->value, context, true, false);
-                    for (auto & i : context)
-                        if (i.at(0) == '!') {
-                            size_t index = i.find("!", 1);
-                            job["constituents"].push_back(std::string(i, index + 1));
-                        }
-
-                    state.forceList(*a->value, a->pos);
+                    state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
                     for (unsigned int n = 0; n < a->value->listSize(); ++n) {
                         auto v = a->value->listElems()[n];
                         state.forceValue(*v, noPos);
                         if (v->type() == nString)
-                            job["namedConstituents"].push_back(state.forceStringNoCtx(*v));
+                            job["namedConstituents"].push_back(v->str());
                     }
                 }
 
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 21a6c331..6baff7df 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -116,12 +116,12 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
        the remote host to substitute missing paths. */
     // FIXME: substitute output pollutes our build log
     to << cmdQueryValidPaths << 1 << useSubstitutes;
-    worker_proto::write(destStore, to, closure);
+    workerProtoWrite(destStore, to, closure);
     to.flush();
 
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = worker_proto::read(destStore, from, Phantom<StorePathSet> {});
+    auto present = WorkerProto<StorePathSet>::read(destStore, from);
 
     if (present.size() == closure.size()) return;
 
@@ -367,7 +367,7 @@ void State::buildRemote(ref<Store> destStore,
             }
         }
         if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
-            worker_proto::read(*localStore, from, Phantom<DrvOutputs> {});
+            WorkerProto<DrvOutputs>::read(*localStore, from);
         }
         switch ((BuildResult::Status) res) {
             case BuildResult::Built:
@@ -444,17 +444,17 @@ void State::buildRemote(ref<Store> destStore,
             std::map<StorePath, ValidPathInfo> infos;
             size_t totalNarSize = 0;
             to << cmdQueryPathInfos;
-            worker_proto::write(*localStore, to, outputs);
+            workerProtoWrite(*localStore, to, outputs);
             to.flush();
             while (true) {
                 auto storePathS = readString(from);
                 if (storePathS == "") break;
                 auto deriver = readString(from); // deriver
-                auto references = worker_proto::read(*localStore, from, Phantom<StorePathSet> {});
+                auto references = WorkerProto<StorePathSet>::read(*localStore, from);
                 readLongLong(from); // download size
                 auto narSize = readLongLong(from);
                 auto narHash = Hash::parseAny(readString(from), htSHA256);
-                auto ca = parseContentAddressOpt(readString(from));
+                auto ca = ContentAddress::parseOpt(readString(from));
                 readStrings<StringSet>(from); // sigs
                 ValidPathInfo info(localStore->parseStorePath(storePathS), narHash);
                 assert(outputs.count(info.path));
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 37022522..89aec323 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -323,7 +323,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                 pqxx::work txn(*conn);
 
                 for (auto & b : direct) {
-                    printMsg(lvlInfo, format("marking build %1% as succeeded") % b->id);
+                    printInfo("marking build %1% as succeeded", b->id);
                     markSucceededBuild(txn, b, res, buildId != b->id || result.isCached,
                         result.startTime, result.stopTime);
                 }
@@ -451,7 +451,7 @@ void State::failStep(
             /* Mark all builds that depend on this derivation as failed. */
             for (auto & build : indirect) {
                 if (build->finishedInDB) continue;
-                printMsg(lvlError, format("marking build %1% as failed") % build->id);
+                printError("marking build %1% as failed", build->id);
                 txn.exec_params0
                     ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0",
                      build->id,
diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index d2bb3c90..1e40fa69 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -52,7 +52,7 @@ void State::dispatcher()
             {
                 auto dispatcherWakeup_(dispatcherWakeup.lock());
                 if (!*dispatcherWakeup_) {
-                    printMsg(lvlDebug, format("dispatcher sleeping for %1%s") %
+                    debug("dispatcher sleeping for %1%s",
                         std::chrono::duration_cast<std::chrono::seconds>(sleepUntil - std::chrono::system_clock::now()).count());
                     dispatcherWakeup_.wait_until(dispatcherWakeupCV, sleepUntil);
                 }
@@ -60,7 +60,7 @@ void State::dispatcher()
             }
 
         } catch (std::exception & e) {
-            printMsg(lvlError, format("dispatcher: %1%") % e.what());
+            printError("dispatcher: %s", e.what());
             sleep(1);
         }
 
@@ -80,8 +80,8 @@ system_time State::doDispatch()
             jobset.second->pruneSteps();
             auto s2 = jobset.second->shareUsed();
             if (s1 != s2)
-                printMsg(lvlDebug, format("pruned scheduling window of ‘%1%:%2%’ from %3% to %4%")
-                    % jobset.first.first % jobset.first.second % s1 % s2);
+                debug("pruned scheduling window of ‘%1%:%2%’ from %3% to %4%",
+                    jobset.first.first, jobset.first.second, s1, s2);
         }
     }
 
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index b84681d5..acf1282e 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -161,9 +161,9 @@ void State::parseMachines(const std::string & contents)
            same name. */
         auto i = oldMachines.find(machine->sshName);
         if (i == oldMachines.end())
-            printMsg(lvlChatty, format("adding new machine ‘%1%’") % machine->sshName);
+            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName);
         else
-            printMsg(lvlChatty, format("updating machine ‘%1%’") % machine->sshName);
+            printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
         machine->state = i == oldMachines.end()
             ? std::make_shared<Machine::State>()
             : i->second->state;
@@ -173,7 +173,7 @@ void State::parseMachines(const std::string & contents)
     for (auto & m : oldMachines)
         if (newMachines.find(m.first) == newMachines.end()) {
             if (m.second->enabled)
-                printMsg(lvlInfo, format("removing machine ‘%1%’") % m.first);
+                printInfo("removing machine ‘%1%’", m.first);
             /* Add a disabled Machine object to make sure stats are
                maintained. */
             auto machine = std::make_shared<Machine>(*(m.second));
@@ -928,7 +928,6 @@ int main(int argc, char * * argv)
         });
 
         settings.verboseBuild = true;
-        settings.lockCPU = false;
 
         State state{metricsAddrOpt};
         if (status)
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 12d55b79..0bb167a2 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -13,7 +13,7 @@ void State::queueMonitor()
         try {
             queueMonitorLoop();
         } catch (std::exception & e) {
-            printMsg(lvlError, format("queue monitor: %1%") % e.what());
+            printError("queue monitor: %s", e.what());
             sleep(10); // probably a DB problem, so don't retry right away
         }
     }
@@ -142,13 +142,13 @@ bool State::getQueuedBuilds(Connection & conn,
 
     createBuild = [&](Build::ptr build) {
         prom.queue_build_loads.Increment();
-        printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
+        printMsg(lvlTalkative, "loading build %1% (%2%)", build->id, build->fullJobName());
         nrAdded++;
         newBuildsByID.erase(build->id);
 
         if (!localStore->isValidPath(build->drvPath)) {
             /* Derivation has been GC'ed prematurely. */
-            printMsg(lvlError, format("aborting GC'ed build %1%") % build->id);
+            printError("aborting GC'ed build %1%", build->id);
             if (!build->finishedInDB) {
                 auto mc = startDbUpdate();
                 pqxx::work txn(conn);
@@ -302,7 +302,7 @@ bool State::getQueuedBuilds(Connection & conn,
 
         /* Add the new runnable build steps to ‘runnable’ and wake up
            the builder threads. */
-        printMsg(lvlChatty, format("got %1% new runnable steps from %2% new builds") % newRunnable.size() % nrAdded);
+        printMsg(lvlChatty, "got %1% new runnable steps from %2% new builds", newRunnable.size(), nrAdded);
         for (auto & r : newRunnable)
             makeRunnable(r);
 
@@ -358,13 +358,13 @@ void State::processQueueChange(Connection & conn)
         for (auto i = builds_->begin(); i != builds_->end(); ) {
             auto b = currentIds.find(i->first);
             if (b == currentIds.end()) {
-                printMsg(lvlInfo, format("discarding cancelled build %1%") % i->first);
+                printInfo("discarding cancelled build %1%", i->first);
                 i = builds_->erase(i);
                 // FIXME: ideally we would interrupt active build steps here.
                 continue;
             }
             if (i->second->globalPriority < b->second) {
-                printMsg(lvlInfo, format("priority of build %1% increased") % i->first);
+                printInfo("priority of build %1% increased", i->first);
                 i->second->globalPriority = b->second;
                 i->second->propagatePriorities();
             }
@@ -654,7 +654,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
         if (r.empty()) continue;
         BuildID id = r[0][0].as<BuildID>();
 
-        printMsg(lvlInfo, format("reusing build %d") % id);
+        printInfo("reusing build %d", id);
 
         BuildOutput res;
         res.failed = r[0][1].as<int>() == bsFailedWithOutput;

From ce001bb1420bb0c774ea08cd21fd624ccea04788 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 23 Jun 2023 15:09:09 +0200
Subject: [PATCH 068/285] Relax time interval checks

I saw one of these failing randomly.
---
 t/Hydra/Plugin/RunCommand/basic.t      |  4 ++--
 t/Hydra/Plugin/RunCommand/errno.t      |  4 ++--
 t/Hydra/Schema/Result/RunCommandLogs.t | 22 +++++++++++-----------
 t/Hydra/Schema/Result/TaskRetries.t    |  4 ++--
 t/Hydra/Schema/ResultSet/TaskRetries.t |  2 +-
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/basic.t b/t/Hydra/Plugin/RunCommand/basic.t
index e9fc730b..2c0eec68 100644
--- a/t/Hydra/Plugin/RunCommand/basic.t
+++ b/t/Hydra/Plugin/RunCommand/basic.t
@@ -57,8 +57,8 @@ subtest "Validate a run log was created" => sub {
     ok($runlog->did_succeed(), "The process did succeed.");
     is($runlog->job_matcher, "*:*:*", "An unspecified job matcher is defaulted to *:*:*");
     is($runlog->command, 'cp "$HYDRA_JSON" "$HYDRA_DATA/joboutput.json"', "The executed command is saved.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is also recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is also recent.");
     is($runlog->exit_code, 0, "This command should have succeeded.");
 
     subtest "Validate the run log file exists" => sub {
diff --git a/t/Hydra/Plugin/RunCommand/errno.t b/t/Hydra/Plugin/RunCommand/errno.t
index 9e06f9bb..6b05d457 100644
--- a/t/Hydra/Plugin/RunCommand/errno.t
+++ b/t/Hydra/Plugin/RunCommand/errno.t
@@ -43,8 +43,8 @@ subtest "Validate a run log was created" => sub {
     ok($runlog->did_fail_with_exec_error(), "The process failed to start due to an exec error.");
     is($runlog->job_matcher, "*:*:*", "An unspecified job matcher is defaulted to *:*:*");
     is($runlog->command, 'invalid-command-this-does-not-exist', "The executed command is saved.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is also recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is also recent.");
     is($runlog->exit_code, undef, "This command should not have executed.");
     is($runlog->error_number, 2, "This command failed to exec.");
 };
diff --git a/t/Hydra/Schema/Result/RunCommandLogs.t b/t/Hydra/Schema/Result/RunCommandLogs.t
index 80589549..f702fcf9 100644
--- a/t/Hydra/Schema/Result/RunCommandLogs.t
+++ b/t/Hydra/Schema/Result/RunCommandLogs.t
@@ -55,7 +55,7 @@ subtest "Starting a process" => sub {
     ok($runlog->is_running(), "The process is running.");
     ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal.");
     ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
     is($runlog->end_time, undef, "The end time is undefined.");
     is($runlog->exit_code, undef, "The exit code is undefined.");
     is($runlog->signal, undef, "The signal is undefined.");
@@ -70,8 +70,8 @@ subtest "The process completed (success)" => sub {
     ok(!$runlog->is_running(), "The process is not running.");
     ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal.");
     ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is recent.");
     is($runlog->error_number, undef, "The error number is undefined");
     is($runlog->exit_code, 0, "The exit code is 0.");
     is($runlog->signal, undef, "The signal is undefined.");
@@ -86,8 +86,8 @@ subtest "The process completed (errored)" => sub {
     ok(!$runlog->is_running(), "The process is not running.");
     ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal.");
     ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is recent.");
     is($runlog->error_number, undef, "The error number is undefined");
     is($runlog->exit_code, 85, "The exit code is 85.");
     is($runlog->signal, undef, "The signal is undefined.");
@@ -102,8 +102,8 @@ subtest "The process completed (status 15, child error 0)" => sub {
     ok(!$runlog->is_running(), "The process is not running.");
     ok($runlog->did_fail_with_signal(), "The process was killed by a signal.");
     ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is recent.");
     is($runlog->error_number, undef, "The error number is undefined");
     is($runlog->exit_code, undef, "The exit code is undefined.");
     is($runlog->signal, 15, "Signal 15 was sent.");
@@ -118,8 +118,8 @@ subtest "The process completed (signaled)" => sub {
     ok(!$runlog->is_running(), "The process is not running.");
     ok($runlog->did_fail_with_signal(), "The process was killed by a signal.");
     ok(!$runlog->did_fail_with_exec_error(), "The process did not fail to start due to an exec error.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is recent.");
     is($runlog->error_number, undef, "The error number is undefined");
     is($runlog->exit_code, undef, "The exit code is undefined.");
     is($runlog->signal, 9, "The signal is 9.");
@@ -134,8 +134,8 @@ subtest "The process failed to start" => sub {
     ok(!$runlog->is_running(), "The process is running.");
     ok(!$runlog->did_fail_with_signal(), "The process was not killed by a signal.");
     ok($runlog->did_fail_with_exec_error(), "The process failed to start due to an exec error.");
-    is($runlog->start_time, within(time() - 1, 2), "The start time is recent.");
-    is($runlog->end_time, within(time() - 1, 2), "The end time is recent.");
+    is($runlog->start_time, within(time() - 1, 5), "The start time is recent.");
+    is($runlog->end_time, within(time() - 1, 5), "The end time is recent.");
     is($runlog->error_number, 2, "The error number is saved");
     is($runlog->exit_code, undef, "The exit code is undefined.");
     is($runlog->signal, undef, "The signal is undefined.");
diff --git a/t/Hydra/Schema/Result/TaskRetries.t b/t/Hydra/Schema/Result/TaskRetries.t
index 0425f11c..a9c9f132 100644
--- a/t/Hydra/Schema/Result/TaskRetries.t
+++ b/t/Hydra/Schema/Result/TaskRetries.t
@@ -25,11 +25,11 @@ subtest "requeue" => sub {
 
     $task->requeue();
     is($task->attempts, 2, "We should have stored a second retry");
-    is($task->retry_at, within(time() + 4, 2), "Delayed two exponential backoff step");
+    is($task->retry_at, within(time() + 4, 5), "Delayed two exponential backoff step");
 
     $task->requeue();
     is($task->attempts, 3, "We should have stored a third retry");
-    is($task->retry_at, within(time() + 8, 2), "Delayed a third exponential backoff step");
+    is($task->retry_at, within(time() + 8, 5), "Delayed a third exponential backoff step");
 };
 
 done_testing;
diff --git a/t/Hydra/Schema/ResultSet/TaskRetries.t b/t/Hydra/Schema/ResultSet/TaskRetries.t
index 4555832c..a9354896 100644
--- a/t/Hydra/Schema/ResultSet/TaskRetries.t
+++ b/t/Hydra/Schema/ResultSet/TaskRetries.t
@@ -101,7 +101,7 @@ subtest "save_task" => sub {
     is($retry->pluginname, "FooPluginName", "Plugin name should match");
     is($retry->payload, "1", "Payload should match");
     is($retry->attempts, 1, "We've had one attempt");
-    is($retry->retry_at, within(time() + 1, 2), "The retry at should be approximately one second away");
+    is($retry->retry_at, within(time() + 1, 5), "The retry at should be approximately one second away");
 };
 
 done_testing;

From 5c35d1be2005ee0471076df5c904c72f4c748d66 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 25 Jun 2023 17:25:43 +0200
Subject: [PATCH 069/285] hydra-queue-runner: fix stats

---
 src/hydra-queue-runner/hydra-queue-runner.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index b84681d5..30fad267 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -618,7 +618,7 @@ void State::dumpStatus(Connection & conn)
         }
 
         {
-            auto jobsets_json = statusJson["jobsets"] = json::object();
+            auto jobsets_json = json::object();
             auto jobsets_(jobsets.lock());
             for (auto & jobset : *jobsets_) {
                 jobsets_json[jobset.first.first + ":" + jobset.first.second] = {
@@ -626,10 +626,11 @@ void State::dumpStatus(Connection & conn)
                     {"seconds", jobset.second->getSeconds()},
                 };
             }
+            statusJson["jobsets"] = jobsets_json;
         }
 
         {
-            auto machineTypesJson = statusJson["machineTypes"] = json::object();
+            auto machineTypesJson = json::object();
             auto machineTypes_(machineTypes.lock());
             for (auto & i : *machineTypes_) {
                 auto machineTypeJson = machineTypesJson[i.first] = {
@@ -642,6 +643,7 @@ void State::dumpStatus(Connection & conn)
                 if (i.second.running == 0)
                     machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive);
             }
+            statusJson["machineTypes"] = machineTypesJson;
         }
 
         auto store = getDestStore();

From 46246dcae3ad05dfd85fcf4f61b44f5c2114634a Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Wed, 19 Jul 2023 15:13:25 +0200
Subject: [PATCH 070/285] Fix docs for  /eval/{id}  endpoint

You  need to pass it an eval-id, not a build-id pretty sure
---
 hydra-api.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index ce7e0f9a..c3068d66 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -533,13 +533,13 @@ paths:
               schema:
                 $ref: '#/components/schemas/Error'
 
-  /eval/{build-id}:
+  /eval/{eval-id}:
     get:
-      summary: Retrieves evaluations identified by build id
+      summary: Retrieves evaluations identified by eval id
       parameters:
-        - name: build-id
+        - name: eval-id
           in: path
-          description: build identifier
+          description: eval identifier
           required: true
           schema:
             type: integer

From a78664f1b5faac4ade5b3a2f3c0e6c6e59a94c51 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Thu, 20 Jul 2023 14:43:03 +0200
Subject: [PATCH 071/285] Fix documentation of defaultpath in api docs

---
 hydra-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index ce7e0f9a..2646a275 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -870,7 +870,7 @@ components:
           description: Size of the produced file
           type: integer
         defaultpath:
-          description: This is a Git/Mercurial commit hash or a Subversion revision number
+          description: if path is a directory, the default file relative to path to be served
           type: string
         'type':
           description: Types of build product (user defined)

From b23431a657d8a9b2f478c95dd81034780751a262 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Fri, 4 Aug 2023 15:53:06 +0200
Subject: [PATCH 072/285] Support Nix 2.17

---
 flake.lock                             |  8 ++++----
 flake.nix                              |  2 +-
 src/hydra-queue-runner/build-remote.cc | 25 +++++++++++++++----------
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/flake.lock b/flake.lock
index ee85f6fa..4b18fbb4 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1686048923,
-        "narHash": "sha256-/XCWa2osNFIpPC5MkxlX6qTZf/DaTLwS3LWN0SRFiuU=",
+        "lastModified": 1690219894,
+        "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "84050709ea18f3285a85d729f40c8f8eddf5008e",
+        "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.16.1",
+        "ref": "2.17.0",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 6bbec9b0..7e7d50e2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.16.1";
+  inputs.nix.url = "github:NixOS/nix/2.17.0";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 6baff7df..6bbd22e2 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -6,6 +6,7 @@
 #include <fcntl.h>
 
 #include "build-result.hh"
+#include "path.hh"
 #include "serve-protocol.hh"
 #include "state.hh"
 #include "util.hh"
@@ -110,18 +111,20 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
     StorePathSet closure;
     destStore.computeFSClosure(paths, closure);
 
+    WorkerProto::WriteConn wconn { .to = to };
+    WorkerProto::ReadConn rconn { .from = from };
     /* Send the "query valid paths" command with the "lock" option
        enabled. This prevents a race where the remote host
        garbage-collect paths that are already there. Optionally, ask
        the remote host to substitute missing paths. */
     // FIXME: substitute output pollutes our build log
-    to << cmdQueryValidPaths << 1 << useSubstitutes;
-    workerProtoWrite(destStore, to, closure);
+    to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
+    WorkerProto::write(destStore, wconn, closure);
     to.flush();
 
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = WorkerProto<StorePathSet>::read(destStore, from);
+    auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, rconn);
 
     if (present.size() == closure.size()) return;
 
@@ -136,7 +139,7 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
     std::unique_lock<std::timed_mutex> sendLock(sendMutex,
         std::chrono::seconds(600));
 
-    to << cmdImportPaths;
+    to << ServeProto::Command::ImportPaths;
     destStore.exportPaths(missing, to);
     to.flush();
 
@@ -223,7 +226,9 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         FdSource from(child.from.get());
+        WorkerProto::ReadConn rconn { .from = from };
         FdSink to(child.to.get());
+        WorkerProto::WriteConn wconn { .to = to };
 
         Finally updateStats([&]() {
             bytesReceived += from.read;
@@ -334,7 +339,7 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssBuilding);
 
-        to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
+        to << ServeProto::Command::BuildDerivation << localStore->printStorePath(step->drvPath);
         writeDerivation(to, *localStore, basicDrv);
         to << maxSilentTime << buildTimeout;
         if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
@@ -367,7 +372,7 @@ void State::buildRemote(ref<Store> destStore,
             }
         }
         if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
-            WorkerProto<DrvOutputs>::read(*localStore, from);
+          WorkerProto::Serialise<DrvOutputs>::read(*localStore, rconn);
         }
         switch ((BuildResult::Status) res) {
             case BuildResult::Built:
@@ -443,14 +448,14 @@ void State::buildRemote(ref<Store> destStore,
             /* Get info about each output path. */
             std::map<StorePath, ValidPathInfo> infos;
             size_t totalNarSize = 0;
-            to << cmdQueryPathInfos;
-            workerProtoWrite(*localStore, to, outputs);
+            to << ServeProto::Command::QueryPathInfos;
+            WorkerProto::write(*localStore, wconn, outputs);
             to.flush();
             while (true) {
                 auto storePathS = readString(from);
                 if (storePathS == "") break;
                 auto deriver = readString(from); // deriver
-                auto references = WorkerProto<StorePathSet>::read(*localStore, from);
+                auto references = WorkerProto::Serialise<StorePathSet>::read(*localStore, rconn);
                 readLongLong(from); // download size
                 auto narSize = readLongLong(from);
                 auto narHash = Hash::parseAny(readString(from), htSHA256);
@@ -494,7 +499,7 @@ void State::buildRemote(ref<Store> destStore,
                        lambda function only gets executed if someone tries to read
                        from source2, we will send the command from here rather
                        than outside the lambda. */
-                    to << cmdDumpStorePath << localStore->printStorePath(path);
+                    to << ServeProto::Command::DumpStorePath << localStore->printStorePath(path);
                     to.flush();
 
                     TeeSource tee(from, sink);

From 9f0427385fa9f306c4cd651dc97377624b90997c Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Sun, 20 Aug 2023 14:55:56 +0200
Subject: [PATCH 073/285] Apply LTO fix suggested by Ericson2314

---
 src/hydra-queue-runner/build-remote.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 6bbd22e2..46c94e60 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -11,6 +11,7 @@
 #include "state.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "finally.hh"
 #include "url.hh"
 

From 35ccc9ebb22796270aafe1b567420e13529e3be5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Aug 2023 17:04:45 +0200
Subject: [PATCH 074/285] Fix indentation

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 46c94e60..56ce1ccf 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -373,7 +373,7 @@ void State::buildRemote(ref<Store> destStore,
             }
         }
         if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
-          WorkerProto::Serialise<DrvOutputs>::read(*localStore, rconn);
+            WorkerProto::Serialise<DrvOutputs>::read(*localStore, rconn);
         }
         switch ((BuildResult::Status) res) {
             case BuildResult::Built:

From 113836ebae04c518b9c787983da9d43af4e27f73 Mon Sep 17 00:00:00 2001
From: sternenseemann <sternenseemann@systemli.org>
Date: Wed, 30 Aug 2023 15:06:48 +0200
Subject: [PATCH 075/285] hydra-api.yaml: name JobsetEval parameter eval-id

This is more accurate since the id space is not shared between build and
eval ids.
---
 hydra-api.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 2646a275..f61a27aa 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -533,13 +533,13 @@ paths:
               schema:
                 $ref: '#/components/schemas/Error'
 
-  /eval/{build-id}:
+  /eval/{eval-id}:
     get:
-      summary: Retrieves evaluations identified by build id
+      summary: Retrieves evaluations identified by eval id
       parameters:
-        - name: build-id
+        - name: eval-id
           in: path
-          description: build identifier
+          description: eval identifier
           required: true
           schema:
             type: integer

From e2195c46d12e1d5ff14dd3d9319e405e2b2b49ab Mon Sep 17 00:00:00 2001
From: sternenseemann <sternenseemann@systemli.org>
Date: Wed, 30 Aug 2023 15:08:11 +0200
Subject: [PATCH 076/285] hydra-api.yaml: document all_builds
 (/eval/{eval-id}/builds)

---
 hydra-api.yaml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index f61a27aa..623c9082 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -551,6 +551,24 @@ paths:
               schema:
                 $ref: '#/components/schemas/JobsetEval'
 
+  /eval/{eval-id}/builds:
+    get:
+      summary: Retrieves all builds belonging to an evaluation identified by eval id
+      parameters:
+        - name: eval-id
+          in: path
+          description: eval identifier
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          description: builds
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/JobsetEvalBuilds'
+
 components:
   schemas:
 
@@ -796,6 +814,13 @@ components:
           additionalProperties:
             $ref: '#/components/schemas/JobsetEvalInput'
 
+    JobsetEvalBuilds:
+      type: array
+      items:
+        type: object
+        additionalProperties:
+          $ref: '#/components/schemas/Build'
+
     JobsetOverview:
       type: array
       items:

From b7c864c515faee58a11a67ea8619e464e883ad34 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Fri, 8 Sep 2023 23:38:30 +0200
Subject: [PATCH 077/285] queue-runner: only re-sort runnables by prio once per
 dispatch cycle
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous implementation was O(N²lg(N)) due to sorting the full
runnables priority list once per runnable being scheduled. While not
confirmed, this is suspected to cause performance issues and
bottlenecking with the queue runner when the runnable list gets large
enough.

This commit changes the dispatcher to instead only sort runnables per
priority once per dispatch cycle. This has the drawback of being less
reactive to runnable priority changes: the previous code would react
immediately, while this might end up using "old" priorities until the
next dispatch cycle. However, dispatch cycles are not supposed to take
very long (seconds, not minutes/hours), so this is not expected to have
much or any practical impact.

Ideally runnables would be maintained in a sorted data structure instead
of the current approach of copying + sorting in the scheduler. This
would however be a much more invasive change to implement, and might
have to wait until we can confirm where the queue runner bottlenecks
actually lie.
---
 src/hydra-queue-runner/dispatcher.cc | 205 ++++++++++++++-------------
 1 file changed, 106 insertions(+), 99 deletions(-)

diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index 1e40fa69..af21dcbd 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -85,12 +85,113 @@ system_time State::doDispatch()
         }
     }
 
+    system_time now = std::chrono::system_clock::now();
+
     /* Start steps until we're out of steps or slots. */
     auto sleepUntil = system_time::max();
     bool keepGoing;
 
+    /* Sort the runnable steps by priority. Priority is establised
+       as follows (in order of precedence):
+
+       - The global priority of the builds that depend on the
+         step. This allows admins to bump a build to the front of
+         the queue.
+
+       - The lowest used scheduling share of the jobsets depending
+         on the step.
+
+       - The local priority of the build, as set via the build's
+         meta.schedulingPriority field. Note that this is not
+         quite correct: the local priority should only be used to
+         establish priority between builds in the same jobset, but
+         here it's used between steps in different jobsets if they
+         happen to have the same lowest used scheduling share. But
+         that's not very likely.
+
+       - The lowest ID of the builds depending on the step;
+         i.e. older builds take priority over new ones.
+
+       FIXME: O(n lg n); obviously, it would be better to keep a
+       runnable queue sorted by priority. */
+    struct StepInfo
+    {
+        Step::ptr step;
+        bool alreadyScheduled = false;
+
+        /* The lowest share used of any jobset depending on this
+           step. */
+        double lowestShareUsed = 1e9;
+
+        /* Info copied from step->state to ensure that the
+           comparator is a partial ordering (see MachineInfo). */
+        int highestGlobalPriority;
+        int highestLocalPriority;
+        BuildID lowestBuildID;
+
+        StepInfo(Step::ptr step, Step::State & step_) : step(step)
+        {
+            for (auto & jobset : step_.jobsets)
+                lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
+            highestGlobalPriority = step_.highestGlobalPriority;
+            highestLocalPriority = step_.highestLocalPriority;
+            lowestBuildID = step_.lowestBuildID;
+        }
+    };
+
+    std::vector<StepInfo> runnableSorted;
+
+    struct RunnablePerType
+    {
+        unsigned int count{0};
+        std::chrono::seconds waitTime{0};
+    };
+
+    std::unordered_map<std::string, RunnablePerType> runnablePerType;
+
+    {
+        auto runnable_(runnable.lock());
+        runnableSorted.reserve(runnable_->size());
+        for (auto i = runnable_->begin(); i != runnable_->end(); ) {
+            auto step = i->lock();
+
+            /* Remove dead steps. */
+            if (!step) {
+                i = runnable_->erase(i);
+                continue;
+            }
+
+            ++i;
+
+            auto & r = runnablePerType[step->systemType];
+            r.count++;
+
+            /* Skip previously failed steps that aren't ready
+               to be retried. */
+            auto step_(step->state.lock());
+            r.waitTime += std::chrono::duration_cast<std::chrono::seconds>(now - step_->runnableSince);
+            if (step_->tries > 0 && step_->after > now) {
+                if (step_->after < sleepUntil)
+                    sleepUntil = step_->after;
+                continue;
+            }
+
+            runnableSorted.emplace_back(step, *step_);
+        }
+    }
+
+    sort(runnableSorted.begin(), runnableSorted.end(),
+        [](const StepInfo & a, const StepInfo & b)
+        {
+            return
+                a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
+                a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
+                a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
+                a.lowestBuildID < b.lowestBuildID;
+        });
+
     do {
-        system_time now = std::chrono::system_clock::now();
+        now = std::chrono::system_clock::now();
 
         /* Copy the currentJobs field of each machine. This is
            necessary to ensure that the sort comparator below is
@@ -138,104 +239,6 @@ system_time State::doDispatch()
                     a.currentJobs > b.currentJobs;
             });
 
-        /* Sort the runnable steps by priority. Priority is establised
-           as follows (in order of precedence):
-
-           - The global priority of the builds that depend on the
-             step. This allows admins to bump a build to the front of
-             the queue.
-
-           - The lowest used scheduling share of the jobsets depending
-             on the step.
-
-           - The local priority of the build, as set via the build's
-             meta.schedulingPriority field. Note that this is not
-             quite correct: the local priority should only be used to
-             establish priority between builds in the same jobset, but
-             here it's used between steps in different jobsets if they
-             happen to have the same lowest used scheduling share. But
-             that's not very likely.
-
-           - The lowest ID of the builds depending on the step;
-             i.e. older builds take priority over new ones.
-
-           FIXME: O(n lg n); obviously, it would be better to keep a
-           runnable queue sorted by priority. */
-        struct StepInfo
-        {
-            Step::ptr step;
-
-            /* The lowest share used of any jobset depending on this
-               step. */
-            double lowestShareUsed = 1e9;
-
-            /* Info copied from step->state to ensure that the
-               comparator is a partial ordering (see MachineInfo). */
-            int highestGlobalPriority;
-            int highestLocalPriority;
-            BuildID lowestBuildID;
-
-            StepInfo(Step::ptr step, Step::State & step_) : step(step)
-            {
-                for (auto & jobset : step_.jobsets)
-                    lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
-                highestGlobalPriority = step_.highestGlobalPriority;
-                highestLocalPriority = step_.highestLocalPriority;
-                lowestBuildID = step_.lowestBuildID;
-            }
-        };
-
-        std::vector<StepInfo> runnableSorted;
-
-        struct RunnablePerType
-        {
-            unsigned int count{0};
-            std::chrono::seconds waitTime{0};
-        };
-
-        std::unordered_map<std::string, RunnablePerType> runnablePerType;
-
-        {
-            auto runnable_(runnable.lock());
-            runnableSorted.reserve(runnable_->size());
-            for (auto i = runnable_->begin(); i != runnable_->end(); ) {
-                auto step = i->lock();
-
-                /* Remove dead steps. */
-                if (!step) {
-                    i = runnable_->erase(i);
-                    continue;
-                }
-
-                ++i;
-
-                auto & r = runnablePerType[step->systemType];
-                r.count++;
-
-                /* Skip previously failed steps that aren't ready
-                   to be retried. */
-                auto step_(step->state.lock());
-                r.waitTime += std::chrono::duration_cast<std::chrono::seconds>(now - step_->runnableSince);
-                if (step_->tries > 0 && step_->after > now) {
-                    if (step_->after < sleepUntil)
-                        sleepUntil = step_->after;
-                    continue;
-                }
-
-                runnableSorted.emplace_back(step, *step_);
-            }
-        }
-
-        sort(runnableSorted.begin(), runnableSorted.end(),
-            [](const StepInfo & a, const StepInfo & b)
-            {
-                return
-                    a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
-                    a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
-                    a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
-                    a.lowestBuildID < b.lowestBuildID;
-            });
-
         /* Find a machine with a free slot and find a step to run
            on it. Once we find such a pair, we restart the outer
            loop because the machine sorting will have changed. */
@@ -245,6 +248,8 @@ system_time State::doDispatch()
             if (mi.machine->state->currentJobs >= mi.machine->maxJobs) continue;
 
             for (auto & stepInfo : runnableSorted) {
+                if (stepInfo.alreadyScheduled) continue;
+
                 auto & step(stepInfo.step);
 
                 /* Can this machine do this step? */
@@ -271,6 +276,8 @@ system_time State::doDispatch()
                     r.count--;
                 }
 
+                stepInfo.alreadyScheduled = true;
+
                 /* Make a slot reservation and start a thread to
                    do the build. */
                 auto builderThread = std::thread(&State::builder, this,

From 6a5fb9efaea35ca29836371307f5083576f421ab Mon Sep 17 00:00:00 2001
From: Stig Palmquist <git@stig.io>
Date: Fri, 20 Oct 2023 00:09:28 +0200
Subject: [PATCH 078/285] Set output length of C::P::Argon2 hashes to 16

Since the default lengths in Crypt::Passphrase::Argon2 changed from 16
to 32 in in 0.009, some tests that expected the passphrase to be
unchanged started failing.
---
 src/lib/Hydra/Schema/Result/Users.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Schema/Result/Users.pm b/src/lib/Hydra/Schema/Result/Users.pm
index b3de6543..c28ae931 100644
--- a/src/lib/Hydra/Schema/Result/Users.pm
+++ b/src/lib/Hydra/Schema/Result/Users.pm
@@ -216,7 +216,7 @@ sub json_hint {
 
 sub _authenticator() {
     my $authenticator = Crypt::Passphrase->new(
-        encoder    => 'Argon2',
+        encoder    => { module => 'Argon2', output_size => 16 },
         validators => [
             (sub {
                 my ($password, $hash) = @_;

From e9da80fff6234fab2458173272ee0bedbe8935c3 Mon Sep 17 00:00:00 2001
From: chayleaf <chayleaf-git@pavluk.org>
Date: Tue, 21 Nov 2023 18:41:52 +0700
Subject: [PATCH 079/285] support nix 2.18

---
 flake.lock                              | 8 ++++----
 flake.nix                               | 2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc  | 7 ++++---
 src/hydra-queue-runner/build-remote.cc  | 6 +++---
 src/hydra-queue-runner/queue-monitor.cc | 4 ++--
 5 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/flake.lock b/flake.lock
index 4b18fbb4..af913ea3 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1690219894,
-        "narHash": "sha256-QMYAkdtU+g9HlZKtoJ+AI6TbWzzovKGnPZJHfZdclc8=",
+        "lastModified": 1696259154,
+        "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "a212300a1d9f9c7b0daf19c00c87fc50480f54f4",
+        "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.17.0",
+        "ref": "2.18.1",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 7e7d50e2..2f2abe62 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.17.0";
+  inputs.nix.url = "github:NixOS/nix/2.18.1";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 79523944..30ab9740 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -7,6 +7,7 @@
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "util.hh"
 #include "get-drvs.hh"
 #include "globals.hh"
@@ -208,13 +209,13 @@ static void worker(
                     for (auto & c : context)
                         std::visit(overloaded {
                             [&](const NixStringContextElem::Built & b) {
-                                job["constituents"].push_back(state.store->printStorePath(b.drvPath));
+                                job["constituents"].push_back(b.drvPath->to_string(*state.store));
                             },
                             [&](const NixStringContextElem::Opaque & o) {
                             },
                             [&](const NixStringContextElem::DrvDeep & d) {
                             },
-                        }, c.raw());
+                        }, c.raw);
 
                     state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
                     for (unsigned int n = 0; n < a->value->listSize(); ++n) {
@@ -516,7 +517,7 @@ int main(int argc, char * * argv)
                     auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
                     auto drv2 = store->readDerivation(drvPath2);
                     job["constituents"].push_back(store->printStorePath(drvPath2));
-                    drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
+                    drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
                 }
 
                 if (brokenJobs.empty()) {
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 56ce1ccf..92438349 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -276,9 +276,9 @@ void State::buildRemote(ref<Store> destStore,
         for (auto & p : step->drv->inputSrcs)
             inputs.insert(p);
 
-        for (auto & input : step->drv->inputDrvs) {
-            auto drv2 = localStore->readDerivation(input.first);
-            for (auto & name : input.second) {
+        for (auto & [drvPath, node] : step->drv->inputDrvs.map) {
+            auto drv2 = localStore->readDerivation(drvPath);
+            for (auto & name : node.value) {
                 if (auto i = get(drv2.outputs, name)) {
                     auto outPath = i->path(*localStore, drv2.name, name);
                     inputs.insert(*outPath);
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 0bb167a2..6c339af6 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -315,7 +315,7 @@ bool State::getQueuedBuilds(Connection & conn,
         if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
             prom.queue_checks_early_exits.Increment();
             break;
-        } 
+        }
     }
 
     prom.queue_checks_finished.Increment();
@@ -561,7 +561,7 @@ Step::ptr State::createStep(ref<Store> destStore,
     printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath));
 
     /* Create steps for the dependencies. */
-    for (auto & i : step->drv->inputDrvs) {
+    for (auto & i : step->drv->inputDrvs.map) {
         auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
         if (dep) {
             auto step_(step->state.lock());

From 5db374cb500b687039ba4701b205ca7dfa67caba Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 30 Nov 2023 10:40:57 -0500
Subject: [PATCH 080/285] Cleanup deps

- `nativeBuildInputs` vs `buildInputs`

- narrow down `with`s for clarity

- use `autoreconfHook` not `bootstrap` script

These sorts of changes have also been done in the Nix repo.
---
 README.md |  2 +-
 bootstrap |  2 --
 flake.nix | 31 +++++++++++++++++--------------
 3 files changed, 18 insertions(+), 17 deletions(-)
 delete mode 100755 bootstrap

diff --git a/README.md b/README.md
index 54cb9a93..2a085325 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ $ nix-build
 You can use the provided shell.nix to get a working development environment:
 ```
 $ nix-shell
-$ ./bootstrap
+$ autoreconfPhase
 $ configurePhase # NOTE: not ./configure
 $ make
 ```
diff --git a/bootstrap b/bootstrap
deleted file mode 100755
index 091b0ee4..00000000
--- a/bootstrap
+++ /dev/null
@@ -1,2 +0,0 @@
-#! /bin/sh -e
-exec autoreconf -vfi
diff --git a/flake.nix b/flake.nix
index 7e7d50e2..ea96ac2e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -61,10 +61,11 @@
 
         };
 
-        hydra = with final; let
-          perlDeps = buildEnv {
+        hydra = let
+          inherit (final) lib stdenv;
+          perlDeps = final.buildEnv {
             name = "hydra-perl-deps";
-            paths = with perlPackages; lib.closePropagation
+            paths = with final.perlPackages; lib.closePropagation
               [
                 AuthenSASL
                 CatalystActionREST
@@ -98,7 +99,7 @@
                 FileSlurper
                 FileWhich
                 final.nix.perl-bindings
-                git
+                final.git
                 IOCompress
                 IPCRun
                 IPCRun3
@@ -141,15 +142,20 @@
 
           src = self;
 
-          buildInputs =
-            [
+          nativeBuildInputs =
+            with final.buildPackages; [
               makeWrapper
-              autoconf
+              autoreconfHook
               automake
               libtool
-              unzip
               nukeReferences
               pkg-config
+              mdbook
+            ];
+
+          buildInputs =
+            with final; [
+              unzip
               libpqxx
               top-git
               mercurial
@@ -162,7 +168,6 @@
               final.nix
               perlDeps
               perl
-              mdbook
               pixz
               boost
               postgresql_13
@@ -172,7 +177,7 @@
               prometheus-cpp
             ];
 
-          checkInputs = [
+          checkInputs = with final; [
             cacert
             foreman
             glibcLocales
@@ -181,7 +186,7 @@
             python3
           ];
 
-          hydraPath = lib.makeBinPath (
+          hydraPath = with final; lib.makeBinPath (
             [
               subversion
               openssh
@@ -203,7 +208,7 @@
             ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
           );
 
-          OPENLDAP_ROOT = openldap;
+          OPENLDAP_ROOT = final.openldap;
 
           shellHook = ''
             pushd $(git rev-parse --show-toplevel) >/dev/null
@@ -218,8 +223,6 @@
             popd >/dev/null
           '';
 
-          preConfigure = "autoreconf -vfi";
-
           NIX_LDFLAGS = [ "-lpthread" ];
 
           enableParallelBuilding = true;

From 2bda7ca642528e1a7c7188d72460fe90c89c4ebf Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 30 Nov 2023 11:31:58 -0500
Subject: [PATCH 081/285] Further use `Machine::Connection` to deduplicate

---
 src/hydra-queue-runner/build-remote.cc | 28 +++++++++++++-------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index a82a95e6..8f8d6532 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -107,27 +107,27 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
 }
 
 
-static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
-    FdSource & from, FdSink & to, const StorePathSet & paths,
+static void copyClosureTo(
+    Machine::Connection & conn,
+    Store & destStore,
+    const StorePathSet & paths,
     bool useSubstitutes = false)
 {
     StorePathSet closure;
     destStore.computeFSClosure(paths, closure);
 
-    WorkerProto::WriteConn wconn { .to = to };
-    WorkerProto::ReadConn rconn { .from = from };
     /* Send the "query valid paths" command with the "lock" option
        enabled. This prevents a race where the remote host
        garbage-collect paths that are already there. Optionally, ask
        the remote host to substitute missing paths. */
     // FIXME: substitute output pollutes our build log
-    to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
-    WorkerProto::write(destStore, wconn, closure);
-    to.flush();
+    conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
+    WorkerProto::write(destStore, conn, closure);
+    conn.to.flush();
 
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, rconn);
+    auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, conn);
 
     if (present.size() == closure.size()) return;
 
@@ -139,14 +139,14 @@ static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
 
     printMsg(lvlDebug, "sending %d missing paths", missing.size());
 
-    std::unique_lock<std::timed_mutex> sendLock(sendMutex,
+    std::unique_lock<std::timed_mutex> sendLock(conn.machine->state->sendLock,
         std::chrono::seconds(600));
 
-    to << ServeProto::Command::ImportPaths;
-    destStore.exportPaths(missing, to);
-    to.flush();
+    conn.to << ServeProto::Command::ImportPaths;
+    destStore.exportPaths(missing, conn.to);
+    conn.to.flush();
 
-    if (readInt(from) != 1)
+    if (readInt(conn.from) != 1)
         throw Error("remote machine failed to import closure");
 }
 
@@ -257,7 +257,7 @@ BasicDerivation sendInputs(
             destStore.computeFSClosure(basicDrv.inputSrcs, closure);
             copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
         } else {
-            copyClosureTo(conn.machine->state->sendLock, destStore, conn.from, conn.to, basicDrv.inputSrcs, true);
+            copyClosureTo(conn, destStore, basicDrv.inputSrcs, true);
         }
 
         auto now2 = std::chrono::steady_clock::now();

From 0917145622c521fff5c9af1b26053c2217bbd02b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 30 Nov 2023 12:19:05 -0500
Subject: [PATCH 082/285] Make new functions not in header `static`

---
 src/hydra-queue-runner/build-remote.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 8f8d6532..a5892016 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -152,7 +152,7 @@ static void copyClosureTo(
 
 
 // FIXME: use Store::topoSortPaths().
-StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths)
+static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths)
 {
     StorePaths sorted;
     StorePathSet visited;
@@ -180,7 +180,7 @@ StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths
     return sorted;
 }
 
-std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
+static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
 {
     std::string base(drvPath.to_string());
     auto logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2);
@@ -193,7 +193,7 @@ std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const Store
     return {std::move(logFile), std::move(logFD)};
 }
 
-void handshake(Machine::Connection & conn, unsigned int repeats)
+static void handshake(Machine::Connection & conn, unsigned int repeats)
 {
     conn.to << SERVE_MAGIC_1 << 0x206;
     conn.to.flush();
@@ -208,7 +208,7 @@ void handshake(Machine::Connection & conn, unsigned int repeats)
         throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName);
 }
 
-BasicDerivation sendInputs(
+static BasicDerivation sendInputs(
     State & state,
     Step & step,
     Store & localStore,
@@ -268,7 +268,7 @@ BasicDerivation sendInputs(
     return basicDrv;
 }
 
-BuildResult performBuild(
+static BuildResult performBuild(
     Machine::Connection & conn,
     Store & localStore,
     StorePath drvPath,
@@ -321,7 +321,7 @@ BuildResult performBuild(
     return result;
 }
 
-std::map<StorePath, ValidPathInfo> queryPathInfos(
+static std::map<StorePath, ValidPathInfo> queryPathInfos(
     Machine::Connection & conn,
     Store & localStore,
     StorePathSet & outputs,
@@ -359,7 +359,7 @@ std::map<StorePath, ValidPathInfo> queryPathInfos(
     return infos;
 }
 
-void copyPathFromRemote(
+static void copyPathFromRemote(
     Machine::Connection & conn,
     NarMemberDatas & narMembers,
     Store & localStore,
@@ -389,7 +389,7 @@ void copyPathFromRemote(
       destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
 }
 
-void copyPathsFromRemote(
+static void copyPathsFromRemote(
     Machine::Connection & conn,
     NarMemberDatas & narMembers,
     Store & localStore,

From e172461e550676a9a295cf917f9b6fab623af3e6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 30 Nov 2023 12:19:20 -0500
Subject: [PATCH 083/285] Use `const` in for loop

As requested by @teh
---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index a5892016..ca508b8b 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -221,7 +221,7 @@ static BasicDerivation sendInputs(
 {
     BasicDerivation basicDrv(*step.drv);
 
-    for (auto & input : step.drv->inputDrvs) {
+    for (const auto & input : step.drv->inputDrvs) {
         auto drv2 = localStore.readDerivation(input.first);
         for (auto & name : input.second) {
             if (auto i = get(drv2.outputs, name)) {

From c922e73c11449759c8e1945ed01fb4cd120ef57d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 30 Nov 2023 14:38:26 -0500
Subject: [PATCH 084/285] Update to Nix 2.19
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/f5f4de6a550327b4b1a06123c2e450f1b92c73b6' (2023-10-02)
  → 'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-11-27)
---
 flake.lock                                   |  8 ++--
 flake.nix                                    |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc       |  8 ++--
 src/hydra-evaluator/hydra-evaluator.cc       |  1 +
 src/hydra-queue-runner/build-remote.cc       | 43 +++++++++++++-------
 src/hydra-queue-runner/build-result.cc       | 12 +++---
 src/hydra-queue-runner/hydra-queue-runner.cc |  3 +-
 src/hydra-queue-runner/nar-extractor.cc      | 12 ++++--
 src/hydra-queue-runner/nar-extractor.hh      |  4 +-
 src/libhydra/db.hh                           |  1 +
 src/libhydra/hydra-config.hh                 |  1 +
 11 files changed, 61 insertions(+), 34 deletions(-)

diff --git a/flake.lock b/flake.lock
index af913ea3..9a9046c8 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1696259154,
-        "narHash": "sha256-WNmifcTsN9aG1ONkv+l2BC4sHZZxtNKy0keqBHXXQ7w=",
+        "lastModified": 1701122567,
+        "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "f5f4de6a550327b4b1a06123c2e450f1b92c73b6",
+        "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.18.1",
+        "ref": "2.19.2",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 2f2abe62..4e8f6dd0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.18.1";
+  inputs.nix.url = "github:NixOS/nix/2.19.2";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 30ab9740..2fe2c80f 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -8,6 +8,8 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
+#include "signals.hh"
+#include "terminal.hh"
 #include "util.hh"
 #include "get-drvs.hh"
 #include "globals.hh"
@@ -54,7 +56,7 @@ using namespace nix;
 static Path gcRootsDir;
 static size_t maxMemorySize;
 
-struct MyArgs : MixEvalArgs, MixCommonArgs
+struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
 {
     Path releaseExpr;
     bool flake = false;
@@ -95,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std:
     rec = [&](Value & v) {
         state.forceValue(v, noPos);
         if (v.type() == nString)
-            res.push_back(v.string.s);
+            res.emplace_back(v.string_view());
         else if (v.isList())
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 rec(*v.listElems()[n]);
@@ -222,7 +224,7 @@ static void worker(
                         auto v = a->value->listElems()[n];
                         state.forceValue(*v, noPos);
                         if (v->type() == nString)
-                            job["namedConstituents"].push_back(v->str());
+                            job["namedConstituents"].push_back(v->string_view());
                     }
                 }
 
diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc
index a1ccf047..75506ff8 100644
--- a/src/hydra-evaluator/hydra-evaluator.cc
+++ b/src/hydra-evaluator/hydra-evaluator.cc
@@ -2,6 +2,7 @@
 #include "hydra-config.hh"
 #include "pool.hh"
 #include "shared.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <thread>
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 92438349..5552e91a 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -9,9 +9,11 @@
 #include "path.hh"
 #include "serve-protocol.hh"
 #include "state.hh"
+#include "current-process.hh"
+#include "processes.hh"
 #include "util.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "finally.hh"
 #include "url.hh"
 
@@ -106,26 +108,32 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
 
 
 static void copyClosureTo(std::timed_mutex & sendMutex, Store & destStore,
-    FdSource & from, FdSink & to, const StorePathSet & paths,
+    FdSource & from, FdSink & to, ServeProto::Version remoteVersion, const StorePathSet & paths,
     bool useSubstitutes = false)
 {
     StorePathSet closure;
     destStore.computeFSClosure(paths, closure);
 
-    WorkerProto::WriteConn wconn { .to = to };
-    WorkerProto::ReadConn rconn { .from = from };
+    ServeProto::WriteConn wconn {
+        .to = to,
+        .version = remoteVersion,
+    };
+    ServeProto::ReadConn rconn {
+        .from = from,
+        .version = remoteVersion,
+    };
     /* Send the "query valid paths" command with the "lock" option
        enabled. This prevents a race where the remote host
        garbage-collect paths that are already there. Optionally, ask
        the remote host to substitute missing paths. */
     // FIXME: substitute output pollutes our build log
     to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
-    WorkerProto::write(destStore, wconn, closure);
+    ServeProto::write(destStore, wconn, closure);
     to.flush();
 
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, rconn);
+    auto present = ServeProto::Serialise<StorePathSet>::read(destStore, rconn);
 
     if (present.size() == closure.size()) return;
 
@@ -227,9 +235,7 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         FdSource from(child.from.get());
-        WorkerProto::ReadConn rconn { .from = from };
         FdSink to(child.to.get());
-        WorkerProto::WriteConn wconn { .to = to };
 
         Finally updateStats([&]() {
             bytesReceived += from.read;
@@ -237,7 +243,7 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         /* Handshake. */
-        unsigned int remoteVersion;
+        ServeProto::Version remoteVersion;
 
         try {
             to << SERVE_MAGIC_1 << 0x206;
@@ -258,6 +264,15 @@ void State::buildRemote(ref<Store> destStore,
             throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
         }
 
+        ServeProto::ReadConn rconn {
+            .from = from,
+            .version = remoteVersion,
+        };
+        ServeProto::WriteConn wconn {
+            .to = to,
+            .version = remoteVersion,
+        };
+
         {
             auto info(machine->state->connectInfo.lock());
             info->consecutiveFailures = 0;
@@ -313,7 +328,7 @@ void State::buildRemote(ref<Store> destStore,
                 destStore->computeFSClosure(inputs, closure);
                 copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
             } else {
-                copyClosureTo(machine->state->sendLock, *destStore, from, to, inputs, true);
+                copyClosureTo(machine->state->sendLock, *destStore, from, to, remoteVersion, inputs, true);
             }
 
             auto now2 = std::chrono::steady_clock::now();
@@ -373,7 +388,7 @@ void State::buildRemote(ref<Store> destStore,
             }
         }
         if (GET_PROTOCOL_MINOR(remoteVersion) >= 6) {
-            WorkerProto::Serialise<DrvOutputs>::read(*localStore, rconn);
+            ServeProto::Serialise<DrvOutputs>::read(*localStore, rconn);
         }
         switch ((BuildResult::Status) res) {
             case BuildResult::Built:
@@ -450,13 +465,13 @@ void State::buildRemote(ref<Store> destStore,
             std::map<StorePath, ValidPathInfo> infos;
             size_t totalNarSize = 0;
             to << ServeProto::Command::QueryPathInfos;
-            WorkerProto::write(*localStore, wconn, outputs);
+            ServeProto::write(*localStore, wconn, outputs);
             to.flush();
             while (true) {
                 auto storePathS = readString(from);
                 if (storePathS == "") break;
                 auto deriver = readString(from); // deriver
-                auto references = WorkerProto::Serialise<StorePathSet>::read(*localStore, rconn);
+                auto references = ServeProto::Serialise<StorePathSet>::read(*localStore, rconn);
                 readLongLong(from); // download size
                 auto narSize = readLongLong(from);
                 auto narHash = Hash::parseAny(readString(from), htSHA256);
diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc
index ea8b4a6a..691c1f19 100644
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@@ -1,7 +1,7 @@
 #include "hydra-build-result.hh"
 #include "store-api.hh"
 #include "util.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 
 #include <regex>
 
@@ -63,7 +63,7 @@ BuildOutput getBuildOutput(
 
         auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
         if (productsFile == narMembers.end() ||
-            productsFile->second.type != FSAccessor::Type::tRegular)
+            productsFile->second.type != SourceAccessor::Type::tRegular)
             continue;
         assert(productsFile->second.contents);
 
@@ -94,7 +94,7 @@ BuildOutput getBuildOutput(
 
             product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);
 
-            if (file->second.type == FSAccessor::Type::tRegular) {
+            if (file->second.type == SourceAccessor::Type::tRegular) {
                 product.isRegular = true;
                 product.fileSize = file->second.fileSize.value();
                 product.sha256hash = file->second.sha256.value();
@@ -117,7 +117,7 @@ BuildOutput getBuildOutput(
 
             auto file = narMembers.find(product.path);
             assert(file != narMembers.end());
-            if (file->second.type == FSAccessor::Type::tDirectory)
+            if (file->second.type == SourceAccessor::Type::tDirectory)
                 res.products.push_back(product);
         }
     }
@@ -126,7 +126,7 @@ BuildOutput getBuildOutput(
     for (auto & output : outputs) {
         auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
         if (file == narMembers.end() ||
-            file->second.type != FSAccessor::Type::tRegular)
+            file->second.type != SourceAccessor::Type::tRegular)
             continue;
         res.releaseName = trim(file->second.contents.value());
         // FIXME: validate release name
@@ -136,7 +136,7 @@ BuildOutput getBuildOutput(
     for (auto & output : outputs) {
         auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
         if (file == narMembers.end() ||
-            file->second.type != FSAccessor::Type::tRegular)
+            file->second.type != SourceAccessor::Type::tRegular)
             continue;
         for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
             auto fields = tokenizeString<std::vector<std::string>>(line);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 91779288..1d54bb93 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -10,6 +10,7 @@
 
 #include <nlohmann/json.hpp>
 
+#include "signals.hh"
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "store-api.hh"
@@ -467,7 +468,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
              product.type,
              product.subtype,
              product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
-             product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
+             product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt,
              product.path,
              product.name,
              product.defaultPath);
diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc
index 9f0eb431..3c6857bf 100644
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -24,13 +24,13 @@ struct Extractor : ParseSink
 
     void createDirectory(const Path & path) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
+        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
     void createRegularFile(const Path & path) override
     {
         curMember = &members.insert_or_assign(prefix + path, NarMemberData {
-            .type = FSAccessor::Type::tRegular,
+            .type = SourceAccessor::Type::tRegular,
             .fileSize = 0,
             .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
         }).first->second;
@@ -66,8 +66,14 @@ struct Extractor : ParseSink
 
     void createSymlink(const Path & path, const std::string & target) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
+        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
     }
+
+    void isExecutable() override
+    { }
+
+    void closeRegularFile() override
+    { }
 };
 
 
diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh
index 45b2706c..2634135b 100644
--- a/src/hydra-queue-runner/nar-extractor.hh
+++ b/src/hydra-queue-runner/nar-extractor.hh
@@ -1,13 +1,13 @@
 #pragma once
 
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "types.hh"
 #include "serialise.hh"
 #include "hash.hh"
 
 struct NarMemberData
 {
-    nix::FSAccessor::Type type;
+    nix::SourceAccessor::Type type;
     std::optional<uint64_t> fileSize;
     std::optional<std::string> contents;
     std::optional<nix::Hash> sha256;
diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh
index 00e8f406..1e927573 100644
--- a/src/libhydra/db.hh
+++ b/src/libhydra/db.hh
@@ -2,6 +2,7 @@
 
 #include <pqxx/pqxx>
 
+#include "environment-variables.hh"
 #include "util.hh"
 
 
diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh
index 1688c278..b1275896 100644
--- a/src/libhydra/hydra-config.hh
+++ b/src/libhydra/hydra-config.hh
@@ -2,6 +2,7 @@
 
 #include <map>
 
+#include "file-system.hh"
 #include "util.hh"
 
 struct HydraConfig

From 622c25e3c4da9a6f4fd44bfe425137597b81705a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 08:56:06 -0500
Subject: [PATCH 085/285] Sedding prior to merge

---
 src/hydra-queue-runner/build-remote.cc | 14 +++++++-------
 src/hydra-queue-runner/state.hh        |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index ca508b8b..f679db0c 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -10,8 +10,8 @@
 #include "serve-protocol.hh"
 #include "state.hh"
 #include "util.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "finally.hh"
 #include "url.hh"
 
@@ -122,12 +122,12 @@ static void copyClosureTo(
        the remote host to substitute missing paths. */
     // FIXME: substitute output pollutes our build log
     conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
-    WorkerProto::write(destStore, conn, closure);
+    ServeProto::write(destStore, conn, closure);
     conn.to.flush();
 
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = WorkerProto::Serialise<StorePathSet>::read(destStore, conn);
+    auto present = ServeProto::Serialise<StorePathSet>::read(destStore, conn);
 
     if (present.size() == closure.size()) return;
 
@@ -315,7 +315,7 @@ static BuildResult performBuild(
         }
     }
     if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
-        WorkerProto::Serialise<DrvOutputs>::read(localStore, conn);
+        ServeProto::Serialise<DrvOutputs>::read(localStore, conn);
     }
 
     return result;
@@ -332,13 +332,13 @@ static std::map<StorePath, ValidPathInfo> queryPathInfos(
     /* Get info about each output path. */
     std::map<StorePath, ValidPathInfo> infos;
     conn.to << ServeProto::Command::QueryPathInfos;
-    WorkerProto::write(localStore, conn, outputs);
+    ServeProto::write(localStore, conn, outputs);
     conn.to.flush();
     while (true) {
         auto storePathS = readString(conn.from);
         if (storePathS == "") break;
         auto deriver = readString(conn.from); // deriver
-        auto references = WorkerProto::Serialise<StorePathSet>::read(localStore, conn);
+        auto references = ServeProto::Serialise<StorePathSet>::read(localStore, conn);
         readLongLong(conn.from); // download size
         auto narSize = readLongLong(conn.from);
         auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index ce795700..dfeaefe7 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -21,7 +21,7 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "nar-extractor.hh"
-#include "worker-protocol.hh"
+#include "serve-protocol.hh"
 
 
 typedef unsigned int BuildID;
@@ -310,12 +310,12 @@ struct Machine
         // Backpointer to the machine
         ptr machine;
 
-        operator nix::WorkerProto::ReadConn ()
+        operator nix::ServeProto::ReadConn ()
         {
             return { .from = from };
         }
 
-        operator nix::WorkerProto::WriteConn ()
+        operator nix::ServeProto::WriteConn ()
         {
             return { .to = to };
         }

From 104baef503df3afcd09615d9b014923696e9ecf9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 09:42:04 -0500
Subject: [PATCH 086/285] Document the connection initialization process

---
 src/hydra-queue-runner/build-remote.cc | 16 ++++++++++++----
 src/hydra-queue-runner/state.hh        |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 032c2733..baf35d86 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -195,6 +195,12 @@ static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, cons
     return {std::move(logFile), std::move(logFD)};
 }
 
+/**
+ * @param conn is not fully initialized; it is this functions job to set
+ * the `remoteVersion` field after the handshake is completed.
+ * Therefore, no `ServeProto::Serialize` functions can be used until
+ * that field is set.
+ */
 static void handshake(Machine::Connection & conn, unsigned int repeats)
 {
     conn.to << SERVE_MAGIC_1 << 0x206;
@@ -204,6 +210,7 @@ static void handshake(Machine::Connection & conn, unsigned int repeats)
     if (magic != SERVE_MAGIC_2)
         throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName);
     conn.remoteVersion = readInt(conn.from);
+    // Now `conn` is initialized.
     if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200)
         throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName);
     if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0)
@@ -512,10 +519,11 @@ void State::buildRemote(ref<Store> destStore,
                process. Meh. */
         });
 
-        Machine::Connection conn;
-        conn.from = child.from.get();
-        conn.to = child.to.get();
-        conn.machine = machine;
+        Machine::Connection conn {
+            .from = child.from.get(),
+            .to = child.to.get(),
+            .machine = machine,
+        };
 
         Finally updateStats([&]() {
             bytesReceived += conn.from.read;
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 03923917..6359063a 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -303,8 +303,8 @@ struct Machine
 
     // A connection to a machine
     struct Connection {
-        nix::FdSink to;
         nix::FdSource from;
+        nix::FdSink to;
         nix::ServeProto::Version remoteVersion;
 
         // Backpointer to the machine

From 162b538912fc940b3c737634c23c99c574adf4ad Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 11:27:39 -0500
Subject: [PATCH 087/285] Remove unused `thisArrow` variable

---
 src/hydra-queue-runner/build-remote.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index baf35d86..f1575170 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -421,8 +421,6 @@ static void copyPathsFromRemote(
 
 void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 {
-    RemoteResult thisArrow;
-
     startTime = buildResult.startTime;
     stopTime = buildResult.stopTime;
     timesBuilt = buildResult.timesBuilt;

From 363604846adf27db017d2d599b373f1e0839dfd6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 11:31:05 -0500
Subject: [PATCH 088/285] Again, use `const` in for loop

As requested by @teh. Was lost in merge with master, now added back.
---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index f1575170..73f46a53 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -230,7 +230,7 @@ static BasicDerivation sendInputs(
 {
     BasicDerivation basicDrv(*step.drv);
 
-    for (auto & [drvPath, node] : step.drv->inputDrvs.map) {
+    for (const auto & [drvPath, node] : step.drv->inputDrvs.map) {
         auto drv2 = localStore.readDerivation(drvPath);
         for (auto & name : node.value) {
             if (auto i = get(drv2.outputs, name)) {

From 9ba4417940ffdd0fadea43f68c61ef948a4b8d39 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 16:05:50 -0500
Subject: [PATCH 089/285] Prepare for CA derivation support with lower impact
 changes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is just C++ changes without any Perl / Frontend / SQL Schema
changes.

The idea is that it should be possible to redeploy Hydra with these
chnages with (a) no schema migration and also (b) no regressions. We
should be able to much more safely deploy these to a staging server and
then production `hydra.nixos.org`.

Extracted from #875

Co-Authored-By: Théophane Hufschmitt <theophane.hufschmitt@tweag.io>
Co-Authored-By: Alexander Sosedkin <monk@unboiled.info>
Co-Authored-By: Andrea Ciceri <andrea.ciceri@autistici.org>
Co-Authored-By: Charlotte 🦝 Delenk Mlotte@chir.rs>
Co-Authored-By: Sandro Jäckel <sandro.jaeckel@gmail.com>
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc       |  16 ++-
 src/hydra-queue-runner/build-remote.cc       | 105 +++++++++++---
 src/hydra-queue-runner/build-result.cc       |  19 ++-
 src/hydra-queue-runner/builder.cc            |   8 +-
 src/hydra-queue-runner/hydra-build-result.hh |   4 +-
 src/hydra-queue-runner/hydra-queue-runner.cc |  27 +++-
 src/hydra-queue-runner/queue-monitor.cc      | 137 ++++++++++---------
 src/hydra-queue-runner/state.hh              |   4 +-
 8 files changed, 213 insertions(+), 107 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 2fe2c80f..d007189d 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -178,7 +178,11 @@ static void worker(
 
             if (auto drv = getDerivation(state, *v, false)) {
 
-                DrvInfo::Outputs outputs = drv->queryOutputs();
+                // CA derivations do not have static output paths, so we
+                // have to defensively not query output paths in case we
+                // encounter one.
+                DrvInfo::Outputs outputs = drv->queryOutputs(
+                    !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
 
                 if (drv->querySystem() == "unknown")
                     throw EvalError("derivation must have a 'system' attribute");
@@ -239,12 +243,12 @@ static void worker(
                 }
 
                 nlohmann::json out;
-                for (auto & j : outputs)
-                    // FIXME: handle CA/impure builds.
-                    if (j.second)
-                        out[j.first] = state.store->printStorePath(*j.second);
+                for (auto & [outputName, optOutputPath] : outputs)
+                    if (optOutputPath)
+                        out[outputName] = state.store->printStorePath(*optOutputPath);
+                    else
+                        out[outputName] = "";
                 job["outputs"] = std::move(out);
-
                 reply["job"] = std::move(job);
             }
 
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 73f46a53..26f6d63f 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -182,6 +182,41 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
     return sorted;
 }
 
+/**
+ * Replace the input derivations by their output paths to send a minimal closure
+ * to the builder.
+ *
+ * If we can afford it, resolve it, so that the newly generated derivation still
+ * has some sensible output paths.
+ */
+BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
+{
+    BasicDerivation ret;
+    auto outputHashes = staticOutputHashes(store, drv);
+    if (!drv.type().hasKnownOutputPaths()) {
+        auto maybeBasicDrv = drv.tryResolve(store);
+        if (!maybeBasicDrv)
+            throw Error(
+                "the derivation '%s' can’t be resolved. It’s probably "
+                "missing some outputs",
+                store.printStorePath(drvPath));
+        ret = *maybeBasicDrv;
+    } else {
+        // If the derivation is a real `InputAddressed` derivation, we must
+        // resolve it manually to keep the original output paths
+        ret = BasicDerivation(drv);
+        for (auto & [drvPath, node] : drv.inputDrvs.map) {
+            auto drv2 = store.readDerivation(drvPath);
+            auto drv2Outputs = drv2.outputsAndOptPaths(store);
+            for (auto & name : node.value) {
+                auto inputPath = drv2Outputs.at(name);
+                ret.inputSrcs.insert(*inputPath.second);
+            }
+        }
+    }
+    return ret;
+}
+
 static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
 {
     std::string base(drvPath.to_string());
@@ -228,17 +263,7 @@ static BasicDerivation sendInputs(
     counter & nrStepsCopyingTo
 )
 {
-    BasicDerivation basicDrv(*step.drv);
-
-    for (const auto & [drvPath, node] : step.drv->inputDrvs.map) {
-        auto drv2 = localStore.readDerivation(drvPath);
-        for (auto & name : node.value) {
-            if (auto i = get(drv2.outputs, name)) {
-                auto outPath = i->path(localStore, drv2.name, name);
-                basicDrv.inputSrcs.insert(*outPath);
-            }
-        }
-    }
+    BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath);
 
     /* Ensure that the inputs exist in the destination store. This is
        a no-op for regular stores, but for the binary cache store,
@@ -323,8 +348,33 @@ static BuildResult performBuild(
             result.stopTime = stop;
         }
     }
+
+    // Get the newly built outputs, either from the remote if it
+    // supports it, or by introspecting the derivation if the remote is
+    // too old.
     if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
-        ServeProto::Serialise<DrvOutputs>::read(localStore, conn);
+        auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(localStore, conn);
+        for (auto && [output, realisation] : builtOutputs)
+            result.builtOutputs.insert_or_assign(
+                std::move(output.outputName),
+                std::move(realisation));
+    } else {
+        // If the remote is too old to handle CA derivations, we can’t get this
+        // far anyways
+        assert(drv.type().hasKnownOutputPaths());
+        DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore);
+        auto outputHashes = staticOutputHashes(localStore, drv);
+        for (auto & [outputName, output] : drvOutputs) {
+            auto outputPath = output.second;
+            // We’ve just asserted that the output paths of the derivation
+            // were known
+            assert(outputPath);
+            auto outputHash = outputHashes.at(outputName);
+            auto drvOutput = DrvOutput { outputHash, outputName };
+            result.builtOutputs.insert_or_assign(
+                std::move(outputName),
+                Realisation { drvOutput, *outputPath });
+        }
     }
 
     return result;
@@ -376,6 +426,7 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
+    for (auto * store : {&destStore, &localStore}) {
       /* Receive the NAR from the remote and add it to the
           destination store. Meanwhile, extract all the info from the
           NAR that getBuildOutput() needs. */
@@ -395,7 +446,8 @@ static void copyPathFromRemote(
           extractNarData(tee, localStore.printStorePath(info.path), narMembers);
       });
 
-      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
+      store->addToStore(info, *source2, NoRepair, NoCheckSigs);
+    }
 }
 
 static void copyPathsFromRemote(
@@ -592,6 +644,10 @@ void State::buildRemote(ref<Store> destStore,
             result.logFile = "";
         }
 
+        StorePathSet outputs;
+        for (auto & [_, realisation] : buildResult.builtOutputs)
+            outputs.insert(realisation.outPath);
+
         /* Copy the output paths. */
         if (!machine->isLocalhost() || localStore != std::shared_ptr<Store>(destStore)) {
             updateStep(ssReceivingOutputs);
@@ -600,12 +656,6 @@ void State::buildRemote(ref<Store> destStore,
 
             auto now1 = std::chrono::steady_clock::now();
 
-            StorePathSet outputs;
-            for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
-                if (i.second.second)
-                   outputs.insert(*i.second.second);
-            }
-
             size_t totalNarSize = 0;
             auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize);
 
@@ -624,6 +674,23 @@ void State::buildRemote(ref<Store> destStore,
             result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
         }
 
+        /* Register the outputs of the newly built drv */
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            auto outputHashes = staticOutputHashes(*localStore, *step->drv);
+            for (auto & [outputName, realisation] : buildResult.builtOutputs) {
+                // Register the resolved drv output
+                localStore->registerDrvOutput(realisation);
+                destStore->registerDrvOutput(realisation);
+
+                // Also register the unresolved one
+                auto unresolvedRealisation = realisation;
+                unresolvedRealisation.signatures.clear();
+                unresolvedRealisation.id.drvHash = outputHashes.at(outputName);
+                localStore->registerDrvOutput(unresolvedRealisation);
+                destStore->registerDrvOutput(unresolvedRealisation);
+            }
+        }
+
         /* Shut down the connection. */
         child.to = -1;
         child.pid.wait();
diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc
index 691c1f19..ffdc37b7 100644
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@@ -11,18 +11,18 @@ using namespace nix;
 BuildOutput getBuildOutput(
     nix::ref<Store> store,
     NarMemberDatas & narMembers,
-    const Derivation & drv)
+    const OutputPathMap derivationOutputs)
 {
     BuildOutput res;
 
     /* Compute the closure size. */
     StorePathSet outputs;
     StorePathSet closure;
-    for (auto & i : drv.outputsAndOptPaths(*store))
-        if (i.second.second) {
-            store->computeFSClosure(*i.second.second, closure);
-            outputs.insert(*i.second.second);
-        }
+    for (auto& [outputName, outputPath] : derivationOutputs) {
+        store->computeFSClosure(outputPath, closure);
+        outputs.insert(outputPath);
+        res.outputs.insert({outputName, outputPath});
+    }
     for (auto & path : closure) {
         auto info = store->queryPathInfo(path);
         res.closureSize += info->narSize;
@@ -107,13 +107,12 @@ BuildOutput getBuildOutput(
     /* If no build products were explicitly declared, then add all
        outputs as a product of type "nix-build". */
     if (!explicitProducts) {
-        for (auto & [name, output] : drv.outputs) {
+        for (auto & [name, output] : derivationOutputs) {
             BuildProduct product;
-            auto outPath = output.path(*store, drv.name, name);
-            product.path = store->printStorePath(*outPath);
+            product.path = store->printStorePath(output);
             product.type = "nix-build";
             product.subtype = name == "out" ? "" : name;
-            product.name = outPath->name();
+            product.name = output.name();
 
             auto file = narMembers.find(product.path);
             assert(file != narMembers.end());
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 307eee8e..42983941 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         if (result.stepStatus == bsSuccess) {
             updateStep(ssPostProcessing);
-            res = getBuildOutput(destStore, narMembers, *step->drv);
+            res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath));
         }
     }
 
@@ -277,9 +277,9 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         assert(stepNr);
 
-        for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
-            if (i.second.second)
-               addRoot(*i.second.second);
+        for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) {
+            if (i.second)
+               addRoot(*i.second);
         }
 
         /* Register success in the database for all Build objects that
diff --git a/src/hydra-queue-runner/hydra-build-result.hh b/src/hydra-queue-runner/hydra-build-result.hh
index a3f71ae9..7d47f67c 100644
--- a/src/hydra-queue-runner/hydra-build-result.hh
+++ b/src/hydra-queue-runner/hydra-build-result.hh
@@ -36,10 +36,12 @@ struct BuildOutput
 
     std::list<BuildProduct> products;
 
+    std::map<std::string, nix::StorePath> outputs;
+
     std::map<std::string, BuildMetric> metrics;
 };
 
 BuildOutput getBuildOutput(
     nix::ref<nix::Store> store,
     NarMemberDatas & narMembers,
-    const nix::Derivation & drv);
+    const nix::OutputPathMap derivationOutputs);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 1d54bb93..2f2c6091 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -312,10 +312,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
 
     if (r.affected_rows() == 0) goto restart;
 
-    for (auto & [name, output] : step->drv->outputs)
+    for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
-            buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name)));
+            buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
 
     if (status == bsBusy)
         txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@@ -352,11 +352,23 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
     assert(result.logFile.find('\t') == std::string::npos);
     txn.exec(fmt("notify step_finished, '%d\t%d\t%s'",
             buildId, stepNr, result.logFile));
+
+    if (result.stepStatus == bsSuccess) {
+        // Update the corresponding `BuildStepOutputs` row to add the output path
+        auto res = txn.exec_params1("select drvPath from BuildSteps where build = $1 and stepnr = $2", buildId, stepNr);
+        assert(res.size());
+        StorePath drvPath = localStore->parseStorePath(res[0].as<std::string>());
+        // If we've finished building, all the paths should be known
+        for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath))
+            txn.exec_params0
+                ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3",
+                  buildId, stepNr, name, localStore->printStorePath(output));
+    }
 }
 
 
 int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
-    Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath)
+    Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath)
 {
  restart:
     auto stepNr = allocBuildStep(txn, build->id);
@@ -457,6 +469,15 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
          res.releaseName != "" ? std::make_optional(res.releaseName) : std::nullopt,
          isCachedBuild ? 1 : 0);
 
+    for (auto & [outputName, outputPath] : res.outputs) {
+        txn.exec_params0
+            ("update BuildOutputs set path = $3 where build = $1 and name = $2",
+             build->id,
+             outputName,
+             localStore->printStorePath(outputPath)
+            );
+    }
+
     txn.exec_params0("delete from BuildProducts where build = $1", build->id);
 
     unsigned int productNr = 1;
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 6c339af6..cee5b6cc 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -192,15 +192,14 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) {
-                        if (i.second.second) {
-                            auto res = txn.exec_params
-                                ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
-                                 localStore->printStorePath(*i.second.second));
-                            if (!res[0][0].is_null()) {
-                                propagatedFrom = res[0][0].as<BuildID>();
-                                break;
-                            }
+                    for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) {
+                          auto res = txn.exec_params
+                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
+                                localStore->printStorePath(ex.step->drvPath),
+                                i.first);
+                          if (!res[0][0].is_null()) {
+                              propagatedFrom = res[0][0].as<BuildID>();
+                              break;
                         }
                     }
                 }
@@ -236,12 +235,10 @@ bool State::getQueuedBuilds(Connection & conn,
         /* If we didn't get a step, it means the step's outputs are
            all valid. So we mark this as a finished, cached build. */
         if (!step) {
-            auto drv = localStore->readDerivation(build->drvPath);
-            BuildOutput res = getBuildOutputCached(conn, destStore, drv);
+            BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath);
 
-            for (auto & i : drv.outputsAndOptPaths(*localStore))
-                if (i.second.second)
-                    addRoot(*i.second.second);
+            for (auto & i : localStore->queryDerivationOutputMap(build->drvPath))
+                addRoot(i.second);
 
             {
             auto mc = startDbUpdate();
@@ -481,26 +478,39 @@ Step::ptr State::createStep(ref<Store> destStore,
         throw PreviousFailure{step};
 
     /* Are all outputs valid? */
+    auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
     bool valid = true;
-    DerivationOutputs missing;
-    for (auto & i : step->drv->outputs)
-        if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) {
-            valid = false;
-            missing.insert_or_assign(i.first, i.second);
+    std::map<DrvOutput, std::optional<StorePath>> missing;
+    for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) {
+        auto outputHash = outputHashes.at(outputName);
+        if (maybeOutputPath.second) {
+            if (!destStore->isValidPath(*maybeOutputPath.second)) {
+                valid = false;
+                missing.insert({{outputHash, outputName}, maybeOutputPath.second});
+            }
+        } else {
+            experimentalFeatureSettings.require(Xp::CaDerivations);
+            if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) {
+                valid = false;
+                missing.insert({{outputHash, outputName}, std::nullopt});
+            }
         }
+    }
 
     /* Try to copy the missing paths from the local store or from
        substitutes. */
     if (!missing.empty()) {
 
         size_t avail = 0;
-        for (auto & i : missing) {
-            auto path = i.second.path(*localStore, step->drv->name, i.first);
-            if (/* localStore != destStore && */ localStore->isValidPath(*path))
+        for (auto & [i, maybePath] : missing) {
+            if ((maybePath && localStore->isValidPath(*maybePath)))
                 avail++;
-            else if (useSubstitutes) {
+            else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) {
+                maybePath = localStore->queryRealisation(i)->outPath;
+                avail++;
+            } else if (useSubstitutes && maybePath) {
                 SubstitutablePathInfos infos;
-                localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
+                localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos);
                 if (infos.size() == 1)
                     avail++;
             }
@@ -508,44 +518,44 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         if (missing.size() == avail) {
             valid = true;
-            for (auto & i : missing) {
-                auto path = i.second.path(*localStore, step->drv->name, i.first);
+            for (auto & [i, path] : missing) {
+                if (path) {
+                    try {
+                        time_t startTime = time(0);
 
-                try {
-                    time_t startTime = time(0);
+                        if (localStore->isValidPath(*path))
+                            printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
+                                localStore->printStorePath(*path),
+                                localStore->printStorePath(drvPath));
+                        else {
+                            printInfo("substituting output ‘%1%’ of ‘%2%’",
+                                localStore->printStorePath(*path),
+                                localStore->printStorePath(drvPath));
+                            localStore->ensurePath(*path);
+                            // FIXME: should copy directly from substituter to destStore.
+                        }
 
-                    if (localStore->isValidPath(*path))
-                        printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
+                        StorePathSet closure;
+                        localStore->computeFSClosure({*path}, closure);
+                        copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute);
+
+                        time_t stopTime = time(0);
+
+                        {
+                            auto mc = startDbUpdate();
+                            pqxx::work txn(conn);
+                            createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path);
+                            txn.commit();
+                        }
+
+                    } catch (Error & e) {
+                        printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
                             localStore->printStorePath(*path),
-                            localStore->printStorePath(drvPath));
-                    else {
-                        printInfo("substituting output ‘%1%’ of ‘%2%’",
-                            localStore->printStorePath(*path),
-                            localStore->printStorePath(drvPath));
-                        localStore->ensurePath(*path);
-                        // FIXME: should copy directly from substituter to destStore.
+                            localStore->printStorePath(drvPath),
+                            e.what());
+                        valid = false;
+                        break;
                     }
-
-                    copyClosure(*localStore, *destStore,
-                        StorePathSet { *path },
-                        NoRepair, CheckSigs, NoSubstitute);
-
-                    time_t stopTime = time(0);
-
-                    {
-                        auto mc = startDbUpdate();
-                        pqxx::work txn(conn);
-                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
-                        txn.commit();
-                    }
-
-                } catch (Error & e) {
-                    printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
-                        localStore->printStorePath(*path),
-                        localStore->printStorePath(drvPath),
-                        e.what());
-                    valid = false;
-                    break;
                 }
             }
         }
@@ -640,17 +650,20 @@ void State::processJobsetSharesChange(Connection & conn)
 }
 
 
-BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::Derivation & drv)
+BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::StorePath & drvPath)
 {
+
+    auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath);
+
     {
     pqxx::work txn(conn);
 
-    for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) {
+    for (auto & [name, output] : derivationOutputs) {
         auto r = txn.exec_params
             ("select id, buildStatus, releaseName, closureSize, size from Builds b "
              "join BuildOutputs o on b.id = o.build "
              "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
-             localStore->printStorePath(*output.second));
+             localStore->printStorePath(output));
         if (r.empty()) continue;
         BuildID id = r[0][0].as<BuildID>();
 
@@ -704,5 +717,5 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
     }
 
     NarMemberDatas narMembers;
-    return getBuildOutput(destStore, narMembers, drv);
+    return getBuildOutput(destStore, narMembers, derivationOutputs);
 }
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 6359063a..5f01de1e 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -520,7 +520,7 @@ private:
         const std::string & machine);
 
     int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
-        Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath);
+        Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath);
 
     void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status);
 
@@ -536,7 +536,7 @@ private:
     void processQueueChange(Connection & conn);
 
     BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
-        const nix::Derivation & drv);
+        const nix::StorePath & drvPath);
 
     Step::ptr createStep(nix::ref<nix::Store> store,
         Connection & conn, Build::ptr build, const nix::StorePath & drvPath,

From 8046ec266851d8b8d3f08900cf9ba905ed344ed8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 16:21:56 -0500
Subject: [PATCH 090/285] Remove unused `outputHashes` variable

This looks like a stray copy paste.
---
 src/hydra-queue-runner/build-remote.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 26f6d63f..e26a8697 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -192,7 +192,6 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
 BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
 {
     BasicDerivation ret;
-    auto outputHashes = staticOutputHashes(store, drv);
     if (!drv.type().hasKnownOutputPaths()) {
         auto maybeBasicDrv = drv.tryResolve(store);
         if (!maybeBasicDrv)

From e3443cd22a3bc20c7d17f4e76d3868ea755b4b37 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 17:41:11 -0500
Subject: [PATCH 091/285] Put back nicer `copyClosure` instead of manual
 closure + copy

It looks like we accidentally got the old code back, probably after a merge
conflict resolution.
---
 src/hydra-queue-runner/queue-monitor.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index cee5b6cc..04917932 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -535,9 +535,9 @@ Step::ptr State::createStep(ref<Store> destStore,
                             // FIXME: should copy directly from substituter to destStore.
                         }
 
-                        StorePathSet closure;
-                        localStore->computeFSClosure({*path}, closure);
-                        copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute);
+                        copyClosure(*localStore, *destStore,
+                            StorePathSet { *path },
+                            NoRepair, CheckSigs, NoSubstitute);
 
                         time_t stopTime = time(0);
 

From 069b7775c565f5999fe33e8c3f28c7b9306039ca Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 5 Dec 2023 11:26:26 -0500
Subject: [PATCH 092/285] hydra-eval-jobs: Ensure we have output path if
 ca-derivations is disabled

Brought up by @thufschmitt in
https://github.com/NixOS/hydra/pull/1316#discussion_r1415111329 . This
makes this closer to what was originally there --- which just dispatched
off the experimental feature rather than the presence/absense of the
output, too.
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index d007189d..b44ccbb3 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -243,11 +243,16 @@ static void worker(
                 }
 
                 nlohmann::json out;
-                for (auto & [outputName, optOutputPath] : outputs)
-                    if (optOutputPath)
+                for (auto & [outputName, optOutputPath] : outputs) {
+                    if (optOutputPath) {
                         out[outputName] = state.store->printStorePath(*optOutputPath);
-                    else
+                    } else {
+                        // See the `queryOutputs` call above; we should
+                        // not encounter missing output paths otherwise.
+                        assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
                         out[outputName] = "";
+                    }
+                }
                 job["outputs"] = std::move(out);
                 reply["job"] = std::move(job);
             }

From 3df8feb3a2b30b733b141f6e6e43572998311524 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 5 Dec 2023 11:27:52 -0500
Subject: [PATCH 093/285] Add TODO about setting `null` instead of empty string
 in JSON

An empty string is a sneaky way to avoid hard failures --- things that
expect strings still get strings, but it does conversely open the door
up to soft failures (spooky-action-at-a-distance ones because the string
did not have the expected invariants).

"Fail fast" with null will ultimately make the system more robust, but
force us to fix more things up front, and I don't want to change this
without also fixing those things up front, especially as this commit is
for now just part of the the preparatory PR for which this is dead code.
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index b44ccbb3..2794cc62 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -250,6 +250,10 @@ static void worker(
                         // See the `queryOutputs` call above; we should
                         // not encounter missing output paths otherwise.
                         assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
+                        // TODO it would be better to set `null` than an
+                        // empty string here, to force the consumer of
+                        // this JSON to more explicitly handle this
+                        // case.
                         out[outputName] = "";
                     }
                 }

From 11f8030b0f4c75ed7640563d68200cf3ec59edec Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 6 Dec 2023 17:59:25 -0500
Subject: [PATCH 094/285] Add comment from GitHub about adding to store as code
 comment

---
 src/hydra-queue-runner/build-remote.cc | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index e26a8697..94a01676 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -425,6 +425,19 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
+    // Why both stores? @thufschmitt says:
+    //
+    // > I think it's an easy (and terribly inefficient 😬) way of
+    // making sure that `localStore.queryRealisations` will succeed
+    // (which we IIRC we need later to get back some metadata about the
+    // path to put it in the db).
+    // >
+    // > To be honest, we shouldn't do that but instead carry the needed
+    // metadata in memory until the point where we need it (but that can
+    // come later once we're confident that this is at least correct)
+    //
+    // TODO make the above change to avoid copying excess data back and
+    // forth.
     for (auto * store : {&destStore, &localStore}) {
       /* Receive the NAR from the remote and add it to the
           destination store. Meanwhile, extract all the info from the

From 86cd5e907683a8c4b0f6cd75d4879648d8313b98 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Feb 2022 20:06:22 +0000
Subject: [PATCH 095/285] `copyClosureTo`: Use `SubstituteFlag` instead of
 `bool`

This matches Nix (in the same serialization logic in
`src/libstore/legacy-ssh-store.cc`) and adds clarity.
---
 src/hydra-queue-runner/build-remote.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 73f46a53..f41585c8 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -113,7 +113,7 @@ static void copyClosureTo(
     Machine::Connection & conn,
     Store & destStore,
     const StorePathSet & paths,
-    bool useSubstitutes = false)
+    SubstituteFlag useSubstitutes = NoSubstitute)
 {
     StorePathSet closure;
     destStore.computeFSClosure(paths, closure);
@@ -266,7 +266,7 @@ static BasicDerivation sendInputs(
             destStore.computeFSClosure(basicDrv.inputSrcs, closure);
             copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
         } else {
-            copyClosureTo(conn, destStore, basicDrv.inputSrcs, true);
+            copyClosureTo(conn, destStore, basicDrv.inputSrcs, Substitute);
         }
 
         auto now2 = std::chrono::steady_clock::now();

From 6a54ab24e26b4e387c41e6d8958e7a9ec36c8398 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 02:00:22 -0500
Subject: [PATCH 096/285] Use factored-out `BuildResult` serializer

For the record, here is the Nix 2.19 version:
https://github.com/NixOS/nix/blob/2.19-maintenance/src/libstore/serve-protocol.cc,
which is what we would initially use.

It is a more complete version of what Hydra has today except for one
thing: it always unconditionally sets the start/stop times.

I think that is correct at the other end seems to unconditionally
measure them, but just to be extra careful, I reproduced the old
behavior of falling back on Hydra's own measurements if `startTime` is
0.

The only difference is that the fallback `stopTime` is now measured from
after the entire `BuildResult` is transferred over the wire, but I think
that should be negligible if it is measurable at all. (And remember,
this is fallback case I already suspect is dead code.)
---
 src/hydra-queue-runner/build-remote.cc | 36 ++++++++++----------------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 73f46a53..5206ae5a 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -286,9 +286,6 @@ static BuildResult performBuild(
     counter & nrStepsBuilding
 )
 {
-
-    BuildResult result;
-
     conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath);
     writeDerivation(conn.to, localStore, drv);
     conn.to << options.maxSilentTime << options.buildTimeout;
@@ -301,30 +298,25 @@ static BuildResult performBuild(
     }
     conn.to.flush();
 
-    result.startTime = time(0);
+    BuildResult result;
 
+    time_t startTime, stopTime;
+
+    startTime = time(0);
     {
         MaintainCount<counter> mc(nrStepsBuilding);
-        result.status = (BuildResult::Status)readInt(conn.from);
+        result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
     }
-    result.stopTime = time(0);
+    stopTime = time(0);
 
-
-    result.errorMsg = readString(conn.from);
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
-        result.timesBuilt = readInt(conn.from);
-        result.isNonDeterministic = readInt(conn.from);
-        auto start = readInt(conn.from);
-        auto stop = readInt(conn.from);
-        if (start && start) {
-            /* Note: this represents the duration of a single
-                round, rather than all rounds. */
-            result.startTime = start;
-            result.stopTime = stop;
-        }
-    }
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
-        ServeProto::Serialise<DrvOutputs>::read(localStore, conn);
+    if (!result.startTime) {
+        // If the builder gave `startTime = 0`, use our measurements
+        // instead of the builder's.
+        //
+        // Note: this represents the duration of a single round, rather
+        // than all rounds.
+        result.startTime = startTime;
+        result.stopTime = stopTime;
     }
 
     return result;

From a45a27851b6cfc22113a660df75e556199c20c84 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 12:20:55 -0500
Subject: [PATCH 097/285] flake.lock: Update Nixpkgs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This will be required for upgrading Nix beyond 2.19.

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/ef0bc3976340dab9a4e087a0bcff661a8b2e87f3' (2023-06-21)
  → 'github:NixOS/nixpkgs/e9f06adb793d1cca5384907b3b8a4071d5d7cb19' (2023-12-03)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 9a9046c8..2871e70a 100644
--- a/flake.lock
+++ b/flake.lock
@@ -58,11 +58,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1687379288,
-        "narHash": "sha256-cSuwfiqYfeVyqzCRkU9AvLTysmEuSal8nh6CYr+xWog=",
+        "lastModified": 1701615100,
+        "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ef0bc3976340dab9a4e087a0bcff661a8b2e87f3",
+        "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19",
         "type": "github"
       },
       "original": {

From 20c8263e3c993d4ba0a2a0bedd531a7f32861441 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 11:40:30 -0500
Subject: [PATCH 098/285] Update to Nix master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The point of this branch is to always track Nix master, so we are
proactively ready to upgrade to the next Nix release when it is ready.

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-11-27)
  → 'github:NixOS/nix/c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5' (2023-12-07)
• Added input 'nix/libgit2':
    'github:libgit2/libgit2/45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5' (2023-10-18)
---
 flake.lock                              | 24 ++++++++++++++++++++----
 flake.nix                               |  2 +-
 src/hydra-queue-runner/build-remote.cc  |  2 +-
 src/hydra-queue-runner/nar-extractor.cc |  2 +-
 src/hydra-queue-runner/queue-monitor.cc |  2 +-
 5 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/flake.lock b/flake.lock
index 2871e70a..ae4f60eb 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "libgit2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "type": "github"
+      }
+    },
     "lowdown-src": {
       "flake": false,
       "locked": {
@@ -35,6 +51,7 @@
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "libgit2": "libgit2",
         "lowdown-src": "lowdown-src",
         "nixpkgs": [
           "nixpkgs"
@@ -42,16 +59,15 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1701122567,
-        "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
+        "lastModified": 1701948782,
+        "narHash": "sha256-rEu4hLHZIy3fgf88BpiaVfl79hNSESRfQNbmxAO5uzg=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
+        "rev": "c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.19.2",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 5a5aef55..fdeaa6ca 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.19.2";
+  inputs.nix.url = "github:NixOS/nix";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index f41585c8..dad414bf 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -350,7 +350,7 @@ static std::map<StorePath, ValidPathInfo> queryPathInfos(
         auto references = ServeProto::Serialise<StorePathSet>::read(localStore, conn);
         readLongLong(conn.from); // download size
         auto narSize = readLongLong(conn.from);
-        auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
+        auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256);
         auto ca = ContentAddress::parseOpt(readString(conn.from));
         readStrings<StringSet>(conn.from); // sigs
         ValidPathInfo info(localStore.parseStorePath(storePathS), narHash);
diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc
index 3c6857bf..ff4aa268 100644
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -42,7 +42,7 @@ struct Extractor : ParseSink
     void preallocateContents(uint64_t size) override
     {
         expectedSize = size;
-        hashSink = std::make_unique<HashSink>(htSHA256);
+        hashSink = std::make_unique<HashSink>(HashAlgorithm::SHA256);
     }
 
     void receiveContents(std::string_view data) override
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 6c339af6..3ce9bba2 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -677,7 +677,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
                 product.fileSize = row[2].as<off_t>();
             }
             if (!row[3].is_null())
-                product.sha256hash = Hash::parseAny(row[3].as<std::string>(), htSHA256);
+                product.sha256hash = Hash::parseAny(row[3].as<std::string>(), HashAlgorithm::SHA256);
             if (!row[4].is_null())
                 product.path = row[4].as<std::string>();
             product.name = row[5].as<std::string>();

From 2ee0068fdca7d9fb8bef23b3671a86b8f0fcff07 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 15:05:03 -0500
Subject: [PATCH 099/285] Do not copy for both stores for now

It has a performance cost, and as the comment says we should be doing
the better solution. We want to land this preparatory change on prod
while the rest is still on staging, so we should just skip it for now.

Skipping it will not affect regular fixed-output and input-addressed
derivations, which are the only ones prod would deal with upon getting
this code.

The main CA derivations support branch will revert this commit so it
still works.
---
 src/hydra-queue-runner/build-remote.cc | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 1d0bcc35..77484244 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -425,20 +425,6 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
-    // Why both stores? @thufschmitt says:
-    //
-    // > I think it's an easy (and terribly inefficient 😬) way of
-    // making sure that `localStore.queryRealisations` will succeed
-    // (which we IIRC we need later to get back some metadata about the
-    // path to put it in the db).
-    // >
-    // > To be honest, we shouldn't do that but instead carry the needed
-    // metadata in memory until the point where we need it (but that can
-    // come later once we're confident that this is at least correct)
-    //
-    // TODO make the above change to avoid copying excess data back and
-    // forth.
-    for (auto * store : {&destStore, &localStore}) {
       /* Receive the NAR from the remote and add it to the
           destination store. Meanwhile, extract all the info from the
           NAR that getBuildOutput() needs. */
@@ -458,8 +444,7 @@ static void copyPathFromRemote(
           extractNarData(tee, localStore.printStorePath(info.path), narMembers);
       });
 
-      store->addToStore(info, *source2, NoRepair, NoCheckSigs);
-    }
+      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
 }
 
 static void copyPathsFromRemote(

From 411e4d0c2458ec5319b8ea88129dff807793f7d7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 11:30:31 -0500
Subject: [PATCH 100/285] Let tests themselves intentionally leak temp dir
 (#1320)

* Let tests themselves intentionally leak temp dir

By default Yath will clean up temporary files, so the result is the
same. But `--keep-dirs` can be passed to `yath test` telling Yath to
*not* clean them up instead. This is very useful for debugging.

* Update t/lib/HydraTestContext.pm

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 t/lib/HydraTestContext.pm | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index 53eaa0f7..a22c3df1 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -39,7 +39,9 @@ use Hydra::Helper::Exec;
 sub new {
     my ($class, %opts) = @_;
 
-    my $dir = File::Temp->newdir();
+    # Cleanup will be managed by yath. By the default it will be cleaned
+    # up, but can be kept to aid in debugging test failures.
+    my $dir = File::Temp->newdir(CLEANUP => 0);
 
     $ENV{'HYDRA_DATA'} = "$dir/hydra-data";
     mkdir $ENV{'HYDRA_DATA'};

From aaa0e128c1e85885ff55f6abd3a3115ab4b194be Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 9 Dec 2023 11:58:09 -0500
Subject: [PATCH 101/285] flake.lock: Update Nix master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5' (2023-12-07)
  → 'github:NixOS/nix/c8458bd731eb1c74159bebe459ea00165e056b65' (2023-12-09)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index ae4f60eb..424ed3b0 100644
--- a/flake.lock
+++ b/flake.lock
@@ -59,11 +59,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1701948782,
-        "narHash": "sha256-rEu4hLHZIy3fgf88BpiaVfl79hNSESRfQNbmxAO5uzg=",
+        "lastModified": 1702090558,
+        "narHash": "sha256-JzuGOltp5Ht9flroZ7g0erD5ud2okNQChw9YlxWDX3c=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "c3827ff6348a4d5199eaddf8dbc2ca2e2ef46ec5",
+        "rev": "c8458bd731eb1c74159bebe459ea00165e056b65",
         "type": "github"
       },
       "original": {

From 3f932a6731cfd40919258d0731aad235b77e470c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 14:10:28 -0500
Subject: [PATCH 102/285] build-remote: Use `std::map<StorePath,
 UnkeyedValidPathInfo>`

It is less denormalized
---
 src/hydra-queue-runner/build-remote.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 05bfdb66..62d321bb 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -154,7 +154,7 @@ static void copyClosureTo(
 
 
 // FIXME: use Store::topoSortPaths().
-static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo> & paths)
+static StorePaths reverseTopoSortPaths(const std::map<StorePath, UnkeyedValidPathInfo> & paths)
 {
     StorePaths sorted;
     StorePathSet visited;
@@ -322,7 +322,7 @@ static BuildResult performBuild(
     return result;
 }
 
-static std::map<StorePath, ValidPathInfo> queryPathInfos(
+static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
     Machine::Connection & conn,
     Store & localStore,
     StorePathSet & outputs,
@@ -331,7 +331,7 @@ static std::map<StorePath, ValidPathInfo> queryPathInfos(
 {
 
     /* Get info about each output path. */
-    std::map<StorePath, ValidPathInfo> infos;
+    std::map<StorePath, UnkeyedValidPathInfo> infos;
     conn.to << ServeProto::Command::QueryPathInfos;
     ServeProto::write(localStore, conn, outputs);
     conn.to.flush();
@@ -395,14 +395,16 @@ static void copyPathsFromRemote(
     NarMemberDatas & narMembers,
     Store & localStore,
     Store & destStore,
-    const std::map<StorePath, ValidPathInfo> & infos
+    const std::map<StorePath, UnkeyedValidPathInfo> & infos
 )
 {
       auto pathsSorted = reverseTopoSortPaths(infos);
 
       for (auto & path : pathsSorted) {
           auto & info = infos.find(path)->second;
-          copyPathFromRemote(conn, narMembers, localStore, destStore, info);
+          copyPathFromRemote(
+              conn, narMembers, localStore, destStore,
+              ValidPathInfo { path, info });
       }
 
 }

From d0d3b0a2986915ab7aa96d3fce8371a5012c9021 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 14:13:06 -0500
Subject: [PATCH 103/285] Use `ServeProto::Serialise<UnkeyedValidPathInfo>` for
 `QueryValidPaths`

Companion to already-merged https://github.com/NixOS/nix/pull/9560
---
 src/hydra-queue-runner/build-remote.cc | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 62d321bb..5818727b 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -338,23 +338,10 @@ static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
     while (true) {
         auto storePathS = readString(conn.from);
         if (storePathS == "") break;
-        auto deriver = readString(conn.from); // deriver
-        auto references = ServeProto::Serialise<StorePathSet>::read(localStore, conn);
-        readLongLong(conn.from); // download size
-        auto narSize = readLongLong(conn.from);
-        auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256);
-        auto ca = ContentAddress::parseOpt(readString(conn.from));
-        readStrings<StringSet>(conn.from); // sigs
-        ValidPathInfo info(localStore.parseStorePath(storePathS), narHash);
-        assert(outputs.count(info.path));
-        info.references = references;
-        info.narSize = narSize;
-        totalNarSize += info.narSize;
-        info.narHash = narHash;
-        info.ca = ca;
-        if (deriver != "")
-            info.deriver = localStore.parseStorePath(deriver);
-        infos.insert_or_assign(info.path, info);
+
+        auto storePath = localStore.parseStorePath(storePathS);
+        auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
+        infos.insert_or_assign(storePath, info);
     }
 
     return infos;

From f6f817926a5ee98af4251a4244d01f5ba1f31695 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 9 Dec 2023 12:11:04 -0500
Subject: [PATCH 104/285] `std::move` the into the path info map

---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 5818727b..a8047182 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -341,7 +341,7 @@ static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
 
         auto storePath = localStore.parseStorePath(storePathS);
         auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
-        infos.insert_or_assign(storePath, info);
+        infos.insert_or_assign(std::move(storePath), std::move(info));
     }
 
     return infos;

From 1d80b72ffb9d9619a0c146d4662d27e3bd45850b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 13:00:43 -0500
Subject: [PATCH 105/285] flake.lock: Update Nix master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/c8458bd731eb1c74159bebe459ea00165e056b65' (2023-12-09)
  → 'github:NixOS/nix/b7e016ab2464ad2e7e2e856ad0f173157135aae0' (2023-12-10)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 424ed3b0..d99954a9 100644
--- a/flake.lock
+++ b/flake.lock
@@ -59,11 +59,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1702090558,
-        "narHash": "sha256-JzuGOltp5Ht9flroZ7g0erD5ud2okNQChw9YlxWDX3c=",
+        "lastModified": 1702228562,
+        "narHash": "sha256-VwvfxT8LZcEm3H+VCMCrKzl7Ip+dHumxPfugAx3tjk8=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "c8458bd731eb1c74159bebe459ea00165e056b65",
+        "rev": "b7e016ab2464ad2e7e2e856ad0f173157135aae0",
         "type": "github"
       },
       "original": {

From 69a5b00e60df0a40ac8f193f33937b7214872327 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 14:18:00 -0500
Subject: [PATCH 106/285] Use `ServeProto::BuildOption`

More deduplication with Nix.
---
 src/hydra-queue-runner/build-remote.cc |  8 ++++----
 src/hydra-queue-runner/builder.cc      | 15 +++++++++------
 src/hydra-queue-runner/state.hh        | 10 ++--------
 3 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index a8047182..1fdd5a23 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -282,7 +282,7 @@ static BuildResult performBuild(
     Store & localStore,
     StorePath drvPath,
     const BasicDerivation & drv,
-    const State::BuildOptions & options,
+    const ServeProto::BuildOptions & options,
     counter & nrStepsBuilding
 )
 {
@@ -293,7 +293,7 @@ static BuildResult performBuild(
         conn.to << options.maxLogSize;
     if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
         conn.to
-            << options.repeats // == build-repeat
+            << options.nrRepeats
             << options.enforceDeterminism;
     }
     conn.to.flush();
@@ -458,7 +458,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
-    const BuildOptions & buildOptions,
+    const ServeProto::BuildOptions & buildOptions,
     RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
     std::function<void(StepState)> updateStep,
     NarMemberDatas & narMembers)
@@ -510,7 +510,7 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         try {
-          build_remote::handshake(conn, buildOptions.repeats);
+          build_remote::handshake(conn, buildOptions.nrRepeats);
         } catch (EndOfFile & e) {
             child.pid.wait();
             std::string s = chomp(readFile(result.logFile));
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 307eee8e..cfa95cef 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -98,10 +98,13 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        it). */
     BuildID buildId;
     std::optional<StorePath> buildDrvPath;
-    BuildOptions buildOptions;
-    buildOptions.repeats = step->isDeterministic ? 1 : 0;
-    buildOptions.maxLogSize = maxLogSize;
-    buildOptions.enforceDeterminism = step->isDeterministic;
+    // Other fields set below
+    nix::ServeProto::BuildOptions buildOptions {
+        .maxLogSize = maxLogSize,
+        .nrRepeats = step->isDeterministic ? 1u : 0u,
+        .enforceDeterminism = step->isDeterministic,
+        .keepFailed = false,
+    };
 
     auto conn(dbPool.get());
 
@@ -136,7 +139,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
             {
                 auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
                 if (i != jobsetRepeats.end())
-                    buildOptions.repeats = std::max(buildOptions.repeats, i->second);
+                    buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second);
             }
         }
         if (!build) build = *dependents.begin();
@@ -147,7 +150,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
         buildOptions.buildTimeout = build->buildTimeout;
 
         printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
-            localStore->printStorePath(step->drvPath), buildOptions.repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
+            localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1));
     }
 
     if (!buildOneDone)
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 6359063a..830f0598 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -459,7 +459,7 @@ private:
 
     /* How often the build steps of a jobset should be repeated in
        order to detect non-determinism. */
-    std::map<std::pair<std::string, std::string>, unsigned int> jobsetRepeats;
+    std::map<std::pair<std::string, std::string>, size_t> jobsetRepeats;
 
     bool uploadLogsToBinaryCache;
 
@@ -488,12 +488,6 @@ private:
 public:
     State(std::optional<std::string> metricsAddrOpt);
 
-    struct BuildOptions {
-        unsigned int maxSilentTime, buildTimeout, repeats;
-        size_t maxLogSize;
-        bool enforceDeterminism;
-    };
-
 private:
 
     nix::MaintainCount<counter> startDbUpdate();
@@ -578,7 +572,7 @@ private:
 
     void buildRemote(nix::ref<nix::Store> destStore,
         Machine::ptr machine, Step::ptr step,
-        const BuildOptions & buildOptions,
+        const nix::ServeProto::BuildOptions & buildOptions,
         RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
         std::function<void(StepState)> updateStep,
         NarMemberDatas & narMembers);

From b56d2383c1e0c39418af35665f7d5b21cf49d381 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 14:30:44 -0500
Subject: [PATCH 107/285] Do not attempt to speak a newer version of the
 protocol

Both sides need to agree on a version (with `std::min`) for anything to
work. Somehow... we've never done this.

With this comment, the next commit succeeds. Without this commit, the
next commit fails. This is because the next commit exposes serializers
which do different things for proto version 2.7, and we're currently
requesting 2.6.

Opened https://github.com/NixOS/nix/issues/9584 to track this issue
---
 src/hydra-queue-runner/build-remote.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 1fdd5a23..659618b7 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -203,7 +203,9 @@ static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, cons
  */
 static void handshake(Machine::Connection & conn, unsigned int repeats)
 {
-    conn.to << SERVE_MAGIC_1 << 0x206;
+    constexpr ServeProto::Version our_version = 0x206;
+
+    conn.to << SERVE_MAGIC_1 << our_version;
     conn.to.flush();
 
     unsigned int magic = readInt(conn.from);
@@ -215,6 +217,9 @@ static void handshake(Machine::Connection & conn, unsigned int repeats)
         throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName);
     if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0)
         throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName);
+
+    // Do not attempt to speak a newer version of the protocol
+    conn.remoteVersion = std::min(conn.remoteVersion, our_version);
 }
 
 static BasicDerivation sendInputs(

From 20f5a2120cf0663bd52df9e82c42d26942017c1a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 9 Dec 2023 12:26:11 -0500
Subject: [PATCH 108/285] Use `ServeProto::Serialise<ServeProto::BuildOptions>`

---
 src/hydra-queue-runner/build-remote.cc | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 659618b7..a4882f5c 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -293,14 +293,7 @@ static BuildResult performBuild(
 {
     conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath);
     writeDerivation(conn.to, localStore, drv);
-    conn.to << options.maxSilentTime << options.buildTimeout;
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
-        conn.to << options.maxLogSize;
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) {
-        conn.to
-            << options.nrRepeats
-            << options.enforceDeterminism;
-    }
+    ServeProto::write(localStore, conn, options);
     conn.to.flush();
 
     BuildResult result;

From 8c10331ee818a21699a8b774cccdd32f476f8ba6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 14:05:26 -0500
Subject: [PATCH 109/285] Fix `totalNarSize` summation

I accidentally removed it in d0d3b0a2986915ab7aa96d3fce8371a5012c9021.
---
 src/hydra-queue-runner/build-remote.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 1fdd5a23..41c7c453 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -341,6 +341,7 @@ static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
 
         auto storePath = localStore.parseStorePath(storePathS);
         auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
+        totalNarSize += info.narSize;
         infos.insert_or_assign(std::move(storePath), std::move(info));
     }
 

From ebfefb9161046c83bbbebd26f4cb37ac69628b40 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 12:46:36 -0500
Subject: [PATCH 110/285] Sync up with some changes done to the main CA branch

---
 flake.lock                                   |   8 +-
 flake.nix                                    |   2 +-
 src/hydra-queue-runner/build-remote.cc       |  55 +++------
 src/hydra-queue-runner/builder.cc            |  11 +-
 src/hydra-queue-runner/hydra-queue-runner.cc |   4 +-
 src/hydra-queue-runner/queue-monitor.cc      | 119 ++++++++++---------
 6 files changed, 94 insertions(+), 105 deletions(-)

diff --git a/flake.lock b/flake.lock
index 2871e70a..e7f9224c 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1701122567,
-        "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
+        "lastModified": 1702314838,
+        "narHash": "sha256-calxK+fZ4/tZy1fbph8qyx4ePUAf4ZdvIugpzWeFIGE=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
+        "rev": "ae451e2247b18be6bd36b9d85e41b632e774f40b",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.19.2",
+        "ref": "2.19-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 5a5aef55..8280e076 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.19.2";
+  inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 377bdb56..65f9f3fb 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -182,40 +182,6 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
     return sorted;
 }
 
-/**
- * Replace the input derivations by their output paths to send a minimal closure
- * to the builder.
- *
- * If we can afford it, resolve it, so that the newly generated derivation still
- * has some sensible output paths.
- */
-BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
-{
-    BasicDerivation ret;
-    if (!drv.type().hasKnownOutputPaths()) {
-        auto maybeBasicDrv = drv.tryResolve(store);
-        if (!maybeBasicDrv)
-            throw Error(
-                "the derivation '%s' can’t be resolved. It’s probably "
-                "missing some outputs",
-                store.printStorePath(drvPath));
-        ret = *maybeBasicDrv;
-    } else {
-        // If the derivation is a real `InputAddressed` derivation, we must
-        // resolve it manually to keep the original output paths
-        ret = BasicDerivation(drv);
-        for (auto & [drvPath, node] : drv.inputDrvs.map) {
-            auto drv2 = store.readDerivation(drvPath);
-            auto drv2Outputs = drv2.outputsAndOptPaths(store);
-            for (auto & name : node.value) {
-                auto inputPath = drv2Outputs.at(name);
-                ret.inputSrcs.insert(*inputPath.second);
-            }
-        }
-    }
-    return ret;
-}
-
 static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
 {
     std::string base(drvPath.to_string());
@@ -262,7 +228,22 @@ static BasicDerivation sendInputs(
     counter & nrStepsCopyingTo
 )
 {
-    BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath);
+    /* Replace the input derivations by their output paths to send a
+       minimal closure to the builder.
+
+       `tryResolve` currently does *not* rewrite input addresses, so it
+       is safe to do this in all cases. (It should probably have a mode
+       to do that, however, but we would not use it here.)
+     */
+    BasicDerivation basicDrv = ({
+        auto maybeBasicDrv = step.drv->tryResolve(destStore, &localStore);
+        if (!maybeBasicDrv)
+            throw Error(
+                "the derivation '%s' can’t be resolved. It’s probably "
+                "missing some outputs",
+                localStore.printStorePath(step.drvPath));
+        *maybeBasicDrv;
+    });
 
     /* Ensure that the inputs exist in the destination store. This is
        a no-op for regular stores, but for the binary cache store,
@@ -351,6 +332,8 @@ static BuildResult performBuild(
         // far anyways
         assert(drv.type().hasKnownOutputPaths());
         DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore);
+        // Since this a `BasicDerivation`, `staticOutputHashes` will not
+        // do any real work.
         auto outputHashes = staticOutputHashes(localStore, drv);
         for (auto & [outputName, output] : drvOutputs) {
             auto outputPath = output.second;
@@ -665,14 +648,12 @@ void State::buildRemote(ref<Store> destStore,
             auto outputHashes = staticOutputHashes(*localStore, *step->drv);
             for (auto & [outputName, realisation] : buildResult.builtOutputs) {
                 // Register the resolved drv output
-                localStore->registerDrvOutput(realisation);
                 destStore->registerDrvOutput(realisation);
 
                 // Also register the unresolved one
                 auto unresolvedRealisation = realisation;
                 unresolvedRealisation.signatures.clear();
                 unresolvedRealisation.id.drvHash = outputHashes.at(outputName);
-                localStore->registerDrvOutput(unresolvedRealisation);
                 destStore->registerDrvOutput(unresolvedRealisation);
             }
         }
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 42983941..b1b58c05 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         if (result.stepStatus == bsSuccess) {
             updateStep(ssPostProcessing);
-            res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath));
+            res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore));
         }
     }
 
@@ -277,9 +277,12 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         assert(stepNr);
 
-        for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) {
-            if (i.second)
-               addRoot(*i.second);
+        for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) {
+            if (!optOutputPath)
+                throw Error(
+                    "Missing output %s for derivation %d which was supposed to have succeeded",
+                    outputName, localStore->printStorePath(step->drvPath));
+            addRoot(*optOutputPath);
         }
 
         /* Register success in the database for all Build objects that
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 2f2c6091..d514ecfa 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -312,7 +312,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
 
     if (r.affected_rows() == 0) goto restart;
 
-    for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath))
+    for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
             buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
@@ -359,7 +359,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
         assert(res.size());
         StorePath drvPath = localStore->parseStorePath(res[0].as<std::string>());
         // If we've finished building, all the paths should be known
-        for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath))
+        for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore))
             txn.exec_params0
                 ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3",
                   buildId, stepNr, name, localStore->printStorePath(output));
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 04917932..a4d9b4dc 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -192,11 +192,11 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) {
+                    for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
                           auto res = txn.exec_params
                               ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
                                 localStore->printStorePath(ex.step->drvPath),
-                                i.first);
+                                outputName);
                           if (!res[0][0].is_null()) {
                               propagatedFrom = res[0][0].as<BuildID>();
                               break;
@@ -237,7 +237,7 @@ bool State::getQueuedBuilds(Connection & conn,
         if (!step) {
             BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath);
 
-            for (auto & i : localStore->queryDerivationOutputMap(build->drvPath))
+            for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore))
                 addRoot(i.second);
 
             {
@@ -481,20 +481,12 @@ Step::ptr State::createStep(ref<Store> destStore,
     auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
     bool valid = true;
     std::map<DrvOutput, std::optional<StorePath>> missing;
-    for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) {
+    for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
         auto outputHash = outputHashes.at(outputName);
-        if (maybeOutputPath.second) {
-            if (!destStore->isValidPath(*maybeOutputPath.second)) {
-                valid = false;
-                missing.insert({{outputHash, outputName}, maybeOutputPath.second});
-            }
-        } else {
-            experimentalFeatureSettings.require(Xp::CaDerivations);
-            if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) {
-                valid = false;
-                missing.insert({{outputHash, outputName}, std::nullopt});
-            }
-        }
+        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
+            continue;
+        valid = false;
+        missing.insert({{outputHash, outputName}, maybeOutputPath});
     }
 
     /* Try to copy the missing paths from the local store or from
@@ -503,14 +495,24 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         size_t avail = 0;
         for (auto & [i, maybePath] : missing) {
-            if ((maybePath && localStore->isValidPath(*maybePath)))
+            // If we don't know the output path from the destination
+            // store, see if the local store can tell us.
+            if (/* localStore != destStore && */ !maybePath && experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
+                if (auto maybeRealisation = localStore->queryRealisation(i))
+                    maybePath = maybeRealisation->outPath;
+
+            if (!maybePath) {
+                // No hope of getting the store object if we don't know
+                // the path.
+                continue;
+            }
+            auto & path = *maybePath;
+
+            if (/* localStore != destStore && */ localStore->isValidPath(path))
                 avail++;
-            else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) {
-                maybePath = localStore->queryRealisation(i)->outPath;
-                avail++;
-            } else if (useSubstitutes && maybePath) {
+            else if (useSubstitutes) {
                 SubstitutablePathInfos infos;
-                localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos);
+                localStore->querySubstitutablePathInfos({{path, {}}}, infos);
                 if (infos.size() == 1)
                     avail++;
             }
@@ -518,44 +520,47 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         if (missing.size() == avail) {
             valid = true;
-            for (auto & [i, path] : missing) {
-                if (path) {
-                    try {
-                        time_t startTime = time(0);
+            for (auto & [i, maybePath] : missing) {
+                // If we found everything, then we should know the path
+                // to every missing store object now.
+                assert(maybePath);
+                auto & path = *maybePath;
 
-                        if (localStore->isValidPath(*path))
-                            printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
-                                localStore->printStorePath(*path),
-                                localStore->printStorePath(drvPath));
-                        else {
-                            printInfo("substituting output ‘%1%’ of ‘%2%’",
-                                localStore->printStorePath(*path),
-                                localStore->printStorePath(drvPath));
-                            localStore->ensurePath(*path);
-                            // FIXME: should copy directly from substituter to destStore.
-                        }
+                try {
+                    time_t startTime = time(0);
 
-                        copyClosure(*localStore, *destStore,
-                            StorePathSet { *path },
-                            NoRepair, CheckSigs, NoSubstitute);
-
-                        time_t stopTime = time(0);
-
-                        {
-                            auto mc = startDbUpdate();
-                            pqxx::work txn(conn);
-                            createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path);
-                            txn.commit();
-                        }
-
-                    } catch (Error & e) {
-                        printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
-                            localStore->printStorePath(*path),
-                            localStore->printStorePath(drvPath),
-                            e.what());
-                        valid = false;
-                        break;
+                    if (localStore->isValidPath(path))
+                        printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
+                            localStore->printStorePath(path),
+                            localStore->printStorePath(drvPath));
+                    else {
+                        printInfo("substituting output ‘%1%’ of ‘%2%’",
+                            localStore->printStorePath(path),
+                            localStore->printStorePath(drvPath));
+                        localStore->ensurePath(path);
+                        // FIXME: should copy directly from substituter to destStore.
                     }
+
+                    copyClosure(*localStore, *destStore,
+                        StorePathSet { path },
+                        NoRepair, CheckSigs, NoSubstitute);
+
+                    time_t stopTime = time(0);
+
+                    {
+                        auto mc = startDbUpdate();
+                        pqxx::work txn(conn);
+                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path);
+                        txn.commit();
+                    }
+
+                } catch (Error & e) {
+                    printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
+                        localStore->printStorePath(path),
+                        localStore->printStorePath(drvPath),
+                        e.what());
+                    valid = false;
+                    break;
                 }
             }
         }

From a6b6c5a5392a7e705825e8dac7cb818a1f9944fe Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 12:55:57 -0500
Subject: [PATCH 111/285] Revert query -- those columns don't exist yet!

---
 src/hydra-queue-runner/queue-monitor.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index a4d9b4dc..fc344244 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -192,11 +192,12 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
+                    for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
+                          // ca-derivations not actually supported yet
+                          assert(optOutputPath);
                           auto res = txn.exec_params
-                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
-                                localStore->printStorePath(ex.step->drvPath),
-                                outputName);
+                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
+                               localStore->printStorePath(*optOutputPath));
                           if (!res[0][0].is_null()) {
                               propagatedFrom = res[0][0].as<BuildID>();
                               break;

From 6e67884ff164d9d1c115aef066e18f8de0c5fc44 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 14:05:18 -0500
Subject: [PATCH 112/285] One more `queryDerivationOutputMap` should use the
 eval store param

---
 src/hydra-queue-runner/queue-monitor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index fc344244..81678fe1 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -658,8 +658,7 @@ void State::processJobsetSharesChange(Connection & conn)
 
 BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::StorePath & drvPath)
 {
-
-    auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath);
+    auto derivationOutputs = destStore->queryDerivationOutputMap(drvPath, &*localStore);
 
     {
     pqxx::work txn(conn);

From 7517c134c57c7ce967a7b603482a23f4952331a0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 14 Dec 2023 00:29:23 -0500
Subject: [PATCH 113/285] flake.lock: Update Nix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Newer 2.19-maintenance has some `restricted-eval` fixes that benefit
Hydra.

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d' (2023-12-11)
  → 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13)
---
 flake.lock | 8 ++++----
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index 2871e70a..66e30323 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1701122567,
-        "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
+        "lastModified": 1702510710,
+        "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
+        "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.19.2",
+        "ref": "2.19-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 5a5aef55..8280e076 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.19.2";
+  inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:

From abd858d3dcd8dbcb2ee2d17ff5b03dbce46821e7 Mon Sep 17 00:00:00 2001
From: Jack Kelly <jack.kelly@bellroy.com>
Date: Tue, 19 Dec 2023 07:54:40 +1000
Subject: [PATCH 114/285] Document the `store_uri` parameter by way of example

---
 doc/manual/src/configuration.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index 02210449..4954040c 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -74,6 +74,30 @@ following:
         }
     }
 
+Populating a Cache
+------------------
+
+A common use for Hydra is to pre-build and cache derivations which
+take a long time to build. While it is possible to direcly access the
+Hydra server's store over SSH, a more scalable option is to upload
+built derivations to a remote store like an [S3-compatible object
+store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting
+the `store_uri` parameter will cause Hydra to sign and upload
+derivations as they are built:
+
+```
+store_uri = s3://cache-bucket-name?compression=zstd&parallel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key
+```
+
+This example uses [Zstandard](https://github.com/facebook/zstd)
+compression on derivations to reduce CPU usage on the server, but
+[Brotli](https://brotli.org/) compression for derivation listings and
+build logs because it has better browser support.
+
+See [`nix help
+stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html)
+for a description of the store URI format.
+
 Statsd Configuration
 --------------------
 

From 75f26f1fc4f114b7f930c57ba37212ea67990de1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 23 Dec 2023 19:10:58 -0500
Subject: [PATCH 115/285] Clean up `std::optional` dereferencing in the queue
 runner

Instead of doing this partial operation a number of times, assert (with
a comment, get a reference to the thing inside, and use that just once.
(This refactor was done twice, "just once" for each time.)
---
 src/hydra-queue-runner/queue-monitor.cc | 26 ++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 6c339af6..09a2871e 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -495,12 +495,14 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         size_t avail = 0;
         for (auto & i : missing) {
-            auto path = i.second.path(*localStore, step->drv->name, i.first);
-            if (/* localStore != destStore && */ localStore->isValidPath(*path))
+            auto pathOpt = i.second.path(*localStore, step->drv->name, i.first);
+            assert(pathOpt); // CA derivations not yet supported
+            auto & path = *pathOpt;
+            if (/* localStore != destStore && */ localStore->isValidPath(path))
                 avail++;
             else if (useSubstitutes) {
                 SubstitutablePathInfos infos;
-                localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
+                localStore->querySubstitutablePathInfos({{path, {}}}, infos);
                 if (infos.size() == 1)
                     avail++;
             }
@@ -509,25 +511,27 @@ Step::ptr State::createStep(ref<Store> destStore,
         if (missing.size() == avail) {
             valid = true;
             for (auto & i : missing) {
-                auto path = i.second.path(*localStore, step->drv->name, i.first);
+                auto pathOpt = i.second.path(*localStore, step->drv->name, i.first);
+                assert(pathOpt); // CA derivations not yet supported
+                auto & path = *pathOpt;
 
                 try {
                     time_t startTime = time(0);
 
-                    if (localStore->isValidPath(*path))
+                    if (localStore->isValidPath(path))
                         printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
-                            localStore->printStorePath(*path),
+                            localStore->printStorePath(path),
                             localStore->printStorePath(drvPath));
                     else {
                         printInfo("substituting output ‘%1%’ of ‘%2%’",
-                            localStore->printStorePath(*path),
+                            localStore->printStorePath(path),
                             localStore->printStorePath(drvPath));
-                        localStore->ensurePath(*path);
+                        localStore->ensurePath(path);
                         // FIXME: should copy directly from substituter to destStore.
                     }
 
                     copyClosure(*localStore, *destStore,
-                        StorePathSet { *path },
+                        StorePathSet { path },
                         NoRepair, CheckSigs, NoSubstitute);
 
                     time_t stopTime = time(0);
@@ -535,13 +539,13 @@ Step::ptr State::createStep(ref<Store> destStore,
                     {
                         auto mc = startDbUpdate();
                         pqxx::work txn(conn);
-                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
+                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path);
                         txn.commit();
                     }
 
                 } catch (Error & e) {
                     printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
-                        localStore->printStorePath(*path),
+                        localStore->printStorePath(path),
                         localStore->printStorePath(drvPath),
                         e.what());
                     valid = false;

From db7aa01b8d90439081ee60dce998a8b30ad5f5b2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 22 Jan 2024 13:49:05 -0500
Subject: [PATCH 116/285] Update to newer Nix master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/b7e016ab2464ad2e7e2e856ad0f173157135aae0' (2023-12-10)
  → 'github:NixOS/nix/74534829f23b668fb9b2f2a14ff6afa4d5e71d4a' (2024-01-22)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/e9f06adb793d1cca5384907b3b8a4071d5d7cb19' (2023-12-03)
  → 'github:NixOS/nixpkgs/a1982c92d8980a0114372973cbdfe0a307f1bdea' (2024-01-12)
• Removed input 'nix/lowdown-src'
---
 flake.lock                             | 31 ++++++--------------------
 flake.nix                              |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc |  4 ++--
 3 files changed, 10 insertions(+), 27 deletions(-)

diff --git a/flake.lock b/flake.lock
index d99954a9..8b15a882 100644
--- a/flake.lock
+++ b/flake.lock
@@ -32,38 +32,21 @@
         "type": "github"
       }
     },
-    "lowdown-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1633514407,
-        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "type": "github"
-      }
-    },
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
         "libgit2": "libgit2",
-        "lowdown-src": "lowdown-src",
         "nixpkgs": [
           "nixpkgs"
         ],
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1702228562,
-        "narHash": "sha256-VwvfxT8LZcEm3H+VCMCrKzl7Ip+dHumxPfugAx3tjk8=",
+        "lastModified": 1705948977,
+        "narHash": "sha256-8ZUbb/vf2R0kKKHJyLTIh0HYKzgkV2pq/unDj0kzKG8=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "b7e016ab2464ad2e7e2e856ad0f173157135aae0",
+        "rev": "74534829f23b668fb9b2f2a14ff6afa4d5e71d4a",
         "type": "github"
       },
       "original": {
@@ -74,16 +57,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1701615100,
-        "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=",
+        "lastModified": 1705033721,
+        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19",
+        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index fdeaa6ca..c9af0174 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,7 +1,7 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
   inputs.nix.url = "github:NixOS/nix";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 2fe2c80f..1c4b5d93 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -89,7 +89,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
 
 static MyArgs myArgs;
 
-static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std::string & name, const std::string & subAttribute)
+static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute)
 {
     Strings res;
     std::function<void(Value & v)> rec;
@@ -178,7 +178,7 @@ static void worker(
 
             if (auto drv = getDerivation(state, *v, false)) {
 
-                DrvInfo::Outputs outputs = drv->queryOutputs();
+                PackageInfo::Outputs outputs = drv->queryOutputs();
 
                 if (drv->querySystem() == "unknown")
                     throw EvalError("derivation must have a 'system' attribute");

From 4ac31c89df85a6a72d82a392b729b0c90bcd5d33 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 20 Feb 2022 19:51:28 +0000
Subject: [PATCH 117/285] Use `nix::serv_proto::BasicConnection` in
 build_remote.cc

- Use the type itself

  This lays the foundation for being able to dedup the protocol code.

- Use `BasicConnection::handshake`, replacing ours.

- Use `BasicConnection::queryValidPaths`

- Use `BasicConnection::putBuildDerivationRequest`
---
 src/hydra-queue-runner/build-remote.cc | 64 ++++++++++----------------
 src/hydra-queue-runner/state.hh        | 23 +--------
 2 files changed, 26 insertions(+), 61 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index f19008dc..058af02c 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -8,6 +8,7 @@
 #include "build-result.hh"
 #include "path.hh"
 #include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "state.hh"
 #include "current-process.hh"
 #include "processes.hh"
@@ -123,13 +124,10 @@ static void copyClosureTo(
        garbage-collect paths that are already there. Optionally, ask
        the remote host to substitute missing paths. */
     // FIXME: substitute output pollutes our build log
-    conn.to << ServeProto::Command::QueryValidPaths << 1 << useSubstitutes;
-    ServeProto::write(destStore, conn, closure);
-    conn.to.flush();
-
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = ServeProto::Serialise<StorePathSet>::read(destStore, conn);
+    auto present = conn.queryValidPaths(
+        destStore, true, closure, useSubstitutes);
 
     if (present.size() == closure.size()) return;
 
@@ -195,33 +193,6 @@ static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, cons
     return {std::move(logFile), std::move(logFD)};
 }
 
-/**
- * @param conn is not fully initialized; it is this functions job to set
- * the `remoteVersion` field after the handshake is completed.
- * Therefore, no `ServeProto::Serialize` functions can be used until
- * that field is set.
- */
-static void handshake(Machine::Connection & conn, unsigned int repeats)
-{
-    constexpr ServeProto::Version our_version = 0x206;
-
-    conn.to << SERVE_MAGIC_1 << our_version;
-    conn.to.flush();
-
-    unsigned int magic = readInt(conn.from);
-    if (magic != SERVE_MAGIC_2)
-        throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%1%’", conn.machine->sshName);
-    conn.remoteVersion = readInt(conn.from);
-    // Now `conn` is initialized.
-    if (GET_PROTOCOL_MAJOR(conn.remoteVersion) != 0x200)
-        throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%1%’", conn.machine->sshName);
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 3 && repeats > 0)
-        throw Error("machine ‘%1%’ does not support repeating a build; please upgrade it to Nix 1.12", conn.machine->sshName);
-
-    // Do not attempt to speak a newer version of the protocol
-    conn.remoteVersion = std::min(conn.remoteVersion, our_version);
-}
-
 static BasicDerivation sendInputs(
     State & state,
     Step & step,
@@ -291,10 +262,7 @@ static BuildResult performBuild(
     counter & nrStepsBuilding
 )
 {
-    conn.to << ServeProto::Command::BuildDerivation << localStore.printStorePath(drvPath);
-    writeDerivation(conn.to, localStore, drv);
-    ServeProto::write(localStore, conn, options);
-    conn.to.flush();
+    conn.putBuildDerivationRequest(localStore, drvPath, drv, options);
 
     BuildResult result;
 
@@ -498,9 +466,13 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         Machine::Connection conn {
-            .from = child.from.get(),
-            .to = child.to.get(),
-            .machine = machine,
+            {
+                .to = child.to.get(),
+                .from = child.from.get(),
+                /* Handshake. */
+                .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
+            },
+            /*.machine =*/ machine,
         };
 
         Finally updateStats([&]() {
@@ -508,14 +480,26 @@ void State::buildRemote(ref<Store> destStore,
             bytesSent += conn.to.written;
         });
 
+        constexpr ServeProto::Version our_version = 0x206;
+
         try {
-          build_remote::handshake(conn, buildOptions.nrRepeats);
+            conn.remoteVersion = decltype(conn)::handshake(
+                conn.to,
+                conn.from,
+                our_version,
+                machine->sshName);
         } catch (EndOfFile & e) {
             child.pid.wait();
             std::string s = chomp(readFile(result.logFile));
             throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
         }
 
+        // Do not attempt to speak a newer version of the protocol.
+        //
+        // Per https://github.com/NixOS/nix/issues/9584 should be handled as
+        // part of `handshake` in upstream nix.
+        conn.remoteVersion = std::min(conn.remoteVersion, our_version);
+
         {
             auto info(machine->state->connectInfo.lock());
             info->consecutiveFailures = 0;
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 830f0598..82481ab5 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -22,6 +22,7 @@
 #include "sync.hh"
 #include "nar-extractor.hh"
 #include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 
 
 typedef unsigned int BuildID;
@@ -302,29 +303,9 @@ struct Machine
     }
 
     // A connection to a machine
-    struct Connection {
-        nix::FdSource from;
-        nix::FdSink to;
-        nix::ServeProto::Version remoteVersion;
-
+    struct Connection : nix::ServeProto::BasicClientConnection {
         // Backpointer to the machine
         ptr machine;
-
-        operator nix::ServeProto::ReadConn ()
-        {
-            return {
-                .from = from,
-                .version = remoteVersion,
-            };
-        }
-
-        operator nix::ServeProto::WriteConn ()
-        {
-            return {
-                .to = to,
-                .version = remoteVersion,
-            };
-        }
     };
 };
 

From 4e8fbaa3d688a4ffa95ccbaa0af889569c918633 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 23 Jan 2024 00:00:23 -0500
Subject: [PATCH 118/285] Replace `Child` with `SSHMaster::Connection`

Nix defines basically an identical struct for the same purpose, so let's
just use that.
---
 src/hydra-queue-runner/build-remote.cc | 30 +++++++++++---------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index b117fdad..a4f3a35c 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -14,19 +14,13 @@
 #include "util.hh"
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
+#include "ssh.hh"
 #include "finally.hh"
 #include "url.hh"
 
 using namespace nix;
 
 
-struct Child
-{
-    Pid pid;
-    AutoCloseFD to, from;
-};
-
-
 static void append(Strings & dst, const Strings & src)
 {
     dst.insert(dst.end(), src.begin(), src.end());
@@ -54,7 +48,7 @@ static Strings extraStoreArgs(std::string & machine)
     return result;
 }
 
-static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Child & child)
+static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child)
 {
     std::string pgmName;
     Pipe to, from;
@@ -84,7 +78,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
         append(argv, extraArgs);
     }
 
-    child.pid = startProcess([&]() {
+    child.sshPid = startProcess([&]() {
         restoreProcessContext();
 
         if (dup2(to.readSide.get(), STDIN_FILENO) == -1)
@@ -104,8 +98,8 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
     to.readSide = -1;
     from.writeSide = -1;
 
-    child.to = to.writeSide.release();
-    child.from = from.readSide.release();
+    child.in = to.writeSide.release();
+    child.out = from.readSide.release();
 }
 
 
@@ -488,13 +482,13 @@ void State::buildRemote(ref<Store> destStore,
         updateStep(ssConnecting);
 
         // FIXME: rewrite to use Store.
-        Child child;
+        SSHMaster::Connection child;
         build_remote::openConnection(machine, tmpDir, logFD.get(), child);
 
         {
             auto activeStepState(activeStep->state_.lock());
             if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = child.pid;
+            activeStepState->pid = child.sshPid;
         }
 
         Finally clearPid([&]() {
@@ -510,8 +504,8 @@ void State::buildRemote(ref<Store> destStore,
         });
 
         Machine::Connection conn {
-            .from = child.from.get(),
-            .to = child.to.get(),
+            .from = child.out.get(),
+            .to = child.in.get(),
             .machine = machine,
         };
 
@@ -523,7 +517,7 @@ void State::buildRemote(ref<Store> destStore,
         try {
           build_remote::handshake(conn, buildOptions.repeats);
         } catch (EndOfFile & e) {
-            child.pid.wait();
+            child.sshPid.wait();
             std::string s = chomp(readFile(result.logFile));
             throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
         }
@@ -617,8 +611,8 @@ void State::buildRemote(ref<Store> destStore,
         }
 
         /* Shut down the connection. */
-        child.to = -1;
-        child.pid.wait();
+        child.in = -1;
+        child.sshPid.wait();
 
     } catch (Error & e) {
         /* Disable this machine until a certain period of time has

From 84c46b6b6865721bb23a38580f4e190548a5a380 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 23 Jan 2024 09:43:43 -0500
Subject: [PATCH 119/285] Update to newer Nix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/74534829f23b668fb9b2f2a14ff6afa4d5e71d4a' (2024-01-22)
  → 'github:NixOS/nix/b6aee9a93f6646bbffd919d362a5c75c37bb9caa' (2024-01-23)
---
 flake.lock                              |  6 +-
 src/hydra-queue-runner/nar-extractor.cc | 91 +++++++++++++------------
 2 files changed, 52 insertions(+), 45 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8b15a882..c78d5203 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1705948977,
-        "narHash": "sha256-8ZUbb/vf2R0kKKHJyLTIh0HYKzgkV2pq/unDj0kzKG8=",
+        "lastModified": 1706016881,
+        "narHash": "sha256-XZz6tfTuZcrrX27vI+jAereNVWmnCrzFhBFhaKwEdoY=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "74534829f23b668fb9b2f2a14ff6afa4d5e71d4a",
+        "rev": "b6aee9a93f6646bbffd919d362a5c75c37bb9caa",
         "type": "github"
       },
       "original": {
diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc
index ff4aa268..61299ecd 100644
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -6,7 +6,46 @@
 
 using namespace nix;
 
-struct Extractor : ParseSink
+
+struct NarMemberConstructor : CreateRegularFileSink
+{
+    NarMemberData & curMember;
+
+    HashSink hashSink = HashSink { HashAlgorithm::SHA256 };
+
+    std::optional<uint64_t> expectedSize;
+
+    NarMemberConstructor(NarMemberData & curMember)
+        : curMember(curMember)
+    { }
+
+    void isExecutable() override
+    {
+    }
+
+    void preallocateContents(uint64_t size) override
+    {
+        expectedSize = size;
+    }
+
+    void operator () (std::string_view data) override
+    {
+        assert(expectedSize);
+        *curMember.fileSize += data.size();
+        hashSink(data);
+        if (curMember.contents) {
+            curMember.contents->append(data);
+        }
+        assert(curMember.fileSize <= expectedSize);
+        if (curMember.fileSize == expectedSize) {
+            auto [hash, len] = hashSink.finish();
+            assert(curMember.fileSize == len);
+            curMember.sha256 = hash;
+        }
+    }
+};
+
+struct Extractor : FileSystemObjectSink
 {
     std::unordered_set<Path> filesToKeep {
         "/nix-support/hydra-build-products",
@@ -15,7 +54,6 @@ struct Extractor : ParseSink
     };
 
     NarMemberDatas & members;
-    NarMemberData * curMember = nullptr;
     Path prefix;
 
     Extractor(NarMemberDatas & members, const Path & prefix)
@@ -27,53 +65,22 @@ struct Extractor : ParseSink
         members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
-    void createRegularFile(const Path & path) override
+    void createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func) override
     {
-        curMember = &members.insert_or_assign(prefix + path, NarMemberData {
-            .type = SourceAccessor::Type::tRegular,
-            .fileSize = 0,
-            .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
-        }).first->second;
-    }
-
-    std::optional<uint64_t> expectedSize;
-    std::unique_ptr<HashSink> hashSink;
-
-    void preallocateContents(uint64_t size) override
-    {
-        expectedSize = size;
-        hashSink = std::make_unique<HashSink>(HashAlgorithm::SHA256);
-    }
-
-    void receiveContents(std::string_view data) override
-    {
-        assert(expectedSize);
-        assert(curMember);
-        assert(hashSink);
-        *curMember->fileSize += data.size();
-        (*hashSink)(data);
-        if (curMember->contents) {
-            curMember->contents->append(data);
-        }
-        assert(curMember->fileSize <= expectedSize);
-        if (curMember->fileSize == expectedSize) {
-            auto [hash, len] = hashSink->finish();
-            assert(curMember->fileSize == len);
-            curMember->sha256 = hash;
-            hashSink.reset();
-        }
+        NarMemberConstructor nmc {
+            members.insert_or_assign(prefix + path, NarMemberData {
+                .type = SourceAccessor::Type::tRegular,
+                .fileSize = 0,
+                .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
+            }).first->second,
+        };
+        func(nmc);
     }
 
     void createSymlink(const Path & path, const std::string & target) override
     {
         members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
     }
-
-    void isExecutable() override
-    { }
-
-    void closeRegularFile() override
-    { }
 };
 
 

From 7386caaecfd68fbf70099eb8f3a6f8e296d1fb5b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 22 Jan 2024 18:38:39 -0500
Subject: [PATCH 120/285] Use Nix's `SSHMaster`

---
 src/hydra-queue-runner/build-remote.cc | 87 +++++++-------------------
 1 file changed, 24 insertions(+), 63 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 68a339f3..7f547a0b 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -21,12 +21,6 @@
 
 using namespace nix;
 
-
-static void append(Strings & dst, const Strings & src)
-{
-    dst.insert(dst.end(), src.begin(), src.end());
-}
-
 namespace nix::build_remote {
 
 static Strings extraStoreArgs(std::string & machine)
@@ -49,58 +43,20 @@ static Strings extraStoreArgs(std::string & machine)
     return result;
 }
 
-static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child)
+static std::unique_ptr<SSHMaster::Connection> openConnection(
+    Machine::ptr machine, SSHMaster & master)
 {
-    std::string pgmName;
-    Pipe to, from;
-    to.create();
-    from.create();
-
-    Strings argv;
+    Strings command = {"nix-store", "--serve", "--write"};
     if (machine->isLocalhost()) {
-        pgmName = "nix-store";
-        argv = {"nix-store", "--builders", "", "--serve", "--write"};
+        command.push_back("--builders");
+        command.push_back("");
     } else {
-        pgmName = "ssh";
-        auto sshName = machine->sshName;
-        Strings extraArgs = extraStoreArgs(sshName);
-        argv = {"ssh", sshName};
-        if (machine->sshKey != "") append(argv, {"-i", machine->sshKey});
-        if (machine->sshPublicHostKey != "") {
-            Path fileName = tmpDir + "/host-key";
-            auto p = machine->sshName.find("@");
-            std::string host = p != std::string::npos ? std::string(machine->sshName, p + 1) : machine->sshName;
-            writeFile(fileName, host + " " + machine->sshPublicHostKey + "\n");
-            append(argv, {"-oUserKnownHostsFile=" + fileName});
-        }
-        append(argv,
-            { "-x", "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
-            , "--", "nix-store", "--serve", "--write" });
-        append(argv, extraArgs);
+        command.splice(command.end(), extraStoreArgs(machine->sshName));
     }
 
-    child.sshPid = startProcess([&]() {
-        restoreProcessContext();
-
-        if (dup2(to.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("cannot dup input pipe to stdin");
-
-        if (dup2(from.writeSide.get(), STDOUT_FILENO) == -1)
-            throw SysError("cannot dup output pipe to stdout");
-
-        if (dup2(stderrFD, STDERR_FILENO) == -1)
-            throw SysError("cannot dup stderr");
-
-        execvp(argv.front().c_str(), (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
-
-        throw SysError("cannot start %s", pgmName);
+    return master.startCommand(std::move(command), {
+        "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
     });
-
-    to.readSide = -1;
-    from.writeSide = -1;
-
-    child.in = to.writeSide.release();
-    child.out = from.readSide.release();
 }
 
 
@@ -430,21 +386,26 @@ void State::buildRemote(ref<Store> destStore,
     AutoDelete logFileDel(logFile, false);
     result.logFile = logFile;
 
-    nix::Path tmpDir = createTempDir();
-    AutoDelete tmpDirDel(tmpDir, true);
-
     try {
 
         updateStep(ssConnecting);
 
+        SSHMaster master {
+            machine->sshName,
+            machine->sshKey,
+            machine->sshPublicHostKey,
+            false, // no SSH master yet
+            false, // no compression yet
+            logFD.get(),
+        };
+
         // FIXME: rewrite to use Store.
-        SSHMaster::Connection child;
-        build_remote::openConnection(machine, tmpDir, logFD.get(), child);
+        auto child = build_remote::openConnection(machine, master);
 
         {
             auto activeStepState(activeStep->state_.lock());
             if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = child.sshPid;
+            activeStepState->pid = child->sshPid;
         }
 
         Finally clearPid([&]() {
@@ -461,8 +422,8 @@ void State::buildRemote(ref<Store> destStore,
 
         Machine::Connection conn {
             {
-                .to = child.in.get(),
-                .from = child.out.get(),
+                .to = child->in.get(),
+                .from = child->out.get(),
                 /* Handshake. */
                 .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
             },
@@ -483,7 +444,7 @@ void State::buildRemote(ref<Store> destStore,
                 our_version,
                 machine->sshName);
         } catch (EndOfFile & e) {
-            child.sshPid.wait();
+            child->sshPid.wait();
             std::string s = chomp(readFile(result.logFile));
             throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
         }
@@ -583,8 +544,8 @@ void State::buildRemote(ref<Store> destStore,
         }
 
         /* Shut down the connection. */
-        child.in = -1;
-        child.sshPid.wait();
+        child->in = -1;
+        child->sshPid.wait();
 
     } catch (Error & e) {
         /* Disable this machine until a certain period of time has

From 2e6ee28f9bef00ef08e6b6125f04af82cfca9fd3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 23 Jan 2024 11:03:19 -0500
Subject: [PATCH 121/285] `Machine` -> `::Machine` so we don't conflict with
 Nix's

---
 src/hydra-queue-runner/build-remote.cc       | 20 ++++++++++----------
 src/hydra-queue-runner/builder.cc            |  2 +-
 src/hydra-queue-runner/dispatcher.cc         |  4 ++--
 src/hydra-queue-runner/hydra-queue-runner.cc |  8 ++++----
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index a4f3a35c..5cc994fc 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -48,7 +48,7 @@ static Strings extraStoreArgs(std::string & machine)
     return result;
 }
 
-static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child)
+static void openConnection(::Machine::ptr machine, Path tmpDir, int stderrFD, SSHMaster::Connection & child)
 {
     std::string pgmName;
     Pipe to, from;
@@ -104,7 +104,7 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, SSHM
 
 
 static void copyClosureTo(
-    Machine::Connection & conn,
+    ::Machine::Connection & conn,
     Store & destStore,
     const StorePathSet & paths,
     SubstituteFlag useSubstitutes = NoSubstitute)
@@ -195,7 +195,7 @@ static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, cons
  * Therefore, no `ServeProto::Serialize` functions can be used until
  * that field is set.
  */
-static void handshake(Machine::Connection & conn, unsigned int repeats)
+static void handshake(::Machine::Connection & conn, unsigned int repeats)
 {
     conn.to << SERVE_MAGIC_1 << 0x206;
     conn.to.flush();
@@ -216,7 +216,7 @@ static BasicDerivation sendInputs(
     Step & step,
     Store & localStore,
     Store & destStore,
-    Machine::Connection & conn,
+    ::Machine::Connection & conn,
     unsigned int & overhead,
     counter & nrStepsWaiting,
     counter & nrStepsCopyingTo
@@ -272,7 +272,7 @@ static BasicDerivation sendInputs(
 }
 
 static BuildResult performBuild(
-    Machine::Connection & conn,
+    ::Machine::Connection & conn,
     Store & localStore,
     StorePath drvPath,
     const BasicDerivation & drv,
@@ -317,7 +317,7 @@ static BuildResult performBuild(
 }
 
 static std::map<StorePath, ValidPathInfo> queryPathInfos(
-    Machine::Connection & conn,
+    ::Machine::Connection & conn,
     Store & localStore,
     StorePathSet & outputs,
     size_t & totalNarSize
@@ -355,7 +355,7 @@ static std::map<StorePath, ValidPathInfo> queryPathInfos(
 }
 
 static void copyPathFromRemote(
-    Machine::Connection & conn,
+    ::Machine::Connection & conn,
     NarMemberDatas & narMembers,
     Store & localStore,
     Store & destStore,
@@ -385,7 +385,7 @@ static void copyPathFromRemote(
 }
 
 static void copyPathsFromRemote(
-    Machine::Connection & conn,
+    ::Machine::Connection & conn,
     NarMemberDatas & narMembers,
     Store & localStore,
     Store & destStore,
@@ -462,7 +462,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 
 
 void State::buildRemote(ref<Store> destStore,
-    Machine::ptr machine, Step::ptr step,
+    ::Machine::ptr machine, Step::ptr step,
     const BuildOptions & buildOptions,
     RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
     std::function<void(StepState)> updateStep,
@@ -503,7 +503,7 @@ void State::buildRemote(ref<Store> destStore,
                process. Meh. */
         });
 
-        Machine::Connection conn {
+        ::Machine::Connection conn {
             .from = child.out.get(),
             .to = child.in.get(),
             .machine = machine,
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 307eee8e..cb343d77 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -400,7 +400,7 @@ void State::failStep(
     Step::ptr step,
     BuildID buildId,
     const RemoteResult & result,
-    Machine::ptr machine,
+    ::Machine::ptr machine,
     bool & stepFinished)
 {
     /* Register failure in the database for all Build objects that
diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index af21dcbd..6d738ded 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -199,7 +199,7 @@ system_time State::doDispatch()
            filter out temporarily disabled machines. */
         struct MachineInfo
         {
-            Machine::ptr machine;
+            ::Machine::ptr machine;
             unsigned long currentJobs;
         };
         std::vector<MachineInfo> machinesSorted;
@@ -435,7 +435,7 @@ void Jobset::pruneSteps()
 }
 
 
-State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine)
+State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine)
     : state(state), step(step), machine(machine)
 {
     machine->state->currentJobs++;
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 1d54bb93..09d6fcb2 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -140,7 +140,7 @@ void State::parseMachines(const std::string & contents)
         if (tokens.size() < 3) continue;
         tokens.resize(8);
 
-        auto machine = std::make_shared<Machine>();
+        auto machine = std::make_shared<::Machine>();
         machine->sshName = tokens[0];
         machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
         machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
@@ -166,7 +166,7 @@ void State::parseMachines(const std::string & contents)
         else
             printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
         machine->state = i == oldMachines.end()
-            ? std::make_shared<Machine::State>()
+            ? std::make_shared<::Machine::State>()
             : i->second->state;
         newMachines[machine->sshName] = machine;
     }
@@ -175,9 +175,9 @@ void State::parseMachines(const std::string & contents)
         if (newMachines.find(m.first) == newMachines.end()) {
             if (m.second->enabled)
                 printInfo("removing machine ‘%1%’", m.first);
-            /* Add a disabled Machine object to make sure stats are
+            /* Add a disabled ::Machine object to make sure stats are
                maintained. */
-            auto machine = std::make_shared<Machine>(*(m.second));
+            auto machine = std::make_shared<::Machine>(*(m.second));
             machine->enabled = false;
             newMachines[m.first] = machine;
         }

From 70e5469303b422bdb4b123be222bdea4d7f9611c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 23 Jan 2024 12:08:05 -0500
Subject: [PATCH 122/285] Use Nix's `Machine` type in a mimimal way

This is *just* using the fields from that type, and only where the types
coincide. (There are two fields with different types, `speedFactor` most
interestingly.) No code is reused, so we can be sure that no behavior is
changed.

Once the types are reconciled on the Nix side, then we can start
carefully actually reusing code.

Progress on #1164
---
 src/hydra-queue-runner/dispatcher.cc         |  6 +--
 src/hydra-queue-runner/hydra-queue-runner.cc | 53 ++++++++++++++------
 src/hydra-queue-runner/state.hh              | 21 +++++---
 3 files changed, 53 insertions(+), 27 deletions(-)

diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index 6d738ded..f5d9d3e3 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -231,11 +231,11 @@ system_time State::doDispatch()
         sort(machinesSorted.begin(), machinesSorted.end(),
             [](const MachineInfo & a, const MachineInfo & b) -> bool
             {
-                float ta = std::round(a.currentJobs / a.machine->speedFactor);
-                float tb = std::round(b.currentJobs / b.machine->speedFactor);
+                float ta = std::round(a.currentJobs / a.machine->speedFactorFloat);
+                float tb = std::round(b.currentJobs / b.machine->speedFactorFloat);
                 return
                     ta != tb ? ta < tb :
-                    a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor :
+                    a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
                     a.currentJobs > b.currentJobs;
             });
 
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 09d6fcb2..66528302 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -1,6 +1,7 @@
 #include <iostream>
 #include <thread>
 #include <optional>
+#include <type_traits>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -140,23 +141,43 @@ void State::parseMachines(const std::string & contents)
         if (tokens.size() < 3) continue;
         tokens.resize(8);
 
-        auto machine = std::make_shared<::Machine>();
-        machine->sshName = tokens[0];
-        machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
-        machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
-        if (tokens[3] != "")
-            machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
-        else
-            machine->maxJobs = 1;
-        machine->speedFactor = atof(tokens[4].c_str());
         if (tokens[5] == "-") tokens[5] = "";
-        machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
+        auto supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
+
         if (tokens[6] == "-") tokens[6] = "";
-        machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
-        for (auto & f : machine->mandatoryFeatures)
-            machine->supportedFeatures.insert(f);
-        if (tokens[7] != "" && tokens[7] != "-")
-            machine->sshPublicHostKey = base64Decode(tokens[7]);
+        auto mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
+
+        for (auto & f : mandatoryFeatures)
+            supportedFeatures.insert(f);
+
+        using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
+
+        auto machine = std::make_shared<::Machine>(nix::Machine {
+            // `storeUri`, not yet used
+            "",
+            // `systemTypes`, not yet used
+            {},
+            // `sshKey`
+            tokens[2] == "-" ? "" : tokens[2],
+            // `maxJobs`
+            tokens[3] != ""
+                ? string2Int<MaxJobs>(tokens[3]).value()
+                : 1,
+            // `speedFactor`, not yet used
+            1,
+            // `supportedFeatures`
+            std::move(supportedFeatures),
+            // `mandatoryFeatures`
+            std::move(mandatoryFeatures),
+            // `sshPublicHostKey`
+            tokens[7] != "" && tokens[7] != "-"
+                ? base64Decode(tokens[7])
+                : "",
+        });
+
+        machine->sshName = tokens[0];
+        machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
+        machine->speedFactorFloat = atof(tokens[4].c_str());
 
         /* Re-use the State object of the previous machine with the
            same name. */
@@ -596,7 +617,7 @@ void State::dumpStatus(Connection & conn)
 
                 json machine = {
                     {"enabled",  m->enabled},
-                    {"systemTypes", m->systemTypes},
+                    {"systemTypes", m->systemTypesSet},
                     {"supportedFeatures", m->supportedFeatures},
                     {"mandatoryFeatures", m->mandatoryFeatures},
                     {"nrStepsDone", s->nrStepsDone.load()},
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 6359063a..b08e4e37 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -22,6 +22,7 @@
 #include "sync.hh"
 #include "nar-extractor.hh"
 #include "serve-protocol.hh"
+#include "machines.hh"
 
 
 typedef unsigned int BuildID;
@@ -234,17 +235,21 @@ void getDependents(Step::ptr step, std::set<Build::ptr> & builds, std::set<Step:
 void visitDependencies(std::function<void(Step::ptr)> visitor, Step::ptr step);
 
 
-struct Machine
+struct Machine : nix::Machine
 {
     typedef std::shared_ptr<Machine> ptr;
 
-    bool enabled{true};
+    /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
+       we are not yet used to, but once we are, we don't need this. */
+    std::string sshName;
 
-    std::string sshName, sshKey;
-    std::set<std::string> systemTypes, supportedFeatures, mandatoryFeatures;
-    unsigned int maxJobs = 1;
-    float speedFactor = 1.0;
-    std::string sshPublicHostKey;
+    /* TODO Get rid once `nix::Machine::systemTypes` is a set not
+       vector. */
+    std::set<std::string> systemTypesSet;
+
+    /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not
+       an `int`. */
+    float speedFactorFloat = 1.0;
 
     struct State {
         typedef std::shared_ptr<State> ptr;
@@ -272,7 +277,7 @@ struct Machine
     {
         /* Check that this machine is of the type required by the
            step. */
-        if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
+        if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
             return false;
 
         /* Check that the step requires all mandatory features of this

From 2bdbf51d7d947c44af159dab391be2a0dfb930d7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 24 Jan 2024 18:46:56 -0500
Subject: [PATCH 123/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/b6aee9a93f6646bbffd919d362a5c75c37bb9caa' (2024-01-23)
  → 'github:NixOS/nix/212ba69e6f995992f8b4e4c0656d19c0156c8714' (2024-01-24)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index c78d5203..bdbc4197 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1706016881,
-        "narHash": "sha256-XZz6tfTuZcrrX27vI+jAereNVWmnCrzFhBFhaKwEdoY=",
+        "lastModified": 1706100941,
+        "narHash": "sha256-YLYxZrQLQ6i22RLZG/rsB4sIfFzhep7XQYlodzCmKy0=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "b6aee9a93f6646bbffd919d362a5c75c37bb9caa",
+        "rev": "212ba69e6f995992f8b4e4c0656d19c0156c8714",
         "type": "github"
       },
       "original": {

From 449eb2d873d3240ec4f6bc3e6679eeda7a5cadfd Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 24 Jan 2024 20:14:31 -0500
Subject: [PATCH 124/285] Use more `nix::Machine` fields

The upstream fields were made to match Hydra, so we can get rid of the
extra fields temporary added in
70e5469303b422bdb4b123be222bdea4d7f9611c.
---
 src/hydra-queue-runner/dispatcher.cc         |  6 +++---
 src/hydra-queue-runner/hydra-queue-runner.cc | 12 +++++-------
 src/hydra-queue-runner/state.hh              | 10 +---------
 3 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index f5d9d3e3..6d738ded 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -231,11 +231,11 @@ system_time State::doDispatch()
         sort(machinesSorted.begin(), machinesSorted.end(),
             [](const MachineInfo & a, const MachineInfo & b) -> bool
             {
-                float ta = std::round(a.currentJobs / a.machine->speedFactorFloat);
-                float tb = std::round(b.currentJobs / b.machine->speedFactorFloat);
+                float ta = std::round(a.currentJobs / a.machine->speedFactor);
+                float tb = std::round(b.currentJobs / b.machine->speedFactor);
                 return
                     ta != tb ? ta < tb :
-                    a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
+                    a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor :
                     a.currentJobs > b.currentJobs;
             });
 
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 5fbcb641..3d48f3d7 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -155,16 +155,16 @@ void State::parseMachines(const std::string & contents)
         auto machine = std::make_shared<::Machine>(nix::Machine {
             // `storeUri`, not yet used
             "",
-            // `systemTypes`, not yet used
-            {},
+            // `systemTypes`
+            tokenizeString<StringSet>(tokens[1], ","),
             // `sshKey`
             tokens[2] == "-" ? "" : tokens[2],
             // `maxJobs`
             tokens[3] != ""
                 ? string2Int<MaxJobs>(tokens[3]).value()
                 : 1,
-            // `speedFactor`, not yet used
-            1,
+            // `speedFactor`
+            atof(tokens[4].c_str()),
             // `supportedFeatures`
             std::move(supportedFeatures),
             // `mandatoryFeatures`
@@ -176,8 +176,6 @@ void State::parseMachines(const std::string & contents)
         });
 
         machine->sshName = tokens[0];
-        machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
-        machine->speedFactorFloat = atof(tokens[4].c_str());
 
         /* Re-use the State object of the previous machine with the
            same name. */
@@ -638,7 +636,7 @@ void State::dumpStatus(Connection & conn)
 
                 json machine = {
                     {"enabled",  m->enabled},
-                    {"systemTypes", m->systemTypesSet},
+                    {"systemTypes", m->systemTypes},
                     {"supportedFeatures", m->supportedFeatures},
                     {"mandatoryFeatures", m->mandatoryFeatures},
                     {"nrStepsDone", s->nrStepsDone.load()},
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index f6d9d5a9..cda238ae 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -244,14 +244,6 @@ struct Machine : nix::Machine
        we are not yet used to, but once we are, we don't need this. */
     std::string sshName;
 
-    /* TODO Get rid once `nix::Machine::systemTypes` is a set not
-       vector. */
-    std::set<std::string> systemTypesSet;
-
-    /* TODO Get rid once `nix::Machine::systemTypes` is a `float` not
-       an `int`. */
-    float speedFactorFloat = 1.0;
-
     struct State {
         typedef std::shared_ptr<State> ptr;
         counter currentJobs{0};
@@ -278,7 +270,7 @@ struct Machine : nix::Machine
     {
         /* Check that this machine is of the type required by the
            step. */
-        if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
+        if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
             return false;
 
         /* Check that the step requires all mandatory features of this

From 07cb5d1b7c74fdc8423a93b14f4b56af1c49e4b3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 24 Jan 2024 21:04:14 -0500
Subject: [PATCH 125/285] Use
 `nix::ParsedDerivation::getRequiredSystemFeatures()`

A slight dedup, and also ensures that floating CA derivations require a
`ca-derivations` experimental feature. This fixes the scheduling issue
that @SuperSandro2000 found.
---
 src/hydra-queue-runner/queue-monitor.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index a3557316..513d399b 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -462,10 +462,7 @@ Step::ptr State::createStep(ref<Store> destStore,
 
     step->systemType = step->drv->platform;
     {
-        auto i = step->drv->env.find("requiredSystemFeatures");
-        StringSet features;
-        if (i != step->drv->env.end())
-            features = step->requiredSystemFeatures = tokenizeString<std::set<std::string>>(i->second);
+        StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
         if (step->preferLocalBuild)
             features.insert("local");
         if (!features.empty()) {

From b1fa6b3aac4ce8225e506317119b32122cfc1edd Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 24 Jan 2024 21:37:13 -0500
Subject: [PATCH 126/285] Use `StoreConfig::getDefaultSystemFeatures` for
 default machine config

We have to oddly make a `StoreConfig` subclass to get it, but
https://github.com/NixOS/nix/pull/9848 will fix that.

The purpose of this is to ensure that, absent an explicit config,
`localhost` includes `ca-derivations` and `recursive-nix` if those
experimental features are enabled.

Very much the complement of #1342, the previous PR.
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 5fbcb641..1cbcafea 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -15,6 +15,7 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "store-api.hh"
+#include "local-store.hh"
 #include "remote-store.hh"
 
 #include "globals.hh"
@@ -226,7 +227,7 @@ void State::monitorMachinesFile()
         parseMachines("localhost " +
             (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get())
             + " - " + std::to_string(settings.maxBuildJobs) + " 1 "
-            + concatStringsSep(",", settings.systemFeatures.get()));
+            + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures()));
         machinesReadyLock.unlock();
         return;
     }

From 6df06b089e1a3d6af59f3ee545c2b54e43356cd2 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Thu, 25 Jan 2024 09:27:46 +0100
Subject: [PATCH 127/285] web: disable Sign in with Google popup

---
 src/root/topbar.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/topbar.tt b/src/root/topbar.tt
index 1771222d..58dd94e3 100644
--- a/src/root/topbar.tt
+++ b/src/root/topbar.tt
@@ -134,7 +134,7 @@
     [% WRAPPER makeSubMenu title="Sign in" id="sign-in-menu" align="right" %]
       [% IF c.config.enable_google_login %]
         <script src="https://accounts.google.com/gsi/client" async defer></script>
-        <div id="g_id_onload" data-client_id="[% c.config.google_client_id %]" data-callback="onGoogleSignIn">
+        <div id="g_id_onload" data-client_id="[% c.config.google_client_id %]" data-auto_prompt="false" data-callback="onGoogleSignIn">
         </div>
         <div class="g_id_signin" data-type="standard"></div>
         <div class="dropdown-divider"></div>

From a876e46894c73973d155b581283d07ef90706c01 Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Thu, 25 Jan 2024 17:12:40 +0100
Subject: [PATCH 128/285] Remove automake, libtool

Those are already part of autoreconfHook
---
 flake.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 8280e076..eabe5518 100644
--- a/flake.nix
+++ b/flake.nix
@@ -146,8 +146,6 @@
             with final.buildPackages; [
               makeWrapper
               autoreconfHook
-              automake
-              libtool
               nukeReferences
               pkg-config
               mdbook

From 1471aacadc8868fff6eab06d2c330abd90ec2867 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 11:20:27 -0500
Subject: [PATCH 129/285] Split out a `package.nix`

Just like we did with Nix.
---
 flake.nix   | 196 +----------------------------------------
 package.nix | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 248 insertions(+), 194 deletions(-)
 create mode 100644 package.nix

diff --git a/flake.nix b/flake.nix
index eabe5518..f7975b27 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,8 +7,6 @@
 
   outputs = { self, nixpkgs, nix }:
     let
-      version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
-
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
 
@@ -61,198 +59,8 @@
 
         };
 
-        hydra = let
-          inherit (final) lib stdenv;
-          perlDeps = final.buildEnv {
-            name = "hydra-perl-deps";
-            paths = with final.perlPackages; lib.closePropagation
-              [
-                AuthenSASL
-                CatalystActionREST
-                CatalystAuthenticationStoreDBIxClass
-                CatalystAuthenticationStoreLDAP
-                CatalystDevel
-                CatalystPluginAccessLog
-                CatalystPluginAuthorizationRoles
-                CatalystPluginCaptcha
-                CatalystPluginPrometheusTiny
-                CatalystPluginSessionStateCookie
-                CatalystPluginSessionStoreFastMmap
-                CatalystPluginStackTrace
-                CatalystTraitForRequestProxyBase
-                CatalystViewDownload
-                CatalystViewJSON
-                CatalystViewTT
-                CatalystXRoleApplicator
-                CatalystXScriptServerStarman
-                CryptPassphrase
-                CryptPassphraseArgon2
-                CryptRandPasswd
-                DataDump
-                DateTime
-                DBDPg
-                DBDSQLite
-                DigestSHA1
-                EmailMIME
-                EmailSender
-                FileLibMagic
-                FileSlurper
-                FileWhich
-                final.nix.perl-bindings
-                final.git
-                IOCompress
-                IPCRun
-                IPCRun3
-                JSON
-                JSONMaybeXS
-                JSONXS
-                ListSomeUtils
-                LWP
-                LWPProtocolHttps
-                ModulePluggable
-                NetAmazonS3
-                NetPrometheus
-                NetStatsd
-                PadWalker
-                ParallelForkManager
-                PerlCriticCommunity
-                PrometheusTinyShared
-                ReadonlyX
-                SetScalar
-                SQLSplitStatement
-                Starman
-                StringCompareConstantTime
-                SysHostnameLong
-                TermSizeAny
-                TermReadKey
-                Test2Harness
-                TestPostgreSQL
-                TextDiff
-                TextTable
-                UUID4Tiny
-                YAML
-                XMLSimple
-              ];
-          };
-
-        in
-        stdenv.mkDerivation {
-
-          name = "hydra-${version}";
-
+        hydra = final.callPackage ./package.nix {
           src = self;
-
-          nativeBuildInputs =
-            with final.buildPackages; [
-              makeWrapper
-              autoreconfHook
-              nukeReferences
-              pkg-config
-              mdbook
-            ];
-
-          buildInputs =
-            with final; [
-              unzip
-              libpqxx
-              top-git
-              mercurial
-              darcs
-              subversion
-              breezy
-              openssl
-              bzip2
-              libxslt
-              final.nix
-              perlDeps
-              perl
-              pixz
-              boost
-              postgresql_13
-              (if lib.versionAtLeast lib.version "20.03pre"
-              then nlohmann_json
-              else nlohmann_json.override { multipleHeaders = true; })
-              prometheus-cpp
-            ];
-
-          checkInputs = with final; [
-            cacert
-            foreman
-            glibcLocales
-            libressl.nc
-            openldap
-            python3
-          ];
-
-          hydraPath = with final; lib.makeBinPath (
-            [
-              subversion
-              openssh
-              final.nix
-              coreutils
-              findutils
-              pixz
-              gzip
-              bzip2
-              xz
-              gnutar
-              unzip
-              git
-              top-git
-              mercurial
-              darcs
-              gnused
-              breezy
-            ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
-          );
-
-          OPENLDAP_ROOT = final.openldap;
-
-          shellHook = ''
-            pushd $(git rev-parse --show-toplevel) >/dev/null
-
-            PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
-            PERL5LIB=$(pwd)/src/lib:$PERL5LIB
-            export HYDRA_HOME="$(pwd)/src/"
-            mkdir -p .hydra-data
-            export HYDRA_DATA="$(pwd)/.hydra-data"
-            export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
-
-            popd >/dev/null
-          '';
-
-          NIX_LDFLAGS = [ "-lpthread" ];
-
-          enableParallelBuilding = true;
-
-          doCheck = true;
-
-          preCheck = ''
-            patchShebangs .
-            export LOGNAME=''${LOGNAME:-foo}
-            # set $HOME for bzr so it can create its trace file
-            export HOME=$(mktemp -d)
-          '';
-
-          postInstall = ''
-            mkdir -p $out/nix-support
-
-            for i in $out/bin/*; do
-                read -n 4 chars < $i
-                if [[ $chars =~ ELF ]]; then continue; fi
-                wrapProgram $i \
-                    --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
-                    --prefix PATH ':' $out/bin:$hydraPath \
-                    --set HYDRA_RELEASE ${version} \
-                    --set HYDRA_HOME $out/libexec/hydra \
-                    --set NIX_RELEASE ${final.nix.name or "unknown"}
-            done
-          '';
-
-          dontStrip = true;
-
-          meta.description = "Build of Hydra on ${final.stdenv.system}";
-          passthru = { inherit perlDeps; inherit (final) nix; };
         };
       };
 
@@ -262,7 +70,7 @@
 
         manual = forEachSystem (system:
           let pkgs = pkgsBySystem.${system}; in
-          pkgs.runCommand "hydra-manual-${version}" { }
+          pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { }
             ''
               mkdir -p $out/share
               cp -prvd ${pkgs.hydra}/share/doc $out/share/
diff --git a/package.nix b/package.nix
new file mode 100644
index 00000000..35e29be1
--- /dev/null
+++ b/package.nix
@@ -0,0 +1,246 @@
+{ stdenv
+, lib
+
+, src
+
+, buildEnv
+
+, perlPackages
+
+, nix
+, git
+
+, makeWrapper
+, autoreconfHook
+, nukeReferences
+, pkg-config
+, mdbook
+
+, unzip
+, libpqxx
+, top-git
+, mercurial
+, darcs
+, subversion
+, breezy
+, openssl
+, bzip2
+, libxslt
+, perl
+, pixz
+, boost
+, postgresql_13
+, nlohmann_json
+, prometheus-cpp
+
+, cacert
+, foreman
+, glibcLocales
+, libressl
+, openldap
+, python3
+
+, openssh
+, coreutils
+, findutils
+, gzip
+, xz
+, gnutar
+, gnused
+
+, rpm
+, dpkg
+, cdrkit
+}:
+
+let
+  perlDeps = buildEnv {
+    name = "hydra-perl-deps";
+    paths = with perlPackages; lib.closePropagation
+      [
+        AuthenSASL
+        CatalystActionREST
+        CatalystAuthenticationStoreDBIxClass
+        CatalystAuthenticationStoreLDAP
+        CatalystDevel
+        CatalystPluginAccessLog
+        CatalystPluginAuthorizationRoles
+        CatalystPluginCaptcha
+        CatalystPluginPrometheusTiny
+        CatalystPluginSessionStateCookie
+        CatalystPluginSessionStoreFastMmap
+        CatalystPluginStackTrace
+        CatalystTraitForRequestProxyBase
+        CatalystViewDownload
+        CatalystViewJSON
+        CatalystViewTT
+        CatalystXRoleApplicator
+        CatalystXScriptServerStarman
+        CryptPassphrase
+        CryptPassphraseArgon2
+        CryptRandPasswd
+        DataDump
+        DateTime
+        DBDPg
+        DBDSQLite
+        DigestSHA1
+        EmailMIME
+        EmailSender
+        FileLibMagic
+        FileSlurper
+        FileWhich
+        # Not Perl
+        nix.perl-bindings
+        git
+        # Perl again
+        IOCompress
+        IPCRun
+        IPCRun3
+        JSON
+        JSONMaybeXS
+        JSONXS
+        ListSomeUtils
+        LWP
+        LWPProtocolHttps
+        ModulePluggable
+        NetAmazonS3
+        NetPrometheus
+        NetStatsd
+        PadWalker
+        ParallelForkManager
+        PerlCriticCommunity
+        PrometheusTinyShared
+        ReadonlyX
+        SetScalar
+        SQLSplitStatement
+        Starman
+        StringCompareConstantTime
+        SysHostnameLong
+        TermSizeAny
+        TermReadKey
+        Test2Harness
+        TestPostgreSQL
+        TextDiff
+        TextTable
+        UUID4Tiny
+        YAML
+        XMLSimple
+      ];
+  };
+
+  version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}";
+in
+stdenv.mkDerivation {
+  pname = "hydra";
+  inherit version;
+
+  inherit src;
+
+  nativeBuildInputs = [
+    makeWrapper
+    autoreconfHook
+    nukeReferences
+    pkg-config
+    mdbook
+  ];
+
+  buildInputs = [
+    unzip
+    libpqxx
+    top-git
+    mercurial
+    darcs
+    subversion
+    breezy
+    openssl
+    bzip2
+    libxslt
+    nix
+    perlDeps
+    perl
+    pixz
+    boost
+    postgresql_13
+    nlohmann_json
+    prometheus-cpp
+  ];
+
+  checkInputs = [
+    cacert
+    foreman
+    glibcLocales
+    libressl.nc
+    openldap
+    python3
+  ];
+
+  hydraPath = lib.makeBinPath (
+    [
+      subversion
+      openssh
+      nix
+      coreutils
+      findutils
+      pixz
+      gzip
+      bzip2
+      xz
+      gnutar
+      unzip
+      git
+      top-git
+      mercurial
+      darcs
+      gnused
+      breezy
+    ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
+  );
+
+  OPENLDAP_ROOT = openldap;
+
+  shellHook = ''
+    pushd $(git rev-parse --show-toplevel) >/dev/null
+
+    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
+    PERL5LIB=$(pwd)/src/lib:$PERL5LIB
+    export HYDRA_HOME="$(pwd)/src/"
+    mkdir -p .hydra-data
+    export HYDRA_DATA="$(pwd)/.hydra-data"
+    export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
+
+    popd >/dev/null
+  '';
+
+  NIX_LDFLAGS = [ "-lpthread" ];
+
+  enableParallelBuilding = true;
+
+  doCheck = true;
+
+  preCheck = ''
+    patchShebangs .
+    export LOGNAME=''${LOGNAME:-foo}
+    # set $HOME for bzr so it can create its trace file
+    export HOME=$(mktemp -d)
+  '';
+
+  postInstall = ''
+    mkdir -p $out/nix-support
+
+    for i in $out/bin/*; do
+        read -n 4 chars < $i
+        if [[ $chars =~ ELF ]]; then continue; fi
+        wrapProgram $i \
+            --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
+            --prefix PATH ':' $out/bin:$hydraPath \
+            --set HYDRA_RELEASE ${version} \
+            --set HYDRA_HOME $out/libexec/hydra \
+            --set NIX_RELEASE ${nix.name or "unknown"}
+    done
+  '';
+
+  dontStrip = true;
+
+  meta.description = "Build of Hydra on ${stdenv.system}";
+  passthru = { inherit perlDeps nix; };
+}

From 1bd195a5138b3c69c52110d713c706cb4908ba16 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 11:32:00 -0500
Subject: [PATCH 130/285] Clean up deps

- `strictDeps`

- Ensure it builds with and without `doCheck`
---
 flake.nix   |  6 ++++++
 package.nix | 28 ++++++++++++++++++----------
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/flake.nix b/flake.nix
index f7975b27..a6b62cfa 100644
--- a/flake.nix
+++ b/flake.nix
@@ -68,6 +68,12 @@
 
         build = forEachSystem (system: packages.${system}.hydra);
 
+        buildNoTests = forEachSystem (system:
+          packages.${system}.hydra.overrideAttrs (_: {
+            doCheck = false;
+          })
+        );
+
         manual = forEachSystem (system:
           let pkgs = pkgsBySystem.${system}; in
           pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { }
diff --git a/package.nix b/package.nix
index 35e29be1..f91e7a24 100644
--- a/package.nix
+++ b/package.nix
@@ -136,41 +136,49 @@ stdenv.mkDerivation {
 
   inherit src;
 
+  strictDeps = true;
+
   nativeBuildInputs = [
     makeWrapper
     autoreconfHook
     nukeReferences
     pkg-config
     mdbook
+    nix
+    perlDeps
+    perl
+    unzip
   ];
 
   buildInputs = [
-    unzip
     libpqxx
-    top-git
-    mercurial
-    darcs
-    subversion
-    breezy
     openssl
-    bzip2
     libxslt
     nix
     perlDeps
     perl
-    pixz
     boost
-    postgresql_13
     nlohmann_json
     prometheus-cpp
   ];
 
+  nativeCheckInputs = [
+    bzip2
+    darcs
+    top-git
+    mercurial
+    subversion
+    breezy
+    openldap
+    postgresql_13
+    pixz
+  ];
+
   checkInputs = [
     cacert
     foreman
     glibcLocales
     libressl.nc
-    openldap
     python3
   ];
 

From d6d6d1b649ea3b83baf44267637e42c4eb1c92c0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 12:03:15 -0500
Subject: [PATCH 131/285] flake.nix: Temporarily add a second Nixpkgs for
 `lib.fileset`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13)
  → 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19)
• Added input 'nixpkgs-for-fileset':
    'github:NixOS/nixpkgs/a77ab169a83a4175169d78684ddd2e54486ac651' (2024-01-24)
---
 flake.lock | 25 +++++++++++++++++++++----
 flake.nix  |  5 +++++
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index 66e30323..8231484c 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1702510710,
-        "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=",
+        "lastModified": 1705666051,
+        "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74",
+        "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88",
         "type": "github"
       },
       "original": {
@@ -72,6 +72,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-for-fileset": {
+      "locked": {
+        "lastModified": 1706098335,
+        "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -91,7 +107,8 @@
     "root": {
       "inputs": {
         "nix": "nix",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-for-fileset": "nixpkgs-for-fileset"
       }
     }
   },
diff --git a/flake.nix b/flake.nix
index a6b62cfa..0ab25018 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,6 +5,11 @@
   inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
+  # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
+  # mered and we upgrade or Nix, so the main `nixpkgs` input is at least
+  # 23.11 and has `lib.fileset`.
+  inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
+
   outputs = { self, nixpkgs, nix }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];

From 4bbc7b8f75409aa8e99a0f3284bdc5f11f804cfb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 12:21:13 -0500
Subject: [PATCH 132/285] Use the Nixpkgs `fileset` library to filter source

Now I can change Nix files without causing rebuilds.
---
 flake.nix   |  5 +++--
 package.nix | 26 +++++++++++++++++++++-----
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/flake.nix b/flake.nix
index 0ab25018..71fd6c6d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -10,7 +10,7 @@
   # 23.11 and has `lib.fileset`.
   inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
 
-  outputs = { self, nixpkgs, nix }:
+  outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
@@ -65,7 +65,8 @@
         };
 
         hydra = final.callPackage ./package.nix {
-          src = self;
+          inherit (nixpkgs-for-fileset.lib) fileset;
+          rawSrc = self;
         };
       };
 
diff --git a/package.nix b/package.nix
index f91e7a24..eff9ed37 100644
--- a/package.nix
+++ b/package.nix
@@ -1,7 +1,8 @@
 { stdenv
 , lib
+, fileset
 
-, src
+, rawSrc
 
 , buildEnv
 
@@ -128,13 +129,28 @@ let
       ];
   };
 
-  version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}";
+  version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}";
 in
-stdenv.mkDerivation {
+stdenv.mkDerivation (finalAttrs: {
   pname = "hydra";
   inherit version;
 
-  inherit src;
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions ([
+      ./version.txt
+      ./configure.ac
+      ./Makefile.am
+      ./src
+      ./doc
+      ./hydra-module.nix
+      # TODO only when `doCheck`
+      ./t
+    ] ++ lib.optionals finalAttrs.doCheck [
+      ./.perlcriticrc
+      ./.yath.rc
+    ]);
+  };
 
   strictDeps = true;
 
@@ -251,4 +267,4 @@ stdenv.mkDerivation {
 
   meta.description = "Build of Hydra on ${stdenv.system}";
   passthru = { inherit perlDeps nix; };
-}
+})

From 73b6c1fb11b7c5679725599c552f4e163bf668f0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 13:27:05 -0500
Subject: [PATCH 133/285] Filter out (mosts test) when `!doCheck`

---
 Makefile.am  |  6 +++++-
 configure.ac | 20 ++++++++++++++------
 package.nix  |  7 +++++--
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index e744cc33..9e06ebdb 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,4 +1,8 @@
-SUBDIRS = src t doc
+SUBDIRS = src doc
+if CAN_DO_CHECK
+  SUBDIRS += t
+endif
+
 BOOTCLEAN_SUBDIRS = $(SUBDIRS)
 DIST_SUBDIRS      = $(SUBDIRS)
 EXTRA_DIST        = hydra-module.nix
diff --git a/configure.ac b/configure.ac
index eec647c3..e5c57d14 100644
--- a/configure.ac
+++ b/configure.ac
@@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
 testPath="$(dirname $(type -p expr))"
 AC_SUBST(testPath)
 
-jobsPath="$(realpath ./t/jobs)"
-AC_SUBST(jobsPath)
-
 CXXFLAGS+=" -include nix/config.h"
 
 AC_CONFIG_FILES([
@@ -71,11 +68,22 @@ AC_CONFIG_FILES([
   src/lib/Makefile
   src/root/Makefile
   src/script/Makefile
-  t/Makefile
-  t/jobs/config.nix
-  t/jobs/declarative/project.json
 ])
 
+# Tests might be filtered out
+AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"])
+AM_COND_IF(
+  [CAN_DO_CHECK],
+  [
+    jobsPath="$(realpath ./t/jobs)"
+    AC_SUBST(jobsPath)
+    AC_CONFIG_FILES([
+      t/Makefile
+      t/jobs/config.nix
+      t/jobs/declarative/project.json
+    ])
+  ])
+
 AC_CONFIG_COMMANDS([executable-scripts], [])
 
 AC_CONFIG_HEADER([hydra-config.h])
diff --git a/package.nix b/package.nix
index eff9ed37..73ef9b3f 100644
--- a/package.nix
+++ b/package.nix
@@ -144,9 +144,12 @@ stdenv.mkDerivation (finalAttrs: {
       ./src
       ./doc
       ./hydra-module.nix
-      # TODO only when `doCheck`
-      ./t
+      # These are always needed to appease Automake
+      ./t/Makefile.am
+      ./t/jobs/config.nix.in
+      ./t/jobs/declarative/project.json.in
     ] ++ lib.optionals finalAttrs.doCheck [
+      ./t
       ./.perlcriticrc
       ./.yath.rc
     ]);

From c5f37eca91bee5bacbe53f826abcd0c0d0f38ef1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 11:54:44 -0500
Subject: [PATCH 134/285] Reorganize hydra modules

---
 .gitignore                                  |  1 +
 Makefile.am                                 |  6 +--
 flake.nix                                   | 50 +++------------------
 nixos-modules/default.nix                   | 49 ++++++++++++++++++++
 hydra-module.nix => nixos-modules/hydra.nix |  0
 package.nix                                 |  2 +-
 6 files changed, 59 insertions(+), 49 deletions(-)
 create mode 100644 nixos-modules/default.nix
 rename hydra-module.nix => nixos-modules/hydra.nix (100%)

diff --git a/.gitignore b/.gitignore
index 799db665..f8bf5718 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@ t/jobs/declarative/project.json
 hydra-config.h
 hydra-config.h.in
 result
+result-*
 outputs
 config
 stamp-h1
diff --git a/Makefile.am b/Makefile.am
index 9e06ebdb..a28e3f33 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -5,8 +5,8 @@ endif
 
 BOOTCLEAN_SUBDIRS = $(SUBDIRS)
 DIST_SUBDIRS      = $(SUBDIRS)
-EXTRA_DIST        = hydra-module.nix
+EXTRA_DIST        = nixos-modules/hydra.nix
 
-install-data-local: hydra-module.nix
+install-data-local: nixos-modules/hydra.nix
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix
-	$(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/
+	$(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix
diff --git a/flake.nix b/flake.nix
index 71fd6c6d..30aa61c0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -15,9 +15,11 @@
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
 
+      overlayList = [ self.overlays.default nix.overlays.default ];
+
       pkgsBySystem = forEachSystem (system: import nixpkgs {
         inherit system;
-        overlays = [ self.overlays.default nix.overlays.default ];
+        overlays = overlayList;
       });
 
       # NixOS configuration used for VM tests.
@@ -386,50 +388,8 @@
         default = pkgsBySystem.${system}.hydra;
       });
 
-      nixosModules.hydra = {
-        imports = [ ./hydra-module.nix ];
-        nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
-      };
-
-      nixosModules.hydraTest = { pkgs, ... }: {
-        imports = [ self.nixosModules.hydra ];
-
-        services.hydra-dev.enable = true;
-        services.hydra-dev.hydraURL = "http://hydra.example.org";
-        services.hydra-dev.notificationSender = "admin@hydra.example.org";
-
-        systemd.services.hydra-send-stats.enable = false;
-
-        services.postgresql.enable = true;
-        services.postgresql.package = pkgs.postgresql_11;
-
-        # The following is to work around the following error from hydra-server:
-        #   [error] Caught exception in engine "Cannot determine local time zone"
-        time.timeZone = "UTC";
-
-        nix.extraOptions = ''
-          allowed-uris = https://github.com/
-        '';
-      };
-
-      nixosModules.hydraProxy = {
-        services.httpd = {
-          enable = true;
-          adminAddr = "hydra-admin@example.org";
-          extraConfig = ''
-            <Proxy *>
-              Order deny,allow
-              Allow from all
-            </Proxy>
-
-            ProxyRequests     Off
-            ProxyPreserveHost On
-            ProxyPass         /apache-errors !
-            ErrorDocument 503 /apache-errors/503.html
-            ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
-            ProxyPassReverse  /       http://127.0.0.1:3000/
-          '';
-        };
+      nixosModules = import ./nixos-modules {
+        overlays = overlayList;
       };
 
       nixosConfigurations.container = nixpkgs.lib.nixosSystem {
diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix
new file mode 100644
index 00000000..6fc19d31
--- /dev/null
+++ b/nixos-modules/default.nix
@@ -0,0 +1,49 @@
+{ overlays }:
+
+rec {
+  hydra = {
+    imports = [ ./hydra.nix ];
+    nixpkgs = { inherit overlays; };
+  };
+
+  hydraTest = { pkgs, ... }: {
+    imports = [ hydra ];
+
+    services.hydra-dev.enable = true;
+    services.hydra-dev.hydraURL = "http://hydra.example.org";
+    services.hydra-dev.notificationSender = "admin@hydra.example.org";
+
+    systemd.services.hydra-send-stats.enable = false;
+
+    services.postgresql.enable = true;
+    services.postgresql.package = pkgs.postgresql_11;
+
+    # The following is to work around the following error from hydra-server:
+    #   [error] Caught exception in engine "Cannot determine local time zone"
+    time.timeZone = "UTC";
+
+    nix.extraOptions = ''
+      allowed-uris = https://github.com/
+    '';
+  };
+
+  hydraProxy = {
+    services.httpd = {
+      enable = true;
+      adminAddr = "hydra-admin@example.org";
+      extraConfig = ''
+        <Proxy *>
+          Order deny,allow
+          Allow from all
+        </Proxy>
+
+        ProxyRequests     Off
+        ProxyPreserveHost On
+        ProxyPass         /apache-errors !
+        ErrorDocument 503 /apache-errors/503.html
+        ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
+        ProxyPassReverse  /       http://127.0.0.1:3000/
+      '';
+    };
+  };
+}
diff --git a/hydra-module.nix b/nixos-modules/hydra.nix
similarity index 100%
rename from hydra-module.nix
rename to nixos-modules/hydra.nix
diff --git a/package.nix b/package.nix
index 73ef9b3f..d5e6bb69 100644
--- a/package.nix
+++ b/package.nix
@@ -143,7 +143,7 @@ stdenv.mkDerivation (finalAttrs: {
       ./Makefile.am
       ./src
       ./doc
-      ./hydra-module.nix
+      ./nixos-modules/hydra.nix
       # These are always needed to appease Automake
       ./t/Makefile.am
       ./t/jobs/config.nix.in

From b5ed0787f7eba4f0a7d900736976e335f443c19e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 14:25:12 -0500
Subject: [PATCH 135/285] Replace "not Perl" and "Perl again" with something
 more self-explanatory

---
 package.nix | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/package.nix b/package.nix
index d5e6bb69..0f8f004d 100644
--- a/package.nix
+++ b/package.nix
@@ -57,8 +57,11 @@
 let
   perlDeps = buildEnv {
     name = "hydra-perl-deps";
-    paths = with perlPackages; lib.closePropagation
-      [
+    paths = lib.closePropagation
+      ([
+        nix.perl-bindings
+        git
+      ] ++ (with perlPackages; [
         AuthenSASL
         CatalystActionREST
         CatalystAuthenticationStoreDBIxClass
@@ -90,10 +93,6 @@ let
         FileLibMagic
         FileSlurper
         FileWhich
-        # Not Perl
-        nix.perl-bindings
-        git
-        # Perl again
         IOCompress
         IPCRun
         IPCRun3
@@ -126,7 +125,7 @@ let
         UUID4Tiny
         YAML
         XMLSimple
-      ];
+      ]));
   };
 
   version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}";

From aed130cd17ae7da41d9f38f830af3826a85a5dc6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 15:57:39 -0500
Subject: [PATCH 136/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19)
  → 'github:NixOS/nix/2c4bb93ba5a97e7078896ebc36385ce172960e4e' (2024-01-25)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8231484c..2ddc7c16 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1705666051,
-        "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=",
+        "lastModified": 1706208340,
+        "narHash": "sha256-wNyHUEIiKKVs6UXrUzhP7RSJQv0A8jckgcuylzftl8k=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88",
+        "rev": "2c4bb93ba5a97e7078896ebc36385ce172960e4e",
         "type": "github"
       },
       "original": {

From c64eed7d07ad432fa9c5ea979d1039a5207f4629 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 14:48:53 -0500
Subject: [PATCH 137/285] Simplify `StoreConfig::getDefaultSystemFeatures` call

That method is now static.
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 1cbcafea..df79e1dc 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -15,7 +15,6 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "store-api.hh"
-#include "local-store.hh"
 #include "remote-store.hh"
 
 #include "globals.hh"
@@ -227,7 +226,7 @@ void State::monitorMachinesFile()
         parseMachines("localhost " +
             (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get())
             + " - " + std::to_string(settings.maxBuildJobs) + " 1 "
-            + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures()));
+            + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures()));
         machinesReadyLock.unlock();
         return;
     }

From fcde5908d8e51f975b883329b34d24a9f30ea4b3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 21:32:22 -0500
Subject: [PATCH 138/285] More CA derivations prep

Again, with care not to change the schema in any way.
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc       |  6 +-----
 src/hydra-queue-runner/hydra-queue-runner.cc |  5 ++++-
 src/hydra-queue-runner/queue-monitor.cc      | 20 ++++++++++++--------
 src/lib/Hydra/Controller/Build.pm            |  6 ++++--
 src/script/hydra-eval-jobset                 |  6 +++++-
 t/lib/HydraTestContext.pm                    |  6 +++++-
 6 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 2794cc62..5cfc3a52 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -250,11 +250,7 @@ static void worker(
                         // See the `queryOutputs` call above; we should
                         // not encounter missing output paths otherwise.
                         assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
-                        // TODO it would be better to set `null` than an
-                        // empty string here, to force the consumer of
-                        // this JSON to more explicitly handle this
-                        // case.
-                        out[outputName] = "";
+                        out[outputName] = nullptr;
                     }
                 }
                 job["outputs"] = std::move(out);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index df79e1dc..0f4e759c 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -336,7 +336,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
     for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
-            buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
+            buildId, stepNr, name,
+            output
+                ? std::optional { localStore->printStorePath(*output)}
+                : std::nullopt);
 
     if (status == bsBusy)
         txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 513d399b..d7214c43 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -193,14 +193,18 @@ bool State::getQueuedBuilds(Connection & conn,
 
                 if (!propagatedFrom) {
                     for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
-                          // ca-derivations not actually supported yet
-                          assert(optOutputPath);
-                          auto res = txn.exec_params
-                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
-                               localStore->printStorePath(*optOutputPath));
-                          if (!res[0][0].is_null()) {
-                              propagatedFrom = res[0][0].as<BuildID>();
-                              break;
+                        constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1";
+                        auto res = optOutputPath
+                            ? txn.exec_params(
+                                std::string { common } + " and path = $1",
+                                localStore->printStorePath(*optOutputPath))
+                            : txn.exec_params(
+                                std::string { common } + " and drvPath = $1 and name = $2",
+                                localStore->printStorePath(ex.step->drvPath),
+                                outputName);
+                        if (!res[0][0].is_null()) {
+                            propagatedFrom = res[0][0].as<BuildID>();
+                            break;
                         }
                     }
                 }
diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index 2d74f86a..c3869838 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -78,14 +78,16 @@ sub build_GET {
 
     $c->stash->{template} = 'build.tt';
     $c->stash->{isLocalStore} = isLocalStore();
+    # XXX: If the derivation is content-addressed then this will always return
+    # false because `$_->path` will be empty
     $c->stash->{available} =
         $c->stash->{isLocalStore}
-        ? all { isValidPath($_->path) } $build->buildoutputs->all
+        ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
         : 1;
     $c->stash->{drvAvailable} = isValidPath $build->drvpath;
 
     if ($build->finished && $build->iscachedbuild) {
-        my $path = ($build->buildoutputs)[0]->path or die;
+        my $path = ($build->buildoutputs)[0]->path or undef;
         my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path);
         if (defined $cachedBuildStep) {
             $c->stash->{cachedBuild} = $cachedBuildStep->build;
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index c6f6c275..7ed7ebe8 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -438,13 +438,17 @@ sub checkBuild {
         # new build to be scheduled if the meta.maintainers field is
         # changed?
         if (defined $prevEval) {
+            my $pathOrDrvConstraint = defined $firstOutputPath
+                ? { path => $firstOutputPath }
+                : { drvPath => $drvPath };
+
             my ($prevBuild) = $prevEval->builds->search(
                 # The "project" and "jobset" constraints are
                 # semantically unnecessary (because they're implied by
                 # the eval), but they give a factor 1000 speedup on
                 # the Nixpkgs jobset with PostgreSQL.
                 { jobset_id => $jobset->get_column('id'), job => $jobName,
-                  name => $firstOutputName, path => $firstOutputPath },
+                  name => $firstOutputName, %$pathOrDrvConstraint },
                 { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] });
             if (defined $prevBuild) {
                 #print STDERR "    already scheduled/built as build ", $prevBuild->id, "\n";
diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index a22c3df1..e1a5b226 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -39,6 +39,8 @@ use Hydra::Helper::Exec;
 sub new {
     my ($class, %opts) = @_;
 
+    my $deststoredir;
+
     # Cleanup will be managed by yath. By the default it will be cleaned
     # up, but can be kept to aid in debugging test failures.
     my $dir = File::Temp->newdir(CLEANUP => 0);
@@ -55,6 +57,7 @@ sub new {
     my $hydra_config = $opts{'hydra_config'} || "";
     $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config;
     if ($opts{'use_external_destination_store'} // 1) {
+        $deststoredir = "$dir/nix/dest-store";
         $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config;
     }
 
@@ -81,7 +84,8 @@ sub new {
         nix_state_dir => $nix_state_dir,
         nix_log_dir => $nix_log_dir,
         testdir => abs_path(dirname(__FILE__) . "/.."),
-        jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
+        jobsdir => abs_path(dirname(__FILE__) . "/../jobs"),
+        deststoredir => $deststoredir,
     }, $class;
 
     if ($opts{'before_init'}) {

From 323b556dc8c348f1f9d5bf5b8a35f2617bfe600c Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 25 Jan 2024 21:03:08 -0500
Subject: [PATCH 139/285] Minimal CA support

This verison has a worse UI, but also chnages the schema less: One
non-null constraint is removed, but no new columns are added.

Co-Authored-By: Andrea Ciceri <andrea.ciceri@autistici.org>
Co-Authored-By: regnat <rg@regnat.ovh>
---
 doc/manual/src/projects.md                    |  7 +++
 src/lib/Hydra/Controller/Root.pm              | 29 +++++++++
 src/lib/Hydra/Schema/Result/BuildOutputs.pm   |  8 +--
 .../Hydra/Schema/Result/BuildStepOutputs.pm   |  8 +--
 src/sql/hydra.sql                             |  4 +-
 t/content-addressed/basic.t                   | 61 +++++++++++++++++++
 .../without-experimental-feature.t            | 28 +++++++++
 t/jobs/config.nix.in                          |  5 ++
 t/jobs/content-addressed.nix                  | 35 +++++++++++
 t/jobs/dir-with-file-builder.sh               |  7 +++
 t/queue-runner/notifications.t                |  2 +-
 11 files changed, 183 insertions(+), 11 deletions(-)
 create mode 100644 t/content-addressed/basic.t
 create mode 100644 t/content-addressed/without-experimental-feature.t
 create mode 100644 t/jobs/content-addressed.nix
 create mode 100755 t/jobs/dir-with-file-builder.sh

diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md
index a399406d..f7c4975f 100644
--- a/doc/manual/src/projects.md
+++ b/doc/manual/src/projects.md
@@ -404,3 +404,10 @@ analogous:
   | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
   | `String value` | `gitea_http_url`    | *Public URL of `gitea`*, optional  |
 
+Content-addressed derivations
+-----------------------------
+
+Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62).
+To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake.
+
+Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point.
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index c6843d29..548cfac3 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -18,6 +18,8 @@ use Net::Prometheus;
 use Types::Standard qw/StrMatch/;
 
 use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
+# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi
+use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$};
 
 # Put this controller at top-level.
 __PACKAGE__->config->{namespace} = '';
@@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) {
 }
 
 
+sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
+    my ($self, $c, $realisation) = @_;
+
+    if (!isLocalStore) {
+        notFound($c, "There is no binary cache here.");
+    }
+
+    else {
+        my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
+        my $rawRealisation = queryRawRealisation($rawDrvOutput);
+
+        if (!$rawRealisation) {
+            $c->response->status(404);
+            $c->response->content_type('text/plain');
+            $c->stash->{plain}->{data} = "does not exist\n";
+            $c->forward('Hydra::View::Plain');
+            setCacheHeaders($c, 60 * 60);
+            return;
+        }
+
+        $c->response->content_type('text/plain');
+        $c->stash->{plain}->{data} = $rawRealisation;
+        $c->forward('Hydra::View::Plain');
+    }
+}
+
+
 sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
     my ($self, $c, $narinfo) = @_;
 
diff --git a/src/lib/Hydra/Schema/Result/BuildOutputs.pm b/src/lib/Hydra/Schema/Result/BuildOutputs.pm
index 9fc4f7c7..3997b497 100644
--- a/src/lib/Hydra/Schema/Result/BuildOutputs.pm
+++ b/src/lib/Hydra/Schema/Result/BuildOutputs.pm
@@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs");
 =head2 path
 
   data_type: 'text'
-  is_nullable: 0
+  is_nullable: 1
 
 =cut
 
@@ -59,7 +59,7 @@ __PACKAGE__->add_columns(
   "name",
   { data_type => "text", is_nullable => 0 },
   "path",
-  { data_type => "text", is_nullable => 0 },
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -94,8 +94,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng
 
 my %hint = (
     columns => [
diff --git a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm
index 016a35fe..6d997a8c 100644
--- a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm
+++ b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm
@@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs");
 =head2 path
 
   data_type: 'text'
-  is_nullable: 0
+  is_nullable: 1
 
 =cut
 
@@ -67,7 +67,7 @@ __PACKAGE__->add_columns(
   "name",
   { data_type => "text", is_nullable => 0 },
   "path",
-  { data_type => "text", is_nullable => 0 },
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -119,8 +119,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q
 
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration
diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index eaae6da3..e9457972 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row
 create table BuildOutputs (
     build         integer not null,
     name          text not null,
-    path          text not null,
+    path          text,
     primary key   (build, name),
     foreign key   (build) references Builds(id) on delete cascade
 );
@@ -303,7 +303,7 @@ create table BuildStepOutputs (
     build         integer not null,
     stepnr        integer not null,
     name          text not null,
-    path          text not null,
+    path          text,
     primary key   (build, stepnr, name),
     foreign key   (build) references Builds(id) on delete cascade,
     foreign key   (build, stepnr) references BuildSteps(build, stepnr) on delete cascade
diff --git a/t/content-addressed/basic.t b/t/content-addressed/basic.t
new file mode 100644
index 00000000..6597e727
--- /dev/null
+++ b/t/content-addressed/basic.t
@@ -0,0 +1,61 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init(
+    nix_config => qq|
+    experimental-features = ca-derivations
+    |,
+);
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use JSON::MaybeXS;
+
+use HTTP::Request::Common;
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
+
+for my $build (queuedBuildsForJobset($jobset)) {
+    ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
+    is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
+
+    my $response = request("/build/".$build->id);
+    ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly");
+
+    if ($newbuild->buildstatus == 0) {
+      my $buildOutputs = $newbuild->buildoutputs;
+      for my $output ($newbuild->buildoutputs) {
+        # XXX: This hardcodes /nix/store/.
+        # It's fine because in practice the nix store for the tests will be of
+        # the form `/some/thing/nix/store/`, but it would be cleaner if there
+        # was a way to query Nix for its store dir?
+        like(
+          $output->path, qr|/nix/store/|,
+          "Output '".$output->name."' of build '".$build->job."' should be a valid store path"
+        );
+      }
+    }
+
+}
+
+isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
+
+done_testing;
+
diff --git a/t/content-addressed/without-experimental-feature.t b/t/content-addressed/without-experimental-feature.t
new file mode 100644
index 00000000..a37d138e
--- /dev/null
+++ b/t/content-addressed/without-experimental-feature.t
@@ -0,0 +1,28 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use JSON::MaybeXS;
+
+use HTTP::Request::Common;
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset),                  "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build");
+
+done_testing;
diff --git a/t/jobs/config.nix.in b/t/jobs/config.nix.in
index 51b6c06f..41776341 100644
--- a/t/jobs/config.nix.in
+++ b/t/jobs/config.nix.in
@@ -6,4 +6,9 @@ rec {
       system = builtins.currentSystem;
       PATH = path;
     } // args);
+  mkContentAddressedDerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } // args);
 }
diff --git a/t/jobs/content-addressed.nix b/t/jobs/content-addressed.nix
new file mode 100644
index 00000000..65496df5
--- /dev/null
+++ b/t/jobs/content-addressed.nix
@@ -0,0 +1,35 @@
+let cfg =  import ./config.nix; in
+rec {
+  empty_dir =
+    cfg.mkContentAddressedDerivation {
+      name = "empty-dir";
+      builder = ./empty-dir-builder.sh;
+    };
+
+  fails =
+    cfg.mkContentAddressedDerivation {
+      name = "fails";
+      builder = ./fail.sh;
+    };
+
+  succeed_with_failed =
+    cfg.mkContentAddressedDerivation {
+      name = "succeed-with-failed";
+      builder = ./succeed-with-failed.sh;
+    };
+
+  caDependingOnCA =
+    cfg.mkContentAddressedDerivation {
+      name = "ca-depending-on-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = empty_dir;
+    };
+
+  nonCaDependingOnCA =
+    cfg.mkDerivation {
+      name = "non-ca-depending-on-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = empty_dir;
+    };
+}
+
diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh
new file mode 100755
index 00000000..e51c6558
--- /dev/null
+++ b/t/jobs/dir-with-file-builder.sh
@@ -0,0 +1,7 @@
+#! /bin/sh
+
+# Workaround for https://github.com/NixOS/nix/pull/6051
+echo "some output"
+
+mkdir $out
+echo foo > $out/a-file
diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t
index 1966cde1..d0e72409 100644
--- a/t/queue-runner/notifications.t
+++ b/t/queue-runner/notifications.t
@@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir();
 
 my $ctx = test_context(
     nix_config => qq|
-    experimental-features = nix-command
+    experimental-features = nix-command ca-derivations
     substituters = file://${binarycachedir}?trusted=1
     |,
     hydra_config => q|

From 5ee0e443e4ea90888e74b90d2e4198494e8eb597 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 Jan 2024 01:08:11 -0500
Subject: [PATCH 140/285] Remove now-unneeded workaround

---
 t/jobs/empty-dir-builder.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/t/jobs/empty-dir-builder.sh b/t/jobs/empty-dir-builder.sh
index 949216e0..addc7ef6 100755
--- a/t/jobs/empty-dir-builder.sh
+++ b/t/jobs/empty-dir-builder.sh
@@ -1,6 +1,3 @@
 #! /bin/sh
 
-# Workaround for https://github.com/NixOS/nix/pull/6051
-echo "some output"
-
 mkdir $out

From c62eaf248f1fe7653f37427e70ce4741cf9a1c2d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 Jan 2024 01:20:07 -0500
Subject: [PATCH 141/285] Remove now-unneeded workaround

---
 t/jobs/dir-with-file-builder.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh
index e51c6558..8592a1e8 100755
--- a/t/jobs/dir-with-file-builder.sh
+++ b/t/jobs/dir-with-file-builder.sh
@@ -1,7 +1,4 @@
 #! /bin/sh
 
-# Workaround for https://github.com/NixOS/nix/pull/6051
-echo "some output"
-
 mkdir $out
 echo foo > $out/a-file

From 8477009310c0cde111af3da9313d3c99902a48c7 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 26 Jan 2024 17:28:07 +0100
Subject: [PATCH 142/285] doc/manual: fix instructions in contribution
 guidelines

In 5db374cb500b687039ba4701b205ca7dfa67caba the `bootstrap` script was
removed, however it's still referenced in the contribution guidelines.
Change that to `autoreconfPhase` as intended by the commit.
---
 doc/manual/src/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 72a74c82..49c17395 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -18,7 +18,7 @@ $ nix-shell
 To build Hydra, you should then do:
 
 ```console
-[nix-shell]$ ./bootstrap
+[nix-shell]$ autoreconfPhase
 [nix-shell]$ configurePhase
 [nix-shell]$ make
 ```

From b4c91b5a6ab91177d2c52c5b68c880277b4fd35f Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 26 Jan 2024 17:30:07 +0100
Subject: [PATCH 143/285] package: move foreman to nativeCheckInputs

In 1bd195a5138b3c69c52110d713c706cb4908ba16 strictDeps was set for the
Hydra package. As a result, `checkInputs` aren't available anymore in
the local dev-shell which is the sole purpose of foreman, to start
services and a database for development.
---
 package.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index 0f8f004d..f8b1849f 100644
--- a/package.nix
+++ b/package.nix
@@ -183,6 +183,7 @@ stdenv.mkDerivation (finalAttrs: {
   nativeCheckInputs = [
     bzip2
     darcs
+    foreman
     top-git
     mercurial
     subversion
@@ -194,7 +195,6 @@ stdenv.mkDerivation (finalAttrs: {
 
   checkInputs = [
     cacert
-    foreman
     glibcLocales
     libressl.nc
     python3

From b503280256bed6ced0c79d32ee17dda72827a381 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 Jan 2024 11:53:58 -0500
Subject: [PATCH 144/285] Add migration to drop non-null constraints

---
 src/sql/upgrade-84.sql | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 src/sql/upgrade-84.sql

diff --git a/src/sql/upgrade-84.sql b/src/sql/upgrade-84.sql
new file mode 100644
index 00000000..bf142b30
--- /dev/null
+++ b/src/sql/upgrade-84.sql
@@ -0,0 +1,4 @@
+-- CA derivations do not have statically known output paths. The values
+-- are only filled in after the build runs.
+ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL;
+ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL;

From 14aabc1cc94b1a51f67bbcb30f901785d47a9872 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 30 Jan 2024 13:31:56 -0500
Subject: [PATCH 145/285] Update to released Nix 2.20
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/8df68a213fc52a57b02a57005b0e06cc8de40ce3' (2024-01-25)
  → 'github:NixOS/nix/8f42912c80c0a03f62f6a3d28a3af05a9762565d' (2024-01-30)
---
 flake.lock | 7 ++++---
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index e3ea8f61..d8b2e4a9 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,15 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1706195509,
-        "narHash": "sha256-1kwfk7H/MWZAcTKHnnWXo/+KlQeOTIRtOIzc4FX3QnE=",
+        "lastModified": 1706637536,
+        "narHash": "sha256-fjx+nCOzuSxGWfhwWWc8hCsLFZAjZLDDUcbBtldRqbk=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "8df68a213fc52a57b02a57005b0e06cc8de40ce3",
+        "rev": "8f42912c80c0a03f62f6a3d28a3af05a9762565d",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
+        "ref": "2.20-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 4dc7d25e..306ed292 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nix.url = "github:NixOS/nix";
+  inputs.nix.url = "github:NixOS/nix/2.20-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is

From 878c0f240e950c4d7bf8ebc60299bc0db80e6686 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 30 Jan 2024 13:50:25 -0500
Subject: [PATCH 146/285] Switch (back) to Nix master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Re-creating `nix-next` after using it in #1354.

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/8df68a213fc52a57b02a57005b0e06cc8de40ce3' (2024-01-25)
  → 'github:NixOS/nix/75ebb90a70f6320c1c7a1fca87a0a8adb0716143' (2024-01-30)
---
 flake.lock | 7 +++----
 flake.nix  | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index d8b2e4a9..f04d657b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,15 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1706637536,
-        "narHash": "sha256-fjx+nCOzuSxGWfhwWWc8hCsLFZAjZLDDUcbBtldRqbk=",
+        "lastModified": 1706629374,
+        "narHash": "sha256-KyAiLGxJ39fSY0cuq8EWAZQ4vaDdqAItSRP4+vjYvq8=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "8f42912c80c0a03f62f6a3d28a3af05a9762565d",
+        "rev": "75ebb90a70f6320c1c7a1fca87a0a8adb0716143",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.20-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 306ed292..4dc7d25e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nix.url = "github:NixOS/nix/2.20-maintenance";
+  inputs.nix.url = "github:NixOS/nix";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is

From ceff5c5cfeaf211691f4d1156f358a940b5ef7b4 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 12 Feb 2024 18:30:03 +0100
Subject: [PATCH 147/285] flake: fix gitea integration test

This is an integration test that confirms that jobset definitions from
git repositories are correctly built and status updates pushed to the
gitea instance. The following things needed to be fixed:

* We're still on 23.05 where gitea is marked as insecure. Not going to
  update nixpkgs right now, but going for the quick fix.
* Since gitea 1.19 tokens have scopes that describe what's possible.
  Not specifying the scope in the DB appears to imply that no
  permissions are granted.
* Apparently we have three status updates now (for three status hooks,
  queued/started/finished). No idea why that was broken before, but the
  behavior still looks correct.
---
 flake.nix | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/flake.nix b/flake.nix
index 306ed292..a6dfb977 100644
--- a/flake.nix
+++ b/flake.nix
@@ -180,13 +180,9 @@
                 root=d7f16a3412e01a43a414535b16007c6931d3a9c7
                 </gitea_authorization>
               '';
+              nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
               nix = {
-                distributedBuilds = true;
-                buildMachines = [{
-                  hostName = "localhost";
-                  systems = [ system ];
-                }];
-                binaryCaches = [ ];
+                settings.substituters = [ ];
               };
               services.gitea = {
                 enable = true;
@@ -202,7 +198,7 @@
             testScript =
               let
                 scripts.mktoken = pkgs.writeText "token.sql" ''
-                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7');
+                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
                 '';
 
                 scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
@@ -357,9 +353,10 @@
 
                 response = json.loads(data)
 
-                assert len(response) == 2, "Expected exactly two status updates for latest commit!"
-                assert response[0]['status'] == "success", "Expected latest status to be success!"
-                assert response[1]['status'] == "pending", "Expected first status to be pending!"
+                assert len(response) == 3, "Expected exactly three status updates for latest commit (queued, started, finished)!"
+                assert response[0]['status'] == "success", "Expected finished status to be success!"
+                assert response[1]['status'] == "pending", "Expected started status to be pending!"
+                assert response[2]['status'] == "pending", "Expected queued status to be pending!"
 
                 machine.shutdown()
               '';

From e499509595859d670432c0d98162119ecd552666 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 12 Feb 2024 17:12:49 +0100
Subject: [PATCH 148/285] Switch to new Nix bindings, update Nix for that

Implements support for Nix's new Perl bindings[1]. The current state
basically does `openStore()`, but always uses `auto` and doesn't support
stores at other URIs.

Even though the stores are cached inside the Perl implementation, I
decided to instantiate those once in the Nix helper module. That way
store openings aren't cluttered across the entire codebase. Also, there
are two stores used later on - MACHINE_LOCAL_STORE for `auto`,
BINARY_CACHE_STORE for the one from `store_uri` in `hydra.conf` - and
using consistent names should make the intent clearer then.

This doesn't contain any behavioral changes, i.e. the build product
availability issue from #1352 isn't fixed. This patch only contains the
migration to the new API.

[1] https://github.com/NixOS/nix/pull/9863
---
 flake.lock                                  |  6 +++---
 src/hydra-eval-jobs/hydra-eval-jobs.cc      |  6 +++---
 src/hydra-evaluator/hydra-evaluator.cc      |  2 +-
 src/hydra-queue-runner/queue-monitor.cc     |  2 +-
 src/lib/Hydra/Base/Controller/NixChannel.pm |  3 +--
 src/lib/Hydra/Controller/Build.pm           | 18 ++++++++----------
 src/lib/Hydra/Controller/Root.pm            |  2 +-
 src/lib/Hydra/Helper/Nix.pm                 |  5 ++++-
 src/lib/Hydra/Plugin/BazaarInput.pm         |  7 +++----
 src/lib/Hydra/Plugin/DarcsInput.pm          |  7 +++----
 src/lib/Hydra/Plugin/GitInput.pm            |  6 +++---
 src/lib/Hydra/Plugin/MercurialInput.pm      |  7 +++----
 src/lib/Hydra/Plugin/PathInput.pm           |  5 ++---
 src/lib/Hydra/Plugin/SubversionInput.pm     |  9 ++++-----
 src/lib/Hydra/View/NARInfo.pm               |  9 ++++-----
 src/script/hydra-eval-jobset                |  8 ++++----
 src/script/hydra-update-gc-roots            |  5 ++---
 t/s3-backup-test.pl                         |  1 -
 18 files changed, 50 insertions(+), 58 deletions(-)

diff --git a/flake.lock b/flake.lock
index f04d657b..4b4f0076 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1706629374,
-        "narHash": "sha256-KyAiLGxJ39fSY0cuq8EWAZQ4vaDdqAItSRP4+vjYvq8=",
+        "lastModified": 1707750141,
+        "narHash": "sha256-9qSzGQs/Rjf2i3UQjyaZUznZzYDHkLYro/1FTT1easg=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "75ebb90a70f6320c1c7a1fca87a0a8adb0716143",
+        "rev": "c4ebb82da4eade975e874da600dc50e9dec610cb",
         "type": "github"
       },
       "original": {
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 1d7de74b..d5619719 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -185,7 +185,7 @@ static void worker(
                     !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
 
                 if (drv->querySystem() == "unknown")
-                    throw EvalError("derivation must have a 'system' attribute");
+                    state.error<EvalError>("derivation must have a 'system' attribute").debugThrow();
 
                 auto drvPath = state.store->printStorePath(drv->requireDrvPath());
 
@@ -208,7 +208,7 @@ static void worker(
                 if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
                     auto a = v->attrs->get(state.symbols.create("constituents"));
                     if (!a)
-                        throw EvalError("derivation must have a ‘constituents’ attribute");
+                        state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
 
                     NixStringContext context;
                     state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
@@ -274,7 +274,7 @@ static void worker(
             else if (v->type() == nNull)
                 ;
 
-            else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v));
+            else state.error<TypeError>("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow();
 
         } catch (EvalError & e) {
             auto msg = e.msg();
diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc
index 75506ff8..9312d085 100644
--- a/src/hydra-evaluator/hydra-evaluator.cc
+++ b/src/hydra-evaluator/hydra-evaluator.cc
@@ -38,7 +38,7 @@ class JobsetId {
     friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
 
     std::string display() const {
-        return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
+        return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id);
     }
 };
 bool operator==(const JobsetId & lhs, const JobsetId & rhs)
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 203f9f1d..06b06512 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -294,7 +294,7 @@ bool State::getQueuedBuilds(Connection & conn,
         try {
             createBuild(build);
         } catch (Error & e) {
-            e.addTrace({}, hintfmt("while loading build %d: ", build->id));
+            e.addTrace({}, HintFmt("while loading build %d: ", build->id));
             throw;
         }
 
diff --git a/src/lib/Hydra/Base/Controller/NixChannel.pm b/src/lib/Hydra/Base/Controller/NixChannel.pm
index 3f8e9609..a5bc2784 100644
--- a/src/lib/Hydra/Base/Controller/NixChannel.pm
+++ b/src/lib/Hydra/Base/Controller/NixChannel.pm
@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use base 'Hydra::Base::Controller::REST';
 use List::SomeUtils qw(any);
-use Nix::Store;
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 
@@ -30,7 +29,7 @@ sub getChannelData {
         my $outputs = {};
         foreach my $output (@outputs) {
             my $outPath = $output->get_column("outpath");
-            next if $checkValidity && !isValidPath($outPath);
+            next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath);
             $outputs->{$output->get_column("outname")} = $outPath;
             push @storePaths, $outPath;
             # Put the system type in the manifest (for top-level
diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index c3869838..4e249d14 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -10,8 +10,6 @@ use File::Basename;
 use File::LibMagic;
 use File::stat;
 use Data::Dump qw(dump);
-use Nix::Store;
-use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
 use JSON::PP;
@@ -82,9 +80,9 @@ sub build_GET {
     # false because `$_->path` will be empty
     $c->stash->{available} =
         $c->stash->{isLocalStore}
-        ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
+        ? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all
         : 1;
-    $c->stash->{drvAvailable} = isValidPath $build->drvpath;
+    $c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
 
     if ($build->finished && $build->iscachedbuild) {
         my $path = ($build->buildoutputs)[0]->path or undef;
@@ -308,7 +306,7 @@ sub output : Chained('buildChain') PathPart Args(1) {
     error($c, "This build is not finished yet.") unless $build->finished;
     my $output = $build->buildoutputs->find({name => $outputName});
     notFound($c, "This build has no output named ‘$outputName’") unless defined $output;
-    gone($c, "Output is no longer available.") unless isValidPath $output->path;
+    gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path);
 
     $c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\"");
     $c->stash->{current_view} = 'NixNAR';
@@ -425,7 +423,7 @@ sub getDependencyGraph {
             };
         $$done{$path} = $node;
         my @refs;
-        foreach my $ref (queryReferences($path)) {
+        foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) {
             next if $ref eq $path;
             next unless $runtime || $ref =~ /\.drv$/;
             getDependencyGraph($self, $c, $runtime, $done, $ref);
@@ -433,7 +431,7 @@ sub getDependencyGraph {
         }
         # Show in reverse topological order to flatten the graph.
         # Should probably do a proper BFS.
-        my @sorted = reverse topoSortPaths(@refs);
+        my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs);
         $node->{refs} = [map { $$done{$_} } @sorted];
     }
 
@@ -446,7 +444,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') {
     my $build = $c->stash->{build};
     my $drvPath = $build->drvpath;
 
-    error($c, "Derivation no longer available.") unless isValidPath $drvPath;
+    error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath);
 
     $c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath);
 
@@ -461,7 +459,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') {
 
     requireLocalStore($c);
 
-    error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths;
+    error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths;
 
     my $done = {};
     $c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ];
@@ -481,7 +479,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
     if (isLocalStore) {
         foreach my $out ($build->buildoutputs) {
             notFound($c, "Path " . $out->path . " is no longer available.")
-                unless isValidPath($out->path);
+                unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         }
     }
 
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index 548cfac3..162f347e 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -395,7 +395,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
         my ($hash) = $narinfo =~ NARINFO_REGEX;
 
         die("Hash length was not 32") if length($hash) != 32;
-        my $path = queryPathFromHashPart($hash);
+        my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash);
 
         if (!$path) {
             $c->response->status(404);
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 71a8a7d7..900514cb 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -40,8 +40,11 @@ our @EXPORT = qw(
     registerRoot
     restartBuilds
     run
+    $MACHINE_LOCAL_STORE
     );
 
+our $MACHINE_LOCAL_STORE = Nix::Store->new();
+
 
 sub getHydraHome {
     my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
@@ -494,7 +497,7 @@ sub restartBuilds {
     $builds = $builds->search({ finished => 1 });
 
     foreach my $build ($builds->search({}, { columns => ["drvpath"] })) {
-        next if !isValidPath($build->drvpath);
+        next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
         registerRoot $build->drvpath;
     }
 
diff --git a/src/lib/Hydra/Plugin/BazaarInput.pm b/src/lib/Hydra/Plugin/BazaarInput.pm
index 230d108b..b35ed7c8 100644
--- a/src/lib/Hydra/Plugin/BazaarInput.pm
+++ b/src/lib/Hydra/Plugin/BazaarInput.pm
@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -38,9 +37,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -58,7 +57,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-bzr and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedBazaarInputs')->create(
diff --git a/src/lib/Hydra/Plugin/DarcsInput.pm b/src/lib/Hydra/Plugin/DarcsInput.pm
index b7f3db55..a8df6396 100644
--- a/src/lib/Hydra/Plugin/DarcsInput.pm
+++ b/src/lib/Hydra/Plugin/DarcsInput.pm
@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -58,7 +57,7 @@ sub fetchInput {
         {uri => $uri, revision => $revision},
         {rows => 1});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -75,8 +74,8 @@ sub fetchInput {
         die "darcs changes --count failed" if $? != 0;
 
         system "rm", "-rf", "$tmpDir/export/_darcs";
-        $storePath = addToStore("$tmpDir/export", 1, "sha256");
-        $sha256 = queryPathHash($storePath);
+        $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256");
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath);
         $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {
diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm
index e5fc7de9..0de02128 100644
--- a/src/lib/Hydra/Plugin/GitInput.pm
+++ b/src/lib/Hydra/Plugin/GitInput.pm
@@ -186,9 +186,9 @@ sub fetchInput {
         {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0},
         {rows => 1});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $revision = $cachedInput->revision;
@@ -217,7 +217,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1);
 
         # FIXME: time window between nix-prefetch-git and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedGitInputs')->update_or_create(
diff --git a/src/lib/Hydra/Plugin/MercurialInput.pm b/src/lib/Hydra/Plugin/MercurialInput.pm
index 921262ad..85bd2c70 100644
--- a/src/lib/Hydra/Plugin/MercurialInput.pm
+++ b/src/lib/Hydra/Plugin/MercurialInput.pm
@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use File::Path;
 use Hydra::Helper::Nix;
 use Hydra::Helper::Exec;
-use Nix::Store;
 use Fcntl qw(:flock);
 
 sub supportedInputTypes {
@@ -68,9 +67,9 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
         {uri => $uri, branch => $branch, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {
@@ -85,7 +84,7 @@ sub fetchInput {
         ($sha256, $storePath) = split ' ', $stdout;
 
         # FIXME: time window between nix-prefetch-hg and addTempRoot.
-        addTempRoot($storePath);
+        $MACHINE_LOCAL_STORE->addTempRoot($storePath);
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedHgInputs')->update_or_create(
diff --git a/src/lib/Hydra/Plugin/PathInput.pm b/src/lib/Hydra/Plugin/PathInput.pm
index d122ff57..c923a03c 100644
--- a/src/lib/Hydra/Plugin/PathInput.pm
+++ b/src/lib/Hydra/Plugin/PathInput.pm
@@ -5,7 +5,6 @@ use warnings;
 use parent 'Hydra::Plugin';
 use POSIX qw(strftime);
 use Hydra::Helper::Nix;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -30,7 +29,7 @@ sub fetchInput {
         {srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
         {rows => 1, order_by => "lastseen DESC"});
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
         $timestamp = $cachedInput->timestamp;
@@ -46,7 +45,7 @@ sub fetchInput {
         }
         chomp $storePath;
 
-        $sha256 = (queryPathInfo($storePath, 0))[1] or die;
+        $sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die;
 
         ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
             {srcpath => $uri, sha256hash => $sha256});
diff --git a/src/lib/Hydra/Plugin/SubversionInput.pm b/src/lib/Hydra/Plugin/SubversionInput.pm
index 456c6892..83c1f39d 100644
--- a/src/lib/Hydra/Plugin/SubversionInput.pm
+++ b/src/lib/Hydra/Plugin/SubversionInput.pm
@@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
 use Hydra::Helper::Exec;
 use Hydra::Helper::Nix;
 use IPC::Run;
-use Nix::Store;
 
 sub supportedInputTypes {
     my ($self, $inputTypes) = @_;
@@ -45,7 +44,7 @@ sub fetchInput {
     (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
         {uri => $uri, revision => $revision});
 
-    addTempRoot($cachedInput->storepath) if defined $cachedInput;
+    $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
     if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
@@ -62,16 +61,16 @@ sub fetchInput {
         die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
 
         if ($type eq "svn-checkout") {
-            $storePath = addToStore($wcPath, 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256");
         } else {
             # Hm, if the Nix Perl bindings supported filters in
             # addToStore(), then we wouldn't need to make a copy here.
             my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
             (system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed";
-            $storePath = addToStore("$tmpDir/source", 1, "sha256");
+            $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256");
         }
 
-        $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
+        $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://;
 
         $self->{db}->txn_do(sub {
             $self->{db}->resultset('CachedSubversionInputs')->update_or_create(
diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm
index 44db78b1..bf8711a4 100644
--- a/src/lib/Hydra/View/NARInfo.pm
+++ b/src/lib/Hydra/View/NARInfo.pm
@@ -6,8 +6,7 @@ use File::Basename;
 use Hydra::Helper::CatalystUtils;
 use MIME::Base64;
 use Nix::Manifest;
-use Nix::Store;
-use Nix::Utils;
+use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;
 
 sub process {
@@ -17,7 +16,7 @@ sub process {
 
     $c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type
 
-    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
+    my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1);
 
     my $info;
     $info .= "StorePath: $storePath\n";
@@ -28,8 +27,8 @@ sub process {
     $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
     if (defined $deriver) {
         $info .= "Deriver: " . basename $deriver . "\n";
-        if (isValidPath($deriver)) {
-            my $drv = derivationFromPath($deriver);
+        if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) {
+            my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver);
             $info .= "System: $drv->{platform}\n";
         }
     }
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 7ed7ebe8..72a386f5 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -85,14 +85,14 @@ sub attrsToSQL {
 # Fetch a store path from 'eval_substituter' if not already present.
 sub getPath {
     my ($path) = @_;
-    return 1 if isValidPath($path);
+    return 1 if $MACHINE_LOCAL_STORE->isValidPath($path);
 
     my $substituter = $config->{eval_substituter};
 
     system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path)
         if defined $substituter;
 
-    return isValidPath($path);
+    return $MACHINE_LOCAL_STORE->isValidPath($path);
 }
 
 
@@ -143,7 +143,7 @@ sub fetchInputBuild {
         , version => $version
         , outputName => $mainOutput->name
         };
-    if (isValidPath($prevBuild->drvpath)) {
+    if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) {
         $result->{drvPath} = $prevBuild->drvpath;
     }
 
@@ -233,7 +233,7 @@ sub fetchInputEval {
         my $out = $build->buildoutputs->find({ name => "out" });
         next unless defined $out;
         # FIXME: Should we fail if the path is not valid?
-        next unless isValidPath($out->path);
+        next unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
         $jobs->{$build->get_column('job')} = $out->path;
     }
 
diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots
index 11eba7a6..130cbb68 100755
--- a/src/script/hydra-update-gc-roots
+++ b/src/script/hydra-update-gc-roots
@@ -5,7 +5,6 @@ use warnings;
 use File::Path;
 use File::stat;
 use File::Basename;
-use Nix::Store;
 use Hydra::Config;
 use Hydra::Schema;
 use Hydra::Helper::Nix;
@@ -47,7 +46,7 @@ sub keepBuild {
         $build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6))
     {
         foreach my $path (split / /, $build->get_column('outpaths')) {
-            if (isValidPath($path)) {
+            if ($MACHINE_LOCAL_STORE->isValidPath($path)) {
                 addRoot $path;
             } else {
                 print STDERR "    warning: output ", $path, " has disappeared\n" if $build->finished;
@@ -55,7 +54,7 @@ sub keepBuild {
         }
     }
     if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) {
-        if (isValidPath($build->drvpath)) {
+        if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) {
             addRoot $build->drvpath;
         } else {
             print STDERR "    warning: derivation ", $build->drvpath, " has disappeared\n";
diff --git a/t/s3-backup-test.pl b/t/s3-backup-test.pl
index 9332ab15..a367981f 100644
--- a/t/s3-backup-test.pl
+++ b/t/s3-backup-test.pl
@@ -3,7 +3,6 @@ use warnings;
 use File::Basename;
 use Hydra::Model::DB;
 use Hydra::Helper::Nix;
-use Nix::Store;
 use Cwd;
 
 my $db = Hydra::Model::DB->new;

From 9db5d0a88daee61b8b4507e517d53cab7377a70d Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Fri, 23 Feb 2024 14:02:36 +0300
Subject: [PATCH 149/285] urlencode drv names when fetching logs

Otherwise names with special characters like + break things.
---
 src/lib/Hydra/Controller/Build.pm | 5 +++--
 src/lib/Hydra/Controller/Root.pm  | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index c3869838..6b25ff80 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -15,6 +15,7 @@ use Nix::Config;
 use List::SomeUtils qw(all);
 use Encode;
 use JSON::PP;
+use WWW::Form::UrlEncoded::PP qw();
 
 use feature 'state';
 
@@ -141,7 +142,7 @@ sub view_nixlog : Chained('buildChain') PathPart('nixlog') {
     $c->stash->{step} = $step;
 
     my $drvPath = $step->drvpath;
-    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]);
+    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))]);
     showLog($c, $mode, $log_uri);
 }
 
@@ -150,7 +151,7 @@ sub view_log : Chained('buildChain') PathPart('log') {
     my ($self, $c, $mode) = @_;
 
     my $drvPath = $c->stash->{build}->drvpath;
-    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [basename($drvPath)]);
+    my $log_uri = $c->uri_for($c->controller('Root')->action_for("log"), [WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath))]);
     showLog($c, $mode, $log_uri);
 }
 
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index 548cfac3..e6c3049f 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -16,6 +16,7 @@ use List::Util qw[min max];
 use List::SomeUtils qw{any};
 use Net::Prometheus;
 use Types::Standard qw/StrMatch/;
+use WWW::Form::UrlEncoded::PP qw();
 
 use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
 # e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi
@@ -553,7 +554,7 @@ sub log :Local :Args(1) {
     my $logPrefix = $c->config->{log_prefix};
 
     if (defined $logPrefix) {
-        $c->res->redirect($logPrefix . "log/" . basename($drvPath));
+        $c->res->redirect($logPrefix . "log/" . WWW::Form::UrlEncoded::PP::url_encode(basename($drvPath)));
     } else {
         notFound($c, "The build log of $drvPath is not available.");
     }

From 669617ab54a667623fdbbc07dfd7354b5d66286b Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 7 Mar 2024 18:44:13 +0100
Subject: [PATCH 150/285] Use `submit` event in login form

It's a pet peeve from me when logging into my personal Hydra that I
always have to press the button rather than hitting Return after entering
my password.

Reason for that is that the form doesn't have a "submit" button, so far
it was always listened to the "click" event. Submit does that and you
can hit Return alternatively.
---
 src/root/auth.tt | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/root/auth.tt b/src/root/auth.tt
index d1539765..d49ba5bd 100644
--- a/src/root/auth.tt
+++ b/src/root/auth.tt
@@ -33,7 +33,7 @@
   <div id="hydra-signin" class="modal hide fade" tabindex="-1" role="dialog" aria-hidden="true">
     <div class="modal-dialog" role="document">
       <div class="modal-content">
-        <form>
+        <form id="signin-form">
           <div class="modal-body">
             <div class="form-group">
               <label for="username" class="col-form-label">User name</label>
@@ -45,7 +45,7 @@
             </div>
           </div>
           <div class="modal-footer">
-            <button id="do-signin" type="button" class="btn btn-primary">Sign in</button>
+            <button type="submit" class="btn btn-primary">Sign in</button>
             <button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
           </div>
         </form>
@@ -57,10 +57,11 @@
 
     function finishSignOut() { }
 
-    $("#do-signin").click(function() {
+    $("#signin-form").submit(function(e) {
+      e.preventDefault();
       requestJSON({
         url: "[% c.uri_for('/login') %]",
-        data: $(this).parents("form").serialize(),
+        data: $(this).serialize(),
         type: 'POST',
         success: function(data) {
           window.location.reload();

From 806c375c338b4e6a1d276b96994018908784bf11 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 8 Mar 2024 11:07:38 +0100
Subject: [PATCH 151/285] Don't send gitea status update when build is started

This was the source of a flaky test because sometimes hydra-notify was
quick enough to send out `buildStarted` and sometimes it apparently
wasn't which was quickly spottable with `nix build --rebuild`.

Removing that status update doesn't make a difference functionally,
gitea doesn't differentiate between "queued" and "running", so we send
the same status ("pending") out on both events, so we'd even safe one
avoidable request.
---
 flake.nix                           | 5 ++---
 src/lib/Hydra/Plugin/GiteaStatus.pm | 4 ----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/flake.nix b/flake.nix
index a6dfb977..31a38416 100644
--- a/flake.nix
+++ b/flake.nix
@@ -353,10 +353,9 @@
 
                 response = json.loads(data)
 
-                assert len(response) == 3, "Expected exactly three status updates for latest commit (queued, started, finished)!"
+                assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
                 assert response[0]['status'] == "success", "Expected finished status to be success!"
-                assert response[1]['status'] == "pending", "Expected started status to be pending!"
-                assert response[2]['status'] == "pending", "Expected queued status to be pending!"
+                assert response[1]['status'] == "pending", "Expected queued status to be pending!"
 
                 machine.shutdown()
               '';
diff --git a/src/lib/Hydra/Plugin/GiteaStatus.pm b/src/lib/Hydra/Plugin/GiteaStatus.pm
index 426c93f5..f3498941 100644
--- a/src/lib/Hydra/Plugin/GiteaStatus.pm
+++ b/src/lib/Hydra/Plugin/GiteaStatus.pm
@@ -88,10 +88,6 @@ sub buildQueued {
     common(@_, [], 0);
 }
 
-sub buildStarted {
-    common(@_, [], 1);
-}
-
 sub buildFinished {
     common(@_, 2);
 }

From 99afff03b06d6b9a3d464ede9724e29a03e91329 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 14 Mar 2024 22:47:37 +0100
Subject: [PATCH 152/285] hydra-queue-runner: drop broken connections from pool

Closes #1336

When restarting postgresql, the connections are still reused in
`hydra-queue-runner` causing errors like this

    main thread: Lost connection to the database server.
    queue monitor: Lost connection to the database server.

and no more builds being processed.

`hydra-evaluator` doesn't have that issue since it crashes right away.
We could let it retry indefinitely as well (see below), but I don't
want to change too much.

If the DB is still unreachable 10s later, the process will stop with a
non-zero exit code because of a missing DB connection. This however
isn't such a big deal because it will be immediately restarted
afterwards. With the current configuration, Hydra will never give up,
but restart (and retry) infinitely. To me that seems reasonable, i.e. to
retry DB connections on a long-running process. If this doesn't work
out, the monitoring should fire anyways because the queue fills up, but
I'm open to discuss that.

Please note that this isn't reproducible with the DB and the queue
runner on the same machine when using `services.hydra-dev`, because of
the `Requires=` dependency `hydra-queue-runner.service` ->
`hydra-init.service` -> `postgresql.service` that causes the queue
runner to be restarted on `systemctl restart postgresql`.

Internally, Hydra uses Nix's pool data structure: it basically has N
slots (here DB connections) and whenever a new one is requested, an idle
slot is provided or a new one is created (when N slots are active, it'll
be waited until one slot is free). The issue in the code here is however
that whenever an error is encountered, the slot is released, however the
same broken connection will be reused the next time. By using
`Pool::Handle::markBad`, Nix will drop a broken slot. This is now being
done when `pqxx::broken_connection` was caught.
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 15 ++++++---
 src/hydra-queue-runner/queue-monitor.cc      | 34 +++++++++++---------
 src/hydra-queue-runner/state.hh              |  2 +-
 3 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 0ee710cb..5ffa7fe6 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -926,10 +926,17 @@ void State::run(BuildID buildOne)
     while (true) {
         try {
             auto conn(dbPool.get());
-            receiver dumpStatus_(*conn, "dump_status");
-            while (true) {
-                conn->await_notification();
-                dumpStatus(*conn);
+            try {
+                receiver dumpStatus_(*conn, "dump_status");
+                while (true) {
+                    conn->await_notification();
+                    dumpStatus(*conn);
+                }
+            } catch (pqxx::broken_connection & connEx) {
+                printMsg(lvlError, "main thread: %s", connEx.what());
+                printMsg(lvlError, "main thread: Reconnecting in 10s");
+                conn.markBad();
+                sleep(10);
             }
         } catch (std::exception & e) {
             printMsg(lvlError, "main thread: %s", e.what());
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 203f9f1d..1bd51f5a 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -10,8 +10,14 @@ using namespace nix;
 void State::queueMonitor()
 {
     while (true) {
+        auto conn(dbPool.get());
         try {
-            queueMonitorLoop();
+            queueMonitorLoop(*conn);
+        } catch (pqxx::broken_connection & e) {
+            printMsg(lvlError, "queue monitor: %s", e.what());
+            printMsg(lvlError, "queue monitor: Reconnecting in 10s");
+            conn.markBad();
+            sleep(10);
         } catch (std::exception & e) {
             printError("queue monitor: %s", e.what());
             sleep(10); // probably a DB problem, so don't retry right away
@@ -20,16 +26,14 @@ void State::queueMonitor()
 }
 
 
-void State::queueMonitorLoop()
+void State::queueMonitorLoop(Connection & conn)
 {
-    auto conn(dbPool.get());
-
-    receiver buildsAdded(*conn, "builds_added");
-    receiver buildsRestarted(*conn, "builds_restarted");
-    receiver buildsCancelled(*conn, "builds_cancelled");
-    receiver buildsDeleted(*conn, "builds_deleted");
-    receiver buildsBumped(*conn, "builds_bumped");
-    receiver jobsetSharesChanged(*conn, "jobset_shares_changed");
+    receiver buildsAdded(conn, "builds_added");
+    receiver buildsRestarted(conn, "builds_restarted");
+    receiver buildsCancelled(conn, "builds_cancelled");
+    receiver buildsDeleted(conn, "builds_deleted");
+    receiver buildsBumped(conn, "builds_bumped");
+    receiver jobsetSharesChanged(conn, "jobset_shares_changed");
 
     auto destStore = getDestStore();
 
@@ -39,17 +43,17 @@ void State::queueMonitorLoop()
     while (!quit) {
         localStore->clearPathInfoCache();
 
-        bool done = getQueuedBuilds(*conn, destStore, lastBuildId);
+        bool done = getQueuedBuilds(conn, destStore, lastBuildId);
 
         if (buildOne && buildOneDone) quit = true;
 
         /* Sleep until we get notification from the database about an
            event. */
         if (done && !quit) {
-            conn->await_notification();
+            conn.await_notification();
             nrQueueWakeups++;
         } else
-            conn->get_notifs();
+            conn.get_notifs();
 
         if (auto lowestId = buildsAdded.get()) {
             lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
@@ -61,11 +65,11 @@ void State::queueMonitorLoop()
         }
         if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
             printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
-            processQueueChange(*conn);
+            processQueueChange(conn);
         }
         if (jobsetSharesChanged.get()) {
             printMsg(lvlTalkative, "got notification: jobset shares changed");
-            processJobsetSharesChange(*conn);
+            processJobsetSharesChange(conn);
         }
     }
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index cda238ae..5d242cdf 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -498,7 +498,7 @@ private:
 
     void queueMonitor();
 
-    void queueMonitorLoop();
+    void queueMonitorLoop(Connection & conn);
 
     /* Check the queue for new builds. */
     bool getQueuedBuilds(Connection & conn,

From 1665aed5e302f213db048a38af44021b5e815d80 Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Wed, 3 Apr 2024 22:45:53 +0200
Subject: [PATCH 153/285] t: content-addressed: add test for
 caDependingOnFailingCA

This uncovers an issue with the front-end.
---
 t/content-addressed/basic.t  | 4 ++--
 t/jobs/content-addressed.nix | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/t/content-addressed/basic.t b/t/content-addressed/basic.t
index 6597e727..4f92b1dc 100644
--- a/t/content-addressed/basic.t
+++ b/t/content-addressed/basic.t
@@ -27,13 +27,13 @@ my $project = $db->resultset('Projects')->create({name => "tests", displayname =
 my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
 
 ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
+is(nrQueuedBuildsForJobset($jobset), 6, "Evaluating jobs/content-addressed.nix should result in 6 builds");
 
 for my $build (queuedBuildsForJobset($jobset)) {
     ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
     my $newbuild = $db->resultset('Builds')->find($build->id);
     is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
-    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : $build->job =~ /FailingCA/ ? 2 : 0;
     is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
 
     my $response = request("/build/".$build->id);
diff --git a/t/jobs/content-addressed.nix b/t/jobs/content-addressed.nix
index 65496df5..03bb56e7 100644
--- a/t/jobs/content-addressed.nix
+++ b/t/jobs/content-addressed.nix
@@ -25,6 +25,13 @@ rec {
       FOO = empty_dir;
     };
 
+  caDependingOnFailingCA =
+    cfg.mkContentAddressedDerivation {
+      name = "ca-depending-on-failing-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = fails;
+    };
+
   nonCaDependingOnCA =
     cfg.mkDerivation {
       name = "non-ca-depending-on-ca";

From 71986632ced0dcaa0bf8262d2f41e52d3216c39d Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Wed, 3 Apr 2024 22:47:22 +0200
Subject: [PATCH 154/285] hydra-server: findLog: fix issue with ca-derivations
 enabled

When content addressed derivations are built on the hydra server,
one may run into an issue where some builds suddenly don't load anymore.

This seems to be caused by outPaths that are NULL (which is
allowed for ca-derivations). Filter them out to prevent querying the
database for them, which is not supported by the database abstraction
layer that's currently in use.

On my instance this appears to resolve the issue.
I feel like I might be doing this at the wrong abstraction layer, but on
the other hand -- it seems to resolve it and it also doesn't really look
like it will hurt anything.

The test added in a previous commit uncovers this issue, and this commit
resolves it. So I'm happy with this patch for now.

The issue I was seeing on my server:

hydra-server[2549]: [error] Couldn't render template "undef error - DBIx::Class::SQLMaker::ClassicExtensions::puke(): Fatal: NULL-within-IN not implemented: The upcoming SQL::Abstract::Classic 2.0 will emit the logically correct SQL instead of raising this exception. at /nix/store/<hash>-hydra-unstable-2024-03-08_nix_2_20/libexec/hydra/lib/Hydra/Helper/Nix.pm line 190

See also short discussion here: https://github.com/NixOS/nixpkgs/pull/297392#issuecomment-2035366263
---
 src/lib/Hydra/Helper/Nix.pm | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 71a8a7d7..7b16b1be 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -187,6 +187,10 @@ sub findLog {
 
     return undef if scalar @outPaths == 0;
 
+    # Filter out any NULLs. Content-addressed derivations
+    # that haven't built yet or failed to build may have a NULL outPath.
+    @outPaths = grep {defined} @outPaths;
+
     my @steps = $c->model('DB::BuildSteps')->search(
         { path => { -in => [@outPaths] } },
         { select => ["drvpath"]

From 3f913a771d1cdfba004a3639fc109e57bf3dae8e Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Wed, 3 Apr 2024 22:55:00 +0200
Subject: [PATCH 155/285] t: content-addressed: add a comment about a
 misleading testcase

---
 t/content-addressed/basic.t | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/t/content-addressed/basic.t b/t/content-addressed/basic.t
index 4f92b1dc..aefb457b 100644
--- a/t/content-addressed/basic.t
+++ b/t/content-addressed/basic.t
@@ -55,6 +55,8 @@ for my $build (queuedBuildsForJobset($jobset)) {
 
 }
 
+# XXX: deststoredir is undefined: Use of uninitialized value $ctx{"deststoredir"} in concatenation (.) or string at t/content-addressed/basic.t line 58.
+# XXX: This test seems to not do what it seems to be doing. See documentation: https://metacpan.org/pod/Test2::V0#isnt($got,-$do_not_want,-$name)
 isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
 
 done_testing;

From ab1f64aa4d8e89225375b5d45c28d7f3492fcc7a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 12 Apr 2024 12:25:49 -0400
Subject: [PATCH 156/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/c4ebb82da4eade975e874da600dc50e9dec610cb' (2024-02-12)
  → 'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/a1982c92d8980a0114372973cbdfe0a307f1bdea' (2024-01-12)
  → 'github:NixOS/nixpkgs/1d6a23f11e44d0fb64b3237569b87658a9eb5643' (2024-04-11)
• Removed input 'nixpkgs-for-fileset'
---
 flake.lock | 34 +++++++++-------------------------
 flake.nix  | 13 ++++---------
 2 files changed, 13 insertions(+), 34 deletions(-)

diff --git a/flake.lock b/flake.lock
index 4b4f0076..966431f9 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,47 +42,32 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1707750141,
-        "narHash": "sha256-9qSzGQs/Rjf2i3UQjyaZUznZzYDHkLYro/1FTT1easg=",
+        "lastModified": 1712849398,
+        "narHash": "sha256-10z/SoidVl9/lh56cMLj7ntJZHtVrumFvmn1YEqXmaM=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "c4ebb82da4eade975e874da600dc50e9dec610cb",
+        "rev": "60824fa97c588a0faf68ea61260a47e388b0a4e5",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
+        "ref": "2.21-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1712848736,
+        "narHash": "sha256-CzZwhqyLlebljv1zFS2KWVH/3byHND0LfaO1jKsGuVo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "1d6a23f11e44d0fb64b3237569b87658a9eb5643",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-for-fileset": {
-      "locked": {
-        "lastModified": 1706098335,
-        "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-23.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -106,8 +91,7 @@
     "root": {
       "inputs": {
         "nix": "nix",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-for-fileset": "nixpkgs-for-fileset"
+        "nixpkgs": "nixpkgs"
       }
     }
   },
diff --git a/flake.nix b/flake.nix
index 4dc7d25e..fd6ad586 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,16 +1,11 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nix.url = "github:NixOS/nix";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
+  inputs.nix.url = "github:NixOS/nix/2.21-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
-  # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
-  # mered and we upgrade or Nix, so the main `nixpkgs` input is at least
-  # 23.11 and has `lib.fileset`.
-  inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
-
-  outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }:
+  outputs = { self, nixpkgs, nix }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
@@ -67,7 +62,7 @@
         };
 
         hydra = final.callPackage ./package.nix {
-          inherit (nixpkgs-for-fileset.lib) fileset;
+          inherit (nixpkgs.lib) fileset;
           rawSrc = self;
         };
       };

From 39a4e4791ea9ae4c29400f322bc1d1b71453cf1d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 12 Apr 2024 17:30:43 -0400
Subject: [PATCH 157/285] Switch (back) to Nix master
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Re-creating `nix-next` after using it in #1375.

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11)
  → 'github:NixOS/nix/aa438b8fbaebbbdb922655127053c4e8ea3e55bb' (2024-04-12)
---
 flake.lock | 7 +++----
 flake.nix  | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index 966431f9..0b5c3bbb 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,15 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1712849398,
-        "narHash": "sha256-10z/SoidVl9/lh56cMLj7ntJZHtVrumFvmn1YEqXmaM=",
+        "lastModified": 1712957033,
+        "narHash": "sha256-/P298re0Qga0eJV3Q3BppvYbVEN0CKXMnxMQX192hcE=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "60824fa97c588a0faf68ea61260a47e388b0a4e5",
+        "rev": "aa438b8fbaebbbdb922655127053c4e8ea3e55bb",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.21-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index c6646200..9d8c7142 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.21-maintenance";
+  inputs.nix.url = "github:NixOS/nix";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:

From b72528be5074f3e62e9ae2c2ae8ef9c07a0b4dd3 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sun, 21 Apr 2024 16:14:24 +0200
Subject: [PATCH 158/285] web: serveFile: also serve a CSP putting served HTML
 in its own origin

---
 src/lib/Hydra/Controller/Build.pm | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index f8587169..de2c204d 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -234,6 +234,9 @@ sub serveFile {
     }
 
     elsif ($ls->{type} eq "regular") {
+        # Have the hosted data considered its own origin to avoid being a giant
+        # XSS hole.
+        $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
 
         $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };

From 1b8154e67fb20f84b7f84049de03204b0df0a366 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Fri, 3 May 2024 14:31:15 +0200
Subject: [PATCH 159/285] Fix doi resolution after #1375

This fixes:

> Caught exception in Hydra::Controller::Root->realisations "Undefined subroutine &Hydra::Controller::Root::queryRawRealisation called at /nix/store/v842xb35ph8ka1yi1xanjhk4xh1pn5nm-hydra-2024-04-22/libexec/hydra/lib/Hydra/Controller/Root.pm line 371."
---
 src/lib/Hydra/Controller/Root.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index 47389251..406dc52e 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -367,7 +367,7 @@ sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
 
     else {
         my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
-        my $rawRealisation = queryRawRealisation($rawDrvOutput);
+        my $rawRealisation = $MACHINE_LOCAL_STORE->queryRawRealisation($rawDrvOutput);
 
         if (!$rawRealisation) {
             $c->response->status(404);

From 4bd687e3e677c3c89603d8738a94c74c51d4f9ba Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 3 May 2024 10:34:56 -0400
Subject: [PATCH 160/285] Update to Nix 2.22
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11)
  → 'github:NixOS/nix/1c8150ac312b5f9ba1b3f6768ff43b09867e5883' (2024-04-23)
• Added input 'nix/flake-parts':
    'github:hercules-ci/flake-parts/9126214d0a59633752a136528f5f3b9aa8565b7d' (2024-04-01)
• Added input 'nix/flake-parts/nixpkgs-lib':
    follows 'nix/nixpkgs'
• Added input 'nix/pre-commit-hooks':
    'github:cachix/pre-commit-hooks.nix/40e6053ecb65fcbf12863338a6dcefb3f55f1bf8' (2024-04-12)
• Added input 'nix/pre-commit-hooks/flake-compat':
    follows 'nix'
• Added input 'nix/pre-commit-hooks/flake-utils':
    'github:numtide/flake-utils/5aed5285a952e0b949eb3ba02c12fa4fcfef535f' (2022-11-02)
• Added input 'nix/pre-commit-hooks/gitignore':
    follows 'nix'
• Added input 'nix/pre-commit-hooks/nixpkgs':
    follows 'nix/nixpkgs'
• Added input 'nix/pre-commit-hooks/nixpkgs-stable':
    follows 'nix/nixpkgs'
---
 flake.lock                             | 80 ++++++++++++++++++++++++--
 flake.nix                              |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 18 +++---
 3 files changed, 85 insertions(+), 15 deletions(-)

diff --git a/flake.lock b/flake.lock
index 966431f9..e828aa7b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,42 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -35,23 +71,25 @@
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
         "libgit2": "libgit2",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "nixpkgs-regression": "nixpkgs-regression"
+        "nixpkgs-regression": "nixpkgs-regression",
+        "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1712849398,
-        "narHash": "sha256-10z/SoidVl9/lh56cMLj7ntJZHtVrumFvmn1YEqXmaM=",
+        "lastModified": 1713874370,
+        "narHash": "sha256-gW1mO/CvsQQ5gvgiwzxsGhPFI/tx30NING+qgF5Do0s=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "60824fa97c588a0faf68ea61260a47e388b0a4e5",
+        "rev": "1c8150ac312b5f9ba1b3f6768ff43b09867e5883",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.21-maintenance",
+        "ref": "2.22-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -88,6 +126,38 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nix"
+        ],
+        "flake-utils": "flake-utils",
+        "gitignore": [
+          "nix"
+        ],
+        "nixpkgs": [
+          "nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "nix": "nix",
diff --git a/flake.nix b/flake.nix
index c6646200..82824733 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.21-maintenance";
+  inputs.nix.url = "github:NixOS/nix/2.22-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index d5619719..5adea42b 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -102,8 +102,8 @@ static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 rec(*v.listElems()[n]);
         else if (v.type() == nAttrs) {
-            auto a = v.attrs->find(state.symbols.create(subAttribute));
-            if (a != v.attrs->end())
+            auto a = v.attrs()->find(state.symbols.create(subAttribute));
+            if (a != v.attrs()->end())
                 res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
         }
     };
@@ -138,12 +138,12 @@ static void worker(
 
         callFlake(state, lockedFlake, *vFlake);
 
-        auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value;
+        auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value;
         state.forceValue(*vOutputs, noPos);
 
-        auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));
+        auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs"));
         if (!aHydraJobs)
-            aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));
+            aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks"));
         if (!aHydraJobs)
             throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);
 
@@ -204,9 +204,9 @@ static void worker(
                 job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);
 
                 /* If this is an aggregate, then get its constituents. */
-                auto a = v->attrs->get(state.symbols.create("_hydraAggregate"));
+                auto a = v->attrs()->get(state.symbols.create("_hydraAggregate"));
                 if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
-                    auto a = v->attrs->get(state.symbols.create("constituents"));
+                    auto a = v->attrs()->get(state.symbols.create("constituents"));
                     if (!a)
                         state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
 
@@ -260,7 +260,7 @@ static void worker(
             else if (v->type() == nAttrs) {
                 auto attrs = nlohmann::json::array();
                 StringSet ss;
-                for (auto & i : v->attrs->lexicographicOrder(state.symbols)) {
+                for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) {
                     std::string name(state.symbols[i->name]);
                     if (name.find(' ') != std::string::npos) {
                         printError("skipping job with illegal name '%s'", name);
@@ -368,7 +368,7 @@ int main(int argc, char * * argv)
                             ]()
                             {
                                 try {
-                                    EvalState state(myArgs.searchPath, openStore());
+                                    EvalState state(myArgs.lookupPath, openStore());
                                     Bindings & autoArgs = *myArgs.getAutoArgs(state);
                                     worker(state, autoArgs, *to, *from);
                                 } catch (Error & e) {

From 92155f9a07f5fe32e0778e474e7313997811e635 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 3 May 2024 11:41:42 -0400
Subject: [PATCH 161/285] Remove `PrometheusTiny` from overlay

It's in Nixpkgs for a good while now.
---
 flake.nix | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/flake.nix b/flake.nix
index 82824733..b3de3d73 100644
--- a/flake.nix
+++ b/flake.nix
@@ -40,27 +40,6 @@
 
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlays.default = final: prev: {
-
-        # Add LDAP dependencies that aren't currently found within nixpkgs.
-        perlPackages = prev.perlPackages // {
-
-          PrometheusTiny = final.perlPackages.buildPerlPackage {
-            pname = "Prometheus-Tiny";
-            version = "0.007";
-            src = final.fetchurl {
-              url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-0.007.tar.gz";
-              sha256 = "0ef8b226a2025cdde4df80129dd319aa29e884e653c17dc96f4823d985c028ec";
-            };
-            buildInputs = with final.perlPackages; [ HTTPMessage Plack TestException ];
-            meta = {
-              homepage = "https://github.com/robn/Prometheus-Tiny";
-              description = "A tiny Prometheus client";
-              license = with final.lib.licenses; [ artistic1 gpl1Plus ];
-            };
-          };
-
-        };
-
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
           rawSrc = self;

From 743795b2b090a5cdfe8bd90120add8db7770086a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 3 May 2024 12:07:05 -0400
Subject: [PATCH 162/285] Factor out NixOS tests, and clean up

Due to newer nixpkgs, there were a number of things that could be
cleaned up in the process.
---
 flake.nix                 | 299 +-----------------------------------
 nixos-modules/default.nix |  11 +-
 nixos-tests.nix           | 309 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 319 insertions(+), 300 deletions(-)
 create mode 100644 nixos-tests.nix

diff --git a/flake.nix b/flake.nix
index b3de3d73..7e2f3f67 100644
--- a/flake.nix
+++ b/flake.nix
@@ -17,24 +17,6 @@
         overlays = overlayList;
       });
 
-      # NixOS configuration used for VM tests.
-      hydraServer =
-        { config, pkgs, ... }:
-        {
-          imports = [ self.nixosModules.hydraTest ];
-
-          virtualisation.memorySize = 1024;
-          virtualisation.writableStore = true;
-
-          environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
-
-          nix = {
-            # Without this nix tries to fetch packages from the default
-            # cache.nixos.org which is not reachable from this sandboxed NixOS test.
-            binaryCaches = [ ];
-          };
-        };
-
     in
     rec {
 
@@ -67,282 +49,9 @@
               echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
             '');
 
-        tests.install = forEachSystem (system:
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          simpleTest {
-            name = "hydra-install";
-            nodes.machine = hydraServer;
-            testScript =
-              ''
-                machine.wait_for_job("hydra-init")
-                machine.wait_for_job("hydra-server")
-                machine.wait_for_job("hydra-evaluator")
-                machine.wait_for_job("hydra-queue-runner")
-                machine.wait_for_open_port(3000)
-                machine.succeed("curl --fail http://localhost:3000/")
-              '';
-          });
-
-        tests.notifications = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          simpleTest {
-            name = "hydra-notifications";
-            nodes.machine = { pkgs, ... }: {
-              imports = [ hydraServer ];
-              services.hydra-dev.extraConfig = ''
-                <influxdb>
-                  url = http://127.0.0.1:8086
-                  db = hydra
-                </influxdb>
-              '';
-              services.influxdb.enable = true;
-            };
-            testScript = ''
-              machine.wait_for_job("hydra-init")
-
-              # Create an admin account and some other state.
-              machine.succeed(
-                  """
-                      su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
-                      mkdir /run/jobset
-                      chmod 755 /run/jobset
-                      cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
-                      chmod 644 /run/jobset/default.nix
-                      chown -R hydra /run/jobset
-              """
-              )
-
-              # Wait until InfluxDB can receive web requests
-              machine.wait_for_job("influxdb")
-              machine.wait_for_open_port(8086)
-
-              # Create an InfluxDB database where hydra will write to
-              machine.succeed(
-                  "curl -XPOST 'http://127.0.0.1:8086/query' "
-                  + "--data-urlencode 'q=CREATE DATABASE hydra'"
-              )
-
-              # Wait until hydra-server can receive HTTP requests
-              machine.wait_for_job("hydra-server")
-              machine.wait_for_open_port(3000)
-
-              # Setup the project and jobset
-              machine.succeed(
-                  "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
-              )
-
-              # Wait until hydra has build the job and
-              # the InfluxDBNotification plugin uploaded its notification to InfluxDB
-              machine.wait_until_succeeds(
-                  "curl -s -H 'Accept: application/csv' "
-                  + "-G 'http://127.0.0.1:8086/query?db=hydra' "
-                  + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
-              )
-            '';
-          });
-
-        tests.gitea = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-          makeTest {
-            name = "hydra-gitea";
-            nodes.machine = { pkgs, ... }: {
-              imports = [ hydraServer ];
-              services.hydra-dev.extraConfig = ''
-                <gitea_authorization>
-                root=d7f16a3412e01a43a414535b16007c6931d3a9c7
-                </gitea_authorization>
-              '';
-              nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
-              nix = {
-                settings.substituters = [ ];
-              };
-              services.gitea = {
-                enable = true;
-                database.type = "postgres";
-                disableRegistration = true;
-                httpPort = 3001;
-              };
-              services.openssh.enable = true;
-              environment.systemPackages = with pkgs; [ gitea git jq gawk ];
-              networking.firewall.allowedTCPPorts = [ 3000 ];
-            };
-            skipLint = true;
-            testScript =
-              let
-                scripts.mktoken = pkgs.writeText "token.sql" ''
-                  INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
-                '';
-
-                scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
-                  set -x
-                  mkdir -p /tmp/repo $HOME/.ssh
-                  cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
-                  chmod 0400 $HOME/.ssh/privk
-                  git -C /tmp/repo init
-                  cp ${smallDrv} /tmp/repo/jobset.nix
-                  git -C /tmp/repo add .
-                  git config --global user.email test@localhost
-                  git config --global user.name test
-                  git -C /tmp/repo commit -m 'Initial import'
-                  git -C /tmp/repo remote add origin gitea@machine:root/repo
-                  GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
-                    git -C /tmp/repo push origin master
-                  git -C /tmp/repo log >&2
-                '';
-
-                scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
-                  set -x
-                  su -l hydra -c "hydra-create-user root --email-address \
-                    'alice@example.org' --password foobar --role admin"
-
-                  URL=http://localhost:3000
-                  USERNAME="root"
-                  PASSWORD="foobar"
-                  PROJECT_NAME="trivial"
-                  JOBSET_NAME="trivial"
-                  mycurl() {
-                    curl --referer $URL -H "Accept: application/json" \
-                      -H "Content-Type: application/json" $@
-                  }
-
-                  cat >data.json <<EOF
-                  { "username": "$USERNAME", "password": "$PASSWORD" }
-                  EOF
-                  mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
-
-                  cat >data.json <<EOF
-                  {
-                    "displayname":"Trivial",
-                    "enabled":"1",
-                    "visible":"1"
-                  }
-                  EOF
-                  mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
-                    -d @data.json -b hydra-cookie.txt
-
-                  cat >data.json <<EOF
-                  {
-                    "description": "Trivial",
-                    "checkinterval": "60",
-                    "enabled": "1",
-                    "visible": "1",
-                    "keepnr": "1",
-                    "enableemail": true,
-                    "emailoverride": "hydra@localhost",
-                    "type": 0,
-                    "nixexprinput": "git",
-                    "nixexprpath": "jobset.nix",
-                    "inputs": {
-                      "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
-                      "gitea_repo_name": {"value": "repo", "type": "string"},
-                      "gitea_repo_owner": {"value": "root", "type": "string"},
-                      "gitea_status_repo": {"value": "git", "type": "string"},
-                      "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
-                    }
-                  }
-                  EOF
-
-                  mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
-                    -d @data.json -b hydra-cookie.txt
-                '';
-
-                api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
-
-                snakeoilKeypair = {
-                  privkey = pkgs.writeText "privkey.snakeoil" ''
-                    -----BEGIN EC PRIVATE KEY-----
-                    MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
-                    AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
-                    r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
-                    -----END EC PRIVATE KEY-----
-                  '';
-
-                  pubkey = pkgs.lib.concatStrings [
-                    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
-                    "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
-                    "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
-                  ];
-                };
-
-                smallDrv = pkgs.writeText "jobset.nix" ''
-                  { trivial = builtins.derivation {
-                      name = "trivial";
-                      system = "${system}";
-                      builder = "/bin/sh";
-                      allowSubstitutes = false;
-                      preferLocalBuild = true;
-                      args = ["-c" "echo success > $out; exit 0"];
-                    };
-                   }
-                '';
-              in
-              ''
-                import json
-
-                machine.start()
-                machine.wait_for_unit("multi-user.target")
-                machine.wait_for_open_port(3000)
-                machine.wait_for_open_port(3001)
-
-                machine.succeed(
-                    "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
-                    + "--username root --password root --email test@localhost'"
-                )
-                machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
-
-                machine.succeed(
-                    "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                    + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
-                )
-
-                machine.succeed(
-                    "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                    + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
-                )
-
-                machine.succeed(
-                    "${scripts.git-setup}"
-                )
-
-                machine.succeed(
-                    "${scripts.hydra-setup}"
-                )
-
-                machine.wait_until_succeeds(
-                    'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
-                    + '|  jq .buildstatus | xargs test 0 -eq'
-                )
-
-                data = machine.succeed(
-                    'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
-                    + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
-                    + f"-H 'Authorization: token ${api_token}'"
-                )
-
-                response = json.loads(data)
-
-                assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
-                assert response[0]['status'] == "success", "Expected finished status to be success!"
-                assert response[1]['status'] == "pending", "Expected queued status to be pending!"
-
-                machine.shutdown()
-              '';
-          });
-
-        tests.validate-openapi = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          pkgs.runCommand "validate-openapi"
-          { buildInputs = [ pkgs.openapi-generator-cli ]; }
-          ''
-            openapi-generator-cli validate -i ${./hydra-api.yaml}
-            touch $out
-          '');
+        tests = import ./nixos-tests.nix {
+          inherit forEachSystem nixpkgs pkgsBySystem nixosModules;
+        };
 
         container = nixosConfigurations.container.config.system.build.toplevel;
       };
@@ -366,6 +75,8 @@
         system = "x86_64-linux";
         modules =
           [
+            self.nixosModules.hydra
+            self.nixosModules.overlayNixpkgsForThisHyydra
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {
diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix
index 6fc19d31..f44d7808 100644
--- a/nixos-modules/default.nix
+++ b/nixos-modules/default.nix
@@ -1,14 +1,13 @@
 { overlays }:
 
-rec {
-  hydra = {
-    imports = [ ./hydra.nix ];
+{
+  hydra = import ./hydra.nix;
+
+  overlayNixpkgsForThisHyydra = {
     nixpkgs = { inherit overlays; };
   };
 
   hydraTest = { pkgs, ... }: {
-    imports = [ hydra ];
-
     services.hydra-dev.enable = true;
     services.hydra-dev.hydraURL = "http://hydra.example.org";
     services.hydra-dev.notificationSender = "admin@hydra.example.org";
@@ -16,7 +15,7 @@ rec {
     systemd.services.hydra-send-stats.enable = false;
 
     services.postgresql.enable = true;
-    services.postgresql.package = pkgs.postgresql_11;
+    services.postgresql.package = pkgs.postgresql_12;
 
     # The following is to work around the following error from hydra-server:
     #   [error] Caught exception in engine "Cannot determine local time zone"
diff --git a/nixos-tests.nix b/nixos-tests.nix
new file mode 100644
index 00000000..3c9dc6c8
--- /dev/null
+++ b/nixos-tests.nix
@@ -0,0 +1,309 @@
+{ forEachSystem, nixpkgs, pkgsBySystem, nixosModules }:
+
+let
+  # NixOS configuration used for VM tests.
+  hydraServer =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        nixosModules.hydra
+        nixosModules.overlayNixpkgsForThisHyydra
+        nixosModules.hydraTest
+      ];
+
+      virtualisation.memorySize = 1024;
+      virtualisation.writableStore = true;
+
+      environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ];
+
+      nix = {
+        # Without this nix tries to fetch packages from the default
+        # cache.nixos.org which is not reachable from this sandboxed NixOS test.
+        settings.substituters = [ ];
+      };
+    };
+
+in
+
+{
+
+  install = forEachSystem (system:
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    simpleTest {
+      name = "hydra-install";
+      nodes.machine = hydraServer;
+      testScript =
+        ''
+          machine.wait_for_job("hydra-init")
+          machine.wait_for_job("hydra-server")
+          machine.wait_for_job("hydra-evaluator")
+          machine.wait_for_job("hydra-queue-runner")
+          machine.wait_for_open_port(3000)
+          machine.succeed("curl --fail http://localhost:3000/")
+        '';
+    });
+
+  notifications = forEachSystem (system:
+    let pkgs = pkgsBySystem.${system}; in
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    simpleTest {
+      name = "hydra-notifications";
+      nodes.machine = { pkgs, ... }: {
+        imports = [ hydraServer ];
+        services.hydra-dev.extraConfig = ''
+          <influxdb>
+            url = http://127.0.0.1:8086
+            db = hydra
+          </influxdb>
+        '';
+        services.influxdb.enable = true;
+      };
+      testScript = ''
+        machine.wait_for_job("hydra-init")
+
+        # Create an admin account and some other state.
+        machine.succeed(
+            """
+                su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin"
+                mkdir /run/jobset
+                chmod 755 /run/jobset
+                cp ${./t/jobs/api-test.nix} /run/jobset/default.nix
+                chmod 644 /run/jobset/default.nix
+                chown -R hydra /run/jobset
+        """
+        )
+
+        # Wait until InfluxDB can receive web requests
+        machine.wait_for_job("influxdb")
+        machine.wait_for_open_port(8086)
+
+        # Create an InfluxDB database where hydra will write to
+        machine.succeed(
+            "curl -XPOST 'http://127.0.0.1:8086/query' "
+            + "--data-urlencode 'q=CREATE DATABASE hydra'"
+        )
+
+        # Wait until hydra-server can receive HTTP requests
+        machine.wait_for_job("hydra-server")
+        machine.wait_for_open_port(3000)
+
+        # Setup the project and jobset
+        machine.succeed(
+            "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+        )
+
+        # Wait until hydra has build the job and
+        # the InfluxDBNotification plugin uploaded its notification to InfluxDB
+        machine.wait_until_succeeds(
+            "curl -s -H 'Accept: application/csv' "
+            + "-G 'http://127.0.0.1:8086/query?db=hydra' "
+            + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"
+        )
+      '';
+    });
+
+  gitea = forEachSystem (system:
+    let pkgs = pkgsBySystem.${system}; in
+    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
+    makeTest {
+      name = "hydra-gitea";
+      nodes.machine = { pkgs, ... }: {
+        imports = [ hydraServer ];
+        services.hydra-dev.extraConfig = ''
+          <gitea_authorization>
+          root=d7f16a3412e01a43a414535b16007c6931d3a9c7
+          </gitea_authorization>
+        '';
+        nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ];
+        nix = {
+          settings.substituters = [ ];
+        };
+        services.gitea = {
+          enable = true;
+          database.type = "postgres";
+          settings = {
+            service.DISABLE_REGISTRATION = true;
+            server.HTTP_PORT = 3001;
+          };
+        };
+        services.openssh.enable = true;
+        environment.systemPackages = with pkgs; [ gitea git jq gawk ];
+        networking.firewall.allowedTCPPorts = [ 3000 ];
+      };
+      skipLint = true;
+      testScript =
+        let
+          scripts.mktoken = pkgs.writeText "token.sql" ''
+            INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');
+          '';
+
+          scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
+            set -x
+            mkdir -p /tmp/repo $HOME/.ssh
+            cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
+            chmod 0400 $HOME/.ssh/privk
+            git -C /tmp/repo init
+            cp ${smallDrv} /tmp/repo/jobset.nix
+            git -C /tmp/repo add .
+            git config --global user.email test@localhost
+            git config --global user.name test
+            git -C /tmp/repo commit -m 'Initial import'
+            git -C /tmp/repo remote add origin gitea@machine:root/repo
+            GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
+              git -C /tmp/repo push origin master
+            git -C /tmp/repo log >&2
+          '';
+
+          scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
+            set -x
+            su -l hydra -c "hydra-create-user root --email-address \
+              'alice@example.org' --password foobar --role admin"
+
+            URL=http://localhost:3000
+            USERNAME="root"
+            PASSWORD="foobar"
+            PROJECT_NAME="trivial"
+            JOBSET_NAME="trivial"
+            mycurl() {
+              curl --referer $URL -H "Accept: application/json" \
+                -H "Content-Type: application/json" $@
+            }
+
+            cat >data.json <<EOF
+            { "username": "$USERNAME", "password": "$PASSWORD" }
+            EOF
+            mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
+
+            cat >data.json <<EOF
+            {
+              "displayname":"Trivial",
+              "enabled":"1",
+              "visible":"1"
+            }
+            EOF
+            mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
+              -d @data.json -b hydra-cookie.txt
+
+            cat >data.json <<EOF
+            {
+              "description": "Trivial",
+              "checkinterval": "60",
+              "enabled": "1",
+              "visible": "1",
+              "keepnr": "1",
+              "enableemail": true,
+              "emailoverride": "hydra@localhost",
+              "type": 0,
+              "nixexprinput": "git",
+              "nixexprpath": "jobset.nix",
+              "inputs": {
+                "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
+                "gitea_repo_name": {"value": "repo", "type": "string"},
+                "gitea_repo_owner": {"value": "root", "type": "string"},
+                "gitea_status_repo": {"value": "git", "type": "string"},
+                "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
+              }
+            }
+            EOF
+
+            mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
+              -d @data.json -b hydra-cookie.txt
+          '';
+
+          api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
+
+          snakeoilKeypair = {
+            privkey = pkgs.writeText "privkey.snakeoil" ''
+              -----BEGIN EC PRIVATE KEY-----
+              MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
+              AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
+              r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
+              -----END EC PRIVATE KEY-----
+            '';
+
+            pubkey = pkgs.lib.concatStrings [
+              "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
+              "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
+              "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
+            ];
+          };
+
+          smallDrv = pkgs.writeText "jobset.nix" ''
+            { trivial = builtins.derivation {
+                name = "trivial";
+                system = "${system}";
+                builder = "/bin/sh";
+                allowSubstitutes = false;
+                preferLocalBuild = true;
+                args = ["-c" "echo success > $out; exit 0"];
+              };
+             }
+          '';
+        in
+        ''
+          import json
+
+          machine.start()
+          machine.wait_for_unit("multi-user.target")
+          machine.wait_for_open_port(3000)
+          machine.wait_for_open_port(3001)
+
+          machine.succeed(
+              "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create "
+              + "--username root --password root --email test@localhost'"
+          )
+          machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
+          )
+
+          machine.succeed(
+              "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+              + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
+          )
+
+          machine.succeed(
+              "${scripts.git-setup}"
+          )
+
+          machine.succeed(
+              "${scripts.hydra-setup}"
+          )
+
+          machine.wait_until_succeeds(
+              'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
+              + '|  jq .buildstatus | xargs test 0 -eq'
+          )
+
+          data = machine.succeed(
+              'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
+              + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+              + f"-H 'Authorization: token ${api_token}'"
+          )
+
+          response = json.loads(data)
+
+          assert len(response) == 2, "Expected exactly three status updates for latest commit (queued, finished)!"
+          assert response[0]['status'] == "success", "Expected finished status to be success!"
+          assert response[1]['status'] == "pending", "Expected queued status to be pending!"
+
+          machine.shutdown()
+        '';
+    });
+
+  validate-openapi = forEachSystem (system:
+    let pkgs = pkgsBySystem.${system}; in
+    pkgs.runCommand "validate-openapi"
+    { buildInputs = [ pkgs.openapi-generator-cli ]; }
+    ''
+      openapi-generator-cli validate -i ${./hydra-api.yaml}
+      touch $out
+    '');
+
+}

From e149da7b9bbc04bd0b1ca03fa0768e958cbcd40e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 3 May 2024 12:41:17 -0400
Subject: [PATCH 163/285] Try again to ensure hydra module is usable

Nixpkgs only contains a `hydra_unstable`, not `hydra`, package, so
adjust the default accordingly, and then override it to our package in
the separate module which does that.
---
 flake.nix                 | 2 +-
 nixos-modules/default.nix | 3 ++-
 nixos-modules/hydra.nix   | 4 ++--
 nixos-tests.nix           | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index 7e2f3f67..1f73e3b3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -76,7 +76,7 @@
         modules =
           [
             self.nixosModules.hydra
-            self.nixosModules.overlayNixpkgsForThisHyydra
+            self.nixosModules.overlayNixpkgsForThisHydra
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {
diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix
index f44d7808..dac705c3 100644
--- a/nixos-modules/default.nix
+++ b/nixos-modules/default.nix
@@ -3,8 +3,9 @@
 {
   hydra = import ./hydra.nix;
 
-  overlayNixpkgsForThisHyydra = {
+  overlayNixpkgsForThisHydra = { pkgs, ... }: {
     nixpkgs = { inherit overlays; };
+    services.hydra.package = pkgs.hydra;
   };
 
   hydraTest = { pkgs, ... }: {
diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 1f0792d7..7a0486c1 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -68,7 +68,7 @@ in
 
       package = mkOption {
         type = types.path;
-        default = pkgs.hydra;
+        default = pkgs.hydra_unstable;
         defaultText = literalExpression "pkgs.hydra";
         description = "The Hydra package.";
       };
@@ -233,7 +233,7 @@ in
       gc-keep-outputs = true;
       gc-keep-derivations = true;
     };
-    
+
     services.hydra-dev.extraConfig =
       ''
         using_frontend_proxy = 1
diff --git a/nixos-tests.nix b/nixos-tests.nix
index 3c9dc6c8..19a9ba35 100644
--- a/nixos-tests.nix
+++ b/nixos-tests.nix
@@ -7,7 +7,7 @@ let
     {
       imports = [
         nixosModules.hydra
-        nixosModules.overlayNixpkgsForThisHyydra
+        nixosModules.overlayNixpkgsForThisHydra
         nixosModules.hydraTest
       ];
 

From 5e910fa2cec113642ded1294c0c47100670ef587 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 8 May 2024 11:25:14 -0400
Subject: [PATCH 164/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/00ca2b05b8fbbef09be5d1e4820857605d4c31b6' (2024-05-03)
  → 'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index aa29e826..8b3c146d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,11 +80,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1714701944,
-        "narHash": "sha256-trTxWfGElp0rkjquqG5I5RYVoxo8foCflxJFUtHwnOQ=",
+        "lastModified": 1715180119,
+        "narHash": "sha256-6tdv6d0lzpS+EeUTRZ15vS0YdcSpkmRBrJ5ABHC0QiU=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "00ca2b05b8fbbef09be5d1e4820857605d4c31b6",
+        "rev": "0930058189f350a3729cd5aef2ffc8dae2ad436e",
         "type": "github"
       },
       "original": {

From f99cdaf5fe1d70a00ea80d3cd6cbe29e6172b7eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <Mic92@users.noreply.github.com>
Date: Wed, 8 May 2024 21:31:32 +0200
Subject: [PATCH 165/285] README: update wiki link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2a085325..58373052 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ You can also interface with Hydra through a JSON API. The API is defined in [hyd
 ## Additional Resources
 
 - [Hydra User's Guide](https://nixos.org/hydra/manual/)
-- [Hydra on the NixOS Wiki](https://nixos.wiki/wiki/Hydra)
+- [Hydra on the NixOS Wiki](https://wiki.nixos.org/wiki/Hydra)
 - [hydra-cli](https://github.com/nlewo/hydra-cli)
 - [Peter Simons - Hydra: Setting up your own build farm (NixOS)](https://www.youtube.com/watch?v=RXV0Y5Bn-QQ)
 

From e4f2c84f8dfd2863e40c74a35fb90b35749fbc94 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 17 May 2024 20:02:54 -0400
Subject: [PATCH 166/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/0930058189f350a3729cd5aef2ffc8dae2ad436e' (2024-05-08)
  → 'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8b3c146d..4155cb06 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,11 +80,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1715180119,
-        "narHash": "sha256-6tdv6d0lzpS+EeUTRZ15vS0YdcSpkmRBrJ5ABHC0QiU=",
+        "lastModified": 1715986148,
+        "narHash": "sha256-mR8FVWkQxl7jmO14WRRYNJdTG0yno9x5VQX/bQp7Mx4=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "0930058189f350a3729cd5aef2ffc8dae2ad436e",
+        "rev": "beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b",
         "type": "github"
       },
       "original": {

From e4552ddf918aa25a89f594413b626dbb64a9214b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 20 May 2024 18:11:37 -0400
Subject: [PATCH 167/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b' (2024-05-17)
  → 'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 4155cb06..cf51e51f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,11 +80,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1715986148,
-        "narHash": "sha256-mR8FVWkQxl7jmO14WRRYNJdTG0yno9x5VQX/bQp7Mx4=",
+        "lastModified": 1716243047,
+        "narHash": "sha256-SfQwsgwgvvGixkRIwZbULsKGKoRUPf9yH+0hdBKkgRo=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "beb3c2bc7ab781c1b8907b647c6e72b72fa9f56b",
+        "rev": "a57abbd143f8ed44e823c3244e93507f64020878",
         "type": "github"
       },
       "original": {

From 71c4e2dc5b834a38e199f7a59b8928cc60c115fd Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 20 May 2024 18:00:16 -0400
Subject: [PATCH 168/285] Dedup more protocol code

Use https://github.com/NixOS/nix/pull/10749
---
 src/hydra-queue-runner/build-remote.cc | 36 +++-----------------------
 1 file changed, 3 insertions(+), 33 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index ad510e1b..b4275ecc 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -267,32 +267,6 @@ static BuildResult performBuild(
     return result;
 }
 
-static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
-    ::Machine::Connection & conn,
-    Store & localStore,
-    StorePathSet & outputs,
-    size_t & totalNarSize
-)
-{
-
-    /* Get info about each output path. */
-    std::map<StorePath, UnkeyedValidPathInfo> infos;
-    conn.to << ServeProto::Command::QueryPathInfos;
-    ServeProto::write(localStore, conn, outputs);
-    conn.to.flush();
-    while (true) {
-        auto storePathS = readString(conn.from);
-        if (storePathS == "") break;
-
-        auto storePath = localStore.parseStorePath(storePathS);
-        auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(localStore, conn);
-        totalNarSize += info.narSize;
-        infos.insert_or_assign(std::move(storePath), std::move(info));
-    }
-
-    return infos;
-}
-
 static void copyPathFromRemote(
     ::Machine::Connection & conn,
     NarMemberDatas & narMembers,
@@ -478,12 +452,6 @@ void State::buildRemote(ref<Store> destStore,
             throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
         }
 
-        // Do not attempt to speak a newer version of the protocol.
-        //
-        // Per https://github.com/NixOS/nix/issues/9584 should be handled as
-        // part of `handshake` in upstream nix.
-        conn.remoteVersion = std::min(conn.remoteVersion, our_version);
-
         {
             auto info(machine->state->connectInfo.lock());
             info->consecutiveFailures = 0;
@@ -552,8 +520,10 @@ void State::buildRemote(ref<Store> destStore,
 
             auto now1 = std::chrono::steady_clock::now();
 
+            auto infos = conn.queryPathInfos(*localStore, outputs);
+
             size_t totalNarSize = 0;
-            auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize);
+            for (auto & [_, info] : infos) totalNarSize += info.narSize;
 
             if (totalNarSize > maxOutputSize) {
                 result.stepStatus = bsNarSizeLimitExceeded;

From 21044bc4d9d0373366ca189f27066124f526b3b8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 May 2024 18:01:30 -0400
Subject: [PATCH 169/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/8f42912c80c0a03f62f6a3d28a3af05a9762565d' (2024-01-30)
  → 'github:NixOS/nix/ab48ea416a203e9ccefb70aa634e27477e4c1ac4' (2024-05-15)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index d8b2e4a9..3c4c0f75 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1706637536,
-        "narHash": "sha256-fjx+nCOzuSxGWfhwWWc8hCsLFZAjZLDDUcbBtldRqbk=",
+        "lastModified": 1715805674,
+        "narHash": "sha256-0CIBMECsA3ISJZrJcOTzi6wa3QENTKGLtOpYIOoxwxo=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "8f42912c80c0a03f62f6a3d28a3af05a9762565d",
+        "rev": "ab48ea416a203e9ccefb70aa634e27477e4c1ac4",
         "type": "github"
       },
       "original": {

From 879ceb5cdc88096004882402d333f27d2afac577 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 May 2024 18:23:35 -0400
Subject: [PATCH 170/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/60824fa97c588a0faf68ea61260a47e388b0a4e5' (2024-04-11)
  → 'github:NixOS/nix/1ebc34e9c54b740ea4f4466443047d709dccf5b2' (2024-05-16)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 966431f9..a4be7433 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1712849398,
-        "narHash": "sha256-10z/SoidVl9/lh56cMLj7ntJZHtVrumFvmn1YEqXmaM=",
+        "lastModified": 1715845907,
+        "narHash": "sha256-1OigUcZGDInTVZJBTioo9vwRt70yvcfAkSRUeAD/mfg=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "60824fa97c588a0faf68ea61260a47e388b0a4e5",
+        "rev": "1ebc34e9c54b740ea4f4466443047d709dccf5b2",
         "type": "github"
       },
       "original": {

From 346badc66ff68bccb20ccf9f48964da7bf60f2cf Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 22 May 2024 22:00:38 -0400
Subject: [PATCH 171/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/a57abbd143f8ed44e823c3244e93507f64020878' (2024-05-20)
  → 'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index cf51e51f..6a6721b7 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,11 +80,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1716243047,
-        "narHash": "sha256-SfQwsgwgvvGixkRIwZbULsKGKoRUPf9yH+0hdBKkgRo=",
+        "lastModified": 1716420255,
+        "narHash": "sha256-F8CpJTkbVl805LO5nvU5ivBdIHjcFUg8uk1EePGPu8s=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "a57abbd143f8ed44e823c3244e93507f64020878",
+        "rev": "5845fd59c34198ad52a7f7bcb6d3ea7176ca437b",
         "type": "github"
       },
       "original": {

From d55bea2a1edf67645ace7221c3be4af192d198f3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 May 2024 13:34:30 -0400
Subject: [PATCH 172/285] Utilize `nix::Machine` more fully

With https://github.com/NixOS/nix/pull/9839, the `storeUri` field is
much better structured, so we can use it while still opening the SSH
connection ourselves.
---
 src/hydra-queue-runner/build-remote.cc       | 60 +++++++++++---------
 src/hydra-queue-runner/builder.cc            | 12 ++--
 src/hydra-queue-runner/dispatcher.cc         |  2 +-
 src/hydra-queue-runner/hydra-queue-runner.cc | 18 +++---
 src/hydra-queue-runner/state.hh              | 13 +----
 5 files changed, 49 insertions(+), 56 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index b4275ecc..617cce44 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -21,28 +21,23 @@
 
 using namespace nix;
 
-namespace nix::build_remote {
-
-static Strings extraStoreArgs(std::string & machine)
+bool ::Machine::isLocalhost() const
 {
-    Strings result;
-    try {
-        auto parsed = parseURL(machine);
-        if (parsed.scheme != "ssh") {
-            throw SysError("Currently, only (legacy-)ssh stores are supported!");
-        }
-        machine = parsed.authority.value_or("");
-        auto remoteStore = parsed.query.find("remote-store");
-        if (remoteStore != parsed.query.end()) {
-            result = {"--store", shellEscape(remoteStore->second)};
-        }
-    } catch (BadURL &) {
-        // We just try to continue with `machine->sshName` here for backwards compat.
-    }
-
-    return result;
+    return storeUri.params.empty() && std::visit(overloaded {
+        [](const StoreReference::Auto &) {
+            return true;
+        },
+        [](const StoreReference::Specified & s) {
+            return
+               (s.scheme == "local" || s.scheme == "unix") ||
+               ((s.scheme == "ssh" || s.scheme == "ssh-ng") &&
+                s.authority == "localhost");
+        },
+    }, storeUri.variant);
 }
 
+namespace nix::build_remote {
+
 static std::unique_ptr<SSHMaster::Connection> openConnection(
     ::Machine::ptr machine, SSHMaster & master)
 {
@@ -51,7 +46,11 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
         command.push_back("--builders");
         command.push_back("");
     } else {
-        command.splice(command.end(), extraStoreArgs(machine->sshName));
+        auto remoteStore = machine->storeUri.params.find("remote-store");
+        if (remoteStore != machine->storeUri.params.end()) {
+            command.push_back("--store");
+            command.push_back(shellEscape(remoteStore->second));
+        }
     }
 
     return master.startCommand(std::move(command), {
@@ -187,7 +186,7 @@ static BasicDerivation sendInputs(
         MaintainCount<counter> mc2(nrStepsCopyingTo);
 
         printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
-            localStore.printStorePath(step.drvPath), conn.machine->sshName);
+            localStore.printStorePath(step.drvPath), conn.machine->storeUri.render());
 
         auto now1 = std::chrono::steady_clock::now();
 
@@ -393,8 +392,13 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssConnecting);
 
+        auto * pSpecified = std::get_if<StoreReference::Specified>(&machine->storeUri.variant);
+        if (!pSpecified || pSpecified->scheme != "ssh") {
+            throw Error("Currently, only (legacy-)ssh stores are supported!");
+        }
+
         SSHMaster master {
-            machine->sshName,
+            pSpecified->authority,
             machine->sshKey,
             machine->sshPublicHostKey,
             false, // no SSH master yet
@@ -445,11 +449,11 @@ void State::buildRemote(ref<Store> destStore,
                 conn.to,
                 conn.from,
                 our_version,
-                machine->sshName);
+                machine->storeUri.render());
         } catch (EndOfFile & e) {
             child->sshPid.wait();
             std::string s = chomp(readFile(result.logFile));
-            throw Error("cannot connect to ‘%1%’: %2%", machine->sshName, s);
+            throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s);
         }
 
         {
@@ -480,7 +484,7 @@ void State::buildRemote(ref<Store> destStore,
         /* Do the build. */
         printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
             localStore->printStorePath(step->drvPath),
-            machine->sshName);
+            machine->storeUri.render());
 
         updateStep(ssBuilding);
 
@@ -503,7 +507,7 @@ void State::buildRemote(ref<Store> destStore,
            get a build log. */
         if (result.isCached) {
             printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
-                localStore->printStorePath(step->drvPath), machine->sshName);
+                localStore->printStorePath(step->drvPath), machine->storeUri.render());
             unlink(result.logFile.c_str());
             result.logFile = "";
         }
@@ -532,7 +536,7 @@ void State::buildRemote(ref<Store> destStore,
 
             /* Copy each path. */
             printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
-                localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
+                localStore->printStorePath(step->drvPath), machine->storeUri.render(), totalNarSize);
 
             build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos);
             auto now2 = std::chrono::steady_clock::now();
@@ -571,7 +575,7 @@ void State::buildRemote(ref<Store> destStore,
             info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
             info->lastFailure = now;
             int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
-            printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
+            printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta);
             info->disabledUntil = now + std::chrono::seconds(delta);
         }
         throw;
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 5269febd..a0773511 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -41,7 +41,7 @@ void State::builder(MachineReservation::ptr reservation)
         } catch (std::exception & e) {
             printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
                 localStore->printStorePath(reservation->step->drvPath),
-                reservation->machine->sshName,
+                reservation->machine->storeUri.render(),
                 e.what());
         }
     }
@@ -150,7 +150,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
         buildOptions.buildTimeout = build->buildTimeout;
 
         printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
-            localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->sshName, buildId, (dependents.size() - 1));
+            localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1));
     }
 
     if (!buildOneDone)
@@ -196,7 +196,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
         {
             auto mc = startDbUpdate();
             pqxx::work txn(*conn);
-            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy);
+            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy);
             txn.commit();
         }
 
@@ -253,7 +253,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
     /* Finish the step in the database. */
     if (stepNr) {
         pqxx::work txn(*conn);
-        finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
+        finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render());
         txn.commit();
     }
 
@@ -261,7 +261,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
        issue). Retry a number of times. */
     if (result.canRetry) {
         printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s",
-            localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg);
+            localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg);
         assert(stepNr);
         bool retry;
         {
@@ -452,7 +452,7 @@ void State::failStep(
                     build->finishedInDB)
                     continue;
                 createBuildStep(txn,
-                    0, build->id, step, machine ? machine->sshName : "",
+                    0, build->id, step, machine ? machine->storeUri.render() : "",
                     result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId);
             }
 
diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index 6d738ded..0f5a65e7 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -255,7 +255,7 @@ system_time State::doDispatch()
                 /* Can this machine do this step? */
                 if (!mi.machine->supportsStep(step)) {
                     debug("machine '%s' does not support step '%s' (system type '%s')",
-                        mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform);
+                        mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform);
                     continue;
                 }
 
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 5ffa7fe6..d7a88a0d 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -153,8 +153,8 @@ void State::parseMachines(const std::string & contents)
         using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
 
         auto machine = std::make_shared<::Machine>(nix::Machine {
-            // `storeUri`, not yet used
-            "",
+            // `storeUri`
+            tokens[0],
             // `systemTypes`
             tokenizeString<StringSet>(tokens[1], ","),
             // `sshKey`
@@ -175,25 +175,23 @@ void State::parseMachines(const std::string & contents)
                 : "",
         });
 
-        machine->sshName = tokens[0];
-
         /* Re-use the State object of the previous machine with the
            same name. */
-        auto i = oldMachines.find(machine->sshName);
+        auto i = oldMachines.find(machine->storeUri.variant);
         if (i == oldMachines.end())
-            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->sshName);
+            printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render());
         else
-            printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
+            printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render());
         machine->state = i == oldMachines.end()
             ? std::make_shared<::Machine::State>()
             : i->second->state;
-        newMachines[machine->sshName] = machine;
+        newMachines[machine->storeUri.variant] = machine;
     }
 
     for (auto & m : oldMachines)
         if (newMachines.find(m.first) == newMachines.end()) {
             if (m.second->enabled)
-                printInfo("removing machine ‘%1%’", m.first);
+                printInfo("removing machine ‘%1%’", m.second->storeUri.render());
             /* Add a disabled ::Machine object to make sure stats are
                maintained. */
             auto machine = std::make_shared<::Machine>(*(m.second));
@@ -657,7 +655,7 @@ void State::dumpStatus(Connection & conn)
                     machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
                     machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
                 }
-                statusJson["machines"][m->sshName] = machine;
+                statusJson["machines"][m->storeUri.render()] = machine;
             }
         }
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 5d242cdf..e2fb0c9c 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -6,7 +6,6 @@
 #include <map>
 #include <memory>
 #include <queue>
-#include <regex>
 
 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -240,10 +239,6 @@ struct Machine : nix::Machine
 {
     typedef std::shared_ptr<Machine> ptr;
 
-    /* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
-       we are not yet used to, but once we are, we don't need this. */
-    std::string sshName;
-
     struct State {
         typedef std::shared_ptr<State> ptr;
         counter currentJobs{0};
@@ -293,11 +288,7 @@ struct Machine : nix::Machine
         return true;
     }
 
-    bool isLocalhost()
-    {
-        std::regex r("^(ssh://|ssh-ng://)?localhost$");
-        return std::regex_search(sshName, r);
-    }
+    bool isLocalhost() const;
 
     // A connection to a machine
     struct Connection : nix::ServeProto::BasicClientConnection {
@@ -357,7 +348,7 @@ private:
 
     /* The build machines. */
     std::mutex machinesReadyLock;
-    typedef std::map<std::string, Machine::ptr> Machines;
+    typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
     nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
 
     /* Various stats. */

From bede2a141a3c1073d89b7852b08ac58c71d6778b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 23 May 2024 09:59:32 -0400
Subject: [PATCH 173/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/5845fd59c34198ad52a7f7bcb6d3ea7176ca437b' (2024-05-22)
  → 'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 6a6721b7..327d44ad 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,11 +80,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1716420255,
-        "narHash": "sha256-F8CpJTkbVl805LO5nvU5ivBdIHjcFUg8uk1EePGPu8s=",
+        "lastModified": 1716472735,
+        "narHash": "sha256-506pEWmMufBGD9hM99WZynH9Vxkv3o3IJJsPiVAs4T4=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "5845fd59c34198ad52a7f7bcb6d3ea7176ca437b",
+        "rev": "2c42e7b8d9ea32e59c01334852599b548b214d31",
         "type": "github"
       },
       "original": {

From 09a1e64ed2d16ffbbcbc0c98458c621f5d939cf0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 23 May 2024 00:32:11 -0400
Subject: [PATCH 174/285] Dedup with nix: use `nix::Machine::parseConfig`

Companion to https://github.com/NixOS/nix/pull/10763
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 41 +-------------------
 1 file changed, 2 insertions(+), 39 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index d7a88a0d..99411f9f 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -135,45 +135,8 @@ void State::parseMachines(const std::string & contents)
         oldMachines = *machines_;
     }
 
-    for (auto line : tokenizeString<Strings>(contents, "\n")) {
-        line = trim(std::string(line, 0, line.find('#')));
-        auto tokens = tokenizeString<std::vector<std::string>>(line);
-        if (tokens.size() < 3) continue;
-        tokens.resize(8);
-
-        if (tokens[5] == "-") tokens[5] = "";
-        auto supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
-
-        if (tokens[6] == "-") tokens[6] = "";
-        auto mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
-
-        for (auto & f : mandatoryFeatures)
-            supportedFeatures.insert(f);
-
-        using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
-
-        auto machine = std::make_shared<::Machine>(nix::Machine {
-            // `storeUri`
-            tokens[0],
-            // `systemTypes`
-            tokenizeString<StringSet>(tokens[1], ","),
-            // `sshKey`
-            tokens[2] == "-" ? "" : tokens[2],
-            // `maxJobs`
-            tokens[3] != ""
-                ? string2Int<MaxJobs>(tokens[3]).value()
-                : 1,
-            // `speedFactor`
-            atof(tokens[4].c_str()),
-            // `supportedFeatures`
-            std::move(supportedFeatures),
-            // `mandatoryFeatures`
-            std::move(mandatoryFeatures),
-            // `sshPublicHostKey`
-            tokens[7] != "" && tokens[7] != "-"
-                ? base64Decode(tokens[7])
-                : "",
-        });
+    for (auto && machine_ : nix::Machine::parseConfig({}, contents)) {
+        auto machine = std::make_shared<::Machine>(std::move(machine_));
 
         /* Re-use the State object of the previous machine with the
            same name. */

From 5728011da1aae308694a7098e00610b78d49451b Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Thu, 22 Feb 2024 14:45:17 +0100
Subject: [PATCH 175/285] queue-runner: try larger pipe buffer sizes

(cherry picked from commit 18466e83261d39b997a73bbd9f0f249c3a91fbeb)
---
 src/hydra-queue-runner/build-remote.cc | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index ad510e1b..1cabd291 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -54,9 +54,20 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
         command.splice(command.end(), extraStoreArgs(machine->sshName));
     }
 
-    return master.startCommand(std::move(command), {
+    auto ret = master.startCommand(std::move(command), {
         "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
     });
+
+    // XXX: determine the actual max value we can use from /proc.
+
+    // FIXME: Should this be upstreamed into `startCommand` in Nix?
+
+    int pipesize = 1024 * 1024;
+
+    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
+    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
+
+    return ret;
 }
 
 

From 2feddd8511cfa7a06a2c2b31da2a8a6707d81511 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 29 May 2024 17:05:41 -0400
Subject: [PATCH 176/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/2c42e7b8d9ea32e59c01334852599b548b214d31' (2024-05-23)
  → 'github:NixOS/nix/ef5c846e257e1e284ad47ed6be4308d190fe6531' (2024-05-29)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 327d44ad..38b9aded 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,11 +80,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1716472735,
-        "narHash": "sha256-506pEWmMufBGD9hM99WZynH9Vxkv3o3IJJsPiVAs4T4=",
+        "lastModified": 1717016009,
+        "narHash": "sha256-RJ0k7fvKjuUGt439F9uFtUw44SM87A+XKBf6sH6qPy8=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "2c42e7b8d9ea32e59c01334852599b548b214d31",
+        "rev": "ef5c846e257e1e284ad47ed6be4308d190fe6531",
         "type": "github"
       },
       "original": {

From bc19e7cd65e55e39e9c304d108010399ef0987a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sat, 20 Jul 2024 23:45:12 +0200
Subject: [PATCH 177/285] renderInputDiff: Increase git hash length 8 -> 12

See investigation on lengths required to be conflict-free in practice:

https://github.com/NixOS/hydra/pull/1258#issuecomment-1321891677
---
 src/root/common.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/common.tt b/src/root/common.tt
index 4487cbe3..869d8856 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -374,7 +374,7 @@ BLOCK renderInputDiff; %]
               [% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %]
                 [% IF bi1.type == "git" %]
                   <tr><td>
-                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 8) _ ' to ' _ bi2.revision.substr(0, 8)) %]</tt>
+                    <b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderDiffUri contents=(bi1.revision.substr(0, 12) _ ' to ' _ bi2.revision.substr(0, 12)) %]</tt>
                   </td></tr>
                 [% ELSE %]
                   <tr><td>

From ada51d70fc3e7fec52eeeef26b937e4d3ae0529b Mon Sep 17 00:00:00 2001
From: marius david <marius@mariusdavid.fr>
Date: Tue, 23 Jul 2024 22:37:32 +0200
Subject: [PATCH 178/285] Document the default user and port in hacking.md

---
 doc/manual/src/hacking.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 49c17395..de826720 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -30,6 +30,8 @@ foreman:
 $ foreman start
 ```
 
+The Hydra interface will be available on port 63333, with an admin user named "alice" with password "foobar"
+
 You can run just the Hydra web server in your source tree as follows:
 
 ```console

From 578a3d22920c2dc319c3c55ff0b63e899aa15588 Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Wed, 31 Jul 2024 01:19:40 +0200
Subject: [PATCH 179/285] t: increase timeouts for slow commands with high load

We've seen many fails on ofborg, at lot of them ultimately appear to come down to
a timeout being hit, resulting in something like this:

Failure executing slapadd -F /<path>/slap.d -b dc=example -l /<path>/load.ldif.

Hopefully this resolves it for most cases.
I've done some endurance testing and this helps a lot.
some other commands also regularly time-out with high load:

- hydra-init
- hydra-create-user
- nix-store --delete

This should address most issues with tests randomly failing.

Used the following script for endurance testing:

```

import os
import subprocess

run_counter = 0
fail_counter = 0

while True:
    try:
        run_counter += 1
        print(f"Starting run {run_counter}")
        env = os.environ
        env["YATH_JOB_COUNT"] = "20"
        result = subprocess.run(["perl", "t/test.pl"], env=env)
        if (result.returncode != 0):
            fail_counter += 1
        print(f"Finish run {run_counter}, total fail count: {fail_counter}")
    except KeyboardInterrupt:
        print(f"Finished {run_counter} runs with {fail_counter} fails")
        break
```

In case someone else wants to do it on their system :).
Note that YATH_JOB_COUNT may need to be changed loosely based on your
cores.
I only have 4 cores (8 threads), so for others higher numbers might
yield better results in hashing out unstable tests.
---
 t/lib/HydraTestContext.pm                          |  2 +-
 t/lib/LDAPContext.pm                               |  6 +++---
 .../build-locally-with-substitutable-path.t        |  2 +-
 t/scripts/hydra-create-user.t                      | 14 +++++++-------
 t/scripts/hydra-init.t                             |  4 ++--
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index e1a5b226..1d6fa909 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -92,7 +92,7 @@ sub new {
         $opts{'before_init'}->($self);
     }
 
-    expectOkay(5, ("hydra-init"));
+    expectOkay(30, ("hydra-init"));
 
     return $self;
 }
diff --git a/t/lib/LDAPContext.pm b/t/lib/LDAPContext.pm
index 2cd1a19d..df1334f0 100644
--- a/t/lib/LDAPContext.pm
+++ b/t/lib/LDAPContext.pm
@@ -70,7 +70,7 @@ sub add_user {
     my $email = $opts{'email'} // "$name\@example";
     my $password = $opts{'password'} // rand_chars();
 
-    my ($res, $stdout, $stderr) = captureStdoutStderr(1, ("slappasswd", "-s", $password));
+    my ($res, $stdout, $stderr) = captureStdoutStderr(5, ("slappasswd", "-s", $password));
     if ($res) {
         die "Failed to execute slappasswd ($res): $stderr, $stdout";
     }
@@ -178,7 +178,7 @@ sub start {
 sub validateConfig {
     my ($self) = @_;
 
-    expectOkay(1, ("slaptest", "-u", "-F", $self->{"_slapd_dir"}));
+    expectOkay(5, ("slaptest", "-u", "-F", $self->{"_slapd_dir"}));
 }
 
 sub _spawn {
@@ -218,7 +218,7 @@ sub load_ldif {
 
     my $path = "${\$self->{'_tmpdir'}}/load.ldif";
     write_file($path, $content);
-    expectOkay(1, ("slapadd", "-F", $self->{"_slapd_dir"}, "-b", $suffix, "-l", $path));
+    expectOkay(5, ("slapadd", "-F", $self->{"_slapd_dir"}, "-b", $suffix, "-l", $path));
     $self->validateConfig();
 }
 
diff --git a/t/queue-runner/build-locally-with-substitutable-path.t b/t/queue-runner/build-locally-with-substitutable-path.t
index e3b31761..6477635a 100644
--- a/t/queue-runner/build-locally-with-substitutable-path.t
+++ b/t/queue-runner/build-locally-with-substitutable-path.t
@@ -39,7 +39,7 @@ subtest "Building, caching, and then garbage collecting the underlying job" => s
 
     ok(unlink(Hydra::Helper::Nix::gcRootFor($path)), "Unlinking the GC root for underlying Dependency succeeds");
 
-    (my $ret, my $stdout, my $stderr) = captureStdoutStderr(5, "nix-store", "--delete", $path);
+    (my $ret, my $stdout, my $stderr) = captureStdoutStderr(15, "nix-store", "--delete", $path);
     is($ret, 0, "Deleting the underlying dependency should succeed");
 };
 
diff --git a/t/scripts/hydra-create-user.t b/t/scripts/hydra-create-user.t
index 71a5eda3..7f943f9d 100644
--- a/t/scripts/hydra-create-user.t
+++ b/t/scripts/hydra-create-user.t
@@ -9,7 +9,7 @@ my $db = $ctx->db();
 
 subtest "Handling password and password hash creation" => sub {
     subtest "Creating a user with a plain text password (insecure) stores the password securely" => sub {
-        my ($res, $stdout, $stderr) = captureStdoutStderr(5, ("hydra-create-user", "plain-text-user", "--password", "foobar"));
+        my ($res, $stdout, $stderr) = captureStdoutStderr(15, ("hydra-create-user", "plain-text-user", "--password", "foobar"));
         is($res, 0, "hydra-create-user should exit zero");
         like($stderr, qr/Submitting plaintext passwords as arguments is deprecated and will be removed/, "Submitting a plain text password is deprecated.");
 
@@ -23,7 +23,7 @@ subtest "Handling password and password hash creation" => sub {
     };
 
     subtest "Creating a user with a sha1 password (still insecure) stores the password as a hashed sha1" => sub {
-        my ($res, $stdout, $stderr) = captureStdoutStderr(5, ("hydra-create-user", "old-password-hash-user", "--password-hash", "8843d7f92416211de9ebb963ff4ce28125932878"));
+        my ($res, $stdout, $stderr) = captureStdoutStderr(15, ("hydra-create-user", "old-password-hash-user", "--password-hash", "8843d7f92416211de9ebb963ff4ce28125932878"));
         is($res, 0, "hydra-create-user should exit zero");
 
         my $user = $db->resultset('Users')->find({ username => "old-password-hash-user" });
@@ -36,7 +36,7 @@ subtest "Handling password and password hash creation" => sub {
     };
 
     subtest "Creating a user with an argon2 password stores the password as given" => sub {
-        my ($res, $stdout, $stderr) = captureStdoutStderr(5, ("hydra-create-user", "argon2-hash-user", "--password-hash", '$argon2id$v=19$m=262144,t=3,p=1$tMnV5paYjmIrUIb6hylaNA$M8/e0i3NGrjhOliVLa5LqQ'));
+        my ($res, $stdout, $stderr) = captureStdoutStderr(15, ("hydra-create-user", "argon2-hash-user", "--password-hash", '$argon2id$v=19$m=262144,t=3,p=1$tMnV5paYjmIrUIb6hylaNA$M8/e0i3NGrjhOliVLa5LqQ'));
         is($res, 0, "hydra-create-user should exit zero");
 
         my $user = $db->resultset('Users')->find({ username => "argon2-hash-user" });
@@ -50,7 +50,7 @@ subtest "Handling password and password hash creation" => sub {
 
     subtest "Creating a user by prompting for the password" => sub {
         subtest "with the same password twice" => sub {
-            my ($res, $stdout, $stderr) = captureStdoutStderrWithStdin(5, ["hydra-create-user", "prompted-pass-user", "--password-prompt"], "my-password\nmy-password\n");
+            my ($res, $stdout, $stderr) = captureStdoutStderrWithStdin(15, ["hydra-create-user", "prompted-pass-user", "--password-prompt"], "my-password\nmy-password\n");
             is($res, 0, "hydra-create-user should exit zero");
 
             my $user = $db->resultset('Users')->find({ username => "prompted-pass-user" });
@@ -62,7 +62,7 @@ subtest "Handling password and password hash creation" => sub {
         };
 
         subtest "With mismatched password confirmation" => sub {
-            my ($res, $stdout, $stderr) = captureStdoutStderrWithStdin(5, ["hydra-create-user", "prompted-pass-user", "--password-prompt"], "my-password\nnot-my-password\n");
+            my ($res, $stdout, $stderr) = captureStdoutStderrWithStdin(15, ["hydra-create-user", "prompted-pass-user", "--password-prompt"], "my-password\nnot-my-password\n");
             isnt($res, 0, "hydra-create-user should exit non-zero");
         };
     };
@@ -76,7 +76,7 @@ subtest "Handling password and password hash creation" => sub {
         );
 
         for my $case (@cases) {
-            my ($res, $stdout, $stderr) = captureStdoutStderr(5, (
+            my ($res, $stdout, $stderr) = captureStdoutStderr(15, (
                 "hydra-create-user", "bogus-password-options", @{$case}));
             like($stderr, qr/please specify only one of --password-prompt or --password-hash/, "We get an error about specifying the password");
             isnt($res, 0, "hydra-create-user should exit non-zero with conflicting " . join(" ", @{$case}));
@@ -84,7 +84,7 @@ subtest "Handling password and password hash creation" => sub {
     };
 
     subtest "A password is not required for creating a Google-based account" => sub {
-        my ($res, $stdout, $stderr) = captureStdoutStderr(5, (
+        my ($res, $stdout, $stderr) = captureStdoutStderr(15, (
             "hydra-create-user", "google-account", "--type", "google"));
         is($res, 0, "hydra-create-user should exit zero");
     };
diff --git a/t/scripts/hydra-init.t b/t/scripts/hydra-init.t
index bd5bd4bf..603aa4a4 100644
--- a/t/scripts/hydra-init.t
+++ b/t/scripts/hydra-init.t
@@ -28,7 +28,7 @@ subtest "hydra-init upgrades user's password hashes from sha1 to sha1 inside Arg
     $janet->setPassword("foobar");
 
     is($alice->password, "8843d7f92416211de9ebb963ff4ce28125932878", "Alices's sha1 is stored in the database");
-    my ($res, $stdout, $stderr) = captureStdoutStderr(5, ("hydra-init"));
+    my ($res, $stdout, $stderr) = captureStdoutStderr(30, ("hydra-init"));
     if ($res != 0) {
         is($stdout, "");
         is($stderr, "");
@@ -55,7 +55,7 @@ subtest "hydra-init upgrades user's password hashes from sha1 to sha1 inside Arg
     };
 
     subtest "Running hydra-init don't break Alice or Janet's passwords" => sub {
-        my ($res, $stdout, $stderr) = captureStdoutStderr(5, ("hydra-init"));
+        my ($res, $stdout, $stderr) = captureStdoutStderr(30, ("hydra-init"));
         is($res, 0, "hydra-init should exit zero");
 
         my $updatedAlice = $db->resultset('Users')->find({ username => "alice" });

From a6b14369ee05c376deb04dd71062a5b95f186096 Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Wed, 31 Jul 2024 17:10:44 +0200
Subject: [PATCH 180/285] t/test.pl: increase event-timeout, set qvf

Only log issues/failures when something's actually up.
It has irked me for a long time that so much output came
out of running the tests, this seems to silence it.
It does hide some warnings, but I think it makes the output
so much more readable that it's worth the tradeoff.

Helps for highly parallel running of jobs, sometimes they'd not give output for a while.
Setting this timeout higher appears to help.
Not completely sure if this is the right place to do it, but it works fine for me.
---
 t/test.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/test.pl b/t/test.pl
index ba7f3781..12284637 100644
--- a/t/test.pl
+++ b/t/test.pl
@@ -21,7 +21,7 @@ if (defined($ENV{"NIX_BUILD_CORES"})
     print STDERR "test.pl: Defaulting \$YATH_JOB_COUNT to \$NIX_BUILD_CORES (${\$ENV{'NIX_BUILD_CORES'}})\n";
 }
 
-system($^X, find_yath(), '-D', 'test', '--default-search' => './', @ARGV);
+system($^X, find_yath(), '-D', 'test', '--qvf', '--event-timeout', 240, '--default-search' => './', @ARGV);
 my $exit = $?;
 
 # This makes sure it works with prove.

From 54002f0fcf4a7cb65baf3e25e665e5325292f609 Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Wed, 31 Jul 2024 17:12:47 +0200
Subject: [PATCH 181/285] t/evaluator/evaluate-oom-job.t: always skip, the test
 always fails

We should look into how to resolve this, but I tried some things and nothing really worked.
Let's put it skipped for now until someone comes along to improve it.
---
 t/evaluator/evaluate-oom-job.t | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/t/evaluator/evaluate-oom-job.t b/t/evaluator/evaluate-oom-job.t
index 6c17d4e4..8f0450c5 100644
--- a/t/evaluator/evaluate-oom-job.t
+++ b/t/evaluator/evaluate-oom-job.t
@@ -31,6 +31,10 @@ if ($sd_res != 0) {
   skip_all("`systemd-run` returned non-zero when executing `true` (expected 0)");
 }
 
+# XXX(Mindavi): We should think about how to fix this.
+#               Note that it was always skipped on ofborg/h.n.o (nixos hydra) since systemd-run is not present in the ambient environment there.
+skip_all("Always fails, an error about 'oom' being a string is logged and the process never OOMs. Needs a way to use more memory.");
+
 my $ctx = test_context();
 
 # Contain the memory usage to 25 MegaBytes using `systemd-run`

From 2c886f51d3554bfd4246c34cc3dab15a5b6efc1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Sun, 12 Jun 2022 17:57:49 +0200
Subject: [PATCH 182/285] CompressLog: Add zstd compression

---
 nixos-modules/hydra.nix             |  1 +
 src/lib/Hydra/Helper/Nix.pm         |  3 +++
 src/lib/Hydra/Plugin/CompressLog.pm | 21 +++++++++++++++++----
 src/lib/Hydra/View/NixLog.pm        |  5 ++++-
 4 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 7a0486c1..576bc553 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -408,6 +408,7 @@ in
         requires = [ "hydra-init.service" ];
         after = [ "hydra-init.service" ];
         restartTriggers = [ hydraConf ];
+        path = [ pkgs.zstd ];
         environment = env // {
           PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
           HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 16637b2b..bff7a5ed 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -174,6 +174,9 @@ sub getDrvLogPath {
     for ($fn . $bucketed, $fn . $bucketed . ".bz2") {
         return $_ if -f $_;
     }
+    for ($fn . $bucketed, $fn . $bucketed . ".zst") {
+        return $_ if -f $_;
+    }
     return undef;
 }
 
diff --git a/src/lib/Hydra/Plugin/CompressLog.pm b/src/lib/Hydra/Plugin/CompressLog.pm
index 10e8f6cc..fe4d33b0 100644
--- a/src/lib/Hydra/Plugin/CompressLog.pm
+++ b/src/lib/Hydra/Plugin/CompressLog.pm
@@ -9,11 +9,24 @@ use Hydra::Helper::CatalystUtils;
 sub stepFinished {
     my ($self, $step, $logPath) = @_;
 
-    my $doCompress = $self->{config}->{'compress_build_logs'} // "1";
+    my $doCompress = $self->{config}->{'compress_build_logs'} // '1';
+    my $silent = $self->{config}->{'compress_build_logs_silent'} // '0';
+    my $compression = $self->{config}->{'compress_build_logs_compression'} // 'bzip2';
 
-    if ($doCompress eq "1" && -e $logPath) {
-        print STDERR "compressing ‘$logPath’...\n";
-        system("bzip2", "--force", $logPath);
+    if (not -e $logPath or $doCompress ne "1") {
+        return;
+    }
+
+    if ($silent ne '1') {
+        print STDERR "compressing '$logPath' with $compression...\n";
+    }
+
+    if ($compression eq 'bzip2') {
+        system('bzip2', '--force', $logPath);
+    } elsif ($compression eq 'zstd') {
+        system('zstd', '--rm', '--quiet', '-T0', $logPath);
+    } else {
+        print STDERR "unknown compression type '$compression'\n";
     }
 }
 
diff --git a/src/lib/Hydra/View/NixLog.pm b/src/lib/Hydra/View/NixLog.pm
index 7f37ae78..fe37d900 100644
--- a/src/lib/Hydra/View/NixLog.pm
+++ b/src/lib/Hydra/View/NixLog.pm
@@ -16,7 +16,10 @@ sub process {
 
     my $tail = int($c->stash->{tail} // "0");
 
-    if ($logPath =~ /\.bz2$/) {
+    if ($logPath =~ /\.zst$/) {
+        my $doTail = $tail ? "| tail -n '$tail'" : "";
+        open($fh, "-|", "zstd -dc < '$logPath' $doTail") or die;
+    } elsif ($logPath =~ /\.bz2$/) {
         my $doTail = $tail ? "| tail -n '$tail'" : "";
         open($fh, "-|", "bzip2 -dc < '$logPath' $doTail") or die;
     } else {

From 99ca560d583239b1bc34485f2f719989642fa697 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Thu, 8 Aug 2024 11:24:16 +0200
Subject: [PATCH 183/285] Use configured compression in hydra-compress-logs
 service

---
 nixos-modules/hydra.nix | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 576bc553..97906cb3 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -459,10 +459,15 @@ in
     # logs automatically after a step finishes, but this doesn't work
     # if the queue runner is stopped prematurely.
     systemd.services.hydra-compress-logs =
-      { path = [ pkgs.bzip2 ];
+      { path = [ pkgs.bzip2 pkgs.zstd ];
         script =
           ''
-            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
+            set -eou pipefail
+            compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
+            if [[ $compression == zstd ]]; then
+              compression="zstd --rm"
+            fi
+            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r $compression --force --quiet
           '';
         startAt = "Sun 01:45";
       };

From b2b2d6e26ce8a212541fca780985216279c3c00c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Thu, 8 Aug 2024 13:33:08 +0200
Subject: [PATCH 184/285] Expand docs with new compression options

---
 doc/manual/src/plugins/README.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/plugins/README.md b/doc/manual/src/plugins/README.md
index 6e46224f..93aa80b4 100644
--- a/doc/manual/src/plugins/README.md
+++ b/doc/manual/src/plugins/README.md
@@ -42,7 +42,7 @@ Sets CircleCI status.
 
 ## Compress build logs
 
-Compresses build logs after a build with bzip2.
+Compresses build logs after a build with bzip2 or zstd.
 
 ### Configuration options
 
@@ -50,6 +50,14 @@ Compresses build logs after a build with bzip2.
 
 Enable log compression
 
+- `compress_build_logs_compression`
+
+Which compression format to use. Valid values are bzip2 (default) and zstd.
+
+- `compress_build_logs_silent`
+
+Whether to compress logs silently.
+
 ### Example
 
 ```xml

From 250780aaf2c185b4b8530ae82dd67dc8e00f4345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 20 Aug 2024 08:15:18 +0200
Subject: [PATCH 185/285] tests: use `like` for testing regexes

This gives us better diagnostics when the test fails.
---
 t/Hydra/Controller/Job/builds.t     | 13 +++++++------
 t/Hydra/Controller/Jobset/http.t    |  2 +-
 t/Hydra/Controller/User/dashboard.t |  2 +-
 t/api-test.t                        |  2 +-
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/t/Hydra/Controller/Job/builds.t b/t/Hydra/Controller/Job/builds.t
index d933fbb8..aa6a36ce 100644
--- a/t/Hydra/Controller/Job/builds.t
+++ b/t/Hydra/Controller/Job/builds.t
@@ -54,13 +54,14 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {
 
 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
     my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
-    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
-    my $metrics = $response->content;
 
-    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
-    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
-    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
-    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
+    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
+
+    my $metrics = $response->content;
+    like($metrics, qr/hydra_job_failed\{.*\} 0/);
+    like($metrics, qr/hydra_job_completion_time\{.*\} [\d]+/);
+    like($metrics, qr/hydra_build_closure_size\{.*\} 96/);
+    like($metrics, qr/hydra_build_output_size\{.*\} 96/);
 };
 
 done_testing;
diff --git a/t/Hydra/Controller/Jobset/http.t b/t/Hydra/Controller/Jobset/http.t
index 4e53949d..1a60715c 100644
--- a/t/Hydra/Controller/Jobset/http.t
+++ b/t/Hydra/Controller/Jobset/http.t
@@ -186,7 +186,7 @@ subtest 'Update jobset "job" to have an invalid input type' => sub {
       })
   );
   ok(!$jobsetupdate->is_success);
-  ok($jobsetupdate->content =~ m/Invalid input type.*valid types:/);
+  like($jobsetupdate->content, qr/Invalid input type.*valid types:/);
 };
 
 
diff --git a/t/Hydra/Controller/User/dashboard.t b/t/Hydra/Controller/User/dashboard.t
index 8a24585d..12d7dd80 100644
--- a/t/Hydra/Controller/User/dashboard.t
+++ b/t/Hydra/Controller/User/dashboard.t
@@ -24,7 +24,7 @@ my $cookie = $login->header("set-cookie");
 my $my_jobs = request(GET '/dashboard/alice/my-jobs-tab', Accept => 'application/json', Cookie => $cookie);
 ok($my_jobs->is_success);
 my $content = $my_jobs->content();
-ok($content =~ /empty_dir/);
+like($content, qr/empty_dir/);
 ok(!($content =~ /fails/));
 ok(!($content =~ /succeed_with_failed/));
 done_testing;
diff --git a/t/api-test.t b/t/api-test.t
index e89d4460..637d412c 100644
--- a/t/api-test.t
+++ b/t/api-test.t
@@ -115,7 +115,7 @@ subtest "evaluation" => sub {
         my $build = decode_json(request_json({ uri => "/build/" . $evals->[0]->{builds}->[0] })->content());
         is($build->{job}, "job", "The build's job name is job");
         is($build->{finished}, 0, "The build isn't finished yet");
-        ok($build->{buildoutputs}->{out}->{path} =~ /\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
+        like($build->{buildoutputs}->{out}->{path}, qr/\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
 
         subtest "search" => sub {
             my $search_project = decode_json(request_json({ uri => "/search/?query=sample" })->content());

From 54a9729a0f257b9bb011973ebcae1a42f6284e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 20 Aug 2024 07:40:22 +0200
Subject: [PATCH 186/285] hacking.md: mention nix develop

---
 doc/manual/src/hacking.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index de826720..1f1761a7 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -15,6 +15,12 @@ and dependencies can be found:
 $ nix-shell
 ```
 
+of when flakes are enabled:
+
+```console
+$ nix develop
+```
+
 To build Hydra, you should then do:
 
 ```console

From 02a514234bcb15e20241d98b2004e0292b4cc160 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 20 Aug 2024 07:40:33 +0200
Subject: [PATCH 187/285] hacking.md: make build parallel

---
 doc/manual/src/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index 1f1761a7..ec96b8c6 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -26,7 +26,7 @@ To build Hydra, you should then do:
 ```console
 [nix-shell]$ autoreconfPhase
 [nix-shell]$ configurePhase
-[nix-shell]$ make
+[nix-shell]$ make -j$(nproc)
 ```
 
 You start a local database, the webserver, and other components with

From b94a7b6d5c56362af9ea85d944f8454d861ec001 Mon Sep 17 00:00:00 2001
From: hacker1024 <hacker1024@users.sourceforge.net>
Date: Wed, 21 Aug 2024 22:13:38 +1000
Subject: [PATCH 188/285] Use Nix::Store and Nix::Utils in NARInfo.pm

These are required for the `signString` and `readFile` subroutines used when signing NARs.
---
 src/lib/Hydra/View/NARInfo.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm
index bf8711a4..801fc06a 100644
--- a/src/lib/Hydra/View/NARInfo.pm
+++ b/src/lib/Hydra/View/NARInfo.pm
@@ -6,6 +6,8 @@ use File::Basename;
 use Hydra::Helper::CatalystUtils;
 use MIME::Base64;
 use Nix::Manifest;
+use Nix::Store;
+use Nix::Utils;
 use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;
 

From 0ead8dc65c5d4505aaf67769f8b89e3a9c359f35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Tue, 27 Aug 2024 17:44:19 +0200
Subject: [PATCH 189/285] default.nix: Drop URL literal

---
 default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/default.nix b/default.nix
index d4c7ec29..b81119c3 100644
--- a/default.nix
+++ b/default.nix
@@ -1,6 +1,6 @@
 # The `default.nix` in flake-compat reads `flake.nix` and `flake.lock` from `src` and
 # returns an attribute set of the shape `{ defaultNix, shellNix }`
 
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
   src = ./.;
 }).defaultNix

From 916531dc9ccee52e6dab256232933fcf6d198158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Tue, 27 Aug 2024 17:34:48 +0200
Subject: [PATCH 190/285] api: Require POST for /api/push

---
 hydra-api.yaml                        | 2 +-
 src/lib/Hydra/Controller/API.pm       | 2 ++
 src/lib/Hydra/Helper/CatalystUtils.pm | 2 +-
 src/root/jobset.tt                    | 1 +
 t/Hydra/Controller/API/checks.t       | 4 ++--
 5 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 623c9082..a2fdea28 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -70,7 +70,7 @@ paths:
                   $ref: '#/components/examples/projects-success'
 
   /api/push:
-    put:
+    post:
       summary: trigger jobsets
       parameters:
         - in: query
diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
index 06f35d4b..cd005551 100644
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -239,6 +239,8 @@ sub triggerJobset {
 sub push : Chained('api') PathPart('push') Args(0) {
     my ($self, $c) = @_;
 
+    requirePost($c);
+
     $c->{stash}->{json}->{jobsetsTriggered} = [];
 
     my $force = exists $c->request->query_params->{force};
diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm
index 2a2ad86f..15d50b1a 100644
--- a/src/lib/Hydra/Helper/CatalystUtils.pm
+++ b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -272,7 +272,7 @@ sub requireAdmin {
 
 sub requirePost {
     my ($c) = @_;
-    error($c, "Request must be POSTed.") if $c->request->method ne "POST";
+    error($c, "Request must be POSTed.", 405) if $c->request->method ne "POST";
 }
 
 
diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 56abdb50..5d8345f9 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -205,6 +205,7 @@
         if (!c) return;
         requestJSON({
           url: "[% HTML.escape(c.uri_for('/api/push', { jobsets = project.name _ ':' _ jobset.name, force = "1" })) %]",
+          type: 'POST',
           success: function(data) {
             bootbox.alert("The jobset has been scheduled for evaluation.");
           }
diff --git a/t/Hydra/Controller/API/checks.t b/t/Hydra/Controller/API/checks.t
index 2b97b489..f0f51f1c 100644
--- a/t/Hydra/Controller/API/checks.t
+++ b/t/Hydra/Controller/API/checks.t
@@ -109,7 +109,7 @@ subtest "/api/push" => sub {
         my $jobsetName = $jobset->name;
         is($jobset->forceeval, undef, "The existing jobset is not set to be forced to eval");
 
-        my $response = request(GET "/api/push?jobsets=$projectName:$jobsetName&force=1");
+        my $response = request(POST "/api/push?jobsets=$projectName:$jobsetName&force=1");
         ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");
 
         my $data = is_json($response);
@@ -128,7 +128,7 @@ subtest "/api/push" => sub {
 
         print STDERR $repo;
 
-        my $response = request(GET "/api/push?repos=$repo&force=1");
+        my $response = request(POST "/api/push?repos=$repo&force=1");
         ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");
 
         my $data = is_json($response);

From f73043378907c2c7e44f633ad764c8bdd1c947d5 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Tue, 27 Aug 2024 17:00:00 +0200
Subject: [PATCH 191/285] Create eval-jobset role and guard /api/push route

---
 doc/manual/src/configuration.md       |  3 ++-
 src/lib/Hydra/Config.pm               |  2 ++
 src/lib/Hydra/Controller/API.pm       |  7 ++++++-
 src/lib/Hydra/Helper/CatalystUtils.pm | 22 ++++++++++++++++++++++
 src/root/user.tt                      |  1 +
 t/Hydra/Config/ldap_role_map.t        |  2 ++
 t/Hydra/Controller/API/checks.t       | 27 ++++++++++++++++++++++++---
 t/Hydra/Controller/User/ldap-legacy.t |  3 ++-
 t/Hydra/Controller/User/ldap.t        |  5 ++++-
 9 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index 4954040c..d370312a 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -208,7 +208,8 @@ Example configuration:
   <role_mapping>
     # Make all users in the hydra_admin group Hydra admins
     hydra_admin = admin
-    # Allow all users in the dev group to restart jobs and cancel builds
+    # Allow all users in the dev group to eval jobsets, restart jobs and cancel builds
+    dev = eval-jobset
     dev = restart-jobs
     dev = cancel-build
   </role_mapping>
diff --git a/src/lib/Hydra/Config.pm b/src/lib/Hydra/Config.pm
index af686fca..6aae5a5e 100644
--- a/src/lib/Hydra/Config.pm
+++ b/src/lib/Hydra/Config.pm
@@ -95,6 +95,7 @@ sub get_legacy_ldap_config {
             "hydra_bump-to-front" => [ "bump-to-front" ],
             "hydra_cancel-build" => [ "cancel-build" ],
             "hydra_create-projects" => [ "create-projects" ],
+            "hydra_eval-jobset" => [ "eval-jobset" ],
             "hydra_restart-jobs" => [ "restart-jobs" ],
         },
     };
@@ -159,6 +160,7 @@ sub valid_roles {
         "bump-to-front",
         "cancel-build",
         "create-projects",
+        "eval-jobset",
         "restart-jobs",
     ];
 }
diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
index cd005551..9f8b7cba 100644
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -248,19 +248,24 @@ sub push : Chained('api') PathPart('push') Args(0) {
     foreach my $s (@jobsets) {
         my ($p, $j) = parseJobsetName($s);
         my $jobset = $c->model('DB::Jobsets')->find($p, $j);
+        requireEvalJobsetPrivileges($c, $jobset->project);
         next unless defined $jobset && ($force || ($jobset->project->enabled && $jobset->enabled));
         triggerJobset($self, $c, $jobset, $force);
     }
 
     my @repos = split /,/, ($c->request->query_params->{repos} // "");
     foreach my $r (@repos) {
-        triggerJobset($self, $c, $_, $force) foreach $c->model('DB::Jobsets')->search(
+        my @jobsets = $c->model('DB::Jobsets')->search(
             { 'project.enabled' => 1, 'me.enabled' => 1 },
             {
                 join => 'project',
                 where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ],
                 order_by => 'me.id DESC'
             });
+        foreach my $jobset (@jobsets) {
+            requireEvalJobsetPrivileges($c, $jobset->project);
+            triggerJobset($self, $c, $jobset, $force)
+        }
     }
 
     $self->status_ok(
diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm
index 15d50b1a..6ccdbc4d 100644
--- a/src/lib/Hydra/Helper/CatalystUtils.pm
+++ b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -15,6 +15,7 @@ our @EXPORT = qw(
     forceLogin requireUser requireProjectOwner requireRestartPrivileges requireAdmin requirePost isAdmin isProjectOwner
     requireBumpPrivileges
     requireCancelBuildPrivileges
+    requireEvalJobsetPrivileges
     trim
     getLatestFinishedEval getFirstEval
     paramToList
@@ -186,6 +187,27 @@ sub isProjectOwner {
          defined $c->model('DB::ProjectMembers')->find({ project => $project, userName => $c->user->username }));
 }
 
+sub hasEvalJobsetRole {
+    my ($c) = @_;
+    return $c->user_exists && $c->check_user_roles("eval-jobset");
+}
+
+sub mayEvalJobset {
+    my ($c, $project) = @_;
+    return
+        $c->user_exists &&
+        (isAdmin($c) ||
+         hasEvalJobsetRole($c) ||
+         isProjectOwner($c, $project));
+}
+
+sub requireEvalJobsetPrivileges {
+    my ($c, $project) = @_;
+    requireUser($c);
+    accessDenied($c, "Only the project members, administrators, and accounts with eval-jobset privileges can perform this operation.")
+        unless mayEvalJobset($c, $project);
+}
+
 sub hasCancelBuildRole {
     my ($c) = @_;
     return $c->user_exists && $c->check_user_roles('cancel-build');
diff --git a/src/root/user.tt b/src/root/user.tt
index 76f85850..04eb6e68 100644
--- a/src/root/user.tt
+++ b/src/root/user.tt
@@ -91,6 +91,7 @@
           [% INCLUDE roleoption mutable=mutable role="restart-jobs" %]
           [% INCLUDE roleoption mutable=mutable role="bump-to-front" %]
           [% INCLUDE roleoption mutable=mutable role="cancel-build" %]
+          [% INCLUDE roleoption mutable=mutable role="eval-jobset" %]
         </p>
       </div>
     </div>
diff --git a/t/Hydra/Config/ldap_role_map.t b/t/Hydra/Config/ldap_role_map.t
index cb1adf46..9287c782 100644
--- a/t/Hydra/Config/ldap_role_map.t
+++ b/t/Hydra/Config/ldap_role_map.t
@@ -57,6 +57,7 @@ subtest "getLDAPConfig" => sub {
                             "hydra_cancel-build" => [ "cancel-build" ],
                             "hydra_create-projects" => [ "create-projects" ],
                             "hydra_restart-jobs" => [ "restart-jobs" ],
+                            "hydra_eval-jobset" => [ "eval-jobset" ],
                         }
                     },
                     "The empty file and set env var make legacy mode active."
@@ -177,6 +178,7 @@ subtest "get_legacy_ldap_config" => sub {
                 "hydra_cancel-build" => [ "cancel-build" ],
                 "hydra_create-projects" => [ "create-projects" ],
                 "hydra_restart-jobs" => [ "restart-jobs" ],
+                "hydra_eval-jobset" => [ "eval-jobset" ],
             }
         },
         "Legacy, default role maps are applied."
diff --git a/t/Hydra/Controller/API/checks.t b/t/Hydra/Controller/API/checks.t
index f0f51f1c..e4c72ff2 100644
--- a/t/Hydra/Controller/API/checks.t
+++ b/t/Hydra/Controller/API/checks.t
@@ -22,9 +22,24 @@ sub is_json {
 }
 
 my $ctx = test_context();
-
 Catalyst::Test->import('Hydra');
 
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'alice@example.com', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+# Login and save cookie for future requests
+my $req = request(POST '/login',
+    Referer => 'http://localhost/',
+    Content => {
+        username => 'alice',
+        password => 'foobar'
+    }
+);
+is($req->code, 302, "The login redirects");
+my $cookie = $req->header("set-cookie");
+
 my $finishedBuilds = $ctx->makeAndEvaluateJobset(
     expression => "one-job.nix",
     build => 1
@@ -109,7 +124,10 @@ subtest "/api/push" => sub {
         my $jobsetName = $jobset->name;
         is($jobset->forceeval, undef, "The existing jobset is not set to be forced to eval");
 
-        my $response = request(POST "/api/push?jobsets=$projectName:$jobsetName&force=1");
+        my $response = request(POST "/api/push?jobsets=$projectName:$jobsetName&force=1",
+            Cookie => $cookie,
+            Referer => 'http://localhost/',
+        );
         ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");
 
         my $data = is_json($response);
@@ -128,7 +146,10 @@ subtest "/api/push" => sub {
 
         print STDERR $repo;
 
-        my $response = request(POST "/api/push?repos=$repo&force=1");
+        my $response = request(POST "/api/push?repos=$repo&force=1",
+            Cookie => $cookie,
+            Referer => 'http://localhost/',
+        );
         ok($response->is_success, "The API enpdoint for triggering jobsets returns 200.");
 
         my $data = is_json($response);
diff --git a/t/Hydra/Controller/User/ldap-legacy.t b/t/Hydra/Controller/User/ldap-legacy.t
index 9cb197c0..19f0c6bf 100644
--- a/t/Hydra/Controller/User/ldap-legacy.t
+++ b/t/Hydra/Controller/User/ldap-legacy.t
@@ -24,6 +24,7 @@ $ldap->add_group("hydra_create-projects", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_restart-jobs", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_bump-to-front", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_cancel-build", $users->{"many_roles"}->{"username"});
+$ldap->add_group("hydra_eval-jobset", $users->{"many_roles"}->{"username"});
 
 my $hydra_ldap_config = "${\$ldap->tmpdir()}/hydra_ldap_config.yaml";
 LDAPContext::write_file($hydra_ldap_config, <<YAML);
@@ -68,7 +69,7 @@ subtest "Valid login attempts" => sub {
         unrelated => [],
          admin => ["admin"],
          not_admin => [],
-         many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build" ],
+         many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build", "eval-jobset" ],
     );
     for my $username (keys %users_to_roles) {
         my $user = $users->{$username};
diff --git a/t/Hydra/Controller/User/ldap.t b/t/Hydra/Controller/User/ldap.t
index 175b66aa..050fde23 100644
--- a/t/Hydra/Controller/User/ldap.t
+++ b/t/Hydra/Controller/User/ldap.t
@@ -24,6 +24,7 @@ $ldap->add_group("hydra_create-projects", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_restart-jobs", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_bump-to-front", $users->{"many_roles"}->{"username"});
 $ldap->add_group("hydra_cancel-build", $users->{"many_roles"}->{"username"});
+$ldap->add_group("hydra_eval-jobset", $users->{"many_roles"}->{"username"});
 
 
 my $ctx = test_context(
@@ -76,10 +77,12 @@ my $ctx = test_context(
                 hydra_cancel-build = cancel-build
                 hydra_bump-to-front = bump-to-front
                 hydra_restart-jobs = restart-jobs
+                hydra_eval-jobset = eval-jobset
 
                 hydra_one_group_many_roles = create-projects
                 hydra_one_group_many_roles = cancel-build
                 hydra_one_group_many_roles = bump-to-front
+                hydra_one_group_many-roles = eval-jobset
             </role_mapping>
         </ldap>
 CFG
@@ -92,7 +95,7 @@ subtest "Valid login attempts" => sub {
         unrelated => [],
         admin => ["admin"],
         not_admin => [],
-        many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build" ],
+        many_roles => [ "create-projects", "restart-jobs", "bump-to-front", "cancel-build", "eval-jobset" ],
         many_roles_one_group => [ "create-projects", "bump-to-front", "cancel-build" ],
     );
     for my $username (keys %users_to_roles) {

From 2dad87ad89ee6e483516c9bc0da21c60977fe024 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Fri, 20 Sep 2024 07:50:24 +0200
Subject: [PATCH 192/285] hydra-queue-runner: fix compilation warning

instead of converting to double, we can convert to float right away.
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 5ffa7fe6..03b322f8 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -164,7 +164,7 @@ void State::parseMachines(const std::string & contents)
                 ? string2Int<MaxJobs>(tokens[3]).value()
                 : 1,
             // `speedFactor`
-            atof(tokens[4].c_str()),
+            std::stof(tokens[4].c_str()),
             // `supportedFeatures`
             std::move(supportedFeatures),
             // `mandatoryFeatures`

From ae787e5799cc637824b1101ca9edb04ade1c0610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Fri, 20 Sep 2024 08:49:32 +0200
Subject: [PATCH 193/285] bump nix to 2.24

---
 flake.lock  | 108 ++++++++++++++++++++++++++--------------------------
 flake.nix   |   5 ++-
 package.nix |   3 +-
 3 files changed, 60 insertions(+), 56 deletions(-)

diff --git a/flake.lock b/flake.lock
index e828aa7b..246b5380 100644
--- a/flake.lock
+++ b/flake.lock
@@ -37,33 +37,50 @@
         "type": "github"
       }
     },
-    "flake-utils": {
+    "git-hooks-nix": {
+      "inputs": {
+        "flake-compat": [
+          "nix"
+        ],
+        "gitignore": [
+          "nix"
+        ],
+        "nixpkgs": [
+          "nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nix",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "lastModified": 1726745158,
+        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
         "type": "github"
       }
     },
     "libgit2": {
       "flake": false,
       "locked": {
-        "lastModified": 1697646580,
-        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "lastModified": 1715853528,
+        "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
         "owner": "libgit2",
         "repo": "libgit2",
-        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
         "type": "github"
       },
       "original": {
         "owner": "libgit2",
+        "ref": "v1.8.1",
         "repo": "libgit2",
         "type": "github"
       }
@@ -72,44 +89,61 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-parts": "flake-parts",
+        "git-hooks-nix": "git-hooks-nix",
         "libgit2": "libgit2",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "nixpkgs-regression": "nixpkgs-regression",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "nixpkgs-23-11": "nixpkgs-23-11",
+        "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1713874370,
-        "narHash": "sha256-gW1mO/CvsQQ5gvgiwzxsGhPFI/tx30NING+qgF5Do0s=",
+        "lastModified": 1726787955,
+        "narHash": "sha256-XFznzb8L4SdUm9u+w3DPpMWJhffuv+/6+aiVl00slns=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "1c8150ac312b5f9ba1b3f6768ff43b09867e5883",
+        "rev": "a7fdef6858dd45b9d7bda7c92324c63faee7f509",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.22-maintenance",
+        "ref": "2.24-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1712848736,
-        "narHash": "sha256-CzZwhqyLlebljv1zFS2KWVH/3byHND0LfaO1jKsGuVo=",
+        "lastModified": 1726688310,
+        "narHash": "sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "1d6a23f11e44d0fb64b3237569b87658a9eb5643",
+        "rev": "dbebdd67a6006bb145d98c8debf9140ac7e651d0",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11-small",
+        "ref": "nixos-24.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "nixpkgs-23-11": {
+      "locked": {
+        "lastModified": 1717159533,
+        "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -126,38 +160,6 @@
         "type": "github"
       }
     },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": [
-          "nix"
-        ],
-        "flake-utils": "flake-utils",
-        "gitignore": [
-          "nix"
-        ],
-        "nixpkgs": [
-          "nix",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712897695,
-        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "nix": "nix",
diff --git a/flake.nix b/flake.nix
index 1f73e3b3..19381efd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,8 +1,8 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.22-maintenance";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
+  inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
@@ -25,6 +25,7 @@
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
           rawSrc = self;
+          nix-perl-bindings = final.nixComponents.nix-perl-bindings;
         };
       };
 
diff --git a/package.nix b/package.nix
index f8b1849f..73dd0f2a 100644
--- a/package.nix
+++ b/package.nix
@@ -9,6 +9,7 @@
 , perlPackages
 
 , nix
+, nix-perl-bindings
 , git
 
 , makeWrapper
@@ -59,7 +60,7 @@ let
     name = "hydra-perl-deps";
     paths = lib.closePropagation
       ([
-        nix.perl-bindings
+        nix-perl-bindings
         git
       ] ++ (with perlPackages; [
         AuthenSASL

From 0231453cc5d21a802a630ca0e0ebaab5738f7b82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Fri, 20 Sep 2024 08:49:46 +0200
Subject: [PATCH 194/285] hydra-eval-jobs: fix build against nix 2.24

---
 src/hydra-eval-jobs/Makefile.am        |  2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 14 +++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am
index 90742a30..fb67f89b 100644
--- a/src/hydra-eval-jobs/Makefile.am
+++ b/src/hydra-eval-jobs/Makefile.am
@@ -1,5 +1,5 @@
 bin_PROGRAMS = hydra-eval-jobs
 
 hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
-hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd
+hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd -lnixflake
 hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 5adea42b..b83cae91 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -6,6 +6,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "eval.hh"
+#include "eval-gc.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "signals.hh"
@@ -125,11 +126,14 @@ static void worker(
     if (myArgs.flake) {
         using namespace flake;
 
-        auto flakeRef = parseFlakeRef(myArgs.releaseExpr);
+        auto [flakeRef, fragment, outputSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(fetchSettings, myArgs.releaseExpr, absPath("."));
 
         auto vFlake = state.allocValue();
 
-        auto lockedFlake = lockFlake(state, flakeRef,
+        auto lockedFlake = lockFlake(
+                flakeSettings,
+                state,
+                flakeRef,
             LockFlags {
                 .updateLockFile = false,
                 .useRegistries = false,
@@ -368,7 +372,11 @@ int main(int argc, char * * argv)
                             ]()
                             {
                                 try {
-                                    EvalState state(myArgs.lookupPath, openStore());
+                                    auto evalStore = myArgs.evalStoreUrl
+                                             ? openStore(*myArgs.evalStoreUrl)
+                                             : openStore();
+                                    EvalState state(myArgs.lookupPath,
+                                            evalStore, fetchSettings, evalSettings);
                                     Bindings & autoArgs = *myArgs.getAutoArgs(state);
                                     worker(state, autoArgs, *to, *from);
                                 } catch (Error & e) {

From b472f55563285ae9ecfa1322530a31a81cc3d42d Mon Sep 17 00:00:00 2001
From: zowoq <59103226+zowoq@users.noreply.github.com>
Date: Sun, 8 Sep 2024 12:15:34 +1000
Subject: [PATCH 195/285] set a default for hydra-compress-logs service

follow up from 99ca560d583239b1bc34485f2f719989642fa697
---
 nixos-modules/hydra.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 97906cb3..d001675d 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -464,10 +464,12 @@ in
           ''
             set -eou pipefail
             compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
-            if [[ $compression == zstd ]]; then
+            if [[ $compression == "" ]]; then
+              compression="bzip2"
+            elif [[ $compression == zstd ]]; then
               compression="zstd --rm"
             fi
-            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r $compression --force --quiet
+            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
           '';
         startAt = "Sun 01:45";
       };

From cc1b6d394e2e65575b29093b6a63af7a7b30e4ec Mon Sep 17 00:00:00 2001
From: Michael Bishop <cleverca22@gmail.com>
Date: Fri, 20 Sep 2024 04:08:22 -0300
Subject: [PATCH 196/285] fix the binary cache calling isValidPath, and 2 other
 files with the same issue

---
 src/lib/Hydra/Controller/Root.pm        | 2 +-
 src/lib/Hydra/Plugin/S3Backup.pm        | 2 +-
 src/lib/Hydra/Plugin/SubversionInput.pm | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index 47389251..75d35ac3 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -329,7 +329,7 @@ sub nar :Local :Args(1) {
     else {
         $path = $Nix::Config::storeDir . "/$path";
 
-        gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+        gone($c, "Path " . $path . " is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($path);
 
         $c->stash->{current_view} = 'NixNAR';
         $c->stash->{storePath} = $path;
diff --git a/src/lib/Hydra/Plugin/S3Backup.pm b/src/lib/Hydra/Plugin/S3Backup.pm
index 98e79747..a0d67851 100644
--- a/src/lib/Hydra/Plugin/S3Backup.pm
+++ b/src/lib/Hydra/Plugin/S3Backup.pm
@@ -92,7 +92,7 @@ sub buildFinished {
         my $hash = substr basename($path), 0, 32;
         my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($path, 0);
         my $system;
-        if (defined $deriver and isValidPath($deriver)) {
+        if (defined $deriver and $MACHINE_LOCAL_STORE->isValidPath($deriver)) {
             $system = derivationFromPath($deriver)->{platform};
         }
         foreach my $reference (@{$refs}) {
diff --git a/src/lib/Hydra/Plugin/SubversionInput.pm b/src/lib/Hydra/Plugin/SubversionInput.pm
index 83c1f39d..d3579c40 100644
--- a/src/lib/Hydra/Plugin/SubversionInput.pm
+++ b/src/lib/Hydra/Plugin/SubversionInput.pm
@@ -46,7 +46,7 @@ sub fetchInput {
 
     $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {

From 1ef6b5e7b467c67c085facca909f3eafc33d9b55 Mon Sep 17 00:00:00 2001
From: hacker1024 <hacker1024@users.sourceforge.net>
Date: Wed, 21 Aug 2024 22:13:38 +1000
Subject: [PATCH 197/285] Use Nix::Store and Nix::Utils in NARInfo.pm

These are required for the `signString` and `readFile` subroutines used when signing NARs.

(cherry picked from commit b94a7b6d5c56362af9ea85d944f8454d861ec001)
---
 src/lib/Hydra/View/NARInfo.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm
index bf8711a4..801fc06a 100644
--- a/src/lib/Hydra/View/NARInfo.pm
+++ b/src/lib/Hydra/View/NARInfo.pm
@@ -6,6 +6,8 @@ use File::Basename;
 use Hydra::Helper::CatalystUtils;
 use MIME::Base64;
 use Nix::Manifest;
+use Nix::Store;
+use Nix::Utils;
 use Hydra::Helper::Nix;
 use base qw/Catalyst::View/;
 

From 029116422d01772bf532988814f529512ab02efb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 24 Sep 2024 11:28:55 -0400
Subject: [PATCH 198/285] Update to Nix 2.23
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/1c8150ac312b5f9ba1b3f6768ff43b09867e5883' (2024-04-23)
  → 'github:NixOS/nix/5ffd239adc9b7fddca7a2a59a8b87da5af14ec4d' (2024-09-23)
---
 flake.lock                      | 8 ++++----
 flake.nix                       | 2 +-
 src/hydra-queue-runner/state.hh | 1 +
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index e828aa7b..535ca096 100644
--- a/flake.lock
+++ b/flake.lock
@@ -80,16 +80,16 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1713874370,
-        "narHash": "sha256-gW1mO/CvsQQ5gvgiwzxsGhPFI/tx30NING+qgF5Do0s=",
+        "lastModified": 1727132408,
+        "narHash": "sha256-esYY9tT7Q79rG4+Xvf32L3AZvVGgdVeT1n0uK6Gd6ls=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "1c8150ac312b5f9ba1b3f6768ff43b09867e5883",
+        "rev": "5ffd239adc9b7fddca7a2a59a8b87da5af14ec4d",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.22-maintenance",
+        "ref": "2.23-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 1f73e3b3..1acb245a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.22-maintenance";
+  inputs.nix.url = "github:NixOS/nix/2.23-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 5d242cdf..839239fe 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -23,6 +23,7 @@
 #include "nar-extractor.hh"
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
+#include "serve-protocol-connection.hh"
 #include "machines.hh"
 
 

From 012cbd43f555c5657168d667b8018e54f22adc5a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 24 Sep 2024 11:51:17 -0400
Subject: [PATCH 199/285] Add missing include

---
 src/hydra-queue-runner/dispatcher.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index 6d738ded..a4c84252 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -2,6 +2,7 @@
 #include <cmath>
 #include <thread>
 #include <unordered_map>
+#include <unordered_set>
 
 #include "state.hh"
 

From ceb8b48cce4cc21ddb8941ed3858f6ed630361c8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 24 Sep 2024 12:14:23 -0400
Subject: [PATCH 200/285] Fix type error with NAR accesssor

---
 src/hydra-queue-runner/nar-extractor.cc | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc
index 61299ecd..9f94af9c 100644
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -60,26 +60,26 @@ struct Extractor : FileSystemObjectSink
         : members(members), prefix(prefix)
     { }
 
-    void createDirectory(const Path & path) override
+    void createDirectory(const CanonPath & path) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
+        members.insert_or_assign(prefix + path.abs(), NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
-    void createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func) override
+    void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
     {
         NarMemberConstructor nmc {
-            members.insert_or_assign(prefix + path, NarMemberData {
+            members.insert_or_assign(prefix + path.abs(), NarMemberData {
                 .type = SourceAccessor::Type::tRegular,
                 .fileSize = 0,
-                .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
+                .contents = filesToKeep.count(path.abs()) ? std::optional("") : std::nullopt,
             }).first->second,
         };
         func(nmc);
     }
 
-    void createSymlink(const Path & path, const std::string & target) override
+    void createSymlink(const CanonPath & path, const std::string & target) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
+        members.insert_or_assign(prefix + path.abs(), NarMemberData { .type = SourceAccessor::Type::tSymlink });
     }
 };
 

From 750275d6e8740ec7e170e7ac98c68ab3e36f37e6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Oct 2024 11:43:58 -0400
Subject: [PATCH 201/285] Avoid trailing slash that broke lookup

---
 src/hydra-queue-runner/nar-extractor.cc | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc
index 9f94af9c..8729dd39 100644
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -54,7 +54,15 @@ struct Extractor : FileSystemObjectSink
     };
 
     NarMemberDatas & members;
-    Path prefix;
+    std::filesystem::path prefix;
+
+    Path toKey(const CanonPath & path)
+    {
+        std::filesystem::path p = prefix;
+        // Conditional to avoid trailing slash
+        if (!path.isRoot()) p /= path.rel();
+        return p;
+    }
 
     Extractor(NarMemberDatas & members, const Path & prefix)
         : members(members), prefix(prefix)
@@ -62,13 +70,13 @@ struct Extractor : FileSystemObjectSink
 
     void createDirectory(const CanonPath & path) override
     {
-        members.insert_or_assign(prefix + path.abs(), NarMemberData { .type = SourceAccessor::Type::tDirectory });
+        members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
     void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
     {
         NarMemberConstructor nmc {
-            members.insert_or_assign(prefix + path.abs(), NarMemberData {
+            members.insert_or_assign(toKey(path), NarMemberData {
                 .type = SourceAccessor::Type::tRegular,
                 .fileSize = 0,
                 .contents = filesToKeep.count(path.abs()) ? std::optional("") : std::nullopt,
@@ -79,7 +87,7 @@ struct Extractor : FileSystemObjectSink
 
     void createSymlink(const CanonPath & path, const std::string & target) override
     {
-        members.insert_or_assign(prefix + path.abs(), NarMemberData { .type = SourceAccessor::Type::tSymlink });
+        members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tSymlink });
     }
 };
 

From 2a7b070da0ef2b2f23e4331282ba6e013690edb0 Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Thu, 17 Oct 2024 22:18:56 +0200
Subject: [PATCH 202/285] S3Backup: fix compilation issue where
 MACHINE_LOCAL_STORE variable is undefined

See https://github.com/NixOS/hydra/pull/1414#issuecomment-2412350929

The variable is defined in src/lib/Hydra/Helper/Nix.pm

Error message without this patch:

```
hydra-evaluator[PID]: Couldn't require Hydra::Plugin::S3Backup : Global symbol "$MACHINE_LOCAL_STORE" requires explicit package name (did you forget to declare "my $MACHINE_LOCAL_STORE"?) at /nix/store/xxx-hydra-0-unstable-2024-09-24/libexec/hydra/lib/Hydra/Plugin/S3Backup.pm line 95.
hydra-evaluator[PID]: Compilation failed in require at /nix/store/xxx-hydra-perl-deps/lib/perl5/site_perl/5.38.2/Module/Runtime.pm line 314.
hydra-evaluator[PID]:  at /nix/store/xxx-hydra-perl-deps/lib/perl5/site_perl/5.38.2/Module/Pluggable.pm line 32.
```
---
 src/lib/Hydra/Plugin/S3Backup.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib/Hydra/Plugin/S3Backup.pm b/src/lib/Hydra/Plugin/S3Backup.pm
index a0d67851..f1f50754 100644
--- a/src/lib/Hydra/Plugin/S3Backup.pm
+++ b/src/lib/Hydra/Plugin/S3Backup.pm
@@ -14,6 +14,7 @@ use Nix::Config;
 use Nix::Store;
 use Hydra::Model::DB;
 use Hydra::Helper::CatalystUtils;
+use Hydra::Helper::Nix;
 
 sub isEnabled {
     my ($self) = @_;

From 8a54924d2aeffa841a3764b2c916211a8219c34c Mon Sep 17 00:00:00 2001
From: Rick van Schijndel <rol3517@gmail.com>
Date: Thu, 17 Oct 2024 22:25:11 +0200
Subject: [PATCH 203/285] nix-prefetch-git: set branch name to suppress hint
 from git

In my system logs I see this every time a new eval starts:

```
hydra-evaluator[PID]: hint: Using 'master' as the name for the initial branch. This default branch name
hydra-evaluator[PID]: hint: is subject to change. To configure the initial branch name to use in all
hydra-evaluator[PID]: hint: of your new repositories, which will suppress this warning, call:
hydra-evaluator[PID]: hint:
hydra-evaluator[PID]: hint:         git config --global init.defaultBranch <name>
hydra-evaluator[PID]: hint:
hydra-evaluator[PID]: hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and
hydra-evaluator[PID]: hint: 'development'. The just-created branch can be renamed via this command:
hydra-evaluator[PID]: hint:
hydra-evaluator[PID]: hint:         git branch -m <name>
```

This ensures this hint is not logged anymore and unclutters the syslog.
I presume it does not really matter what name is chosen for the branch.
---
 src/script/nix-prefetch-git | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/script/nix-prefetch-git b/src/script/nix-prefetch-git
index 3fe399c8..316eb9b0 100755
--- a/src/script/nix-prefetch-git
+++ b/src/script/nix-prefetch-git
@@ -78,7 +78,7 @@ fi
 
 init_remote(){
     local url=$1;
-    git init;
+    git init --initial-branch=trunk;
     git remote add origin $url;
 }
 

From 1222ba03a66811564c7ef4a9117c09e166087a26 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <mweinelt@users.noreply.github.com>
Date: Sat, 19 Oct 2024 17:30:59 +0200
Subject: [PATCH 204/285] Make the in-tree package the default package

There is an overlay for the `hydra` name, but `hydra_unstable` was used, which can refer to the nixpkgs package and lead to and outdated hydra version and requires configuring the correct package attribute downstream.
---
 nixos-modules/hydra.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index d001675d..6a2390c4 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -68,7 +68,7 @@ in
 
       package = mkOption {
         type = types.path;
-        default = pkgs.hydra_unstable;
+        default = pkgs.hydra;
         defaultText = literalExpression "pkgs.hydra";
         description = "The Hydra package.";
       };

From a9a5b143317dfa343348ecdc80f04e1b4b118d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 19 Oct 2024 18:05:04 +0200
Subject: [PATCH 205/285] make nixos module hydra from this repository by
 default

When people reach out to the git repository they probably want to use
hydra from the same source.
This also removes the need for an overlay with simpler and more
performant direct use of the nixpkgs passed in. Before it was
re-importing nixpkgs.

test
---
 flake.nix                 | 33 +++++++++++++++------------------
 nixos-modules/default.nix | 11 +++++------
 nixos-modules/hydra.nix   |  2 --
 nixos-tests.nix           | 14 ++++++--------
 4 files changed, 26 insertions(+), 34 deletions(-)

diff --git a/flake.nix b/flake.nix
index 19381efd..78024656 100644
--- a/flake.nix
+++ b/flake.nix
@@ -9,14 +9,6 @@
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
-
-      overlayList = [ self.overlays.default nix.overlays.default ];
-
-      pkgsBySystem = forEachSystem (system: import nixpkgs {
-        inherit system;
-        overlays = overlayList;
-      });
-
     in
     rec {
 
@@ -30,7 +22,6 @@
       };
 
       hydraJobs = {
-
         build = forEachSystem (system: packages.${system}.hydra);
 
         buildNoTests = forEachSystem (system:
@@ -39,19 +30,21 @@
           })
         );
 
-        manual = forEachSystem (system:
-          let pkgs = pkgsBySystem.${system}; in
-          pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { }
+        manual = forEachSystem (system: let
+          pkgs = nixpkgs.legacyPackages.${system};
+          hydra = self.packages.${pkgs.hostPlatform.system}.hydra;
+        in
+          pkgs.runCommand "hydra-manual-${hydra.version}" { }
             ''
               mkdir -p $out/share
-              cp -prvd ${pkgs.hydra}/share/doc $out/share/
+              cp -prvd ${hydra}/share/doc $out/share/
 
               mkdir $out/nix-support
               echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
             '');
 
         tests = import ./nixos-tests.nix {
-          inherit forEachSystem nixpkgs pkgsBySystem nixosModules;
+          inherit forEachSystem nixpkgs nixosModules;
         };
 
         container = nixosConfigurations.container.config.system.build.toplevel;
@@ -64,12 +57,17 @@
       });
 
       packages = forEachSystem (system: {
-        hydra = pkgsBySystem.${system}.hydra;
-        default = pkgsBySystem.${system}.hydra;
+        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
+          inherit (nixpkgs.lib) fileset;
+          rawSrc = self;
+          nix = nix.packages.${system}.nix;
+          nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
+        };
+        default = self.packages.${system}.hydra;
       });
 
       nixosModules = import ./nixos-modules {
-        overlays = overlayList;
+        inherit self;
       };
 
       nixosConfigurations.container = nixpkgs.lib.nixosSystem {
@@ -77,7 +75,6 @@
         modules =
           [
             self.nixosModules.hydra
-            self.nixosModules.overlayNixpkgsForThisHydra
             self.nixosModules.hydraTest
             self.nixosModules.hydraProxy
             {
diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix
index dac705c3..62b18406 100644
--- a/nixos-modules/default.nix
+++ b/nixos-modules/default.nix
@@ -1,11 +1,10 @@
-{ overlays }:
+{ self }:
 
 {
-  hydra = import ./hydra.nix;
-
-  overlayNixpkgsForThisHydra = { pkgs, ... }: {
-    nixpkgs = { inherit overlays; };
-    services.hydra.package = pkgs.hydra;
+  hydra = { pkgs, lib,... }: {
+    _file = ./default.nix;
+    imports = [ ./hydra.nix ];
+    services.hydra-dev.package = lib.mkDefault self.packages.${pkgs.hostPlatform.system}.hydra;
   };
 
   hydraTest = { pkgs, ... }: {
diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 6a2390c4..c471b0ef 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -68,8 +68,6 @@ in
 
       package = mkOption {
         type = types.path;
-        default = pkgs.hydra;
-        defaultText = literalExpression "pkgs.hydra";
         description = "The Hydra package.";
       };
 
diff --git a/nixos-tests.nix b/nixos-tests.nix
index 19a9ba35..9efe68c8 100644
--- a/nixos-tests.nix
+++ b/nixos-tests.nix
@@ -1,13 +1,12 @@
-{ forEachSystem, nixpkgs, pkgsBySystem, nixosModules }:
+{ forEachSystem, nixpkgs, nixosModules }:
 
 let
   # NixOS configuration used for VM tests.
   hydraServer =
-    { config, pkgs, ... }:
+    { pkgs, ... }:
     {
       imports = [
         nixosModules.hydra
-        nixosModules.overlayNixpkgsForThisHydra
         nixosModules.hydraTest
       ];
 
@@ -44,11 +43,10 @@ in
     });
 
   notifications = forEachSystem (system:
-    let pkgs = pkgsBySystem.${system}; in
     with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
     simpleTest {
       name = "hydra-notifications";
-      nodes.machine = { pkgs, ... }: {
+      nodes.machine = {
         imports = [ hydraServer ];
         services.hydra-dev.extraConfig = ''
           <influxdb>
@@ -89,7 +87,7 @@ in
 
         # Setup the project and jobset
         machine.succeed(
-            "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+            "su - hydra -c 'perl -I ${config.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
         )
 
         # Wait until hydra has build the job and
@@ -103,7 +101,7 @@ in
     });
 
   gitea = forEachSystem (system:
-    let pkgs = pkgsBySystem.${system}; in
+    let pkgs = nixpkgs.legacyPackages.${system}; in
     with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
     makeTest {
       name = "hydra-gitea";
@@ -298,7 +296,7 @@ in
     });
 
   validate-openapi = forEachSystem (system:
-    let pkgs = pkgsBySystem.${system}; in
+    let pkgs = nixpkgs.legacyPackages.${system}; in
     pkgs.runCommand "validate-openapi"
     { buildInputs = [ pkgs.openapi-generator-cli ]; }
     ''

From f442d74f6e6ea3fdc4109b4051cced263058f09d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 19 Oct 2024 16:46:45 +0000
Subject: [PATCH 206/285] remove unused nix dev flake inputs

---
 flake.lock | 115 +++++------------------------------------------------
 flake.nix  |  12 +++++-
 2 files changed, 20 insertions(+), 107 deletions(-)

diff --git a/flake.lock b/flake.lock
index 246b5380..897785ed 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,73 +1,5 @@
 {
   "nodes": {
-    "flake-compat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1673956053,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712014858,
-        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "git-hooks-nix": {
-      "inputs": {
-        "flake-compat": [
-          "nix"
-        ],
-        "gitignore": [
-          "nix"
-        ],
-        "nixpkgs": [
-          "nix",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1726745158,
-        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "type": "github"
-      }
-    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -87,15 +19,17 @@
     },
     "nix": {
       "inputs": {
-        "flake-compat": "flake-compat",
-        "flake-parts": "flake-parts",
-        "git-hooks-nix": "git-hooks-nix",
-        "libgit2": "libgit2",
+        "flake-compat": [],
+        "flake-parts": [],
+        "git-hooks-nix": [],
+        "libgit2": [
+          "libgit2"
+        ],
         "nixpkgs": [
           "nixpkgs"
         ],
-        "nixpkgs-23-11": "nixpkgs-23-11",
-        "nixpkgs-regression": "nixpkgs-regression"
+        "nixpkgs-23-11": [],
+        "nixpkgs-regression": []
       },
       "locked": {
         "lastModified": 1726787955,
@@ -128,40 +62,9 @@
         "type": "github"
       }
     },
-    "nixpkgs-23-11": {
-      "locked": {
-        "lastModified": 1717159533,
-        "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
-        "type": "github"
-      }
-    },
-    "nixpkgs-regression": {
-      "locked": {
-        "lastModified": 1643052045,
-        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
+        "libgit2": "libgit2",
         "nix": "nix",
         "nixpkgs": "nixpkgs"
       }
diff --git a/flake.nix b/flake.nix
index 78024656..c06b5f62 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,10 +2,20 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
+
+  inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
   inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
+  inputs.nix.inputs.libgit2.follows = "libgit2";
 
-  outputs = { self, nixpkgs, nix }:
+  # hide nix dev tooling from our lock file
+  inputs.nix.inputs.flake-parts.follows = "";
+  inputs.nix.inputs.git-hooks-nix.follows = "";
+  inputs.nix.inputs.nixpkgs-regression.follows = "";
+  inputs.nix.inputs.nixpkgs-23-11.follows = "";
+  inputs.nix.inputs.flake-compat.follows = "";
+
+  outputs = { self, nixpkgs, nix, ... }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;

From 4e2c06ec2c74e0387825e8025ee8eb2121b8433b Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Sun, 20 Oct 2024 01:57:07 +0200
Subject: [PATCH 207/285] queue-runner: don't decode base64 hostkey in hydra

Nix expects a base64 encoded hostkey in SSHMaster, so make sure we don't
decode this prematurely in hydra.

Reported-By: Puck Meerburg <puck@puck.moe>
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 5ffa7fe6..bc2974b2 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -171,7 +171,7 @@ void State::parseMachines(const std::string & contents)
             std::move(mandatoryFeatures),
             // `sshPublicHostKey`
             tokens[7] != "" && tokens[7] != "-"
-                ? base64Decode(tokens[7])
+                ? tokens[7]
                 : "",
         });
 

From 182a48c9fbcd6e6063bd6166c314174a8e79294c Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Mon, 22 Jul 2024 15:54:29 +0200
Subject: [PATCH 208/285] autotools -> meson

Original commit message:

> There are some known regressions regarding local testing setups - since
> everything was kinda half written with the expectation that build dir =
> source dir (which should not be true anymore). But everything builds and
> the test suite runs fine, after several hours spent debugging random
> crashes in libpqxx with MALLOC_PERTURB_...

I have not experienced regressions with local testing.

(cherry picked from commit 4b886d9c45cd2d7fe9b0a8dbc05c7318d46f615d)
---
 .gitignore                         | 42 +-------------
 .yath.rc                           |  2 -
 Makefile.am                        | 12 ----
 configure.ac                       | 91 ------------------------------
 doc/Makefile.am                    |  4 --
 doc/manual/Makefile.am             |  6 --
 doc/manual/meson.build             | 36 ++++++++++++
 flake.nix                          |  2 +-
 meson.build                        | 40 +++++++++++++
 nixos-modules/meson.build          |  4 ++
 package.nix                        | 37 ++++++------
 src/Makefile.am                    |  3 -
 src/hydra-eval-jobs/Makefile.am    |  5 --
 src/hydra-eval-jobs/meson.build    |  8 +++
 src/hydra-evaluator/Makefile.am    |  5 --
 src/hydra-evaluator/meson.build    |  9 +++
 src/hydra-queue-runner/Makefile.am |  8 ---
 src/hydra-queue-runner/meson.build | 22 ++++++++
 src/lib/Makefile.am                | 22 --------
 src/libhydra/meson.build           |  5 ++
 src/meson.build                    | 86 ++++++++++++++++++++++++++++
 src/root/Makefile.am               | 39 -------------
 src/script/Makefile.am             | 19 -------
 src/sql/Makefile.am                |  9 ---
 src/sql/meson.build                | 90 +++++++++++++++++++++++++++++
 src/ttf/Makefile.am                |  4 --
 src/ttf/meson.build                |  5 ++
 t/Makefile.am                      | 39 -------------
 t/lib/HydraTestContext.pm          | 23 +++++++-
 t/meson.build                      | 43 ++++++++++++++
 30 files changed, 391 insertions(+), 329 deletions(-)
 delete mode 100644 .yath.rc
 delete mode 100644 Makefile.am
 delete mode 100644 configure.ac
 delete mode 100644 doc/Makefile.am
 delete mode 100644 doc/manual/Makefile.am
 create mode 100644 doc/manual/meson.build
 create mode 100644 meson.build
 create mode 100644 nixos-modules/meson.build
 delete mode 100644 src/Makefile.am
 delete mode 100644 src/hydra-eval-jobs/Makefile.am
 create mode 100644 src/hydra-eval-jobs/meson.build
 delete mode 100644 src/hydra-evaluator/Makefile.am
 create mode 100644 src/hydra-evaluator/meson.build
 delete mode 100644 src/hydra-queue-runner/Makefile.am
 create mode 100644 src/hydra-queue-runner/meson.build
 delete mode 100644 src/lib/Makefile.am
 create mode 100644 src/libhydra/meson.build
 create mode 100644 src/meson.build
 delete mode 100644 src/root/Makefile.am
 delete mode 100644 src/script/Makefile.am
 delete mode 100644 src/sql/Makefile.am
 create mode 100644 src/sql/meson.build
 delete mode 100644 src/ttf/Makefile.am
 create mode 100644 src/ttf/meson.build
 delete mode 100644 t/Makefile.am
 create mode 100644 t/meson.build

diff --git a/.gitignore b/.gitignore
index f8bf5718..ddcbadc4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,48 +1,8 @@
-/.pls_cache
-*.o
 *~
-Makefile
-Makefile.in
-.deps
-.hydra-data
-/config.guess
-/config.log
-/config.status
-/config.sub
-/configure
-/depcomp
-/libtool
-/ltmain.sh
-/autom4te.cache
-/aclocal.m4
-/missing
-/install-sh
+.test_info.*
 /src/sql/hydra-postgresql.sql
 /src/sql/hydra-sqlite.sql
 /src/sql/tmp.sqlite
-/src/hydra-eval-jobs/hydra-eval-jobs
-/src/root/static/bootstrap
-/src/root/static/js/flot
-/tests
-/doc/manual/images
-/doc/manual/manual.html
-/doc/manual/manual.pdf
-/t/.bzr*
-/t/.git*
-/t/.hg*
-/t/nix
-/t/data
-/t/jobs/config.nix
-t/jobs/declarative/project.json
-/inst
-hydra-config.h
-hydra-config.h.in
 result
 result-*
 outputs
-config
-stamp-h1
-src/hydra-evaluator/hydra-evaluator
-src/hydra-queue-runner/hydra-queue-runner
-src/root/static/fontawesome/
-src/root/static/bootstrap*/
diff --git a/.yath.rc b/.yath.rc
deleted file mode 100644
index 19bb35af..00000000
--- a/.yath.rc
+++ /dev/null
@@ -1,2 +0,0 @@
-[test]
--I=rel(t/lib)
diff --git a/Makefile.am b/Makefile.am
deleted file mode 100644
index a28e3f33..00000000
--- a/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-SUBDIRS = src doc
-if CAN_DO_CHECK
-  SUBDIRS += t
-endif
-
-BOOTCLEAN_SUBDIRS = $(SUBDIRS)
-DIST_SUBDIRS      = $(SUBDIRS)
-EXTRA_DIST        = nixos-modules/hydra.nix
-
-install-data-local: nixos-modules/hydra.nix
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix
-	$(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix
diff --git a/configure.ac b/configure.ac
deleted file mode 100644
index e5c57d14..00000000
--- a/configure.ac
+++ /dev/null
@@ -1,91 +0,0 @@
-AC_INIT([Hydra], [m4_esyscmd([echo -n $(cat ./version.txt)$VERSION_SUFFIX])])
-AC_CONFIG_AUX_DIR(config)
-AM_INIT_AUTOMAKE([foreign serial-tests])
-
-AC_LANG([C++])
-
-AC_PROG_CC
-AC_PROG_INSTALL
-AC_PROG_LN_S
-AC_PROG_LIBTOOL
-AC_PROG_CXX
-
-AC_PATH_PROG([XSLTPROC], [xsltproc])
-
-AC_ARG_WITH([docbook-xsl],
-  [AS_HELP_STRING([--with-docbook-xsl=PATH],
-    [path of the DocBook XSL stylesheets])],
-  [docbookxsl="$withval"],
-  [docbookxsl="/docbook-xsl-missing"])
-AC_SUBST([docbookxsl])
-
-
-AC_DEFUN([NEED_PROG],
-[
-AC_PATH_PROG($1, $2)
-if test -z "$$1"; then
-    AC_MSG_ERROR([$2 is required])
-fi
-])
-
-NEED_PROG(perl, perl)
-
-NEED_PROG([NIX_STORE_PROGRAM], [nix-store])
-
-AC_MSG_CHECKING([whether $NIX_STORE_PROGRAM is recent enough])
-if test -n "$NIX_STORE" -a -n "$TMPDIR"
-then
-    # This may be executed from within a build chroot, so pacify
-    # `nix-store' instead of letting it choke while trying to mkdir
-    # /nix/var.
-    NIX_STATE_DIR="$TMPDIR"
-    export NIX_STATE_DIR
-fi
-if NIX_REMOTE=daemon PAGER=cat "$NIX_STORE_PROGRAM" --timeout 123 -q; then
-    AC_MSG_RESULT([yes])
-else
-    AC_MSG_RESULT([no])
-    AC_MSG_ERROR([`$NIX_STORE_PROGRAM' doesn't support `--timeout'; please use a newer version.])
-fi
-
-PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
-
-testPath="$(dirname $(type -p expr))"
-AC_SUBST(testPath)
-
-CXXFLAGS+=" -include nix/config.h"
-
-AC_CONFIG_FILES([
-  Makefile
-  doc/Makefile
-  doc/manual/Makefile
-  src/Makefile
-  src/hydra-evaluator/Makefile
-  src/hydra-eval-jobs/Makefile
-  src/hydra-queue-runner/Makefile
-  src/sql/Makefile
-  src/ttf/Makefile
-  src/lib/Makefile
-  src/root/Makefile
-  src/script/Makefile
-])
-
-# Tests might be filtered out
-AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"])
-AM_COND_IF(
-  [CAN_DO_CHECK],
-  [
-    jobsPath="$(realpath ./t/jobs)"
-    AC_SUBST(jobsPath)
-    AC_CONFIG_FILES([
-      t/Makefile
-      t/jobs/config.nix
-      t/jobs/declarative/project.json
-    ])
-  ])
-
-AC_CONFIG_COMMANDS([executable-scripts], [])
-
-AC_CONFIG_HEADER([hydra-config.h])
-
-AC_OUTPUT
diff --git a/doc/Makefile.am b/doc/Makefile.am
deleted file mode 100644
index 9ac91d24..00000000
--- a/doc/Makefile.am
+++ /dev/null
@@ -1,4 +0,0 @@
-SUBDIRS = manual
-BOOTCLEAN_SUBDIRS = $(SUBDIRS)
-DIST_SUBDIRS      = $(SUBDIRS)
-
diff --git a/doc/manual/Makefile.am b/doc/manual/Makefile.am
deleted file mode 100644
index ec732166..00000000
--- a/doc/manual/Makefile.am
+++ /dev/null
@@ -1,6 +0,0 @@
-MD_FILES = src/*.md
-
-EXTRA_DIST = $(MD_FILES)
-
-install: $(MD_FILES)
-	mdbook build . -d $(docdir)
diff --git a/doc/manual/meson.build b/doc/manual/meson.build
new file mode 100644
index 00000000..11178809
--- /dev/null
+++ b/doc/manual/meson.build
@@ -0,0 +1,36 @@
+srcs = files(
+  'src/SUMMARY.md',
+  'src/about.md',
+  'src/api.md',
+  'src/configuration.md',
+  'src/hacking.md',
+  'src/installation.md',
+  'src/introduction.md',
+  'src/jobs.md',
+  'src/monitoring/README.md',
+  'src/notifications.md',
+  'src/plugins/README.md',
+  'src/plugins/RunCommand.md',
+  'src/plugins/declarative-projects.md',
+  'src/projects.md',
+  'src/webhooks.md',
+)
+
+manual = custom_target(
+  'manual',
+  command: [
+    mdbook,
+    'build',
+    '@SOURCE_ROOT@/doc/manual',
+    '-d', meson.current_build_dir() / 'html'
+  ],
+  depend_files: srcs,
+  output: ['html'],
+  build_by_default: true,
+)
+
+install_subdir(
+  manual.full_path(),
+  install_dir: get_option('datadir') / 'doc/hydra',
+  strip_directory: true,
+)
diff --git a/flake.nix b/flake.nix
index c06b5f62..fccd45b9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -47,7 +47,7 @@
           pkgs.runCommand "hydra-manual-${hydra.version}" { }
             ''
               mkdir -p $out/share
-              cp -prvd ${hydra}/share/doc $out/share/
+              cp -prvd ${hydra.doc}/share/doc $out/share/
 
               mkdir $out/nix-support
               echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products
diff --git a/meson.build b/meson.build
new file mode 100644
index 00000000..b9327d51
--- /dev/null
+++ b/meson.build
@@ -0,0 +1,40 @@
+project('hydra', 'cpp',
+  version: files('version.txt'),
+  license: 'GPL-3.0',
+  default_options: [
+    'debug=true',
+    'optimization=2',
+    'cpp_std=c++20',
+  ],
+)
+
+nix_store_dep = dependency('nix-store', required: true)
+nix_main_dep = dependency('nix-main', required: true)
+nix_expr_dep = dependency('nix-expr', required: true)
+nix_flake_dep = dependency('nix-flake', required: true)
+nix_cmd_dep = dependency('nix-cmd', required: true)
+
+# Nix need extra flags not provided in its pkg-config files.
+nix_dep = declare_dependency(
+  dependencies: [
+    nix_store_dep,
+    nix_main_dep,
+    nix_expr_dep,
+    nix_flake_dep,
+    nix_cmd_dep,
+  ],
+  compile_args: ['-include', 'nix/config.h'],
+)
+
+pqxx_dep = dependency('libpqxx', required: true)
+
+prom_cpp_core_dep = dependency('prometheus-cpp-core', required: true)
+prom_cpp_pull_dep = dependency('prometheus-cpp-pull', required: true)
+
+mdbook = find_program('mdbook', native: true)
+perl = find_program('perl', native: true)
+
+subdir('doc/manual')
+subdir('nixos-modules')
+subdir('src')
+subdir('t')
diff --git a/nixos-modules/meson.build b/nixos-modules/meson.build
new file mode 100644
index 00000000..95c47e9f
--- /dev/null
+++ b/nixos-modules/meson.build
@@ -0,0 +1,4 @@
+install_data('hydra.nix',
+  install_dir: get_option('datadir') / 'nix',
+  rename: ['hydra-module.nix'],
+)
diff --git a/package.nix b/package.nix
index 73dd0f2a..f944fe2b 100644
--- a/package.nix
+++ b/package.nix
@@ -13,7 +13,8 @@
 , git
 
 , makeWrapper
-, autoreconfHook
+, meson
+, ninja
 , nukeReferences
 , pkg-config
 , mdbook
@@ -91,6 +92,7 @@ let
         DigestSHA1
         EmailMIME
         EmailSender
+        FileCopyRecursive
         FileLibMagic
         FileSlurper
         FileWhich
@@ -138,28 +140,24 @@ stdenv.mkDerivation (finalAttrs: {
   src = fileset.toSource {
     root = ./.;
     fileset = fileset.unions ([
-      ./version.txt
-      ./configure.ac
-      ./Makefile.am
-      ./src
       ./doc
-      ./nixos-modules/hydra.nix
-      # These are always needed to appease Automake
-      ./t/Makefile.am
-      ./t/jobs/config.nix.in
-      ./t/jobs/declarative/project.json.in
-    ] ++ lib.optionals finalAttrs.doCheck [
+      ./meson.build
+      ./nixos-modules
+      ./src
       ./t
+      ./version.txt
       ./.perlcriticrc
-      ./.yath.rc
     ]);
   };
 
+  outputs = [ "out" "doc" ];
+
   strictDeps = true;
 
   nativeBuildInputs = [
     makeWrapper
-    autoreconfHook
+    meson
+    ninja
     nukeReferences
     pkg-config
     mdbook
@@ -225,6 +223,12 @@ stdenv.mkDerivation (finalAttrs: {
 
   OPENLDAP_ROOT = openldap;
 
+  mesonBuildType = "release";
+
+  postPatch = ''
+    patchShebangs .
+  '';
+
   shellHook = ''
     pushd $(git rev-parse --show-toplevel) >/dev/null
 
@@ -238,14 +242,11 @@ stdenv.mkDerivation (finalAttrs: {
     popd >/dev/null
   '';
 
-  NIX_LDFLAGS = [ "-lpthread" ];
-
-  enableParallelBuilding = true;
-
   doCheck = true;
 
+  mesonCheckFlags = [ "--verbose" ];
+
   preCheck = ''
-    patchShebangs .
     export LOGNAME=''${LOGNAME:-foo}
     # set $HOME for bzr so it can create its trace file
     export HOME=$(mktemp -d)
diff --git a/src/Makefile.am b/src/Makefile.am
deleted file mode 100644
index a28780b6..00000000
--- a/src/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = hydra-evaluator hydra-eval-jobs hydra-queue-runner sql script lib root ttf
-BOOTCLEAN_SUBDIRS = $(SUBDIRS)
-DIST_SUBDIRS      = $(SUBDIRS)
diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am
deleted file mode 100644
index fb67f89b..00000000
--- a/src/hydra-eval-jobs/Makefile.am
+++ /dev/null
@@ -1,5 +0,0 @@
-bin_PROGRAMS = hydra-eval-jobs
-
-hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
-hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd -lnixflake
-hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra
diff --git a/src/hydra-eval-jobs/meson.build b/src/hydra-eval-jobs/meson.build
new file mode 100644
index 00000000..916212e1
--- /dev/null
+++ b/src/hydra-eval-jobs/meson.build
@@ -0,0 +1,8 @@
+hydra_eval_jobs = executable('hydra-eval-jobs',
+  'hydra-eval-jobs.cc',
+  dependencies: [
+    libhydra_dep,
+    nix_dep,
+  ],
+  install: true,
+)
diff --git a/src/hydra-evaluator/Makefile.am b/src/hydra-evaluator/Makefile.am
deleted file mode 100644
index 73638cfe..00000000
--- a/src/hydra-evaluator/Makefile.am
+++ /dev/null
@@ -1,5 +0,0 @@
-bin_PROGRAMS = hydra-evaluator
-
-hydra_evaluator_SOURCES = hydra-evaluator.cc
-hydra_evaluator_LDADD = $(NIX_LIBS) -lpqxx
-hydra_evaluator_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations
diff --git a/src/hydra-evaluator/meson.build b/src/hydra-evaluator/meson.build
new file mode 100644
index 00000000..b3dc542e
--- /dev/null
+++ b/src/hydra-evaluator/meson.build
@@ -0,0 +1,9 @@
+hydra_evaluator = executable('hydra-evaluator',
+  'hydra-evaluator.cc',
+  dependencies: [
+    libhydra_dep,
+    nix_dep,
+    pqxx_dep,
+  ],
+  install: true,
+)
diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
deleted file mode 100644
index 117112f6..00000000
--- a/src/hydra-queue-runner/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-bin_PROGRAMS = hydra-queue-runner
-
-hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
- builder.cc build-result.cc build-remote.cc \
- hydra-build-result.hh counter.hh state.hh db.hh \
- nar-extractor.cc nar-extractor.hh
-hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lprometheus-cpp-pull -lprometheus-cpp-core
-hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations
diff --git a/src/hydra-queue-runner/meson.build b/src/hydra-queue-runner/meson.build
new file mode 100644
index 00000000..1c20299f
--- /dev/null
+++ b/src/hydra-queue-runner/meson.build
@@ -0,0 +1,22 @@
+srcs = files(
+  'builder.cc',
+  'build-remote.cc',
+  'build-result.cc',
+  'dispatcher.cc',
+  'hydra-queue-runner.cc',
+  'nar-extractor.cc',
+  'queue-monitor.cc',
+)
+
+hydra_queue_runner = executable('hydra-queue-runner',
+  'hydra-queue-runner.cc',
+  srcs,
+  dependencies: [
+    libhydra_dep,
+    nix_dep,
+    pqxx_dep,
+    prom_cpp_core_dep,
+    prom_cpp_pull_dep,
+  ],
+  install: true,
+)
diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
deleted file mode 100644
index 434868e0..00000000
--- a/src/lib/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-PERL_MODULES =					\
-  $(wildcard *.pm)				\
-  $(wildcard Hydra/*.pm)			\
-  $(wildcard Hydra/Helper/*.pm)			\
-  $(wildcard Hydra/Model/*.pm)			\
-  $(wildcard Hydra/View/*.pm)			\
-  $(wildcard Hydra/Schema/*.pm)			\
-  $(wildcard Hydra/Schema/Result/*.pm)			\
-  $(wildcard Hydra/Schema/ResultSet/*.pm)			\
-  $(wildcard Hydra/Controller/*.pm)		\
-  $(wildcard Hydra/Base/*.pm)			\
-  $(wildcard Hydra/Base/Controller/*.pm) 	\
-  $(wildcard Hydra/Script/*.pm)			\
-  $(wildcard Hydra/Component/*.pm)		\
-  $(wildcard Hydra/Event/*.pm)		\
-  $(wildcard Hydra/Plugin/*.pm)
-
-EXTRA_DIST = \
-  $(PERL_MODULES)
-
-hydradir = $(libexecdir)/hydra/lib
-nobase_hydra_DATA = $(PERL_MODULES)
diff --git a/src/libhydra/meson.build b/src/libhydra/meson.build
new file mode 100644
index 00000000..1866233c
--- /dev/null
+++ b/src/libhydra/meson.build
@@ -0,0 +1,5 @@
+libhydra_inc = include_directories('.')
+
+libhydra_dep = declare_dependency(
+  include_directories: [libhydra_inc],
+)
diff --git a/src/meson.build b/src/meson.build
new file mode 100644
index 00000000..8c7562ed
--- /dev/null
+++ b/src/meson.build
@@ -0,0 +1,86 @@
+# Native code
+subdir('libhydra')
+subdir('hydra-eval-jobs')
+subdir('hydra-evaluator')
+subdir('hydra-queue-runner')
+
+hydra_libexecdir = get_option('libexecdir') / 'hydra'
+
+# Data and interpreted
+foreach dir : ['lib', 'root']
+  install_subdir(dir,
+    install_dir: hydra_libexecdir,
+  )
+endforeach
+subdir('sql')
+subdir('ttf')
+
+# Static files for website
+
+hydra_libexecdir_static = hydra_libexecdir / 'root' / 'static'
+
+## Bootstrap
+
+bootstrap_name = 'bootstrap-4.3.1-dist'
+bootstrap = custom_target(
+  'extract-bootstrap',
+  input: 'root' / (bootstrap_name + '.zip'),
+  output: bootstrap_name,
+  command: ['unzip', '-u', '-d', '@OUTDIR@', '@INPUT@'],
+)
+custom_target(
+  'name-bootstrap',
+  input: bootstrap,
+  output: 'bootstrap',
+  command: ['cp', '-r', '@INPUT@' , '@OUTPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static,
+)
+
+## Flot
+
+custom_target(
+  'extract-flot',
+  input: 'root' / 'flot-0.8.3.zip',
+  output: 'flot',
+  command: ['unzip', '-u', '-d', '@OUTDIR@', '@INPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static / 'js',
+)
+
+## Fontawesome
+
+fontawesome_name = 'fontawesome-free-5.10.2-web'
+fontawesome = custom_target(
+  'extract-fontawesome',
+  input: 'root' / (fontawesome_name + '.zip'),
+  output: fontawesome_name,
+  command: ['unzip', '-u', '-d', '@OUTDIR@', '@INPUT@'],
+)
+custom_target(
+  'name-fontawesome-css',
+  input: fontawesome,
+  output: 'css',
+  command: ['cp', '-r', '@INPUT@/css', '@OUTPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static / 'fontawesome',
+)
+custom_target(
+  'name-fontawesome-webfonts',
+  input: fontawesome,
+  output: 'webfonts',
+  command: ['cp', '-r', '@INPUT@/webfonts', '@OUTPUT@'],
+  install: true,
+  install_dir: hydra_libexecdir_static / 'fontawesome',
+)
+
+# Scripts
+
+install_subdir('script',
+  install_dir: get_option('bindir'),
+  exclude_files: [
+    'hydra-dev-server',
+  ],
+  install_mode: 'rwxr-xr-x',
+  strip_directory: true,
+)
diff --git a/src/root/Makefile.am b/src/root/Makefile.am
deleted file mode 100644
index 163b96e0..00000000
--- a/src/root/Makefile.am
+++ /dev/null
@@ -1,39 +0,0 @@
-TEMPLATES = $(wildcard *.tt)
-STATIC = \
-  $(wildcard static/images/*) \
-  $(wildcard static/css/*) \
-  static/js/bootbox.min.js \
-  static/js/popper.min.js \
-  static/js/common.js \
-  static/js/jquery/jquery-3.4.1.min.js \
-  static/js/jquery/jquery-ui-1.10.4.min.js
-
-FLOT = flot-0.8.3.zip
-BOOTSTRAP = bootstrap-4.3.1-dist.zip
-FONTAWESOME = fontawesome-free-5.10.2-web.zip
-
-ZIPS = $(FLOT) $(BOOTSTRAP) $(FONTAWESOME)
-
-EXTRA_DIST = $(TEMPLATES) $(STATIC) $(ZIPS)
-
-hydradir = $(libexecdir)/hydra/root
-nobase_hydra_DATA = $(EXTRA_DIST)
-
-all:
-	mkdir -p $(srcdir)/static/js
-	unzip -u -d $(srcdir)/static $(BOOTSTRAP)
-	rm -rf $(srcdir)/static/bootstrap
-	mv $(srcdir)/static/$(basename $(BOOTSTRAP)) $(srcdir)/static/bootstrap
-	unzip -u -d $(srcdir)/static/js $(FLOT)
-	unzip -u -d $(srcdir)/static $(FONTAWESOME)
-	rm -rf $(srcdir)/static/fontawesome
-	mv $(srcdir)/static/$(basename $(FONTAWESOME)) $(srcdir)/static/fontawesome
-
-install-data-local: $(ZIPS)
-	mkdir -p $(hydradir)/static/js
-	cp -prvd $(srcdir)/static/js/* $(hydradir)/static/js
-	mkdir -p $(hydradir)/static/bootstrap
-	cp -prvd $(srcdir)/static/bootstrap/* $(hydradir)/static/bootstrap
-	mkdir -p $(hydradir)/static/fontawesome/{css,webfonts}
-	cp -prvd $(srcdir)/static/fontawesome/css/* $(hydradir)/static/fontawesome/css
-	cp -prvd $(srcdir)/static/fontawesome/webfonts/* $(hydradir)/static/fontawesome/webfonts
diff --git a/src/script/Makefile.am b/src/script/Makefile.am
deleted file mode 100644
index 466d3153..00000000
--- a/src/script/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-EXTRA_DIST =					\
-  $(distributable_scripts)
-
-distributable_scripts =				\
-  hydra-backfill-ids				\
-  hydra-init					\
-  hydra-eval-jobset				\
-  hydra-server					\
-  hydra-update-gc-roots				\
-  hydra-s3-backup-collect-garbage		\
-  hydra-create-user				\
-  hydra-notify					\
-  hydra-send-stats				\
-  nix-prefetch-git				\
-  nix-prefetch-bzr				\
-  nix-prefetch-hg
-
-bin_SCRIPTS =					\
-  $(distributable_scripts)
diff --git a/src/sql/Makefile.am b/src/sql/Makefile.am
deleted file mode 100644
index a6b96886..00000000
--- a/src/sql/Makefile.am
+++ /dev/null
@@ -1,9 +0,0 @@
-sqldir = $(libexecdir)/hydra/sql
-nobase_dist_sql_DATA =				\
-  hydra.sql					\
-  test.sql					\
-  upgrade-*.sql					\
-  update-dbix.pl
-
-update-dbix: hydra.sql
-	./update-dbix-harness.sh
diff --git a/src/sql/meson.build b/src/sql/meson.build
new file mode 100644
index 00000000..2da35b3c
--- /dev/null
+++ b/src/sql/meson.build
@@ -0,0 +1,90 @@
+sql_files = files(
+  'hydra.sql',
+  'test.sql',
+  'update-dbix.pl',
+  'upgrade-2.sql',
+  'upgrade-3.sql',
+  'upgrade-4.sql',
+  'upgrade-5.sql',
+  'upgrade-6.sql',
+  'upgrade-7.sql',
+  'upgrade-8.sql',
+  'upgrade-9.sql',
+  'upgrade-10.sql',
+  'upgrade-11.sql',
+  'upgrade-12.sql',
+  'upgrade-13.sql',
+  'upgrade-14.sql',
+  'upgrade-15.sql',
+  'upgrade-16.sql',
+  'upgrade-17.sql',
+  'upgrade-18.sql',
+  'upgrade-19.sql',
+  'upgrade-20.sql',
+  'upgrade-21.sql',
+  'upgrade-22.sql',
+  'upgrade-23.sql',
+  'upgrade-24.sql',
+  'upgrade-25.sql',
+  'upgrade-26.sql',
+  'upgrade-27.sql',
+  'upgrade-28.sql',
+  'upgrade-29.sql',
+  'upgrade-30.sql',
+  'upgrade-31.sql',
+  'upgrade-32.sql',
+  'upgrade-33.sql',
+  'upgrade-34.sql',
+  'upgrade-35.sql',
+  'upgrade-36.sql',
+  'upgrade-37.sql',
+  'upgrade-38.sql',
+  'upgrade-39.sql',
+  'upgrade-40.sql',
+  'upgrade-41.sql',
+  'upgrade-42.sql',
+  'upgrade-43.sql',
+  'upgrade-44.sql',
+  'upgrade-45.sql',
+  'upgrade-46.sql',
+  'upgrade-47.sql',
+  'upgrade-48.sql',
+  'upgrade-49.sql',
+  'upgrade-50.sql',
+  'upgrade-51.sql',
+  'upgrade-52.sql',
+  'upgrade-53.sql',
+  'upgrade-54.sql',
+  'upgrade-55.sql',
+  'upgrade-56.sql',
+  'upgrade-57.sql',
+  'upgrade-58.sql',
+  'upgrade-59.sql',
+  'upgrade-60.sql',
+  'upgrade-61.sql',
+  'upgrade-62.sql',
+  'upgrade-63.sql',
+  'upgrade-64.sql',
+  'upgrade-65.sql',
+  'upgrade-66.sql',
+  'upgrade-67.sql',
+  'upgrade-68.sql',
+  'upgrade-69.sql',
+  'upgrade-70.sql',
+  'upgrade-71.sql',
+  'upgrade-72.sql',
+  'upgrade-73.sql',
+  'upgrade-74.sql',
+  'upgrade-75.sql',
+  'upgrade-76.sql',
+  'upgrade-77.sql',
+  'upgrade-78.sql',
+  'upgrade-79.sql',
+  'upgrade-80.sql',
+  'upgrade-81.sql',
+  'upgrade-82.sql',
+  'upgrade-83.sql',
+  'upgrade-84.sql',
+)
+
+install_data(sql_files, install_dir: hydra_libexecdir / 'sql')
diff --git a/src/ttf/Makefile.am b/src/ttf/Makefile.am
deleted file mode 100644
index eba78239..00000000
--- a/src/ttf/Makefile.am
+++ /dev/null
@@ -1,4 +0,0 @@
-EXTRA_DIST = COPYING.LIB StayPuft.ttf
-
-ttfdir = $(libexecdir)/hydra/ttf
-nobase_ttf_DATA = $(EXTRA_DIST)
diff --git a/src/ttf/meson.build b/src/ttf/meson.build
new file mode 100644
index 00000000..260439e5
--- /dev/null
+++ b/src/ttf/meson.build
@@ -0,0 +1,5 @@
+data_files = files(
+  'StayPuft.ttf',
+  'COPYING.LIB',
+)
+install_data(data_files, install_dir: hydra_libexecdir / 'ttf')
diff --git a/t/Makefile.am b/t/Makefile.am
deleted file mode 100644
index 9c14c1e3..00000000
--- a/t/Makefile.am
+++ /dev/null
@@ -1,39 +0,0 @@
-TESTS_ENVIRONMENT = \
-  BZR_HOME="$(abs_builddir)/data"			\
-  HYDRA_DBI="dbi:Pg:dbname=hydra-test-suite;port=6433"	\
-  HYDRA_DATA="$(abs_builddir)/data"			\
-  HYDRA_HOME="$(top_srcdir)/src"			\
-  HYDRA_CONFIG=						\
-  NIX_REMOTE=						\
-  NIX_REMOTE_SYSTEMS=					\
-  NIX_CONF_DIR="$(abs_builddir)/nix/etc/nix"		\
-  NIX_STATE_DIR="$(abs_builddir)/nix/var/nix"		\
-  NIX_STORE_DIR="$(abs_builddir)/nix/store"			\
-  NIX_LOG_DIR="$(abs_builddir)/nix/var/log/nix"		\
-  PGHOST=/tmp	\
-  PERL5LIB="$(srcdir):$(abs_top_srcdir)/src/lib:$$PERL5LIB"	\
-  PYTHONPATH= \
-  PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \
-  perl -w
-
-EXTRA_DIST = \
-  $(wildcard *.pm) \
-  $(wildcard jobs/*.nix) \
-  $(wildcard jobs/*.sh) \
-  $(TESTS)
-
-TESTS = \
-  perlcritic.pl \
-  test.pl
-
-check_SCRIPTS = repos
-
-repos: dirs
-
-dirs:
-	mkdir -p data
-	touch data/hydra.conf
-	mkdir -p nix
-	mkdir -p nix/etc/nix
-	mkdir -p nix/store
-	mkdir -p nix/var
diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index 1d6fa909..d1de2212 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -4,6 +4,8 @@ use warnings;
 package HydraTestContext;
 use File::Path qw(make_path);
 use File::Basename;
+use File::Copy::Recursive qw(rcopy);
+use File::Which qw(which);
 use Cwd qw(abs_path getcwd);
 use CliRunners;
 use Hydra::Helper::Exec;
@@ -77,6 +79,13 @@ sub new {
     );
     $ENV{'HYDRA_DBI'} = $pgsql->dsn;
 
+    my $jobsdir = "$dir/jobs";
+    rcopy(abs_path(dirname(__FILE__) . "/../jobs"), $jobsdir);
+
+    my $coreutils_path = dirname(which 'install');
+    replace_variable_in_file($jobsdir . "/config.nix", '@testPath@', $coreutils_path);
+    replace_variable_in_file($jobsdir . "/declarative/project.json", '@jobsPath@', $jobsdir);
+
     my $self = bless {
         _db => undef,
         db_handle => $pgsql,
@@ -84,7 +93,7 @@ sub new {
         nix_state_dir => $nix_state_dir,
         nix_log_dir => $nix_log_dir,
         testdir => abs_path(dirname(__FILE__) . "/.."),
-        jobsdir => abs_path(dirname(__FILE__) . "/../jobs"),
+        jobsdir => $jobsdir,
         deststoredir => $deststoredir,
     }, $class;
 
@@ -243,6 +252,18 @@ sub write_file {
     close $fh;
 }
 
+sub replace_variable_in_file {
+    my ($fn, $var, $val) = @_;
+
+    open (my $input, '<', "$fn.in") or die $!;
+    open (my $output, '>', $fn) or die $!;
+
+    while (my $line = <$input>) {
+        $line =~ s/$var/$val/g;
+        print $output $line;
+    }
+}
+
 sub rand_chars {
     return sprintf("t%08X", rand(0xFFFFFFFF));
 }
diff --git a/t/meson.build b/t/meson.build
new file mode 100644
index 00000000..11044a03
--- /dev/null
+++ b/t/meson.build
@@ -0,0 +1,43 @@
+fs = import('fs')
+
+test('perlcritic',
+  perl,
+  args: ['-w', files('perlcritic.pl')],
+  workdir: meson.project_source_root(),
+  timeout: -1,
+)
+
+testenv = environment(
+  {
+    'BZR_HOME': meson.current_build_dir() / 'data',
+    'HYDRA_DBI': 'dbi:Pg:dbname=hydra-test-suite;port=6433',
+    'HYDRA_DATA': meson.current_build_dir() / 'data',
+    'HYDRA_HOME': meson.project_source_root() / 'src',
+    'PGHOST': '/tmp',
+    'PYTHONPATH': '',
+
+    # libpqxx seems to randomly crash with certain values of MALLOC_PERTURB_,
+    # set by default by Meson's test(). Very promising, high quality software.
+    'MALLOC_PERTURB_': '0',
+  },
+)
+testenv.prepend('PERL5LIB',
+  meson.current_source_dir(),
+  meson.project_source_root() / 'src/lib',
+  separator: ':'
+)
+testenv.prepend('PATH',
+  fs.parent(hydra_eval_jobs.full_path()),
+  fs.parent(hydra_evaluator.full_path()),
+  fs.parent(hydra_queue_runner.full_path()),
+  meson.project_source_root() / 'src/script',
+  separator: ':'
+)
+
+test('testsuite',
+  perl,
+  args: ['-I', meson.current_source_dir() / 'lib', '-w', files('test.pl')],
+  env: testenv,
+  workdir: meson.current_source_dir(),
+  timeout: -1,
+)

From 6456c1d7d636ed9d046f24c81873b00d226b76da Mon Sep 17 00:00:00 2001
From: Chatnoir Miki <cmiki@amono.me>
Date: Mon, 25 Nov 2024 11:35:08 +0800
Subject: [PATCH 209/285] reproduce.tt: Use realpath for tmpDir to fix macOS
 compatibility

---
 src/root/reproduce.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/reproduce.tt b/src/root/reproduce.tt
index dc88edfa..d8a77518 100644
--- a/src/root/reproduce.tt
+++ b/src/root/reproduce.tt
@@ -7,7 +7,7 @@ main() {
 
 set -e
 
-tmpDir=${TMPDIR:-/tmp}/build-[% build.id +%]
+tmpDir=$(realpath "${TMPDIR:-/tmp}")/build-[% build.id +%]
 declare -a args extraArgs
 
 

From 9de9cb0ad8e23f935eaa24d7e197c0deba885da9 Mon Sep 17 00:00:00 2001
From: Aaron Honeycutt <aaronhoneycutt@protonmail.com>
Date: Tue, 26 Nov 2024 00:52:24 -0700
Subject: [PATCH 210/285] Update README (#1271)

* Update version in example

* Update docs to fix invalid indentifier when using 'hello'

* fix build issue for hello example

---------

Co-authored-by: Aaron Honeycutt <aaronhoneycutt@proton.me>
---
 README.md          | 8 ++++----
 examples/hello.nix | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 58373052..8ac18ac9 100644
--- a/README.md
+++ b/README.md
@@ -39,16 +39,16 @@ In order to evaluate and build anything you need to create _projects_ that conta
 #### Creating A Project
 Log in as administrator, click "_Admin_" and select "_Create project_". Fill the form as follows:
 
-- **Identifier**: `hello`
+- **Identifier**: `hello-project`
 - **Display name**: `hello`
 - **Description**: `hello project`
 
 Click "_Create project_".
 
 #### Creating A Jobset
-After creating a project you are forwarded to the project page. Click "_Actions_" and choose "_Create jobset_". Fill the form with the following values:
+After creating a project you are forwarded to the project page. Click "_Actions_" and choose "_Create jobset_". Change **Type** to Legacy for the example below. Fill the form with the following values:
 
-- **Identifier**: `hello`
+- **Identifier**: `hello-project`
 - **Nix expression**: `examples/hello.nix` in `hydra`
 - **Check interval**: 60
 - **Scheduling shares**: 1
@@ -57,7 +57,7 @@ We have to add two inputs for this jobset. One for _nixpkgs_ and one for _hydra_
 
 - **Input name**: `nixpkgs`
 - **Type**: `Git checkout`
-- **Value**: `https://github.com/nixos/nixpkgs-channels nixos-20.03`
+- **Value**: `https://github.com/NixOS/nixpkgs nixos-24.05`
 
 - **Input name**: `hydra`
 - **Type**: `Git checkout`
diff --git a/examples/hello.nix b/examples/hello.nix
index 5a5d2585..84707025 100644
--- a/examples/hello.nix
+++ b/examples/hello.nix
@@ -1,5 +1,5 @@
 #
-# jobset example file. This file canbe referenced as Nix expression
+# jobset example file. This file can be referenced as Nix expression
 # in a jobset configuration along with inputs for nixpkgs and the
 # repository containing this file.
 #

From efadb6a26c694cc23380406ab471bc035b89ca44 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Tue, 3 Dec 2024 01:44:55 +0100
Subject: [PATCH 211/285] Make hydra-queue-runner want network-online.target

Just ordering yourself after network-online.target will not guarantee
that it will be loaded. You'll have to either want or require it. Hence
the following trace on recent nixpkgs versions:

evaluation warning: hydra-queue-runner.service is ordered after 'network-online.target' but doesn't depend on it
---
 nixos-modules/hydra.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index c471b0ef..4fc2d311 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -338,6 +338,7 @@ in
     systemd.services.hydra-queue-runner =
       { wantedBy = [ "multi-user.target" ];
         requires = [ "hydra-init.service" ];
+        wants = [ "network-online.target" ];
         after = [ "hydra-init.service" "network.target" "network-online.target" ];
         path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
         restartTriggers = [ hydraConf ];

From 8a8ac148772d19188e8e99900cf50ae7b1fc085d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 6 Feb 2025 21:30:49 -0500
Subject: [PATCH 212/285] Test using Hydra with flakes

It seemed there was no self-contained end-to-end test actually doing
this?!

Among other things, this will help ensure that the switch-over to
`nix-eval-jobs` is correct.
---
 t/evaluator/evaluate-flake.t     | 67 ++++++++++++++++++++++++++++++++
 t/jobs/flake-checks/flake.nix    |  6 +++
 t/jobs/flake-hydraJobs/flake.nix |  6 +++
 t/lib/HydraTestContext.pm        | 42 +++++++++++++++-----
 4 files changed, 112 insertions(+), 9 deletions(-)
 create mode 100644 t/evaluator/evaluate-flake.t
 create mode 100644 t/jobs/flake-checks/flake.nix
 create mode 100644 t/jobs/flake-hydraJobs/flake.nix

diff --git a/t/evaluator/evaluate-flake.t b/t/evaluator/evaluate-flake.t
new file mode 100644
index 00000000..d884f25a
--- /dev/null
+++ b/t/evaluator/evaluate-flake.t
@@ -0,0 +1,67 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use File::Copy qw(cp);
+
+my $ctx = test_context(
+    nix_config => qq|
+    experimental-features = nix-command flakes
+    |,
+    hydra_config => q|
+    <runcommand>
+      evaluator_pure_eval = false
+    </runcommand>
+    |
+);
+
+sub checkFlake {
+    my ($flake) = @_;
+
+    cp($ctx->jobsdir . "/basic.nix", $ctx->jobsdir . "/" . $flake);
+    cp($ctx->jobsdir . "/config.nix", $ctx->jobsdir . "/" . $flake);
+    cp($ctx->jobsdir . "/empty-dir-builder.sh", $ctx->jobsdir . "/" . $flake);
+    cp($ctx->jobsdir . "/fail.sh", $ctx->jobsdir . "/" . $flake);
+    cp($ctx->jobsdir . "/succeed-with-failed.sh", $ctx->jobsdir . "/" . $flake);
+
+    chmod 0755, $ctx->jobsdir . "/" . $flake . "/empty-dir-builder.sh";
+    chmod 0755, $ctx->jobsdir . "/" . $flake . "/fail.sh";
+    chmod 0755, $ctx->jobsdir . "/" . $flake . "/succeed-with-failed.sh";
+
+    my $builds = $ctx->makeAndEvaluateJobset(
+        flake => 'path:' . $ctx->jobsdir . "/" . $flake,
+        build => 1
+    );
+
+    subtest "Build: succeed_with_failed" => sub {
+        my $build = $builds->{"succeed_with_failed"};
+
+        is($build->finished, 1, "Build should be finished.");
+        is($build->buildstatus, 6, "succeeeded-but-failed should have buildstatus 6.");
+    };
+
+    subtest "Build: empty_dir" => sub {
+        my $build = $builds->{"empty_dir"};
+
+        is($build->finished, 1, "Build should be finished.");
+        is($build->buildstatus, 0, "Should have succeeded.");
+    };
+
+    subtest "Build: fails" => sub {
+        my $build = $builds->{"fails"};
+
+        is($build->finished, 1, "Build should be finished.");
+        is($build->buildstatus, 1, "Should have failed.");
+    };
+}
+
+subtest "Flake using `checks`" => sub {
+    checkFlake 'flake-checks'
+};
+
+subtest "Flake using `hydraJobs`" => sub {
+    checkFlake 'flake-hydraJobs'
+};
+
+done_testing;
diff --git a/t/jobs/flake-checks/flake.nix b/t/jobs/flake-checks/flake.nix
new file mode 100644
index 00000000..489fa9ec
--- /dev/null
+++ b/t/jobs/flake-checks/flake.nix
@@ -0,0 +1,6 @@
+{
+  outputs = { ... }: {
+    checks =
+      import ./basic.nix;
+  };
+}
diff --git a/t/jobs/flake-hydraJobs/flake.nix b/t/jobs/flake-hydraJobs/flake.nix
new file mode 100644
index 00000000..c02ccddd
--- /dev/null
+++ b/t/jobs/flake-hydraJobs/flake.nix
@@ -0,0 +1,6 @@
+{
+  outputs = { ... }: {
+    hydraJobs =
+      import ./basic.nix;
+  };
+}
diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index d1de2212..34d41eb2 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -165,14 +165,25 @@ sub nix_state_dir {
 sub makeAndEvaluateJobset {
     my ($self, %opts) = @_;
 
-    my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeAndEvaluateJobset.\n";
+    my $expression = $opts{'expression'};
+    my $flake = $opts{'flake'};
+    if (not $expression and not $flake) {
+        die "One of 'expression' or 'flake' must be passed to makeEvaluateJobset.\n";
+    }
+
     my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
     my $should_build = $opts{'build'} // 0;
 
-    my $jobsetCtx = $self->makeJobset(
-        expression => $expression,
+    my %args = (
         jobsdir => $jobsdir,
     );
+    if ($expression) {
+        $args{expression} = $expression;
+    }
+    if ($flake) {
+        $args{flake} = $flake;
+    }
+    my $jobsetCtx = $self->makeJobset(%args);
     my $jobset = $jobsetCtx->{"jobset"};
 
     evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0.\n";
@@ -195,7 +206,7 @@ sub makeAndEvaluateJobset {
 #
 # In return, you get a hash of the user, project, and jobset records.
 #
-# This always uses an `expression` from the `jobsdir` directory.
+# This always uses an `expression` or `flake` from the `jobsdir` directory.
 #
 # Hash Parameters:
 #
@@ -204,7 +215,12 @@ sub makeAndEvaluateJobset {
 sub makeJobset {
     my ($self, %opts) = @_;
 
-    my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeJobset.\n";
+    my $expression = $opts{'expression'};
+    my $flake = $opts{'flake'};
+    if (not $expression and not $flake) {
+        die "One of 'expression' or 'flake' must be passed to makeJobset.\n";
+    }
+
     my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
 
     # Create a new user for this test
@@ -222,12 +238,20 @@ sub makeJobset {
     });
 
     # Create a new jobset for this test and set up the inputs
-    my $jobset = $project->jobsets->create({
+    my %args = (
         name => rand_chars(),
-        nixexprinput => "jobs",
-        nixexprpath => $expression,
         emailoverride => ""
-    });
+    );
+    if ($expression) {
+        $args{type} = 0;
+        $args{nixexprinput} = "jobs";
+        $args{nixexprpath} = $expression;
+    }
+    if ($flake) {
+        $args{type} = 1;
+        $args{flake} = $flake;
+    }
+    my $jobset = $project->jobsets->create(\%args);
     my $jobsetinput = $jobset->jobsetinputs->create({name => "jobs", type => "path"});
     $jobsetinput->jobsetinputalts->create({altnr => 0, value => $jobsdir});
 

From 141b5fd0b5053230e1823f43257fa0ab58b39373 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 7 Feb 2025 16:30:14 -0500
Subject: [PATCH 213/285] Improve tests around constituents

- Test how shorter names are preferred when multiple jobs resolve to the
  same derivation.

- Test the exact aggregate map we get, by looking in the DB.
---
 t/evaluator/evaluate-constituents-gc.t | 49 ++++++++++++++++++++++++--
 t/jobs/constituents.nix                |  4 ++-
 t/lib/HydraTestContext.pm              | 19 ++++++++--
 3 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/t/evaluator/evaluate-constituents-gc.t b/t/evaluator/evaluate-constituents-gc.t
index a9b23e6c..2a5337b2 100644
--- a/t/evaluator/evaluate-constituents-gc.t
+++ b/t/evaluator/evaluate-constituents-gc.t
@@ -5,13 +5,58 @@ use Test2::V0;
 
 my $ctx = test_context();
 
-my $builds = $ctx->makeAndEvaluateJobset(
-    expression => 'constituents.nix',
+my $expression = 'constituents.nix';
+my $jobsetCtx = $ctx->makeJobset(
+    expression => $expression,
+);
+my $builds = $ctx->evaluateJobset(
+    jobset => $jobsetCtx->{"jobset"},
+    expression => $expression,
+    build => 0,
 );
 
 my $constituentA = $builds->{"constituentA"};
 my $directAggregate = $builds->{"direct_aggregate"};
 my $indirectAggregate = $builds->{"indirect_aggregate"};
+my $mixedAggregate = $builds->{"mixed_aggregate"};
+
+# Ensure that we get exactly the aggregates we expect
+my %expected_constituents = (
+    'direct_aggregate' => {
+        'constituentA' => 1,
+    },
+    'indirect_aggregate' => {
+        'constituentA' => 1,
+    },
+    'mixed_aggregate' => {
+        # Note that `constituentA_alias` becomes `constituentA`, because
+        # the shorter name is preferred
+        'constituentA' => 1,
+        'constituentB' => 1,
+    },
+);
+
+my $rs = $ctx->db->resultset('AggregateConstituents')->search(
+    {},
+    {
+        join     => [ 'aggregate', 'constituent' ],  # Use correct relationship names
+        columns  => [],
+        '+select' => [ 'aggregate.job', 'constituent.job' ],
+        '+as'     => [ 'aggregate_job', 'constituent_job' ],
+    }
+);
+
+my %actual_constituents;
+while (my $row = $rs->next) {
+    my $aggregate_job   = $row->get_column('aggregate_job');
+    my $constituent_job = $row->get_column('constituent_job');
+    $actual_constituents{$aggregate_job} //= {};
+    $actual_constituents{$aggregate_job}{$constituent_job} = 1;
+}
+
+is(\%actual_constituents, \%expected_constituents, "Exact aggregate constituents as expected");
+
+# Check that deletion also doesn't work accordingly
 
 is(system('nix-store', '--delete', $constituentA->drvpath), 256, "Deleting a constituent derivation fails");
 is(system('nix-store', '--delete', $directAggregate->drvpath), 256, "Deleting the direct aggregate derivation fails");
diff --git a/t/jobs/constituents.nix b/t/jobs/constituents.nix
index 5b7106b9..b8b88702 100644
--- a/t/jobs/constituents.nix
+++ b/t/jobs/constituents.nix
@@ -5,6 +5,8 @@ rec {
     builder = ./empty-dir-builder.sh;
   };
 
+  constituentA_alias = constituentA;
+
   constituentB = mkDerivation {
     name = "empty-dir-B";
     builder = ./empty-dir-builder.sh;
@@ -32,7 +34,7 @@ rec {
     name = "mixed_aggregate";
     _hydraAggregate = true;
     constituents = [
-      "constituentA"
+      "constituentA_alias"
       constituentB
     ];
     builder = ./empty-dir-builder.sh;
diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index 34d41eb2..27b0be0a 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -172,7 +172,6 @@ sub makeAndEvaluateJobset {
     }
 
     my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
-    my $should_build = $opts{'build'} // 0;
 
     my %args = (
         jobsdir => $jobsdir,
@@ -184,12 +183,28 @@ sub makeAndEvaluateJobset {
         $args{flake} = $flake;
     }
     my $jobsetCtx = $self->makeJobset(%args);
-    my $jobset = $jobsetCtx->{"jobset"};
+
+    return $self->evaluateJobset(
+        jobset => $jobsetCtx->{"jobset"},
+        expression => $expression,
+        flake => $flake,
+        build => $opts{"build"} // 0,
+    )
+}
+
+sub evaluateJobset {
+    my ($self, %opts) = @_;
+
+    my $jobset = $opts{'jobset'};
+
+    my $expression = $opts{'expression'} // $opts{'flake'};
 
     evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0.\n";
 
     my $builds = {};
 
+    my $should_build = $opts{'build'};
+
     for my $build ($jobset->builds) {
         if ($should_build) {
             runBuild($build) or die "Build '".$build->job."' from jobs/$expression should exit with return code 0.\n";

From 0c9726af59f5a19b5fe2928a5542ed3fa8ec2849 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Tue, 16 Jul 2024 04:01:14 +0200
Subject: [PATCH 214/285] flake: add nix-eval-jobs as input

(cherry picked from commit 684cc50d86608cccf7500ce00af89ea34c488473)
---
 flake.lock | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 flake.nix  |  6 +++++
 2 files changed, 73 insertions(+)

diff --git a/flake.lock b/flake.lock
index 897785ed..8055275b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,26 @@
 {
   "nodes": {
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -46,6 +67,30 @@
         "type": "github"
       }
     },
+    "nix-eval-jobs": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nix-github-actions": [],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1733814344,
+        "narHash": "sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU=",
+        "owner": "nix-community",
+        "repo": "nix-eval-jobs",
+        "rev": "889ea1406736b53cf165b6c28398aae3969418d1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "release-2.24",
+        "repo": "nix-eval-jobs",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1726688310,
@@ -66,8 +111,30 @@
       "inputs": {
         "libgit2": "libgit2",
         "nix": "nix",
+        "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": "nixpkgs"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1723303070,
+        "narHash": "sha256-krGNVA30yptyRonohQ+i9cnK+CfCpedg6z3qzqVJcTs=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "14c092e0326de759e16b37535161b3cb9770cea3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index fccd45b9..d25f451c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -8,6 +8,9 @@
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
   inputs.nix.inputs.libgit2.follows = "libgit2";
 
+  inputs.nix-eval-jobs.url = "github:nix-community/nix-eval-jobs/release-2.24";
+  inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
+
   # hide nix dev tooling from our lock file
   inputs.nix.inputs.flake-parts.follows = "";
   inputs.nix.inputs.git-hooks-nix.follows = "";
@@ -15,6 +18,9 @@
   inputs.nix.inputs.nixpkgs-23-11.follows = "";
   inputs.nix.inputs.flake-compat.follows = "";
 
+  # hide nix-eval-jobs dev tooling from our lock file
+  inputs.nix-eval-jobs.inputs.nix-github-actions.follows = "";
+
   outputs = { self, nixpkgs, nix, ... }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];

From d84ff32ce600204c6473889a3ff16cd6053533c9 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Tue, 16 Jul 2024 04:22:41 +0200
Subject: [PATCH 215/285] hydra-eval-jobset: Use `nix-eval-jobs` instead of
 `hydra-eval-jobs`

incrementally ingest eval results

nix-eval-jobs streams output, unlike hydra-eval-jobs. Now that we've
migrated, we can use this to:

1. Use less RAM by avoiding buffering a whole eval's worth of metadata
   into a Perl string and an array of JSON objects.
2. Make evals latency a bit lower by allowing the queue runner to start
   ingesting builds faster.

Also use the newly-restored constituents support in `nix-eval-jobs`

Note, we pass --workers and --max-memory-size to n-e-j

Lost in the h-e-j -> n-e-j migration, causing evaluation to always be
single threaded and limited to 4GiB RAM. Follow the config settings like
h-e-j used to do (via C++ code).

`nix-eval-jobs` should check `hydraJobs` and then `checks` with flakes

(cherry picked from commit 6d4ccff43c41adaf6e4b2b9bced7243bc2f6e97b)
(cherry picked from commit b0e9b4b2f99f9d8f5c4e780e89f955c394b5ced4)
(cherry picked from commit cdfc5c81e8037d3e4818a3e459d0804b2c157ea9)
(cherry picked from commit 4b107e6ff36bd89958fba36e0fe0340903e7cd13)

Co-Authored-By: Maximilian Bosch <maximilian@mbosch.me>
---
 flake.nix                                  |   4 +-
 package.nix                                |   3 +
 src/script/hydra-eval-jobset               | 190 +++++++++++++--------
 t/evaluator/evaluate-constituents-broken.t |   6 +-
 t/evaluator/evaluate-meta.t                |  22 +++
 t/jobs/meta.nix                            |  17 ++
 t/queue-runner/constituents.t              |   4 +-
 7 files changed, 167 insertions(+), 79 deletions(-)
 create mode 100644 t/evaluator/evaluate-meta.t
 create mode 100644 t/jobs/meta.nix

diff --git a/flake.nix b/flake.nix
index d25f451c..f12b8c2f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -21,7 +21,7 @@
   # hide nix-eval-jobs dev tooling from our lock file
   inputs.nix-eval-jobs.inputs.nix-github-actions.follows = "";
 
-  outputs = { self, nixpkgs, nix, ... }:
+  outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:
     let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
@@ -32,6 +32,7 @@
       overlays.default = final: prev: {
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
+          nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default;
           rawSrc = self;
           nix-perl-bindings = final.nixComponents.nix-perl-bindings;
         };
@@ -75,6 +76,7 @@
       packages = forEachSystem (system: {
         hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
+          nix-eval-jobs = nix-eval-jobs.packages.${system}.default;
           rawSrc = self;
           nix = nix.packages.${system}.nix;
           nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
diff --git a/package.nix b/package.nix
index f944fe2b..ceb2c706 100644
--- a/package.nix
+++ b/package.nix
@@ -50,6 +50,7 @@
 , xz
 , gnutar
 , gnused
+, nix-eval-jobs
 
 , rpm
 , dpkg
@@ -190,6 +191,7 @@ stdenv.mkDerivation (finalAttrs: {
     openldap
     postgresql_13
     pixz
+    nix-eval-jobs
   ];
 
   checkInputs = [
@@ -218,6 +220,7 @@ stdenv.mkDerivation (finalAttrs: {
       darcs
       gnused
       breezy
+      nix-eval-jobs
     ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
   );
 
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 72a386f5..9d500a13 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -17,6 +17,7 @@ use Hydra::Helper::Nix;
 use Hydra::Model::DB;
 use Hydra::Plugin;
 use Hydra::Schema;
+use IPC::Run;
 use JSON::MaybeXS;
 use Net::Statsd;
 use Nix::Store;
@@ -357,22 +358,32 @@ sub evalJobs {
     my @cmd;
 
     if (defined $flakeRef) {
-        @cmd = ("hydra-eval-jobs",
-                "--flake", $flakeRef,
-                "--gc-roots-dir", getGCRootsDir,
-                "--max-jobs", 1);
+        my $nix_expr =
+            "let " .
+            "flake = builtins.getFlake (toString \"$flakeRef\"); " .
+            "in " .
+            "flake.hydraJobs " .
+            "or flake.checks " .
+            "or (throw \"flake '$flakeRef' does not provide any Hydra jobs or checks\")";
+
+        @cmd = ("nix-eval-jobs", "--expr", $nix_expr);
     } else {
         my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
             or die "cannot find the input containing the job expression\n";
 
-        @cmd = ("hydra-eval-jobs",
+        @cmd = ("nix-eval-jobs",
                 "<" . $nixExprInputName . "/" . $nixExprPath . ">",
-                "--gc-roots-dir", getGCRootsDir,
-                "--max-jobs", 1,
                 inputsToArgs($inputInfo));
     }
 
-    push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true";
+    push @cmd, ("--gc-roots-dir", getGCRootsDir);
+    push @cmd, ("--max-jobs", 1);
+    push @cmd, "--meta";
+    push @cmd, "--constituents";
+    push @cmd, "--force-recurse";
+    push @cmd, ("--option", "allow-import-from-derivation", "false") if $config->{allow_import_from_derivation} // "true" ne "true";
+    push @cmd, ("--workers", $config->{evaluator_workers} // 1);
+    push @cmd, ("--max-memory-size", $config->{evaluator_max_memory_size} // 4096);
 
     if (defined $ENV{'HYDRA_DEBUG'}) {
         sub escape {
@@ -384,14 +395,33 @@ sub evalJobs {
         print STDERR "evaluator: @escaped\n";
     }
 
-    (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd);
-    die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
-        . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
-        if $res;
+    my $evalProc = IPC::Run::start \@cmd,
+            '>', IPC::Run::new_chunker, \my $out,
+            '2>', \my $err;
 
-    print STDERR "$stderr";
+    return sub {
+        while (1) {
+            $evalProc->pump;
+            if (!defined $out && !defined $err) {
+                $evalProc->finish;
+                if ($?) {
+                    die "nix-eval-jobs returned " . ($? & 127 ? "signal $?" : "exit code " . ($? >> 8)) . "\n";
+                }
+                return;
+            }
 
-    return decode_json($jobsJSON);
+            if (defined $err) {
+                print STDERR "$err";
+                undef $err;
+            }
+
+            if (defined $out && $out ne '') {
+                my $job = decode_json($out);
+                undef $out;
+                return $job;
+            }
+        }
+    };
 }
 
 
@@ -420,7 +450,7 @@ sub checkBuild {
     my $firstOutputName = $outputNames[0];
     my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
 
-    my $jobName = $buildInfo->{jobName} or die;
+    my $jobName = $buildInfo->{attr} or die;
     my $drvPath = $buildInfo->{drvPath} or die;
 
     my $build;
@@ -474,9 +504,30 @@ sub checkBuild {
 
         my $time = time();
 
-        sub null {
-            my ($s) = @_;
-            return $s eq "" ? undef : $s;
+        sub getMeta {
+            my ($s, $def) = @_;
+            return ($s || "") eq "" ? $def : $s;
+        }
+
+        sub getMetaStrings {
+            my ($v, $k, $acc) = @_;
+            my $t = ref $v;
+
+            if ($t eq 'HASH') {
+                push @$acc, $v->{$k} if exists $v->{$k};
+            } elsif ($t eq 'ARRAY') {
+                getMetaStrings($_, $k, $acc) foreach @$v;
+            } elsif (defined $v) {
+                push @$acc, $v;
+            }
+        }
+
+        sub getMetaConcatStrings {
+            my ($v, $k) = @_;
+
+            my @strings;
+            getMetaStrings($v, $k, \@strings);
+            return join(", ", @strings) || undef;
         }
 
         # Add the build to the database.
@@ -484,19 +535,19 @@ sub checkBuild {
             { timestamp => $time
             , jobset_id => $jobset->id
             , job => $jobName
-            , description => null($buildInfo->{description})
-            , license => null($buildInfo->{license})
-            , homepage => null($buildInfo->{homepage})
-            , maintainers => null($buildInfo->{maintainers})
-            , maxsilent => $buildInfo->{maxSilent}
-            , timeout => $buildInfo->{timeout}
-            , nixname => $buildInfo->{nixName}
+            , description => getMeta($buildInfo->{meta}->{description}, undef)
+            , license => getMetaConcatStrings($buildInfo->{meta}->{license}, "shortName")
+            , homepage => getMeta($buildInfo->{meta}->{homepage}, undef)
+            , maintainers => getMetaConcatStrings($buildInfo->{meta}->{maintainers}, "email")
+            , maxsilent => getMeta($buildInfo->{meta}->{maxSilent}, 7200)
+            , timeout => getMeta($buildInfo->{meta}->{timeout}, 36000)
+            , nixname => $buildInfo->{name}
             , drvpath => $drvPath
             , system => $buildInfo->{system}
-            , priority => $buildInfo->{schedulingPriority}
+            , priority => getMeta($buildInfo->{meta}->{schedulingPriority}, 100)
             , finished => 0
             , iscurrent => 1
-            , ischannel => $buildInfo->{isChannel}
+            , ischannel => getMeta($buildInfo->{meta}->{isChannel}, 0)
             });
 
         $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
@@ -665,7 +716,7 @@ sub checkJobsetWrapped {
         return;
     }
 
-    # Hash the arguments to hydra-eval-jobs and check the
+    # Hash the arguments to nix-eval-jobs and check the
     # JobsetInputHashes to see if the previous evaluation had the same
     # inputs.  If so, bail out.
     my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo));
@@ -687,19 +738,12 @@ sub checkJobsetWrapped {
 
     # Evaluate the job expression.
     my $evalStart = clock_gettime(CLOCK_MONOTONIC);
-    my $jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
-    my $evalStop = clock_gettime(CLOCK_MONOTONIC);
-
-    if ($jobsetsJobset) {
-        my @keys = keys %$jobs;
-        die "The .jobsets jobset must only have a single job named 'jobsets'"
-            unless (scalar @keys) == 1 && $keys[0] eq "jobsets";
-    }
-    Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
+    my $evalStop;
+    my $jobsIter = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
 
     if ($dryRun) {
-        foreach my $name (keys %{$jobs}) {
-            my $job = $jobs->{$name};
+        while (defined(my $job = $jobsIter->())) {
+            my $name = $job->{attr};
             if (defined $job->{drvPath}) {
                 print STDERR "good job $name: $job->{drvPath}\n";
             } else {
@@ -709,36 +753,20 @@ sub checkJobsetWrapped {
         return;
     }
 
-    die "Jobset contains a job with an empty name. Make sure the jobset evaluates to an attrset of jobs.\n"
-        if defined $jobs->{""};
-
-    $jobs->{$_}->{jobName} = $_ for keys %{$jobs};
-
-    my $jobOutPathMap = {};
-    my $jobsetChanged = 0;
-    my $dbStart = clock_gettime(CLOCK_MONOTONIC);
-
-
     # Store the error messages for jobs that failed to evaluate.
     my $evaluationErrorTime = time;
     my $evaluationErrorMsg = "";
-    foreach my $job (values %{$jobs}) {
-        next unless defined $job->{error};
-        $evaluationErrorMsg .=
-            ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
-            ":\n" . $job->{error} . "\n\n";
-    }
-    setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
-
     my $evaluationErrorRecord = $db->resultset('EvaluationErrors')->create(
         { errormsg => $evaluationErrorMsg
         , errortime => $evaluationErrorTime
         }
     );
 
+    my $jobOutPathMap = {};
+    my $jobsetChanged = 0;
     my %buildMap;
-    $db->txn_do(sub {
 
+    $db->txn_do(sub {
         my $prevEval = getPrevJobsetEval($db, $jobset, 1);
 
         # Clear the "current" flag on all builds.  Since we're in a
@@ -751,7 +779,7 @@ sub checkJobsetWrapped {
             , evaluationerror => $evaluationErrorRecord
             , timestamp => time
             , checkouttime => abs(int($checkoutStop - $checkoutStart))
-            , evaltime => abs(int($evalStop - $evalStart))
+            , evaltime => 0
             , hasnewbuilds => 0
             , nrbuilds => 0
             , flake => $flakeRef
@@ -759,11 +787,24 @@ sub checkJobsetWrapped {
             , nixexprpath => $jobset->nixexprpath
             });
 
-        # Schedule each successfully evaluated job.
-        foreach my $job (permute(values %{$jobs})) {
-            next if defined $job->{error};
-            #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
-            checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
+        my @jobsWithConstituents;
+
+        while (defined(my $job = $jobsIter->())) {
+            if ($jobsetsJobset) {
+                die "The .jobsets jobset must only have a single job named 'jobsets'"
+                    unless $job->{attr} eq "jobsets";
+            }
+
+            $evaluationErrorMsg .=
+                ($job->{attr} ne "" ? "in job ‘$job->{attr}’" : "at top-level") .
+                ":\n" . $job->{error} . "\n\n" if defined $job->{error};
+
+            checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins)
+                unless defined $job->{error};
+
+            if (defined $job->{constituents}) {
+                push @jobsWithConstituents, $job;
+            }
         }
 
         # Have any builds been added or removed since last time?
@@ -801,21 +842,20 @@ sub checkJobsetWrapped {
                 $drvPathToId{$x->{drvPath}} = $x;
             }
 
-            foreach my $job (values %{$jobs}) {
-                next unless $job->{constituents};
-
+            foreach my $job (values @jobsWithConstituents) {
+                next unless defined $job->{constituents};
                 if (defined $job->{error}) {
-                    die "aggregate job ‘$job->{jobName}’ failed with the error: $job->{error}\n";
+                    die "aggregate job ‘$job->{attr}’ failed with the error: $job->{error}\n";
                 }
 
                 my $x = $drvPathToId{$job->{drvPath}} or
-                    die "aggregate job ‘$job->{jobName}’ has no corresponding build record.\n";
+                    die "aggregate job ‘$job->{attr}’ has no corresponding build record.\n";
                 foreach my $drvPath (@{$job->{constituents}}) {
                     my $constituent = $drvPathToId{$drvPath};
                     if (defined $constituent) {
                         $db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}});
                     } else {
-                        warn "aggregate job ‘$job->{jobName}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n";
+                        warn "aggregate job ‘$job->{attr}’ has a constituent ‘$drvPath’ that doesn't correspond to a Hydra build\n";
                     }
                 }
             }
@@ -857,11 +897,15 @@ sub checkJobsetWrapped {
         $jobset->update({ enabled => 0 }) if $jobset->enabled == 2;
 
         $jobset->update({ lastcheckedtime => time, forceeval => undef });
+
+        $evaluationErrorRecord->update({ errormsg => $evaluationErrorMsg });
+        setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
+
+        $evalStop = clock_gettime(CLOCK_MONOTONIC);
+        $ev->update({ evaltime => abs(int($evalStop - $evalStart)) });
     });
 
-    my $dbStop = clock_gettime(CLOCK_MONOTONIC);
-
-    Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
+    Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
     Net::Statsd::increment("hydra.evaluator.evals");
     Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
 }
diff --git a/t/evaluator/evaluate-constituents-broken.t b/t/evaluator/evaluate-constituents-broken.t
index ed25d192..0e5960bf 100644
--- a/t/evaluator/evaluate-constituents-broken.t
+++ b/t/evaluator/evaluate-constituents-broken.t
@@ -18,14 +18,14 @@ isnt($res, 0, "hydra-eval-jobset exits non-zero");
 ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
 like(
     $stderr,
-    qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
+    qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
     "The stderr record includes a relevant error message"
 );
 
-$jobset->discard_changes;  # refresh from DB
+$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
 like(
     $jobset->errormsg,
-    qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
+    qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
     "The jobset records a relevant error message"
 );
 
diff --git a/t/evaluator/evaluate-meta.t b/t/evaluator/evaluate-meta.t
new file mode 100644
index 00000000..9f546a7f
--- /dev/null
+++ b/t/evaluator/evaluate-meta.t
@@ -0,0 +1,22 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+my $ctx = test_context();
+
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "meta.nix",
+    build => 1
+);
+
+my $build = $builds->{"full-of-meta"};
+
+is($build->finished, 1, "Build should be finished.");
+is($build->description, "This is the description of the job.", "Wrong description extracted from the build.");
+is($build->license, "MIT, BSD", "Wrong licenses extracted from the build.");
+is($build->homepage, "https://example.com/", "Wrong homepage extracted from the build.");
+is($build->maintainers, 'alice@example.com, bob@not.found', "Wrong maintainers extracted from the build.");
+
+done_testing;
diff --git a/t/jobs/meta.nix b/t/jobs/meta.nix
new file mode 100644
index 00000000..9204e384
--- /dev/null
+++ b/t/jobs/meta.nix
@@ -0,0 +1,17 @@
+with import ./config.nix;
+{
+  full-of-meta =
+    mkDerivation {
+      name = "full-of-meta";
+      builder = ./empty-dir-builder.sh;
+
+      meta = {
+        description = "This is the description of the job.";
+        license = [ { shortName = "MIT"; } "BSD" ];
+        homepage = "https://example.com/";
+        maintainers = [ "alice@example.com" { email = "bob@not.found"; } ];
+
+        outPath = "${placeholder "out"}";
+      };
+    };
+}
diff --git a/t/queue-runner/constituents.t b/t/queue-runner/constituents.t
index c6333642..e1b8d733 100644
--- a/t/queue-runner/constituents.t
+++ b/t/queue-runner/constituents.t
@@ -22,11 +22,11 @@ is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix
 
 like(
     $jobset->errormsg,
-    qr/^does-not-exist: does not exist$/m,
+    qr/^"does-not-exist": does not exist$/m,
     "Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
 like(
     $jobset->errormsg,
-    qr/^does-not-evaluate: error: assertion 'false' failed$/m,
+    qr/^"does-not-evaluate": "error: assertion 'false' failed/m,
     "Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
 
 done_testing;

From 2f92846e5a37b284ffe3df0f6082911dbfc6f0e2 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Tue, 16 Jul 2024 04:04:08 +0200
Subject: [PATCH 216/285] hydra-eval-jobs: remove, replaced by nix-eval-jobs

(cherry picked from commit ed7c58708cd3affd62a598a22a500ed2adf318bf)
---
 package.nix                            |   2 +-
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 587 -------------------------
 src/hydra-eval-jobs/meson.build        |   8 -
 src/meson.build                        |   1 -
 t/meson.build                          |   1 -
 5 files changed, 1 insertion(+), 598 deletions(-)
 delete mode 100644 src/hydra-eval-jobs/hydra-eval-jobs.cc
 delete mode 100644 src/hydra-eval-jobs/meson.build

diff --git a/package.nix b/package.nix
index ceb2c706..a9ec12c8 100644
--- a/package.nix
+++ b/package.nix
@@ -235,7 +235,7 @@ stdenv.mkDerivation (finalAttrs: {
   shellHook = ''
     pushd $(git rev-parse --show-toplevel) >/dev/null
 
-    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
+    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH
     PERL5LIB=$(pwd)/src/lib:$PERL5LIB
     export HYDRA_HOME="$(pwd)/src/"
     mkdir -p .hydra-data
diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
deleted file mode 100644
index b83cae91..00000000
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ /dev/null
@@ -1,587 +0,0 @@
-#include <iostream>
-#include <thread>
-#include <optional>
-#include <unordered_map>
-
-#include "shared.hh"
-#include "store-api.hh"
-#include "eval.hh"
-#include "eval-gc.hh"
-#include "eval-inline.hh"
-#include "eval-settings.hh"
-#include "signals.hh"
-#include "terminal.hh"
-#include "util.hh"
-#include "get-drvs.hh"
-#include "globals.hh"
-#include "common-eval-args.hh"
-#include "flake/flakeref.hh"
-#include "flake/flake.hh"
-#include "attr-path.hh"
-#include "derivations.hh"
-#include "local-fs-store.hh"
-
-#include "hydra-config.hh"
-
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <sys/resource.h>
-
-#include <nlohmann/json.hpp>
-
-void check_pid_status_nonblocking(pid_t check_pid)
-{
-    // Only check 'initialized' and known PID's
-    if (check_pid <= 0) { return; }
-
-    int wstatus = 0;
-    pid_t pid = waitpid(check_pid, &wstatus, WNOHANG);
-    // -1 = failure, WNOHANG: 0 = no change
-    if (pid <= 0) { return; }
-
-    std::cerr << "child process (" << pid << ") ";
-
-    if (WIFEXITED(wstatus)) {
-        std::cerr << "exited with status=" << WEXITSTATUS(wstatus) << std::endl;
-    } else if (WIFSIGNALED(wstatus)) {
-        std::cerr << "killed by signal=" << WTERMSIG(wstatus) << std::endl;
-    } else if (WIFSTOPPED(wstatus)) {
-        std::cerr << "stopped by signal=" << WSTOPSIG(wstatus) << std::endl;
-    } else if (WIFCONTINUED(wstatus)) {
-        std::cerr << "continued" << std::endl;
-    }
-}
-
-using namespace nix;
-
-static Path gcRootsDir;
-static size_t maxMemorySize;
-
-struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
-{
-    Path releaseExpr;
-    bool flake = false;
-    bool dryRun = false;
-
-    MyArgs() : MixCommonArgs("hydra-eval-jobs")
-    {
-        addFlag({
-            .longName = "gc-roots-dir",
-            .description = "garbage collector roots directory",
-            .labels = {"path"},
-            .handler = {&gcRootsDir}
-        });
-
-        addFlag({
-            .longName = "dry-run",
-            .description = "don't create store derivations",
-            .handler = {&dryRun, true}
-        });
-
-        addFlag({
-            .longName = "flake",
-            .description = "build a flake",
-            .handler = {&flake, true}
-        });
-
-        expectArg("expr", &releaseExpr);
-    }
-};
-
-static MyArgs myArgs;
-
-static std::string queryMetaStrings(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & subAttribute)
-{
-    Strings res;
-    std::function<void(Value & v)> rec;
-
-    rec = [&](Value & v) {
-        state.forceValue(v, noPos);
-        if (v.type() == nString)
-            res.emplace_back(v.string_view());
-        else if (v.isList())
-            for (unsigned int n = 0; n < v.listSize(); ++n)
-                rec(*v.listElems()[n]);
-        else if (v.type() == nAttrs) {
-            auto a = v.attrs()->find(state.symbols.create(subAttribute));
-            if (a != v.attrs()->end())
-                res.push_back(std::string(state.forceString(*a->value, a->pos, "while evaluating meta attributes")));
-        }
-    };
-
-    Value * v = drv.queryMeta(name);
-    if (v) rec(*v);
-
-    return concatStringsSep(", ", res);
-}
-
-static void worker(
-    EvalState & state,
-    Bindings & autoArgs,
-    AutoCloseFD & to,
-    AutoCloseFD & from)
-{
-    Value vTop;
-
-    if (myArgs.flake) {
-        using namespace flake;
-
-        auto [flakeRef, fragment, outputSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(fetchSettings, myArgs.releaseExpr, absPath("."));
-
-        auto vFlake = state.allocValue();
-
-        auto lockedFlake = lockFlake(
-                flakeSettings,
-                state,
-                flakeRef,
-            LockFlags {
-                .updateLockFile = false,
-                .useRegistries = false,
-                .allowUnlocked = false,
-            });
-
-        callFlake(state, lockedFlake, *vFlake);
-
-        auto vOutputs = vFlake->attrs()->get(state.symbols.create("outputs"))->value;
-        state.forceValue(*vOutputs, noPos);
-
-        auto aHydraJobs = vOutputs->attrs()->get(state.symbols.create("hydraJobs"));
-        if (!aHydraJobs)
-            aHydraJobs = vOutputs->attrs()->get(state.symbols.create("checks"));
-        if (!aHydraJobs)
-            throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);
-
-        vTop = *aHydraJobs->value;
-
-    } else {
-        state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop);
-    }
-
-    auto vRoot = state.allocValue();
-    state.autoCallFunction(autoArgs, vTop, *vRoot);
-
-    while (true) {
-        /* Wait for the master to send us a job name. */
-        writeLine(to.get(), "next");
-
-        auto s = readLine(from.get());
-        if (s == "exit") break;
-        if (!hasPrefix(s, "do ")) abort();
-        std::string attrPath(s, 3);
-
-        debug("worker process %d at '%s'", getpid(), attrPath);
-
-        /* Evaluate it and send info back to the master. */
-        nlohmann::json reply;
-
-        try {
-            auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first;
-
-            auto v = state.allocValue();
-            state.autoCallFunction(autoArgs, *vTmp, *v);
-
-            if (auto drv = getDerivation(state, *v, false)) {
-
-                // CA derivations do not have static output paths, so we
-                // have to defensively not query output paths in case we
-                // encounter one.
-                PackageInfo::Outputs outputs = drv->queryOutputs(
-                    !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
-
-                if (drv->querySystem() == "unknown")
-                    state.error<EvalError>("derivation must have a 'system' attribute").debugThrow();
-
-                auto drvPath = state.store->printStorePath(drv->requireDrvPath());
-
-                nlohmann::json job;
-
-                job["nixName"] = drv->queryName();
-                job["system"] =drv->querySystem();
-                job["drvPath"] = drvPath;
-                job["description"] = drv->queryMetaString("description");
-                job["license"] = queryMetaStrings(state, *drv, "license", "shortName");
-                job["homepage"] = drv->queryMetaString("homepage");
-                job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email");
-                job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100);
-                job["timeout"] = drv->queryMetaInt("timeout", 36000);
-                job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200);
-                job["isChannel"] = drv->queryMetaBool("isHydraChannel", false);
-
-                /* If this is an aggregate, then get its constituents. */
-                auto a = v->attrs()->get(state.symbols.create("_hydraAggregate"));
-                if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
-                    auto a = v->attrs()->get(state.symbols.create("constituents"));
-                    if (!a)
-                        state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
-
-                    NixStringContext context;
-                    state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
-                    for (auto & c : context)
-                        std::visit(overloaded {
-                            [&](const NixStringContextElem::Built & b) {
-                                job["constituents"].push_back(b.drvPath->to_string(*state.store));
-                            },
-                            [&](const NixStringContextElem::Opaque & o) {
-                            },
-                            [&](const NixStringContextElem::DrvDeep & d) {
-                            },
-                        }, c.raw);
-
-                    state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
-                    for (unsigned int n = 0; n < a->value->listSize(); ++n) {
-                        auto v = a->value->listElems()[n];
-                        state.forceValue(*v, noPos);
-                        if (v->type() == nString)
-                            job["namedConstituents"].push_back(v->string_view());
-                    }
-                }
-
-                /* Register the derivation as a GC root.  !!! This
-                   registers roots for jobs that we may have already
-                   done. */
-                auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>();
-                if (gcRootsDir != "" && localStore) {
-                    Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
-                    if (!pathExists(root))
-                        localStore->addPermRoot(localStore->parseStorePath(drvPath), root);
-                }
-
-                nlohmann::json out;
-                for (auto & [outputName, optOutputPath] : outputs) {
-                    if (optOutputPath) {
-                        out[outputName] = state.store->printStorePath(*optOutputPath);
-                    } else {
-                        // See the `queryOutputs` call above; we should
-                        // not encounter missing output paths otherwise.
-                        assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
-                        out[outputName] = nullptr;
-                    }
-                }
-                job["outputs"] = std::move(out);
-                reply["job"] = std::move(job);
-            }
-
-            else if (v->type() == nAttrs) {
-                auto attrs = nlohmann::json::array();
-                StringSet ss;
-                for (auto & i : v->attrs()->lexicographicOrder(state.symbols)) {
-                    std::string name(state.symbols[i->name]);
-                    if (name.find(' ') != std::string::npos) {
-                        printError("skipping job with illegal name '%s'", name);
-                        continue;
-                    }
-                    attrs.push_back(name);
-                }
-                reply["attrs"] = std::move(attrs);
-            }
-
-            else if (v->type() == nNull)
-                ;
-
-            else state.error<TypeError>("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow();
-
-        } catch (EvalError & e) {
-            auto msg = e.msg();
-            // Transmits the error we got from the previous evaluation
-            // in the JSON output.
-            reply["error"] = filterANSIEscapes(msg, true);
-            // Don't forget to print it into the STDERR log, this is
-            // what's shown in the Hydra UI.
-            printError(msg);
-        }
-
-        writeLine(to.get(), reply.dump());
-
-        /* If our RSS exceeds the maximum, exit. The master will
-           start a new process. */
-        struct rusage r;
-        getrusage(RUSAGE_SELF, &r);
-        if ((size_t) r.ru_maxrss > maxMemorySize * 1024) break;
-    }
-
-    writeLine(to.get(), "restart");
-}
-
-int main(int argc, char * * argv)
-{
-    /* Prevent undeclared dependencies in the evaluation via
-       $NIX_PATH. */
-    unsetenv("NIX_PATH");
-
-    return handleExceptions(argv[0], [&]() {
-
-        auto config = std::make_unique<HydraConfig>();
-
-        auto nrWorkers = config->getIntOption("evaluator_workers", 1);
-        maxMemorySize = config->getIntOption("evaluator_max_memory_size", 4096);
-
-        initNix();
-        initGC();
-
-        myArgs.parseCmdline(argvToStrings(argc, argv));
-
-        auto pureEval = config->getBoolOption("evaluator_pure_eval", myArgs.flake);
-
-        /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
-        settings.builders = "";
-
-        /* Prevent access to paths outside of the Nix search path and
-           to the environment. */
-        evalSettings.restrictEval = true;
-
-        /* When building a flake, use pure evaluation (no access to
-           'getEnv', 'currentSystem' etc. */
-        evalSettings.pureEval = pureEval;
-
-        if (myArgs.dryRun) settings.readOnlyMode = true;
-
-        if (myArgs.releaseExpr == "") throw UsageError("no expression specified");
-
-        if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");
-
-        struct State
-        {
-            std::set<std::string> todo{""};
-            std::set<std::string> active;
-            nlohmann::json jobs;
-            std::exception_ptr exc;
-        };
-
-        std::condition_variable wakeup;
-
-        Sync<State> state_;
-
-        /* Start a handler thread per worker process. */
-        auto handler = [&]()
-        {
-            pid_t pid = -1;
-            try {
-                AutoCloseFD from, to;
-
-                while (true) {
-
-                    /* Start a new worker process if necessary. */
-                    if (pid == -1) {
-                        Pipe toPipe, fromPipe;
-                        toPipe.create();
-                        fromPipe.create();
-                        pid = startProcess(
-                            [&,
-                             to{std::make_shared<AutoCloseFD>(std::move(fromPipe.writeSide))},
-                             from{std::make_shared<AutoCloseFD>(std::move(toPipe.readSide))}
-                            ]()
-                            {
-                                try {
-                                    auto evalStore = myArgs.evalStoreUrl
-                                             ? openStore(*myArgs.evalStoreUrl)
-                                             : openStore();
-                                    EvalState state(myArgs.lookupPath,
-                                            evalStore, fetchSettings, evalSettings);
-                                    Bindings & autoArgs = *myArgs.getAutoArgs(state);
-                                    worker(state, autoArgs, *to, *from);
-                                } catch (Error & e) {
-                                    nlohmann::json err;
-                                    auto msg = e.msg();
-                                    err["error"] = filterANSIEscapes(msg, true);
-                                    printError(msg);
-                                    writeLine(to->get(), err.dump());
-                                    // Don't forget to print it into the STDERR log, this is
-                                    // what's shown in the Hydra UI.
-                                    writeLine(to->get(), "restart");
-                                }
-                            },
-                            ProcessOptions { .allowVfork = false });
-                        from = std::move(fromPipe.readSide);
-                        to = std::move(toPipe.writeSide);
-                        debug("created worker process %d", pid);
-                    }
-
-                    /* Check whether the existing worker process is still there. */
-                    auto s = readLine(from.get());
-                    if (s == "restart") {
-                        pid = -1;
-                        continue;
-                    } else if (s != "next") {
-                        auto json = nlohmann::json::parse(s);
-                        throw Error("worker error: %s", (std::string) json["error"]);
-                    }
-
-                    /* Wait for a job name to become available. */
-                    std::string attrPath;
-
-                    while (true) {
-                        checkInterrupt();
-                        auto state(state_.lock());
-                        if ((state->todo.empty() && state->active.empty()) || state->exc) {
-                            writeLine(to.get(), "exit");
-                            return;
-                        }
-                        if (!state->todo.empty()) {
-                            attrPath = *state->todo.begin();
-                            state->todo.erase(state->todo.begin());
-                            state->active.insert(attrPath);
-                            break;
-                        } else
-                            state.wait(wakeup);
-                    }
-
-                    /* Tell the worker to evaluate it. */
-                    writeLine(to.get(), "do " + attrPath);
-
-                    /* Wait for the response. */
-                    auto response = nlohmann::json::parse(readLine(from.get()));
-
-                    /* Handle the response. */
-                    StringSet newAttrs;
-
-                    if (response.find("job") != response.end()) {
-                        auto state(state_.lock());
-                        state->jobs[attrPath] = response["job"];
-                    }
-
-                    if (response.find("attrs") != response.end()) {
-                        for (auto & i : response["attrs"]) {
-                            std::string path = i;
-                            if (path.find(".") != std::string::npos){
-                                path = "\"" + path  + "\"";
-                            }
-                            auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) path;
-                            newAttrs.insert(s);
-                        }
-                    }
-
-                    if (response.find("error") != response.end()) {
-                        auto state(state_.lock());
-                        state->jobs[attrPath]["error"] = response["error"];
-                    }
-
-                    /* Add newly discovered job names to the queue. */
-                    {
-                        auto state(state_.lock());
-                        state->active.erase(attrPath);
-                        for (auto & s : newAttrs)
-                            state->todo.insert(s);
-                        wakeup.notify_all();
-                    }
-                }
-            } catch (...) {
-                check_pid_status_nonblocking(pid);
-                auto state(state_.lock());
-                state->exc = std::current_exception();
-                wakeup.notify_all();
-            }
-        };
-
-        std::vector<std::thread> threads;
-        for (size_t i = 0; i < nrWorkers; i++)
-            threads.emplace_back(std::thread(handler));
-
-        for (auto & thread : threads)
-            thread.join();
-
-        auto state(state_.lock());
-
-        if (state->exc)
-            std::rethrow_exception(state->exc);
-
-        /* For aggregate jobs that have named consistuents
-           (i.e. constituents that are a job name rather than a
-           derivation), look up the referenced job and add it to the
-           dependencies of the aggregate derivation. */
-        auto store = openStore();
-
-        for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) {
-            auto jobName = i.key();
-            auto & job = i.value();
-
-            auto named = job.find("namedConstituents");
-            if (named == job.end()) continue;
-
-            std::unordered_map<std::string, std::string> brokenJobs;
-            auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state](
-                    const std::string & childJobName) -> std::optional<nlohmann::json> {
-                auto childJob = state->jobs.find(childJobName);
-                if (childJob == state->jobs.end()) {
-                    printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);
-                    brokenJobs[childJobName] = "does not exist";
-                    return std::nullopt;
-                }
-                if (childJob->find("error") != childJob->end()) {
-                    std::string error = (*childJob)["error"];
-                    printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error);
-                    brokenJobs[childJobName] = error;
-                    return std::nullopt;
-                }
-                return *childJob;
-            };
-
-            if (myArgs.dryRun) {
-                for (std::string jobName2 : *named) {
-                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
-                    if (!job2) {
-                        continue;
-                    }
-                    std::string drvPath2 = (*job2)["drvPath"];
-                    job["constituents"].push_back(drvPath2);
-                }
-            } else {
-                auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
-                auto drv = store->readDerivation(drvPath);
-
-                for (std::string jobName2 : *named) {
-                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
-                    if (!job2) {
-                        continue;
-                    }
-                    auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
-                    auto drv2 = store->readDerivation(drvPath2);
-                    job["constituents"].push_back(store->printStorePath(drvPath2));
-                    drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
-                }
-
-                if (brokenJobs.empty()) {
-                    std::string drvName(drvPath.name());
-                    assert(hasSuffix(drvName, drvExtension));
-                    drvName.resize(drvName.size() - drvExtension.size());
-
-                    auto hashModulo = hashDerivationModulo(*store, drv, true);
-                    if (hashModulo.kind != DrvHash::Kind::Regular) continue;
-                    auto h = hashModulo.hashes.find("out");
-                    if (h == hashModulo.hashes.end()) continue;
-                    auto outPath = store->makeOutputPath("out", h->second, drvName);
-                    drv.env["out"] = store->printStorePath(outPath);
-                    drv.outputs.insert_or_assign("out", DerivationOutput::InputAddressed { .path = outPath });
-                    auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));
-
-                    debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
-
-                    job["drvPath"] = newDrvPath;
-                    job["outputs"]["out"] = store->printStorePath(outPath);
-                }
-            }
-
-            job.erase("namedConstituents");
-
-            /* Register the derivation as a GC root.  !!! This
-                registers roots for jobs that we may have already
-                done. */
-            auto localStore = store.dynamic_pointer_cast<LocalFSStore>();
-            if (gcRootsDir != "" && localStore) {
-                auto drvPath = job["drvPath"].get<std::string>();
-                Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
-                if (!pathExists(root))
-                    localStore->addPermRoot(localStore->parseStorePath(drvPath), root);
-            }
-
-            if (!brokenJobs.empty()) {
-                std::stringstream ss;
-                for (const auto& [jobName, error] : brokenJobs) {
-                    ss << jobName << ": " << error << "\n";
-                }
-                job["error"] = ss.str();
-            }
-        }
-
-        std::cout << state->jobs.dump(2) << "\n";
-    });
-}
diff --git a/src/hydra-eval-jobs/meson.build b/src/hydra-eval-jobs/meson.build
deleted file mode 100644
index 916212e1..00000000
--- a/src/hydra-eval-jobs/meson.build
+++ /dev/null
@@ -1,8 +0,0 @@
-hydra_eval_jobs = executable('hydra-eval-jobs',
-  'hydra-eval-jobs.cc',
-  dependencies: [
-    libhydra_dep,
-    nix_dep,
-  ],
-  install: true,
-)
diff --git a/src/meson.build b/src/meson.build
index 8c7562ed..52b821bc 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -1,6 +1,5 @@
 # Native code
 subdir('libhydra')
-subdir('hydra-eval-jobs')
 subdir('hydra-evaluator')
 subdir('hydra-queue-runner')
 
diff --git a/t/meson.build b/t/meson.build
index 11044a03..c3c58458 100644
--- a/t/meson.build
+++ b/t/meson.build
@@ -27,7 +27,6 @@ testenv.prepend('PERL5LIB',
   separator: ':'
 )
 testenv.prepend('PATH',
-  fs.parent(hydra_eval_jobs.full_path()),
   fs.parent(hydra_evaluator.full_path()),
   fs.parent(hydra_queue_runner.full_path()),
   meson.project_source_root() / 'src/script',

From 85383b952222e62baeda050b282682c68a8d2581 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 10 Dec 2024 11:31:40 -0500
Subject: [PATCH 217/285] Render the `nix-eval-jobs` version too

---
 package.nix                      | 3 ++-
 src/lib/Hydra/Controller/Root.pm | 1 +
 src/root/layout.tt               | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/package.nix b/package.nix
index a9ec12c8..e0046024 100644
--- a/package.nix
+++ b/package.nix
@@ -266,7 +266,8 @@ stdenv.mkDerivation (finalAttrs: {
             --prefix PATH ':' $out/bin:$hydraPath \
             --set HYDRA_RELEASE ${version} \
             --set HYDRA_HOME $out/libexec/hydra \
-            --set NIX_RELEASE ${nix.name or "unknown"}
+            --set NIX_RELEASE ${nix.name or "unknown"} \
+            --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
     done
   '';
 
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index aa1ad5ab..a231d7c0 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -51,6 +51,7 @@ sub begin :Private {
     $c->stash->{curUri} = $c->request->uri;
     $c->stash->{version} = $ENV{"HYDRA_RELEASE"} || "<devel>";
     $c->stash->{nixVersion} = $ENV{"NIX_RELEASE"} || "<devel>";
+    $c->stash->{nixEvalJobsVersion} = $ENV{"NIX_EVAL_JOBS_RELEASE"} || "<devel>";
     $c->stash->{curTime} = time;
     $c->stash->{logo} = defined $c->config->{hydra_logo} ? "/logo" : "";
     $c->stash->{tracker} = defined $c->config->{tracker} ? $c->config->{tracker} : "";
diff --git a/src/root/layout.tt b/src/root/layout.tt
index d67ff1b8..399962b4 100644
--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@@ -93,7 +93,7 @@
       <footer class="navbar">
         <hr />
         <small>
-          <em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %]).</em>
+          <em><a href="http://nixos.org/hydra" target="_blank" class="squiggle">Hydra</a> [% HTML.escape(version) %] (using [% HTML.escape(nixVersion) %] and [% HTML.escape(nixEvalJobsVersion) %]).</em>
           [% IF c.user_exists %]
           You are signed in as <tt>[% HTML.escape(c.user.username) %]</tt>
             [%- IF c.user.type == 'google' %] via Google[% END %].

From 1dbc7f584580ba839bd11432c1210e95d0038593 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 11 Feb 2025 22:18:53 -0500
Subject: [PATCH 218/285] Log malformed JSON received from `nix-eval-jobs`

---
 src/script/hydra-eval-jobset | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 9d500a13..5b4026a5 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -416,9 +416,16 @@ sub evalJobs {
             }
 
             if (defined $out && $out ne '') {
-                my $job = decode_json($out);
+                my $job;
+                try {
+                    $job = decode_json($out);
+                } catch {
+                    warn "nix-eval-jobs sent invalid JSON.\n  parse error: $_\n  invalid json: $out\n";
+                };
                 undef $out;
-                return $job;
+                if (defined $job) {
+                    return $job;
+                }
             }
         }
     };

From 9a6928d93b54d175268878066d12904a74ee3b20 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 23 May 2024 13:02:34 -0400
Subject: [PATCH 219/285] Use new `CommonSSHStoreConfig::createSSHMaster`

This avoids some duplicated code, leveraging the same `StoreReference`
type that also undergirds the machine file dedup we just did prior.

By using `LegacySSHStoreConfig`, we're also taking a baby step towards
using the store interface rather than messing around with the protocol
internals.
---
 src/hydra-queue-runner/build-remote.cc | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 5b7b81b7..77bde2c4 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -7,6 +7,7 @@
 
 #include "build-result.hh"
 #include "path.hh"
+#include "legacy-ssh-store.hh"
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
 #include "state.hh"
@@ -403,20 +404,23 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssConnecting);
 
-        auto * pSpecified = std::get_if<StoreReference::Specified>(&machine->storeUri.variant);
+        auto storeRef = machine->completeStoreReference();
+
+        auto * pSpecified = std::get_if<StoreReference::Specified>(&storeRef.variant);
         if (!pSpecified || pSpecified->scheme != "ssh") {
             throw Error("Currently, only (legacy-)ssh stores are supported!");
         }
 
-        SSHMaster master {
+        LegacySSHStoreConfig storeConfig {
+            pSpecified->scheme,
             pSpecified->authority,
-            machine->sshKey,
-            machine->sshPublicHostKey,
-            false, // no SSH master yet
-            false, // no compression yet
-            logFD.get(),
+            storeRef.params
         };
 
+        auto master = storeConfig.createSSHMaster(
+            false, // no SSH master yet
+            logFD.get());
+
         // FIXME: rewrite to use Store.
         auto child = build_remote::openConnection(machine, master);
 

From 8835cbd10fe5659bde8be0504d267a66781268ad Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Feb 2025 18:46:36 -0500
Subject: [PATCH 220/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/a7fdef6858dd45b9d7bda7c92324c63faee7f509?narHash=sha256-XFznzb8L4SdUm9u%2Bw3DPpMWJhffuv%2B/6%2BaiVl00slns%3D' (2024-09-19)
  → 'github:NixOS/nix/d652513e4519ed4eb48c92f8670e5a71c7793fc3?narHash=sha256-mIpJgIwPS4o4xYhN1B%2B/fHESEXoxpu6nVoZTzZ0MfTg%3D' (2025-02-12)
• Updated input 'nix-eval-jobs':
    'github:nix-community/nix-eval-jobs/889ea1406736b53cf165b6c28398aae3969418d1?narHash=sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU%3D' (2024-12-10)
  → 'github:nix-community/nix-eval-jobs/6d4fd5a93d7bc953ffa4dcd6d53ad7056a71eff7?narHash=sha256-1dZLPw%2BnlFQzzswfyTxW%2B8VF1AJ4ZvoYvLTjlHiz1SA%3D' (2025-02-13)
---
 flake.lock | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8055275b..baa5e8be 100644
--- a/flake.lock
+++ b/flake.lock
@@ -53,16 +53,16 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1726787955,
-        "narHash": "sha256-XFznzb8L4SdUm9u+w3DPpMWJhffuv+/6+aiVl00slns=",
+        "lastModified": 1739390454,
+        "narHash": "sha256-mIpJgIwPS4o4xYhN1B+/fHESEXoxpu6nVoZTzZ0MfTg=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "a7fdef6858dd45b9d7bda7c92324c63faee7f509",
+        "rev": "d652513e4519ed4eb48c92f8670e5a71c7793fc3",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.24-maintenance",
+        "ref": "2.25-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -77,16 +77,15 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1733814344,
-        "narHash": "sha256-3wwtKpS5tUBdjaGeSia7CotonbiRB6K5Kp0dsUt3nzU=",
+        "lastModified": 1739426028,
+        "narHash": "sha256-1dZLPw+nlFQzzswfyTxW+8VF1AJ4ZvoYvLTjlHiz1SA=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "889ea1406736b53cf165b6c28398aae3969418d1",
+        "rev": "6d4fd5a93d7bc953ffa4dcd6d53ad7056a71eff7",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "release-2.24",
         "repo": "nix-eval-jobs",
         "type": "github"
       }

From 4347833f45027d13172bf41c21653911795cfac7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Feb 2025 19:03:37 -0500
Subject: [PATCH 221/285] Rework to synchronize deps

Bypass `nix-eval-job`'s flake, and just call-package it, for
fine-grained control.
---
 flake.nix | 47 +++++++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/flake.nix b/flake.nix
index f12b8c2f..5b48879f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,23 +3,35 @@
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
 
-  inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
-  inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
-  inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
-  inputs.nix.inputs.libgit2.follows = "libgit2";
+  inputs.libgit2 = {
+    url = "github:libgit2/libgit2/v1.8.1";
+    flake = false;
+  };
 
-  inputs.nix-eval-jobs.url = "github:nix-community/nix-eval-jobs/release-2.24";
-  inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
+  inputs.nix = {
+    url = "github:NixOS/nix/2.25-maintenance";
+    inputs.nixpkgs.follows = "nixpkgs";
+    inputs.libgit2.follows = "libgit2";
 
-  # hide nix dev tooling from our lock file
-  inputs.nix.inputs.flake-parts.follows = "";
-  inputs.nix.inputs.git-hooks-nix.follows = "";
-  inputs.nix.inputs.nixpkgs-regression.follows = "";
-  inputs.nix.inputs.nixpkgs-23-11.follows = "";
-  inputs.nix.inputs.flake-compat.follows = "";
+    # hide nix dev tooling from our lock file
+    inputs.flake-parts.follows = "";
+    inputs.git-hooks-nix.follows = "";
+    inputs.nixpkgs-regression.follows = "";
+    inputs.nixpkgs-23-11.follows = "";
+    inputs.flake-compat.follows = "";
+  };
 
-  # hide nix-eval-jobs dev tooling from our lock file
-  inputs.nix-eval-jobs.inputs.nix-github-actions.follows = "";
+  inputs.nix-eval-jobs = {
+    url = "github:nix-community/nix-eval-jobs";
+    # We want to control the deps precisely
+    flake = false;
+
+    # Hide nix-eval-jobs dev tooling from our lock file.
+    #
+    # TODO why is this needed with `flake = false`?
+    inputs.nixpkgs.follows = "nixpkgs";
+    inputs.nix-github-actions.follows = "";
+  };
 
   outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:
     let
@@ -30,9 +42,9 @@
 
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlays.default = final: prev: {
+        nix-eval-jobs = final.callPackage nix-eval-jobs {};
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
-          nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default;
           rawSrc = self;
           nix-perl-bindings = final.nixComponents.nix-perl-bindings;
         };
@@ -74,9 +86,12 @@
       });
 
       packages = forEachSystem (system: {
+        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
+          nix = nix.packages.${system}.nix;
+        };
         hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
-          nix-eval-jobs = nix-eval-jobs.packages.${system}.default;
+          inherit (self.packages.${system}) nix-eval-jobs;
           rawSrc = self;
           nix = nix.packages.${system}.nix;
           nix-perl-bindings = nix.hydraJobs.perlBindings.${system};

From 80241fc8bedced8dc0af534284715f7fbbccb197 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Feb 2025 19:10:09 -0500
Subject: [PATCH 222/285] Make code change necessary for building with Nix 2.25

---
 src/hydra-queue-runner/builder.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index a0773511..4bc00f0c 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -178,7 +178,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                     unlink(result.logFile.c_str());
                 }
             } catch (...) {
-                ignoreException();
+                ignoreExceptionInDestructor();
             }
         }
     });

From 81d21979ef8b167eff264efdf9316431dbca6fd4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Feb 2025 20:21:06 -0500
Subject: [PATCH 223/285] Clean up flake lockfile stuff
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The `flake = false;` for `nix-eval-jobs` didn't fully take before.

Flake lock file updates:

• Removed input 'nix-eval-jobs/flake-parts'
• Removed input 'nix-eval-jobs/flake-parts/nixpkgs-lib'
• Removed input 'nix-eval-jobs/nix-github-actions'
• Removed input 'nix-eval-jobs/nixpkgs'
• Removed input 'nix-eval-jobs/treefmt-nix'
• Removed input 'nix-eval-jobs/treefmt-nix/nixpkgs'
---
 flake.lock | 51 +--------------------------------------------------
 flake.nix  |  6 ------
 2 files changed, 1 insertion(+), 56 deletions(-)

diff --git a/flake.lock b/flake.lock
index baa5e8be..1d78e0c1 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,26 +1,5 @@
 {
   "nodes": {
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
     "libgit2": {
       "flake": false,
       "locked": {
@@ -68,14 +47,7 @@
       }
     },
     "nix-eval-jobs": {
-      "inputs": {
-        "flake-parts": "flake-parts",
-        "nix-github-actions": [],
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "treefmt-nix": "treefmt-nix"
-      },
+      "flake": false,
       "locked": {
         "lastModified": 1739426028,
         "narHash": "sha256-1dZLPw+nlFQzzswfyTxW+8VF1AJ4ZvoYvLTjlHiz1SA=",
@@ -113,27 +85,6 @@
         "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": "nixpkgs"
       }
-    },
-    "treefmt-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1723303070,
-        "narHash": "sha256-krGNVA30yptyRonohQ+i9cnK+CfCpedg6z3qzqVJcTs=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "14c092e0326de759e16b37535161b3cb9770cea3",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index 5b48879f..6e2d51a2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -25,12 +25,6 @@
     url = "github:nix-community/nix-eval-jobs";
     # We want to control the deps precisely
     flake = false;
-
-    # Hide nix-eval-jobs dev tooling from our lock file.
-    #
-    # TODO why is this needed with `flake = false`?
-    inputs.nixpkgs.follows = "nixpkgs";
-    inputs.nix-github-actions.follows = "";
   };
 
   outputs = { self, nixpkgs, nix, nix-eval-jobs, ... }:

From 4dc0f11379d74740656edecb8938aac0de07290f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Feb 2025 20:27:11 -0500
Subject: [PATCH 224/285] Update flake.nix for Nix 2.26
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Removed input 'libgit2'
• Updated input 'nix':
    'github:NixOS/nix/d652513e4519ed4eb48c92f8670e5a71c7793fc3?narHash=sha256-mIpJgIwPS4o4xYhN1B%2B/fHESEXoxpu6nVoZTzZ0MfTg%3D' (2025-02-12)
  → 'github:NixOS/nix/970942f45836172fda410a638853382952189eb9?narHash=sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0%3D' (2025-02-12)
• Removed input 'nix/libgit2'
• Updated input 'nix-eval-jobs':
    'github:nix-community/nix-eval-jobs/6d4fd5a93d7bc953ffa4dcd6d53ad7056a71eff7?narHash=sha256-1dZLPw%2BnlFQzzswfyTxW%2B8VF1AJ4ZvoYvLTjlHiz1SA%3D' (2025-02-13)
  → 'github:nix-community/nix-eval-jobs/4b392b284877d203ae262e16af269f702df036bc?narHash=sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY%3D' (2025-02-14)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/dbebdd67a6006bb145d98c8debf9140ac7e651d0?narHash=sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0%3D' (2024-09-18)
  → 'github:NixOS/nixpkgs/97a719c9f0a07923c957cf51b20b329f9fb9d43f?narHash=sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX%2BLv6I39zGRzKM%3D' (2025-02-13)
---
 flake.lock | 43 +++++++++++--------------------------------
 flake.nix  | 10 ++--------
 2 files changed, 13 insertions(+), 40 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1d78e0c1..ec9a00e6 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,30 +1,10 @@
 {
   "nodes": {
-    "libgit2": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1715853528,
-        "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
-        "type": "github"
-      },
-      "original": {
-        "owner": "libgit2",
-        "ref": "v1.8.1",
-        "repo": "libgit2",
-        "type": "github"
-      }
-    },
     "nix": {
       "inputs": {
         "flake-compat": [],
         "flake-parts": [],
         "git-hooks-nix": [],
-        "libgit2": [
-          "libgit2"
-        ],
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -32,16 +12,16 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1739390454,
-        "narHash": "sha256-mIpJgIwPS4o4xYhN1B+/fHESEXoxpu6nVoZTzZ0MfTg=",
+        "lastModified": 1739393420,
+        "narHash": "sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "d652513e4519ed4eb48c92f8670e5a71c7793fc3",
+        "rev": "970942f45836172fda410a638853382952189eb9",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.25-maintenance",
+        "ref": "2.26-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -49,11 +29,11 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1739426028,
-        "narHash": "sha256-1dZLPw+nlFQzzswfyTxW+8VF1AJ4ZvoYvLTjlHiz1SA=",
+        "lastModified": 1739500569,
+        "narHash": "sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "6d4fd5a93d7bc953ffa4dcd6d53ad7056a71eff7",
+        "rev": "4b392b284877d203ae262e16af269f702df036bc",
         "type": "github"
       },
       "original": {
@@ -64,23 +44,22 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1726688310,
-        "narHash": "sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0=",
+        "lastModified": 1739461644,
+        "narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "dbebdd67a6006bb145d98c8debf9140ac7e651d0",
+        "rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.05-small",
+        "ref": "nixos-24.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "libgit2": "libgit2",
         "nix": "nix",
         "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": "nixpkgs"
diff --git a/flake.nix b/flake.nix
index 6e2d51a2..da753c5f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,17 +1,11 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
-
-  inputs.libgit2 = {
-    url = "github:libgit2/libgit2/v1.8.1";
-    flake = false;
-  };
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
 
   inputs.nix = {
-    url = "github:NixOS/nix/2.25-maintenance";
+    url = "github:NixOS/nix/2.26-maintenance";
     inputs.nixpkgs.follows = "nixpkgs";
-    inputs.libgit2.follows = "libgit2";
 
     # hide nix dev tooling from our lock file
     inputs.flake-parts.follows = "";

From 341b2f130926a0da999d7ff4869a201bfafd5ab6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Feb 2025 20:50:11 -0500
Subject: [PATCH 225/285] Update build system to depend on Nix 2.26

---
 flake.nix     |  7 ++++++-
 meson.build   | 14 +++++++-------
 package.nix   | 18 ++++++++++++------
 t/meson.build |  1 +
 4 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/flake.nix b/flake.nix
index da753c5f..dc3aaf5c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -81,7 +81,12 @@
           inherit (nixpkgs.lib) fileset;
           inherit (self.packages.${system}) nix-eval-jobs;
           rawSrc = self;
-          nix = nix.packages.${system}.nix;
+          inherit (nix.packages.${system})
+            nix-util
+            nix-store
+            nix-main
+            nix-cli
+            ;
           nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
         };
         default = self.packages.${system}.hydra;
diff --git a/meson.build b/meson.build
index b9327d51..9c9c09a1 100644
--- a/meson.build
+++ b/meson.build
@@ -8,22 +8,22 @@ project('hydra', 'cpp',
   ],
 )
 
+nix_util_dep = dependency('nix-util', required: true)
 nix_store_dep = dependency('nix-store', required: true)
 nix_main_dep = dependency('nix-main', required: true)
-nix_expr_dep = dependency('nix-expr', required: true)
-nix_flake_dep = dependency('nix-flake', required: true)
-nix_cmd_dep = dependency('nix-cmd', required: true)
 
 # Nix need extra flags not provided in its pkg-config files.
 nix_dep = declare_dependency(
   dependencies: [
+    nix_util_dep,
     nix_store_dep,
     nix_main_dep,
-    nix_expr_dep,
-    nix_flake_dep,
-    nix_cmd_dep,
   ],
-  compile_args: ['-include', 'nix/config.h'],
+  compile_args: [
+    '-include', 'nix/config-util.hh',
+    '-include', 'nix/config-store.hh',
+    '-include', 'nix/config-main.hh',
+  ],
 )
 
 pqxx_dep = dependency('libpqxx', required: true)
diff --git a/package.nix b/package.nix
index e0046024..12fac1d8 100644
--- a/package.nix
+++ b/package.nix
@@ -8,7 +8,10 @@
 
 , perlPackages
 
-, nix
+, nix-util
+, nix-store
+, nix-main
+, nix-cli
 , nix-perl-bindings
 , git
 
@@ -162,7 +165,7 @@ stdenv.mkDerivation (finalAttrs: {
     nukeReferences
     pkg-config
     mdbook
-    nix
+    nix-cli
     perlDeps
     perl
     unzip
@@ -172,7 +175,9 @@ stdenv.mkDerivation (finalAttrs: {
     libpqxx
     openssl
     libxslt
-    nix
+    nix-util
+    nix-store
+    nix-main
     perlDeps
     perl
     boost
@@ -199,13 +204,14 @@ stdenv.mkDerivation (finalAttrs: {
     glibcLocales
     libressl.nc
     python3
+    nix-cli
   ];
 
   hydraPath = lib.makeBinPath (
     [
       subversion
       openssh
-      nix
+      nix-cli
       coreutils
       findutils
       pixz
@@ -266,7 +272,7 @@ stdenv.mkDerivation (finalAttrs: {
             --prefix PATH ':' $out/bin:$hydraPath \
             --set HYDRA_RELEASE ${version} \
             --set HYDRA_HOME $out/libexec/hydra \
-            --set NIX_RELEASE ${nix.name or "unknown"} \
+            --set NIX_RELEASE ${nix-cli.name or "unknown"} \
             --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
     done
   '';
@@ -274,5 +280,5 @@ stdenv.mkDerivation (finalAttrs: {
   dontStrip = true;
 
   meta.description = "Build of Hydra on ${stdenv.system}";
-  passthru = { inherit perlDeps nix; };
+  passthru = { inherit perlDeps; };
 })
diff --git a/t/meson.build b/t/meson.build
index c3c58458..dd930dbd 100644
--- a/t/meson.build
+++ b/t/meson.build
@@ -27,6 +27,7 @@ testenv.prepend('PERL5LIB',
   separator: ':'
 )
 testenv.prepend('PATH',
+  fs.parent(find_program('nix').full_path()),
   fs.parent(hydra_evaluator.full_path()),
   fs.parent(hydra_queue_runner.full_path()),
   meson.project_source_root() / 'src/script',

From df07670a21a48993c8c3392446300bb915cebf0e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 16 Feb 2025 18:44:26 -0500
Subject: [PATCH 226/285] Bump to newer 2.26.* Nix version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/970942f45836172fda410a638853382952189eb9?narHash=sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0%3D' (2025-02-12)
  → 'github:NixOS/nix/674a87462cb93f605d4fbeef607d3453e7e5a7d8?narHash=sha256-TBoHqnIdVWhsBcL05vO2B1hSl9m//5Mz2NU%2BPMk3h3Y%3D' (2025-02-16)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index ec9a00e6..8b1cbf2e 100644
--- a/flake.lock
+++ b/flake.lock
@@ -12,11 +12,11 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1739393420,
-        "narHash": "sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0=",
+        "lastModified": 1739746614,
+        "narHash": "sha256-TBoHqnIdVWhsBcL05vO2B1hSl9m//5Mz2NU+PMk3h3Y=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "970942f45836172fda410a638853382952189eb9",
+        "rev": "674a87462cb93f605d4fbeef607d3453e7e5a7d8",
         "type": "github"
       },
       "original": {

From af72b694d8f0fcca90f464e2d716bca65d100b6d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 18 Feb 2025 11:51:51 -0500
Subject: [PATCH 227/285] Bump to newer 2.26.* Nix version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Needed one more thing before trying out using `LegacySSHStore` directly.

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/674a87462cb93f605d4fbeef607d3453e7e5a7d8?narHash=sha256-TBoHqnIdVWhsBcL05vO2B1hSl9m//5Mz2NU%2BPMk3h3Y%3D' (2025-02-16)
  → 'github:NixOS/nix/e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918?narHash=sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs%3D' (2025-02-18)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8b1cbf2e..c47a3b88 100644
--- a/flake.lock
+++ b/flake.lock
@@ -12,11 +12,11 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1739746614,
-        "narHash": "sha256-TBoHqnIdVWhsBcL05vO2B1hSl9m//5Mz2NU+PMk3h3Y=",
+        "lastModified": 1739899400,
+        "narHash": "sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "674a87462cb93f605d4fbeef607d3453e7e5a7d8",
+        "rev": "e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918",
         "type": "github"
       },
       "original": {

From 4a4a0f901c70676ee47f830d2ff6a72789ba1baf Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 20 May 2024 16:22:19 -0400
Subject: [PATCH 228/285] Use `LegacySSHStore`

In https://github.com/NixOS/nix/pull/10748 it is extended with
everything we need.
---
 src/hydra-queue-runner/build-remote.cc | 178 +++++++++----------------
 src/hydra-queue-runner/state.hh        |   8 +-
 2 files changed, 68 insertions(+), 118 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 77bde2c4..39970bd3 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -9,13 +9,10 @@
 #include "path.hh"
 #include "legacy-ssh-store.hh"
 #include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
 #include "state.hh"
 #include "current-process.hh"
 #include "processes.hh"
 #include "util.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
 #include "ssh.hh"
 #include "finally.hh"
 #include "url.hh"
@@ -39,38 +36,6 @@ bool ::Machine::isLocalhost() const
 
 namespace nix::build_remote {
 
-static std::unique_ptr<SSHMaster::Connection> openConnection(
-    ::Machine::ptr machine, SSHMaster & master)
-{
-    Strings command = {"nix-store", "--serve", "--write"};
-    if (machine->isLocalhost()) {
-        command.push_back("--builders");
-        command.push_back("");
-    } else {
-        auto remoteStore = machine->storeUri.params.find("remote-store");
-        if (remoteStore != machine->storeUri.params.end()) {
-            command.push_back("--store");
-            command.push_back(shellEscape(remoteStore->second));
-        }
-    }
-
-    auto ret = master.startCommand(std::move(command), {
-        "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
-    });
-
-    // XXX: determine the actual max value we can use from /proc.
-
-    // FIXME: Should this be upstreamed into `startCommand` in Nix?
-
-    int pipesize = 1024 * 1024;
-
-    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
-    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
-
-    return ret;
-}
-
-
 static void copyClosureTo(
     ::Machine::Connection & conn,
     Store & destStore,
@@ -87,8 +52,8 @@ static void copyClosureTo(
     // FIXME: substitute output pollutes our build log
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = conn.queryValidPaths(
-        destStore, true, closure, useSubstitutes);
+    auto present = conn.store->queryValidPaths(
+        closure, true, useSubstitutes);
 
     if (present.size() == closure.size()) return;
 
@@ -103,12 +68,7 @@ static void copyClosureTo(
     std::unique_lock<std::timed_mutex> sendLock(conn.machine->state->sendLock,
         std::chrono::seconds(600));
 
-    conn.to << ServeProto::Command::ImportPaths;
-    destStore.exportPaths(missing, conn.to);
-    conn.to.flush();
-
-    if (readInt(conn.from) != 1)
-        throw Error("remote machine failed to import closure");
+    conn.store->addMultipleToStoreLegacy(destStore, missing);
 }
 
 
@@ -228,7 +188,7 @@ static BuildResult performBuild(
     counter & nrStepsBuilding
 )
 {
-    conn.putBuildDerivationRequest(localStore, drvPath, drv, options);
+    auto kont = conn.store->buildDerivationAsync(drvPath, drv, options);
 
     BuildResult result;
 
@@ -237,7 +197,10 @@ static BuildResult performBuild(
     startTime = time(0);
     {
         MaintainCount<counter> mc(nrStepsBuilding);
-        result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
+        result = kont();
+        // Without proper call-once functions, we need to manually
+        // delete after calling.
+        kont = {};
     }
     stopTime = time(0);
 
@@ -253,7 +216,7 @@ static BuildResult performBuild(
 
     // If the protocol was too old to give us `builtOutputs`, initialize
     // it manually by introspecting the derivation.
-    if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 6)
+    if (GET_PROTOCOL_MINOR(conn.store->getProtocol()) < 6)
     {
         // If the remote is too old to handle CA derivations, we can’t get this
         // far anyways
@@ -286,26 +249,25 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
-      /* Receive the NAR from the remote and add it to the
-          destination store. Meanwhile, extract all the info from the
-          NAR that getBuildOutput() needs. */
-      auto source2 = sinkToSource([&](Sink & sink)
-      {
-          /* Note: we should only send the command to dump the store
-              path to the remote if the NAR is actually going to get read
-              by the destination store, which won't happen if this path
-              is already valid on the destination store. Since this
-              lambda function only gets executed if someone tries to read
-              from source2, we will send the command from here rather
-              than outside the lambda. */
-          conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path);
-          conn.to.flush();
+    /* Receive the NAR from the remote and add it to the
+        destination store. Meanwhile, extract all the info from the
+        NAR that getBuildOutput() needs. */
+    auto source2 = sinkToSource([&](Sink & sink)
+    {
+        /* Note: we should only send the command to dump the store
+            path to the remote if the NAR is actually going to get read
+            by the destination store, which won't happen if this path
+            is already valid on the destination store. Since this
+            lambda function only gets executed if someone tries to read
+            from source2, we will send the command from here rather
+            than outside the lambda. */
+        conn.store->narFromPath(info.path, [&](Source & source) {
+            TeeSource tee{source, sink};
+            extractNarData(tee, conn.store->printStorePath(info.path), narMembers);
+        });
+    });
 
-          TeeSource tee(conn.from, sink);
-          extractNarData(tee, localStore.printStorePath(info.path), narMembers);
-      });
-
-      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
+    destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
 }
 
 static void copyPathsFromRemote(
@@ -404,30 +366,39 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssConnecting);
 
-        auto storeRef = machine->completeStoreReference();
-
-        auto * pSpecified = std::get_if<StoreReference::Specified>(&storeRef.variant);
-        if (!pSpecified || pSpecified->scheme != "ssh") {
-            throw Error("Currently, only (legacy-)ssh stores are supported!");
-        }
-
-        LegacySSHStoreConfig storeConfig {
-            pSpecified->scheme,
-            pSpecified->authority,
-            storeRef.params
-        };
-
-        auto master = storeConfig.createSSHMaster(
-            false, // no SSH master yet
-            logFD.get());
-
         // FIXME: rewrite to use Store.
-        auto child = build_remote::openConnection(machine, master);
+        ::Machine::Connection conn {
+            .machine = machine,
+            .store = [&]{
+                auto * pSpecified = std::get_if<StoreReference::Specified>(&machine->storeUri.variant);
+                if (!pSpecified || pSpecified->scheme != "ssh") {
+                    throw Error("Currently, only (legacy-)ssh stores are supported!");
+                }
+
+                auto remoteStore = machine->openStore().dynamic_pointer_cast<LegacySSHStore>();
+                assert(remoteStore);
+
+                remoteStore->connPipeSize = 1024 * 1024;
+
+                if (machine->isLocalhost()) {
+                    auto rp_new = remoteStore->remoteProgram.get();
+                    rp_new.push_back("--builders");
+                    rp_new.push_back("");
+                    const_cast<nix::Setting<Strings> &>(remoteStore->remoteProgram).assign(rp_new);
+                }
+                remoteStore->extraSshArgs = {
+                    "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+                };
+                const_cast<nix::Setting<int> &>(remoteStore->logFD).assign(logFD.get());
+
+                return nix::ref{remoteStore};
+            }(),
+        };
 
         {
             auto activeStepState(activeStep->state_.lock());
             if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = child->sshPid;
+            activeStepState->pid = conn.store->getConnectionPid();
         }
 
         Finally clearPid([&]() {
@@ -442,35 +413,12 @@ void State::buildRemote(ref<Store> destStore,
                process. Meh. */
         });
 
-        ::Machine::Connection conn {
-            {
-                .to = child->in.get(),
-                .from = child->out.get(),
-                /* Handshake. */
-                .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
-            },
-            /*.machine =*/ machine,
-        };
-
         Finally updateStats([&]() {
-            bytesReceived += conn.from.read;
-            bytesSent += conn.to.written;
+            auto stats = conn.store->getConnectionStats();
+            bytesReceived += stats.bytesReceived;
+            bytesSent += stats.bytesSent;
         });
 
-        constexpr ServeProto::Version our_version = 0x206;
-
-        try {
-            conn.remoteVersion = decltype(conn)::handshake(
-                conn.to,
-                conn.from,
-                our_version,
-                machine->storeUri.render());
-        } catch (EndOfFile & e) {
-            child->sshPid.wait();
-            std::string s = chomp(readFile(result.logFile));
-            throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s);
-        }
-
         {
             auto info(machine->state->connectInfo.lock());
             info->consecutiveFailures = 0;
@@ -539,7 +487,7 @@ void State::buildRemote(ref<Store> destStore,
 
             auto now1 = std::chrono::steady_clock::now();
 
-            auto infos = conn.queryPathInfos(*localStore, outputs);
+            auto infos = conn.store->queryPathInfosUncached(outputs);
 
             size_t totalNarSize = 0;
             for (auto & [_, info] : infos) totalNarSize += info.narSize;
@@ -574,9 +522,11 @@ void State::buildRemote(ref<Store> destStore,
             }
         }
 
-        /* Shut down the connection. */
-        child->in = -1;
-        child->sshPid.wait();
+        /* Shut down the connection done by RAII.
+
+           Only difference is kill() instead of wait() (i.e. send signal
+           then wait())
+         */
 
     } catch (Error & e) {
         /* Disable this machine until a certain period of time has
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 30e01c74..e2d31434 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -20,9 +20,7 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "nar-extractor.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
-#include "serve-protocol-connection.hh"
+#include "legacy-ssh-store.hh"
 #include "machines.hh"
 
 
@@ -292,9 +290,11 @@ struct Machine : nix::Machine
     bool isLocalhost() const;
 
     // A connection to a machine
-    struct Connection : nix::ServeProto::BasicClientConnection {
+    struct Connection {
         // Backpointer to the machine
         ptr machine;
+        // Opened store
+        nix::ref<nix::LegacySSHStore> store;
     };
 };
 

From d22d0305033b8c18cead3e9736288c3c69a70dc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Josef=20Kemetm=C3=BCller?= <josef.kemetmueller@gmail.com>
Date: Wed, 26 Feb 2025 16:32:42 +0100
Subject: [PATCH 229/285] Fix rendering of metrics with special characters

My main motivation here is to get metrics with brackets to work in order
to support "pytest" test names:

- test_foo.py::test_bar[1]
- test_foo.py::test_bar[2]

I couldn't find an "HTML escape"-style function that would generate
valid html `id` attribute names from random strings, so I went with a
hash digest instead.
---
 src/lib/Hydra/View/TT.pm    | 7 +++++++
 src/root/job-metrics-tab.tt | 3 +--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/View/TT.pm b/src/lib/Hydra/View/TT.pm
index 84fcf3e9..241787e0 100644
--- a/src/lib/Hydra/View/TT.pm
+++ b/src/lib/Hydra/View/TT.pm
@@ -6,6 +6,7 @@ use base 'Catalyst::View::TT';
 use Template::Plugin::HTML;
 use Hydra::Helper::Nix;
 use Time::Seconds;
+use Digest::SHA qw(sha1_hex);
 
 __PACKAGE__->config(
     TEMPLATE_EXTENSION => '.tt',
@@ -25,8 +26,14 @@ __PACKAGE__->config(
     makeNameTextForJobset
     relativeDuration
     stripSSHUser
+    metricDivId
     /]);
 
+sub metricDivId {
+    my ($self, $c, $text) = @_;
+    return "metric-" . sha1_hex($text);
+}
+
 sub buildLogExists {
     my ($self, $c, $build) = @_;
     return 1 if defined $c->config->{log_prefix};
diff --git a/src/root/job-metrics-tab.tt b/src/root/job-metrics-tab.tt
index 23d8ffa3..123a00f1 100644
--- a/src/root/job-metrics-tab.tt
+++ b/src/root/job-metrics-tab.tt
@@ -18,8 +18,7 @@
 
     <h3>Metric: <a [% HTML.attributes(href => c.uri_for('/job' project.name jobset.name job 'metric' metric.name)) %]><tt>[%HTML.escape(metric.name)%]</tt></a></h3>
 
-    [% id = "metric-" _ metric.name;
-       id = id.replace('\.', '_');
+    [% id = metricDivId(metric.name);
        INCLUDE createChart dataUrl=c.uri_for('/job' project.name jobset.name job 'metric' metric.name); %]
 
   [% END %]

From f1deb22c028baf170f871a5b5d8c1936d04a5875 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Sun, 2 Mar 2025 03:08:26 +0100
Subject: [PATCH 230/285] Fix race condition in hydra-compress-logs

---
 nixos-modules/hydra.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 4fc2d311..79d639e6 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -468,7 +468,7 @@ in
             elif [[ $compression == zstd ]]; then
               compression="zstd --rm"
             fi
-            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
+            find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
           '';
         startAt = "Sun 01:45";
       };

From 9a5bd39d4c89ee7bb1607143a8755d5ab68e7f2f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 3 Mar 2025 10:10:04 -0500
Subject: [PATCH 231/285] Revert "Use `LegacySSHStore`"

There were some hangs caused by this. Need to fix them, ideally
reproducing the issue in a test, before trying this again.

This reverts commit 4a4a0f901c70676ee47f830d2ff6a72789ba1baf.
---
 src/hydra-queue-runner/build-remote.cc | 174 ++++++++++++++++---------
 src/hydra-queue-runner/state.hh        |   8 +-
 2 files changed, 116 insertions(+), 66 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 39970bd3..77bde2c4 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -9,10 +9,13 @@
 #include "path.hh"
 #include "legacy-ssh-store.hh"
 #include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "state.hh"
 #include "current-process.hh"
 #include "processes.hh"
 #include "util.hh"
+#include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "ssh.hh"
 #include "finally.hh"
 #include "url.hh"
@@ -36,6 +39,38 @@ bool ::Machine::isLocalhost() const
 
 namespace nix::build_remote {
 
+static std::unique_ptr<SSHMaster::Connection> openConnection(
+    ::Machine::ptr machine, SSHMaster & master)
+{
+    Strings command = {"nix-store", "--serve", "--write"};
+    if (machine->isLocalhost()) {
+        command.push_back("--builders");
+        command.push_back("");
+    } else {
+        auto remoteStore = machine->storeUri.params.find("remote-store");
+        if (remoteStore != machine->storeUri.params.end()) {
+            command.push_back("--store");
+            command.push_back(shellEscape(remoteStore->second));
+        }
+    }
+
+    auto ret = master.startCommand(std::move(command), {
+        "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
+    });
+
+    // XXX: determine the actual max value we can use from /proc.
+
+    // FIXME: Should this be upstreamed into `startCommand` in Nix?
+
+    int pipesize = 1024 * 1024;
+
+    fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
+    fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
+
+    return ret;
+}
+
+
 static void copyClosureTo(
     ::Machine::Connection & conn,
     Store & destStore,
@@ -52,8 +87,8 @@ static void copyClosureTo(
     // FIXME: substitute output pollutes our build log
     /* Get back the set of paths that are already valid on the remote
        host. */
-    auto present = conn.store->queryValidPaths(
-        closure, true, useSubstitutes);
+    auto present = conn.queryValidPaths(
+        destStore, true, closure, useSubstitutes);
 
     if (present.size() == closure.size()) return;
 
@@ -68,7 +103,12 @@ static void copyClosureTo(
     std::unique_lock<std::timed_mutex> sendLock(conn.machine->state->sendLock,
         std::chrono::seconds(600));
 
-    conn.store->addMultipleToStoreLegacy(destStore, missing);
+    conn.to << ServeProto::Command::ImportPaths;
+    destStore.exportPaths(missing, conn.to);
+    conn.to.flush();
+
+    if (readInt(conn.from) != 1)
+        throw Error("remote machine failed to import closure");
 }
 
 
@@ -188,7 +228,7 @@ static BuildResult performBuild(
     counter & nrStepsBuilding
 )
 {
-    auto kont = conn.store->buildDerivationAsync(drvPath, drv, options);
+    conn.putBuildDerivationRequest(localStore, drvPath, drv, options);
 
     BuildResult result;
 
@@ -197,10 +237,7 @@ static BuildResult performBuild(
     startTime = time(0);
     {
         MaintainCount<counter> mc(nrStepsBuilding);
-        result = kont();
-        // Without proper call-once functions, we need to manually
-        // delete after calling.
-        kont = {};
+        result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
     }
     stopTime = time(0);
 
@@ -216,7 +253,7 @@ static BuildResult performBuild(
 
     // If the protocol was too old to give us `builtOutputs`, initialize
     // it manually by introspecting the derivation.
-    if (GET_PROTOCOL_MINOR(conn.store->getProtocol()) < 6)
+    if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 6)
     {
         // If the remote is too old to handle CA derivations, we can’t get this
         // far anyways
@@ -249,25 +286,26 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
-    /* Receive the NAR from the remote and add it to the
-        destination store. Meanwhile, extract all the info from the
-        NAR that getBuildOutput() needs. */
-    auto source2 = sinkToSource([&](Sink & sink)
-    {
-        /* Note: we should only send the command to dump the store
-            path to the remote if the NAR is actually going to get read
-            by the destination store, which won't happen if this path
-            is already valid on the destination store. Since this
-            lambda function only gets executed if someone tries to read
-            from source2, we will send the command from here rather
-            than outside the lambda. */
-        conn.store->narFromPath(info.path, [&](Source & source) {
-            TeeSource tee{source, sink};
-            extractNarData(tee, conn.store->printStorePath(info.path), narMembers);
-        });
-    });
+      /* Receive the NAR from the remote and add it to the
+          destination store. Meanwhile, extract all the info from the
+          NAR that getBuildOutput() needs. */
+      auto source2 = sinkToSource([&](Sink & sink)
+      {
+          /* Note: we should only send the command to dump the store
+              path to the remote if the NAR is actually going to get read
+              by the destination store, which won't happen if this path
+              is already valid on the destination store. Since this
+              lambda function only gets executed if someone tries to read
+              from source2, we will send the command from here rather
+              than outside the lambda. */
+          conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path);
+          conn.to.flush();
 
-    destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
+          TeeSource tee(conn.from, sink);
+          extractNarData(tee, localStore.printStorePath(info.path), narMembers);
+      });
+
+      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
 }
 
 static void copyPathsFromRemote(
@@ -366,39 +404,30 @@ void State::buildRemote(ref<Store> destStore,
 
         updateStep(ssConnecting);
 
-        // FIXME: rewrite to use Store.
-        ::Machine::Connection conn {
-            .machine = machine,
-            .store = [&]{
-                auto * pSpecified = std::get_if<StoreReference::Specified>(&machine->storeUri.variant);
-                if (!pSpecified || pSpecified->scheme != "ssh") {
-                    throw Error("Currently, only (legacy-)ssh stores are supported!");
-                }
+        auto storeRef = machine->completeStoreReference();
 
-                auto remoteStore = machine->openStore().dynamic_pointer_cast<LegacySSHStore>();
-                assert(remoteStore);
+        auto * pSpecified = std::get_if<StoreReference::Specified>(&storeRef.variant);
+        if (!pSpecified || pSpecified->scheme != "ssh") {
+            throw Error("Currently, only (legacy-)ssh stores are supported!");
+        }
 
-                remoteStore->connPipeSize = 1024 * 1024;
-
-                if (machine->isLocalhost()) {
-                    auto rp_new = remoteStore->remoteProgram.get();
-                    rp_new.push_back("--builders");
-                    rp_new.push_back("");
-                    const_cast<nix::Setting<Strings> &>(remoteStore->remoteProgram).assign(rp_new);
-                }
-                remoteStore->extraSshArgs = {
-                    "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
-                };
-                const_cast<nix::Setting<int> &>(remoteStore->logFD).assign(logFD.get());
-
-                return nix::ref{remoteStore};
-            }(),
+        LegacySSHStoreConfig storeConfig {
+            pSpecified->scheme,
+            pSpecified->authority,
+            storeRef.params
         };
 
+        auto master = storeConfig.createSSHMaster(
+            false, // no SSH master yet
+            logFD.get());
+
+        // FIXME: rewrite to use Store.
+        auto child = build_remote::openConnection(machine, master);
+
         {
             auto activeStepState(activeStep->state_.lock());
             if (activeStepState->cancelled) throw Error("step cancelled");
-            activeStepState->pid = conn.store->getConnectionPid();
+            activeStepState->pid = child->sshPid;
         }
 
         Finally clearPid([&]() {
@@ -413,12 +442,35 @@ void State::buildRemote(ref<Store> destStore,
                process. Meh. */
         });
 
+        ::Machine::Connection conn {
+            {
+                .to = child->in.get(),
+                .from = child->out.get(),
+                /* Handshake. */
+                .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
+            },
+            /*.machine =*/ machine,
+        };
+
         Finally updateStats([&]() {
-            auto stats = conn.store->getConnectionStats();
-            bytesReceived += stats.bytesReceived;
-            bytesSent += stats.bytesSent;
+            bytesReceived += conn.from.read;
+            bytesSent += conn.to.written;
         });
 
+        constexpr ServeProto::Version our_version = 0x206;
+
+        try {
+            conn.remoteVersion = decltype(conn)::handshake(
+                conn.to,
+                conn.from,
+                our_version,
+                machine->storeUri.render());
+        } catch (EndOfFile & e) {
+            child->sshPid.wait();
+            std::string s = chomp(readFile(result.logFile));
+            throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s);
+        }
+
         {
             auto info(machine->state->connectInfo.lock());
             info->consecutiveFailures = 0;
@@ -487,7 +539,7 @@ void State::buildRemote(ref<Store> destStore,
 
             auto now1 = std::chrono::steady_clock::now();
 
-            auto infos = conn.store->queryPathInfosUncached(outputs);
+            auto infos = conn.queryPathInfos(*localStore, outputs);
 
             size_t totalNarSize = 0;
             for (auto & [_, info] : infos) totalNarSize += info.narSize;
@@ -522,11 +574,9 @@ void State::buildRemote(ref<Store> destStore,
             }
         }
 
-        /* Shut down the connection done by RAII.
-
-           Only difference is kill() instead of wait() (i.e. send signal
-           then wait())
-         */
+        /* Shut down the connection. */
+        child->in = -1;
+        child->sshPid.wait();
 
     } catch (Error & e) {
         /* Disable this machine until a certain period of time has
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index e2d31434..30e01c74 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -20,7 +20,9 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "nar-extractor.hh"
-#include "legacy-ssh-store.hh"
+#include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
+#include "serve-protocol-connection.hh"
 #include "machines.hh"
 
 
@@ -290,11 +292,9 @@ struct Machine : nix::Machine
     bool isLocalhost() const;
 
     // A connection to a machine
-    struct Connection {
+    struct Connection : nix::ServeProto::BasicClientConnection {
         // Backpointer to the machine
         ptr machine;
-        // Opened store
-        nix::ref<nix::LegacySSHStore> store;
     };
 };
 

From 987dad3371bb661e31c1c4da90a8cc5081d34002 Mon Sep 17 00:00:00 2001
From: Robin Stumm <serverkorken@gmail.com>
Date: Wed, 26 Mar 2025 20:23:26 +0100
Subject: [PATCH 232/285] hydra-eval-jobset: do not wait on n-e-j inside
 transaction

fixes #1429
---
 src/script/hydra-eval-jobset | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 5b4026a5..cf3fa294 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -773,6 +773,9 @@ sub checkJobsetWrapped {
     my $jobsetChanged = 0;
     my %buildMap;
 
+    my @jobs;
+    push @jobs, $_ while defined($_ = $jobsIter->());
+
     $db->txn_do(sub {
         my $prevEval = getPrevJobsetEval($db, $jobset, 1);
 
@@ -796,7 +799,7 @@ sub checkJobsetWrapped {
 
         my @jobsWithConstituents;
 
-        while (defined(my $job = $jobsIter->())) {
+        foreach my $job (@jobs) {
             if ($jobsetsJobset) {
                 die "The .jobsets jobset must only have a single job named 'jobsets'"
                     unless $job->{attr} eq "jobsets";

From 4bcbed2f1b4bb517a4ec02328d934f3c5c0cc5da Mon Sep 17 00:00:00 2001
From: zowoq <59103226+zowoq@users.noreply.github.com>
Date: Fri, 28 Mar 2025 11:11:26 +1000
Subject: [PATCH 233/285] hydraTest: remove outdated postgresql version

error: postgresql_12 has been removed since it reached its EOL upstream
---
 nixos-modules/default.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix
index 62b18406..d12d8338 100644
--- a/nixos-modules/default.nix
+++ b/nixos-modules/default.nix
@@ -15,7 +15,6 @@
     systemd.services.hydra-send-stats.enable = false;
 
     services.postgresql.enable = true;
-    services.postgresql.package = pkgs.postgresql_12;
 
     # The following is to work around the following error from hydra-server:
     #   [error] Caught exception in engine "Cannot determine local time zone"

From feebb618978a39b07ca040417d1e6fcb63b402ec Mon Sep 17 00:00:00 2001
From: zowoq <59103226+zowoq@users.noreply.github.com>
Date: Fri, 28 Mar 2025 11:07:25 +1000
Subject: [PATCH 234/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix-eval-jobs':
    'github:nix-community/nix-eval-jobs/4b392b284877d203ae262e16af269f702df036bc?narHash=sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY%3D' (2025-02-14)
  → 'github:nix-community/nix-eval-jobs/f7418fc1fa45b96d37baa95ff3c016dd5be3876b?narHash=sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0%3D' (2025-03-26)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index c47a3b88..6f1b1347 100644
--- a/flake.lock
+++ b/flake.lock
@@ -29,11 +29,11 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1739500569,
-        "narHash": "sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY=",
+        "lastModified": 1743008255,
+        "narHash": "sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "4b392b284877d203ae262e16af269f702df036bc",
+        "rev": "f7418fc1fa45b96d37baa95ff3c016dd5be3876b",
         "type": "github"
       },
       "original": {

From 9911f0107fc0a7a8142f3b9b194e83d3cbcd08bd Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 16 Jan 2025 15:50:00 +0100
Subject: [PATCH 235/285] Reimplement (named) constituent jobs (+globbing)
 based on nix-eval-jobs

Depends on https://github.com/nix-community/nix-eval-jobs/pull/349 & #1421.

Almost equivalent to #1425, but with a small change: when having e.g. an
aggregate job with a glob that matches nothing, the jobset evaluation is
failed now. This was the intended behavior before (hydra-eval-jobset
fails hard if an aggregate is broken), the code-path was never reached
however since the aggregate was never marked as broken in this case
before.
---
 t/evaluator/evaluate-constituents-broken.t   |  68 ++++++---
 t/evaluator/evaluate-constituents-globbing.t | 138 +++++++++++++++++++
 t/jobs/config.nix                            |  14 ++
 t/jobs/constituents-cycle-glob.nix           |  34 +++++
 t/jobs/constituents-cycle.nix                |  21 +++
 t/jobs/constituents-glob-all.nix             |  22 +++
 t/jobs/constituents-glob.nix                 |  31 +++++
 t/jobs/constituents-no-matches.nix           |  20 +++
 t/jobs/declarative/project.json              |  24 ++++
 t/queue-runner/constituents.t                |   4 +-
 10 files changed, 354 insertions(+), 22 deletions(-)
 create mode 100644 t/evaluator/evaluate-constituents-globbing.t
 create mode 100644 t/jobs/config.nix
 create mode 100644 t/jobs/constituents-cycle-glob.nix
 create mode 100644 t/jobs/constituents-cycle.nix
 create mode 100644 t/jobs/constituents-glob-all.nix
 create mode 100644 t/jobs/constituents-glob.nix
 create mode 100644 t/jobs/constituents-no-matches.nix
 create mode 100644 t/jobs/declarative/project.json

diff --git a/t/evaluator/evaluate-constituents-broken.t b/t/evaluator/evaluate-constituents-broken.t
index 0e5960bf..1391b618 100644
--- a/t/evaluator/evaluate-constituents-broken.t
+++ b/t/evaluator/evaluate-constituents-broken.t
@@ -6,27 +6,55 @@ use Hydra::Helper::Exec;
 
 my $ctx = test_context();
 
-my $jobsetCtx = $ctx->makeJobset(
-    expression => 'constituents-broken.nix',
-);
-my $jobset = $jobsetCtx->{"jobset"};
+subtest "broken constituents expression" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-broken.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
 
-my ($res, $stdout, $stderr) = captureStdoutStderr(60,
-    ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
-);
-isnt($res, 0, "hydra-eval-jobset exits non-zero");
-ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
-like(
-    $stderr,
-    qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
-    "The stderr record includes a relevant error message"
-);
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+    isnt($res, 0, "hydra-eval-jobset exits non-zero");
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+    like(
+        $stderr,
+        qr/aggregate job 'mixed_aggregate' references non-existent job 'constituentA'/,
+        "The stderr record includes a relevant error message"
+    );
 
-$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
-like(
-    $jobset->errormsg,
-    qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
-    "The jobset records a relevant error message"
-);
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
+        "The jobset records a relevant error message"
+    );
+};
+
+subtest "no matches" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-no-matches.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+    isnt($res, 0, "hydra-eval-jobset exits non-zero");
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+    like(
+        $stderr,
+        qr/aggregate job 'non_match_aggregate' references constituent glob pattern 'tests\.\*' with no matches/,
+        "The stderr record includes a relevant error message"
+    );
+
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/aggregate job ‘non_match_aggregate’ failed with the error: tests\.\*: constituent glob pattern had no matches/,
+        qr/in job ‘non_match_aggregate’:\ntests\.\*: constituent glob pattern had no matches/,
+        "The jobset records a relevant error message"
+    );
+};
 
 done_testing;
diff --git a/t/evaluator/evaluate-constituents-globbing.t b/t/evaluator/evaluate-constituents-globbing.t
new file mode 100644
index 00000000..49315d58
--- /dev/null
+++ b/t/evaluator/evaluate-constituents-globbing.t
@@ -0,0 +1,138 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Helper::Exec;
+use Data::Dumper;
+
+my $ctx = test_context();
+
+subtest "general glob testing" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-glob.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+    is($res, 0, "hydra-eval-jobset exits zero");
+
+    my $builds = {};
+    for my $build ($jobset->builds) {
+        $builds->{$build->job} = $build;
+    }
+
+    subtest "basic globbing works" => sub {
+        ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
+        my @constituents = $builds->{"ok_aggregate"}->constituents->all;
+        is(2, scalar @constituents, "'ok_aggregate' has two constituents");
+
+        my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
+
+        is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
+        is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
+    };
+
+    subtest "transitivity is OK" => sub {
+        ok(defined $builds->{"indirect_aggregate"}, "'indirect_aggregate' is part of the jobset evaluation");
+        my @constituents = $builds->{"indirect_aggregate"}->constituents->all;
+        is(1, scalar @constituents, "'indirect_aggregate' has one constituent");
+        is($constituents[0]->nixname, "direct_aggregate", "'indirect_aggregate' has 'direct_aggregate' as single constituent");
+    };
+};
+
+subtest "* selects all except current aggregate" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-glob-all.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+
+    subtest "no eval errors" => sub {
+        ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+        ok(
+            $stderr !~ "aggregate job ‘ok_aggregate’ has a constituent .* that doesn't correspond to a Hydra build",
+            "Catchall wildcard must not select itself as constituent"
+        );
+
+        $jobset->discard_changes;  # refresh from DB
+        is(
+            $jobset->errormsg,
+            "",
+            "eval-errors non-empty"
+        );
+    };
+
+    my $builds = {};
+    for my $build ($jobset->builds) {
+        $builds->{$build->job} = $build;
+    }
+
+    subtest "two constituents" => sub {
+        ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
+        my @constituents = $builds->{"ok_aggregate"}->constituents->all;
+        is(2, scalar @constituents, "'ok_aggregate' has two constituents");
+
+        my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
+
+        is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
+        is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
+    };
+};
+
+subtest "trivial cycle check" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-cycle.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+
+    ok(
+        $stderr =~ "Found dependency cycle between jobs 'indirect_aggregate' and 'ok_aggregate'",
+        "Dependency cycle error is on stderr"
+    );
+
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+
+    $jobset->discard_changes;  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/Dependency cycle: indirect_aggregate <-> ok_aggregate/,
+        "eval-errors non-empty"
+    );
+
+    is(0, $jobset->builds->count, "No builds should be scheduled");
+};
+
+subtest "cycle check with globbing" => sub {
+    my $jobsetCtx = $ctx->makeJobset(
+        expression => 'constituents-cycle-glob.nix',
+    );
+    my $jobset = $jobsetCtx->{"jobset"};
+
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+        ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+    );
+
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+
+    $jobset->discard_changes;  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/aggregate job ‘indirect_aggregate’ failed with the error: Dependency cycle: indirect_aggregate <-> packages.constituentA/,
+        "packages.constituentA error missing"
+    );
+
+    # on this branch of Hydra, hydra-eval-jobset fails hard if an aggregate
+    # job is broken.
+    is(0, $jobset->builds->count, "Zero jobs are scheduled");
+};
+
+done_testing;
diff --git a/t/jobs/config.nix b/t/jobs/config.nix
new file mode 100644
index 00000000..91fd1d1a
--- /dev/null
+++ b/t/jobs/config.nix
@@ -0,0 +1,14 @@
+rec {
+  path = "/nix/store/l9mg93sgx50y88p5rr6x1vib6j1rjsds-coreutils-9.1/bin";
+
+  mkDerivation = args:
+    derivation ({
+      system = builtins.currentSystem;
+      PATH = path;
+    } // args);
+  mkContentAddressedDerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } // args);
+}
diff --git a/t/jobs/constituents-cycle-glob.nix b/t/jobs/constituents-cycle-glob.nix
new file mode 100644
index 00000000..efc152ce
--- /dev/null
+++ b/t/jobs/constituents-cycle-glob.nix
@@ -0,0 +1,34 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [ "*_aggregate" ];
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "packages.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}
diff --git a/t/jobs/constituents-cycle.nix b/t/jobs/constituents-cycle.nix
new file mode 100644
index 00000000..7e086aa1
--- /dev/null
+++ b/t/jobs/constituents-cycle.nix
@@ -0,0 +1,21 @@
+with import ./config.nix;
+{
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "indirect_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}
diff --git a/t/jobs/constituents-glob-all.nix b/t/jobs/constituents-glob-all.nix
new file mode 100644
index 00000000..d671fd70
--- /dev/null
+++ b/t/jobs/constituents-glob-all.nix
@@ -0,0 +1,22 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}
diff --git a/t/jobs/constituents-glob.nix b/t/jobs/constituents-glob.nix
new file mode 100644
index 00000000..f566dbfd
--- /dev/null
+++ b/t/jobs/constituents-glob.nix
@@ -0,0 +1,31 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "packages.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}
diff --git a/t/jobs/constituents-no-matches.nix b/t/jobs/constituents-no-matches.nix
new file mode 100644
index 00000000..699cad91
--- /dev/null
+++ b/t/jobs/constituents-no-matches.nix
@@ -0,0 +1,20 @@
+with import ./config.nix;
+{
+  non_match_aggregate = mkDerivation {
+    name = "mixed_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "tests.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  # Without a second job no jobset is attempted to be created
+  # (the only job would be broken)
+  # and thus the constituent validation is never reached.
+  dummy = mkDerivation {
+    name = "dummy";
+    builder = ./empty-dir-builder.sh;
+  };
+}
diff --git a/t/jobs/declarative/project.json b/t/jobs/declarative/project.json
new file mode 100644
index 00000000..47d6ecf2
--- /dev/null
+++ b/t/jobs/declarative/project.json
@@ -0,0 +1,24 @@
+{
+    "enabled": 1,
+    "hidden": false,
+    "description": "declarative-jobset-example",
+    "nixexprinput": "src",
+    "nixexprpath": "declarative/generator.nix",
+    "checkinterval": 300,
+    "schedulingshares": 100,
+    "enableemail": false,
+    "emailoverride": "",
+    "keepnr": 3,
+    "inputs": {
+        "src": {
+            "type": "path",
+            "value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
+            "emailresponsible": false
+        },
+        "jobspath": {
+            "type": "string",
+            "value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
+            "emailresponsible": false
+        }
+    }
+}
diff --git a/t/queue-runner/constituents.t b/t/queue-runner/constituents.t
index e1b8d733..8e048a73 100644
--- a/t/queue-runner/constituents.t
+++ b/t/queue-runner/constituents.t
@@ -22,11 +22,11 @@ is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix
 
 like(
     $jobset->errormsg,
-    qr/^"does-not-exist": does not exist$/m,
+    qr/^does-not-exist: does not exist$/m,
     "Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
 like(
     $jobset->errormsg,
-    qr/^"does-not-evaluate": "error: assertion 'false' failed/m,
+    qr/^does-not-evaluate: error: assertion 'false' failed/m,
     "Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
 
 done_testing;

From ae18a7b3ae44051161f92e7ae54acc72f338089b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 29 Mar 2025 11:09:32 +0000
Subject: [PATCH 236/285] fix development workflow after switching to
 meson-based build

---
 README.md                 | 27 ++++++++++++++-------------
 doc/manual/src/hacking.md | 27 +++++++--------------------
 package.nix               |  2 +-
 3 files changed, 22 insertions(+), 34 deletions(-)

diff --git a/README.md b/README.md
index 8ac18ac9..54b95549 100644
--- a/README.md
+++ b/README.md
@@ -72,17 +72,16 @@ Make sure **State** at the top of the page is set to "_Enabled_" and click on "_
 You can build Hydra via `nix-build` using the provided [default.nix](./default.nix):
 
 ```
-$ nix-build
+$ nix build
 ```
 
 ### Development Environment
 
 You can use the provided shell.nix to get a working development environment:
 ```
-$ nix-shell
-$ autoreconfPhase
-$ configurePhase # NOTE: not ./configure
-$ make
+$ nix develop
+$ mesonConfigurePhase
+$ ninja
 ```
 
 ### Executing Hydra During Development
@@ -91,9 +90,9 @@ When working on new features or bug fixes you need to be able to run Hydra from
 can be done using [foreman](https://github.com/ddollar/foreman):
 
 ```
-$ nix-shell
+$ nix develop
 $ # hack hack
-$ make
+$ ninja -C build
 $ foreman start
 ```
 
@@ -115,22 +114,24 @@ Start by following the steps in [Development Environment](#development-environme
 Then, you can run the tests and the perlcritic linter together with:
 
 ```console
-$ nix-shell
-$ make check
+$ nix develop
+$ ninja -C build test
 ```
 
 You can run a single test with:
 
 ```
-$ nix-shell
-$ yath test ./t/foo/bar.t
+$ nix develop
+$ cd build
+$ meson test --test-args=../t/Hydra/Event.t testsuite
 ```
 
 And you can run just perlcritic with:
 
 ```
-$ nix-shell
-$ make perlcritic
+$ nix develop
+$ cd build
+$ meson test perlcritic
 ```
 
 ### JSON API
diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md
index ec96b8c6..8b2b13ba 100644
--- a/doc/manual/src/hacking.md
+++ b/doc/manual/src/hacking.md
@@ -11,12 +11,6 @@ $ cd hydra
 To enter a shell in which all environment variables (such as `PERL5LIB`)
 and dependencies can be found:
 
-```console
-$ nix-shell
-```
-
-of when flakes are enabled:
-
 ```console
 $ nix develop
 ```
@@ -24,15 +18,15 @@ $ nix develop
 To build Hydra, you should then do:
 
 ```console
-[nix-shell]$ autoreconfPhase
-[nix-shell]$ configurePhase
-[nix-shell]$ make -j$(nproc)
+$ mesonConfigurePhase
+$ ninja
 ```
 
 You start a local database, the webserver, and other components with
 foreman:
 
 ```console
+$ ninja -C build
 $ foreman start
 ```
 
@@ -47,18 +41,11 @@ $ ./src/script/hydra-server
 You can run Hydra's test suite with the following:
 
 ```console
-[nix-shell]$ make check
-[nix-shell]$ # to run as many tests as you have cores:
-[nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES
-[nix-shell]$ # or run yath directly:
-[nix-shell]$ yath test
-[nix-shell]$ # to run as many tests as you have cores:
-[nix-shell]$ yath test -j $NIX_BUILD_CORES
+$ meson test
+# to run as many tests as you have cores:
+$ YATH_JOB_COUNT=$NIX_BUILD_CORES meson test
 ```
 
-When using `yath` instead of `make check`, ensure you have run `make`
-in the root of the repository at least once.
-
 **Warning**: Currently, the tests can fail
 if run with high parallelism [due to an issue in
 `Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40)
@@ -75,7 +62,7 @@ will reload the page every time you save.
 To build Hydra and its dependencies:
 
 ```console
-$ nix-build release.nix -A build.x86_64-linux
+$ nix build .#packages.x86_64-linux.default
 ```
 
 ## Development Tasks
diff --git a/package.nix b/package.nix
index 12fac1d8..07701bc9 100644
--- a/package.nix
+++ b/package.nix
@@ -241,7 +241,7 @@ stdenv.mkDerivation (finalAttrs: {
   shellHook = ''
     pushd $(git rev-parse --show-toplevel) >/dev/null
 
-    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH
+    PATH=$(pwd)/build/src/hydra-evaluator:$(pwd)/build/src/script:$(pwd)/build/src/hydra-queue-runner:$PATH
     PERL5LIB=$(pwd)/src/lib:$PERL5LIB
     export HYDRA_HOME="$(pwd)/src/"
     mkdir -p .hydra-data

From 3fef32b3643e04626f4fe6903eb7afc3ba383fe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 29 Mar 2025 11:10:17 +0000
Subject: [PATCH 237/285] gitignore hydra-data as created by foreman

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ddcbadc4..12df926f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 /src/sql/hydra-postgresql.sql
 /src/sql/hydra-sqlite.sql
 /src/sql/tmp.sqlite
+.hydra-data
 result
 result-*
 outputs

From 6b97e3ab7b662862cf8e13e872f7e42a0d5260a5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Apr 2025 10:58:22 -0400
Subject: [PATCH 238/285] flake.lock: Update to nix and nix-eval-jobs 2.27
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918?narHash=sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs%3D' (2025-02-18)
  → 'github:NixOS/nix/d0f98c76f962147610489e84c10033ca92e9c532?narHash=sha256-u6RhBWQ1XohTZ4Ub5ml1PTcaxQgtqFNng6Sohy1rojw%3D' (2025-04-07)
• Updated input 'nix-eval-jobs':
    'github:nix-community/nix-eval-jobs/f7418fc1fa45b96d37baa95ff3c016dd5be3876b?narHash=sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0%3D' (2025-03-26)
  → 'github:nix-community/nix-eval-jobs/62f9c9e8d00d2ff6ab27a6197ab459a8e0808e59?narHash=sha256-PypQspB7h7EENe4RQQUQj2Ay8J1%2BO49AKNO9JbAU4Ek%3D' (2025-04-07)
---
 flake.lock | 15 ++++++++-------
 flake.nix  |  4 ++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/flake.lock b/flake.lock
index 6f1b1347..f1744039 100644
--- a/flake.lock
+++ b/flake.lock
@@ -12,16 +12,16 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1739899400,
-        "narHash": "sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs=",
+        "lastModified": 1744029599,
+        "narHash": "sha256-u6RhBWQ1XohTZ4Ub5ml1PTcaxQgtqFNng6Sohy1rojw=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918",
+        "rev": "d0f98c76f962147610489e84c10033ca92e9c532",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.26-maintenance",
+        "ref": "2.27-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -29,15 +29,16 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1743008255,
-        "narHash": "sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0=",
+        "lastModified": 1744009874,
+        "narHash": "sha256-PypQspB7h7EENe4RQQUQj2Ay8J1+O49AKNO9JbAU4Ek=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "f7418fc1fa45b96d37baa95ff3c016dd5be3876b",
+        "rev": "62f9c9e8d00d2ff6ab27a6197ab459a8e0808e59",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
+        "ref": "v2.27.0",
         "repo": "nix-eval-jobs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index dc3aaf5c..5a305ca0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,7 +4,7 @@
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
 
   inputs.nix = {
-    url = "github:NixOS/nix/2.26-maintenance";
+    url = "github:NixOS/nix/2.27-maintenance";
     inputs.nixpkgs.follows = "nixpkgs";
 
     # hide nix dev tooling from our lock file
@@ -16,7 +16,7 @@
   };
 
   inputs.nix-eval-jobs = {
-    url = "github:nix-community/nix-eval-jobs";
+    url = "github:nix-community/nix-eval-jobs/v2.27.0";
     # We want to control the deps precisely
     flake = false;
   };

From 1cb1e139c43101df18295c16bd0038aab91589f2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Apr 2025 11:12:12 -0400
Subject: [PATCH 239/285] Fix build (due to C++ API changes)

---
 src/hydra-queue-runner/queue-monitor.cc | 10 +++++++---
 src/hydra-queue-runner/state.hh         |  5 +++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index bc956453..7d747d3e 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -1,6 +1,7 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
 #include "globals.hh"
+#include "parsed-derivations.hh"
 
 #include <cstring>
 
@@ -463,14 +464,17 @@ Step::ptr State::createStep(ref<Store> destStore,
        it's not runnable yet, and other threads won't make it
        runnable while step->created == false. */
     step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
-    step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
+    {
+        auto parsedDrv = ParsedDerivation{drvPath, *step->drv};
+        step->drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(parsedDrv));
+    }
 
-    step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore);
+    step->preferLocalBuild = step->drvOptions->willBuildLocally(*localStore, *step->drv);
     step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1";
 
     step->systemType = step->drv->platform;
     {
-        StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
+        StringSet features = step->requiredSystemFeatures = step->drvOptions->getRequiredSystemFeatures(*step->drv);
         if (step->preferLocalBuild)
             features.insert("local");
         if (!features.empty()) {
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 30e01c74..e6c5f7df 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -13,7 +13,8 @@
 
 #include "db.hh"
 
-#include "parsed-derivations.hh"
+#include "derivations.hh"
+#include "derivation-options.hh"
 #include "pathlocks.hh"
 #include "pool.hh"
 #include "build-result.hh"
@@ -167,7 +168,7 @@ struct Step
 
     nix::StorePath drvPath;
     std::unique_ptr<nix::Derivation> drv;
-    std::unique_ptr<nix::ParsedDerivation> parsedDrv;
+    std::unique_ptr<nix::DerivationOptions> drvOptions;
     std::set<std::string> requiredSystemFeatures;
     bool preferLocalBuild;
     bool isDeterministic;

From 1595064beeb791bf47e10215097138178753ebcb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Apr 2025 11:15:55 -0400
Subject: [PATCH 240/285] flake.lock: Update to nix and nix-eval-jobs 2.28
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/d0f98c76f962147610489e84c10033ca92e9c532?narHash=sha256-u6RhBWQ1XohTZ4Ub5ml1PTcaxQgtqFNng6Sohy1rojw%3D' (2025-04-07)
  → 'github:NixOS/nix/a4962f73b5fc874d4b16baef47921daf349addfc?narHash=sha256-r%2BpsCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I%3D' (2025-04-07)
• Updated input 'nix-eval-jobs':
    'github:nix-community/nix-eval-jobs/62f9c9e8d00d2ff6ab27a6197ab459a8e0808e59?narHash=sha256-PypQspB7h7EENe4RQQUQj2Ay8J1%2BO49AKNO9JbAU4Ek%3D' (2025-04-07)
  → 'github:nix-community/nix-eval-jobs/cba718bafe5dc1607c2b6761ecf53c641a6f3b21?narHash=sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA%3D' (2025-04-07)
---
 flake.lock | 15 +++++++--------
 flake.nix  |  4 ++--
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/flake.lock b/flake.lock
index f1744039..3a7a1672 100644
--- a/flake.lock
+++ b/flake.lock
@@ -12,16 +12,16 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1744029599,
-        "narHash": "sha256-u6RhBWQ1XohTZ4Ub5ml1PTcaxQgtqFNng6Sohy1rojw=",
+        "lastModified": 1744030329,
+        "narHash": "sha256-r+psCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "d0f98c76f962147610489e84c10033ca92e9c532",
+        "rev": "a4962f73b5fc874d4b16baef47921daf349addfc",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.27-maintenance",
+        "ref": "2.28-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -29,16 +29,15 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1744009874,
-        "narHash": "sha256-PypQspB7h7EENe4RQQUQj2Ay8J1+O49AKNO9JbAU4Ek=",
+        "lastModified": 1744018595,
+        "narHash": "sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "62f9c9e8d00d2ff6ab27a6197ab459a8e0808e59",
+        "rev": "cba718bafe5dc1607c2b6761ecf53c641a6f3b21",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "v2.27.0",
         "repo": "nix-eval-jobs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 5a305ca0..e208c40b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,7 +4,7 @@
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
 
   inputs.nix = {
-    url = "github:NixOS/nix/2.27-maintenance";
+    url = "github:NixOS/nix/2.28-maintenance";
     inputs.nixpkgs.follows = "nixpkgs";
 
     # hide nix dev tooling from our lock file
@@ -16,7 +16,7 @@
   };
 
   inputs.nix-eval-jobs = {
-    url = "github:nix-community/nix-eval-jobs/v2.27.0";
+    url = "github:nix-community/nix-eval-jobs";
     # We want to control the deps precisely
     flake = false;
   };

From 6534a54ee5030252521b224e269410544ddbabf0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Apr 2025 11:27:40 -0400
Subject: [PATCH 241/285] Fix Nix code

Can now at least enter dev shell, but build is still broken.
---
 flake.nix   | 27 +++++++++++++++++----------
 package.nix | 22 +++++++++-------------
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/flake.nix b/flake.nix
index e208c40b..dc1d1b8a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -34,7 +34,6 @@
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
           rawSrc = self;
-          nix-perl-bindings = final.nixComponents.nix-perl-bindings;
         };
       };
 
@@ -73,21 +72,29 @@
         validate-openapi = hydraJobs.tests.validate-openapi.${system};
       });
 
-      packages = forEachSystem (system: {
-        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
-          nix = nix.packages.${system}.nix;
-        };
-        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
-          inherit (nixpkgs.lib) fileset;
-          inherit (self.packages.${system}) nix-eval-jobs;
-          rawSrc = self;
+      packages = forEachSystem (system: let
+        nixComponents = {
           inherit (nix.packages.${system})
             nix-util
             nix-store
+            nix-expr
+            nix-fetchers
+            nix-flake
             nix-main
+            nix-cmd
             nix-cli
+            nix-perl-bindings
             ;
-          nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
+        };
+      in {
+        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
+          inherit nixComponents;
+        };
+        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
+          inherit (nixpkgs.lib) fileset;
+          inherit nixComponents;
+          inherit (self.packages.${system}) nix-eval-jobs;
+          rawSrc = self;
         };
         default = self.packages.${system}.hydra;
       });
diff --git a/package.nix b/package.nix
index 07701bc9..8bf6a199 100644
--- a/package.nix
+++ b/package.nix
@@ -8,11 +8,7 @@
 
 , perlPackages
 
-, nix-util
-, nix-store
-, nix-main
-, nix-cli
-, nix-perl-bindings
+, nixComponents
 , git
 
 , makeWrapper
@@ -65,7 +61,7 @@ let
     name = "hydra-perl-deps";
     paths = lib.closePropagation
       ([
-        nix-perl-bindings
+        nixComponents.nix-perl-bindings
         git
       ] ++ (with perlPackages; [
         AuthenSASL
@@ -165,7 +161,7 @@ stdenv.mkDerivation (finalAttrs: {
     nukeReferences
     pkg-config
     mdbook
-    nix-cli
+    nixComponents.nix-cli
     perlDeps
     perl
     unzip
@@ -175,9 +171,9 @@ stdenv.mkDerivation (finalAttrs: {
     libpqxx
     openssl
     libxslt
-    nix-util
-    nix-store
-    nix-main
+    nixComponents.nix-util
+    nixComponents.nix-store
+    nixComponents.nix-main
     perlDeps
     perl
     boost
@@ -204,14 +200,14 @@ stdenv.mkDerivation (finalAttrs: {
     glibcLocales
     libressl.nc
     python3
-    nix-cli
+    nixComponents.nix-cli
   ];
 
   hydraPath = lib.makeBinPath (
     [
       subversion
       openssh
-      nix-cli
+      nixComponents.nix-cli
       coreutils
       findutils
       pixz
@@ -272,7 +268,7 @@ stdenv.mkDerivation (finalAttrs: {
             --prefix PATH ':' $out/bin:$hydraPath \
             --set HYDRA_RELEASE ${version} \
             --set HYDRA_HOME $out/libexec/hydra \
-            --set NIX_RELEASE ${nix-cli.name or "unknown"} \
+            --set NIX_RELEASE ${nixComponents.nix-cli.name or "unknown"} \
             --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
     done
   '';

From d6a5df25bfe25e87152414e518d9c14de3072334 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Apr 2025 11:36:59 -0400
Subject: [PATCH 242/285] Fix the build

---
 meson.build                                  | 14 -----------
 src/hydra-evaluator/hydra-evaluator.cc       |  6 ++---
 src/hydra-evaluator/meson.build              |  3 ++-
 src/hydra-queue-runner/build-remote.cc       | 26 ++++++++++----------
 src/hydra-queue-runner/build-result.cc       |  6 ++---
 src/hydra-queue-runner/builder.cc            |  4 +--
 src/hydra-queue-runner/hydra-build-result.hh |  6 ++---
 src/hydra-queue-runner/hydra-queue-runner.cc | 12 ++++-----
 src/hydra-queue-runner/meson.build           |  4 ++-
 src/hydra-queue-runner/nar-extractor.cc      |  2 +-
 src/hydra-queue-runner/nar-extractor.hh      |  8 +++---
 src/hydra-queue-runner/queue-monitor.cc      |  4 +--
 src/hydra-queue-runner/state.hh              | 22 ++++++++---------
 src/libhydra/db.hh                           |  4 +--
 src/libhydra/hydra-config.hh                 |  4 +--
 15 files changed, 57 insertions(+), 68 deletions(-)

diff --git a/meson.build b/meson.build
index 9c9c09a1..c1eb577b 100644
--- a/meson.build
+++ b/meson.build
@@ -12,20 +12,6 @@ nix_util_dep = dependency('nix-util', required: true)
 nix_store_dep = dependency('nix-store', required: true)
 nix_main_dep = dependency('nix-main', required: true)
 
-# Nix need extra flags not provided in its pkg-config files.
-nix_dep = declare_dependency(
-  dependencies: [
-    nix_util_dep,
-    nix_store_dep,
-    nix_main_dep,
-  ],
-  compile_args: [
-    '-include', 'nix/config-util.hh',
-    '-include', 'nix/config-store.hh',
-    '-include', 'nix/config-main.hh',
-  ],
-)
-
 pqxx_dep = dependency('libpqxx', required: true)
 
 prom_cpp_core_dep = dependency('prometheus-cpp-core', required: true)
diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc
index 9312d085..52664188 100644
--- a/src/hydra-evaluator/hydra-evaluator.cc
+++ b/src/hydra-evaluator/hydra-evaluator.cc
@@ -1,8 +1,8 @@
 #include "db.hh"
 #include "hydra-config.hh"
-#include "pool.hh"
-#include "shared.hh"
-#include "signals.hh"
+#include <nix/util/pool.hh>
+#include <nix/main/shared.hh>
+#include <nix/util/signals.hh>
 
 #include <algorithm>
 #include <thread>
diff --git a/src/hydra-evaluator/meson.build b/src/hydra-evaluator/meson.build
index b3dc542e..53ddc354 100644
--- a/src/hydra-evaluator/meson.build
+++ b/src/hydra-evaluator/meson.build
@@ -2,7 +2,8 @@ hydra_evaluator = executable('hydra-evaluator',
   'hydra-evaluator.cc',
   dependencies: [
     libhydra_dep,
-    nix_dep,
+    nix_util_dep,
+    nix_main_dep,
     pqxx_dep,
   ],
   install: true,
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 77bde2c4..79c32d46 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -5,20 +5,20 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
-#include "build-result.hh"
-#include "path.hh"
-#include "legacy-ssh-store.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
+#include <nix/store/build-result.hh>
+#include <nix/store/path.hh>
+#include <nix/store/legacy-ssh-store.hh>
+#include <nix/store/serve-protocol.hh>
+#include <nix/store/serve-protocol-impl.hh>
 #include "state.hh"
-#include "current-process.hh"
-#include "processes.hh"
-#include "util.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
-#include "ssh.hh"
-#include "finally.hh"
-#include "url.hh"
+#include <nix/util/current-process.hh>
+#include <nix/util/processes.hh>
+#include <nix/util/util.hh>
+#include <nix/store/serve-protocol.hh>
+#include <nix/store/serve-protocol-impl.hh>
+#include <nix/store/ssh.hh>
+#include <nix/util/finally.hh>
+#include <nix/util/url.hh>
 
 using namespace nix;
 
diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc
index ffdc37b7..b0695e8b 100644
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@@ -1,7 +1,7 @@
 #include "hydra-build-result.hh"
-#include "store-api.hh"
-#include "util.hh"
-#include "source-accessor.hh"
+#include <nix/store/store-api.hh>
+#include <nix/util/util.hh>
+#include <nix/util/source-accessor.hh>
 
 #include <regex>
 
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 4bc00f0c..018215a5 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -2,8 +2,8 @@
 
 #include "state.hh"
 #include "hydra-build-result.hh"
-#include "finally.hh"
-#include "binary-cache-store.hh"
+#include <nix/util/finally.hh>
+#include <nix/store/binary-cache-store.hh>
 
 using namespace nix;
 
diff --git a/src/hydra-queue-runner/hydra-build-result.hh b/src/hydra-queue-runner/hydra-build-result.hh
index 7d47f67c..654bf1be 100644
--- a/src/hydra-queue-runner/hydra-build-result.hh
+++ b/src/hydra-queue-runner/hydra-build-result.hh
@@ -2,9 +2,9 @@
 
 #include <memory>
 
-#include "hash.hh"
-#include "derivations.hh"
-#include "store-api.hh"
+#include <nix/util/hash.hh>
+#include <nix/store/derivations.hh>
+#include <nix/store/store-api.hh>
 #include "nar-extractor.hh"
 
 struct BuildProduct
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 99411f9f..64a98797 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -11,16 +11,16 @@
 
 #include <nlohmann/json.hpp>
 
-#include "signals.hh"
+#include <nix/util/signals.hh>
 #include "state.hh"
 #include "hydra-build-result.hh"
-#include "store-api.hh"
-#include "remote-store.hh"
+#include <nix/store/store-api.hh>
+#include <nix/store/remote-store.hh>
 
-#include "globals.hh"
+#include <nix/store/globals.hh>
 #include "hydra-config.hh"
-#include "s3-binary-cache-store.hh"
-#include "shared.hh"
+#include <nix/store/s3-binary-cache-store.hh>
+#include <nix/main/shared.hh>
 
 using namespace nix;
 using nlohmann::json;
diff --git a/src/hydra-queue-runner/meson.build b/src/hydra-queue-runner/meson.build
index 1c20299f..27dad2c0 100644
--- a/src/hydra-queue-runner/meson.build
+++ b/src/hydra-queue-runner/meson.build
@@ -13,7 +13,9 @@ hydra_queue_runner = executable('hydra-queue-runner',
   srcs,
   dependencies: [
     libhydra_dep,
-    nix_dep,
+    nix_util_dep,
+    nix_store_dep,
+    nix_main_dep,
     pqxx_dep,
     prom_cpp_core_dep,
     prom_cpp_pull_dep,
diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc
index 8729dd39..3bf06ef3 100644
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@@ -1,6 +1,6 @@
 #include "nar-extractor.hh"
 
-#include "archive.hh"
+#include <nix/util/archive.hh>
 
 #include <unordered_set>
 
diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh
index 2634135b..0060efe2 100644
--- a/src/hydra-queue-runner/nar-extractor.hh
+++ b/src/hydra-queue-runner/nar-extractor.hh
@@ -1,9 +1,9 @@
 #pragma once
 
-#include "source-accessor.hh"
-#include "types.hh"
-#include "serialise.hh"
-#include "hash.hh"
+#include <nix/util/source-accessor.hh>
+#include <nix/util/types.hh>
+#include <nix/util/serialise.hh>
+#include <nix/util/hash.hh>
 
 struct NarMemberData
 {
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 7d747d3e..81bda873 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -1,7 +1,7 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
-#include "globals.hh"
-#include "parsed-derivations.hh"
+#include <nix/store/globals.hh>
+#include <nix/store/parsed-derivations.hh>
 
 #include <cstring>
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index e6c5f7df..8933720d 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -13,18 +13,18 @@
 
 #include "db.hh"
 
-#include "derivations.hh"
-#include "derivation-options.hh"
-#include "pathlocks.hh"
-#include "pool.hh"
-#include "build-result.hh"
-#include "store-api.hh"
-#include "sync.hh"
+#include <nix/store/derivations.hh>
+#include <nix/store/derivation-options.hh>
+#include <nix/store/pathlocks.hh>
+#include <nix/util/pool.hh>
+#include <nix/store/build-result.hh>
+#include <nix/store/store-api.hh>
+#include <nix/util/sync.hh>
 #include "nar-extractor.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
-#include "serve-protocol-connection.hh"
-#include "machines.hh"
+#include <nix/store/serve-protocol.hh>
+#include <nix/store/serve-protocol-impl.hh>
+#include <nix/store/serve-protocol-connection.hh>
+#include <nix/store/machines.hh>
 
 
 typedef unsigned int BuildID;
diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh
index 1e927573..c664a01d 100644
--- a/src/libhydra/db.hh
+++ b/src/libhydra/db.hh
@@ -2,8 +2,8 @@
 
 #include <pqxx/pqxx>
 
-#include "environment-variables.hh"
-#include "util.hh"
+#include <nix/util/environment-variables.hh>
+#include <nix/util/util.hh>
 
 
 struct Connection : pqxx::connection
diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh
index b1275896..85c58746 100644
--- a/src/libhydra/hydra-config.hh
+++ b/src/libhydra/hydra-config.hh
@@ -2,8 +2,8 @@
 
 #include <map>
 
-#include "file-system.hh"
-#include "util.hh"
+#include <nix/util/file-system.hh>
+#include <nix/util/util.hh>
 
 struct HydraConfig
 {

From 0159135fc71785b570a314d734d8cb90d8305f08 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Thu, 11 Apr 2024 17:12:47 +0200
Subject: [PATCH 243/285] web: include current step status on /machines

---
 src/root/build.tt          | 16 +---------------
 src/root/common.tt         | 19 +++++++++++++++++++
 src/root/machine-status.tt |  2 ++
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/root/build.tt b/src/root/build.tt
index 93a02e0f..18ff6f01 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -61,21 +61,7 @@ END;
             <td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
             <td class="step-status">
               [% IF step.busy != 0 %]
-                [% IF step.busy == 1 %]
-                  <strong>Preparing</strong>
-                [% ELSIF step.busy == 10 %]
-                  <strong>Connecting</strong>
-                [% ELSIF step.busy == 20 %]
-                  <strong>Sending inputs</strong>
-                [% ELSIF step.busy == 30 %]
-                  <strong>Building</strong>
-                [% ELSIF step.busy == 40 %]
-                  <strong>Receiving outputs</strong>
-                [% ELSIF step.busy == 50 %]
-                  <strong>Post-processing</strong>
-                [% ELSE %]
-                  <strong>Unknown state</strong>
-                [% END %]
+                [% INCLUDE renderBusyStatus %]
               [% ELSIF step.status == 0 %]
                 [% IF step.isnondeterministic %]
                   <span class="warn">Succeeded with non-determistic result</span>
diff --git a/src/root/common.tt b/src/root/common.tt
index 869d8856..814d08ca 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -245,6 +245,25 @@ BLOCK renderBuildStatusIcon;
 END;
 
 
+BLOCK renderBusyStatus;
+  IF step.busy == 1 %]
+    <strong>Preparing</strong>
+  [% ELSIF step.busy == 10 %]
+    <strong>Connecting</strong>
+  [% ELSIF step.busy == 20 %]
+    <strong>Sending inputs</strong>
+  [% ELSIF step.busy == 30 %]
+    <strong>Building</strong>
+  [% ELSIF step.busy == 40 %]
+    <strong>Receiving outputs</strong>
+  [% ELSIF step.busy == 50 %]
+    <strong>Post-processing</strong>
+  [% ELSE %]
+    <strong>Unknown state</strong>
+  [% END;
+END;
+
+
 BLOCK renderStatus;
   IF build.finished;
     buildstatus = build.buildstatus;
diff --git a/src/root/machine-status.tt b/src/root/machine-status.tt
index 4195c178..3af5073c 100644
--- a/src/root/machine-status.tt
+++ b/src/root/machine-status.tt
@@ -10,6 +10,7 @@
       <th>Build</th>
       <th>Step</th>
       <th>What</th>
+      <th>Status</th>
       <th>Since</th>
     </tr>
   </thead>
@@ -44,6 +45,7 @@
             <td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
             <td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
             <td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
+            <td>[% INCLUDE renderBusyStatus %]</td>
             <td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
           </tr>
         [% END %]

From a816e8e22c32553850d456057b9c67404d16ec78 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 12 Mar 2024 22:32:58 +0100
Subject: [PATCH 244/285] Make "timed out" and "log limit exceeded" builds
 aborted

In 73694087a088ed4481b4ab268a03351b1bcaac3c I gave builds that failed
because of a timeout or exceeded log limit a stop sign and I stand by
that reasoning: with that it's possible to distinguish between actual
build failures and rather transient things such as timeouts.

Back then I considered it a feature that these are shown in a different
tab, but I don't think that's a good idea anymore. When using a jobset to
e.g. track the regressions from a mass rebuild (like a compiler or gcc
update), "Newly failed builds" should exclusively display regressions (and
flaky builds of course, not much I can do about that).

Also, when a bunch of builds fail in such a jobset because of e.g. a
broken connection to a builder that results in a timeout, I want to be
able to restart them all w/o rebuilding actual regressions.

To make it clear that we not only have "Aborted" builds in the tab, I
renamed the label to "Aborted / Timed out".
---
 src/lib/Hydra/Helper/BuildDiff.pm | 13 +++++++++++--
 src/root/jobset-eval.tt           |  2 +-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Helper/BuildDiff.pm b/src/lib/Hydra/Helper/BuildDiff.pm
index cd8c7691..65dad17c 100644
--- a/src/lib/Hydra/Helper/BuildDiff.pm
+++ b/src/lib/Hydra/Helper/BuildDiff.pm
@@ -37,7 +37,16 @@ sub buildDiff {
 
     my $n = 0;
     foreach my $build (@{$builds}) {
-        my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
+        my $aborted = $build->finished != 0 && (
+            # aborted
+            $build->buildstatus == 3
+            # cancelled
+            || $build->buildstatus == 4
+            # timeout
+            || $build->buildstatus == 7
+            # log limit exceeded
+            || $build->buildstatus == 10
+        );
         my $d;
         my $found = 0;
         while ($n < scalar(@{$builds2})) {
@@ -79,4 +88,4 @@ sub buildDiff {
     return $ret;
 }
 
-1;
\ No newline at end of file
+1;
diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt
index 8a8d92e4..fd0d4846 100644
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
   [% END %]
 
   [% IF aborted.size > 0 %]
-    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
+    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
   [% END %]
   [% IF nowFail.size > 0 %]
     <li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>

From 33b982f4080f4b9a91becb92f6b4b42f29392d8d Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 16 Mar 2024 17:53:02 +0100
Subject: [PATCH 245/285] Running builds view: show build step names

When using Hydra to build machine configurations, you'll often see
"nixosConfigurations.foo" five times, i.e. for each build step being
run. This isn't very helpful I think because in such a case, a single
build step can also be compiling the Linux kernel.

This change also fetches the `drvpath` and `type` from the `buildsteps`
relation. We're already joining it, so this doesn't make much difference
(confirmed via query logging that this doesn't cause extra SQL queries).

Unfortunately build steps don't have a human readable name, so I'm
deriving it from the drvpath by stripping away the hash (assuming that
it'll never contain a `-` and that `/nix/store/` is used as prefix). I
decided against using the Nix bindings for that to avoid too much
overhead due to store operations for each build step.
---
 src/lib/Hydra/Controller/Root.pm |  2 +-
 src/root/common.tt               | 16 +++++++++++++++-
 src/root/status.tt               |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index a231d7c0..adb5ad44 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -162,7 +162,7 @@ sub status_GET {
             { "buildsteps.busy" => { '!=', 0 } },
             { order_by => ["globalpriority DESC", "id"],
               join => "buildsteps",
-              columns => [@buildListColumns]
+              columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
             })]
     );
 }
diff --git a/src/root/common.tt b/src/root/common.tt
index 814d08ca..b96cf99c 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -91,6 +91,15 @@ BLOCK renderDuration;
   duration % 60 %]s[%
 END;
 
+BLOCK renderDrvInfo;
+  drvname = step.drvpath
+    .substr(11) # strip `/nix/store/`
+    .split('-').slice(1).join("-") # strip hash part
+    .substr(0, -4); # strip `.drv`
+  IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
+  IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
+END;
+
 
 BLOCK renderBuildListHeader %]
   <table class="table table-striped table-condensed clickable-rows">
@@ -131,7 +140,12 @@ BLOCK renderBuildListBody;
       [% END %]
       <td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
       [% IF !hideJobName %]
-        <td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
+        <td>
+          <a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
+          [% IF showStepName %]
+            [% INCLUDE renderDrvInfo step=build.buildsteps %]
+          [% END %]
+        </td>
       [% END %]
       <td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
       <td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
diff --git a/src/root/status.tt b/src/root/status.tt
index f1ec70b9..e6a07bb7 100644
--- a/src/root/status.tt
+++ b/src/root/status.tt
@@ -7,7 +7,7 @@
 
 [% ELSE %]
 
-  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
+  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]
 
 [% END %]
 

From 9d8f30affe90764d10e537d43e28ce5aa66807ab Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 17 Mar 2024 12:57:59 +0100
Subject: [PATCH 246/285] Only show stepname if it doesn't equal the name of
 the drv

When building e.g. nixpkgs, the "Running builds" view will mostly look
like this

    hello.x86_64-linux (Build of hello-X.Y)
    exa.x86_64-linux (Build of exa-X.Y)
    ...

This doesn't provide any useful information. Showing the step name only
makes sense if it's not a child of the job's derivation. With this
patch, that information will only be shown if the drv name (i.e. w/o
`/nix/store/` prefix, .drv ext & hash) is not equal to the drv name of
the job itself (build.nixname).
---
 src/root/common.tt | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/root/common.tt b/src/root/common.tt
index b96cf99c..842ad109 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -96,8 +96,10 @@ BLOCK renderDrvInfo;
     .substr(11) # strip `/nix/store/`
     .split('-').slice(1).join("-") # strip hash part
     .substr(0, -4); # strip `.drv`
-  IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
-  IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
+  IF drvname != releasename;
+    IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
+    IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
+  END;
 END;
 
 
@@ -143,7 +145,7 @@ BLOCK renderBuildListBody;
         <td>
           <a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
           [% IF showStepName %]
-            [% INCLUDE renderDrvInfo step=build.buildsteps %]
+            [% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
           [% END %]
         </td>
       [% END %]

From 99359c251a1e8be2fb245eea7513c1e39bae4762 Mon Sep 17 00:00:00 2001
From: ajs124 <git@ajs124.de>
Date: Fri, 16 Feb 2024 16:50:33 +0100
Subject: [PATCH 247/285] lazy-load evaluation errors

Closes #1362
---
 src/lib/Hydra/Controller/Jobset.pm     |  9 +++++++++
 src/lib/Hydra/Controller/JobsetEval.pm |  9 +++++++++
 src/root/eval-error.tt                 | 26 ++++++++++++++++++++++++++
 src/root/jobset-eval.tt                | 10 +---------
 src/root/jobset.tt                     |  3 +--
 src/root/layout.tt                     | 26 +-------------------------
 src/root/static/js/common.js           |  6 ++++++
 src/root/style.tt                      | 24 ++++++++++++++++++++++++
 t/Hydra/Controller/Jobset/evals.t      |  5 +++++
 t/Hydra/Controller/JobsetEval/fetch.t  |  4 ++++
 10 files changed, 86 insertions(+), 36 deletions(-)
 create mode 100644 src/root/eval-error.tt
 create mode 100644 src/root/style.tt

diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm
index eeb4232a..20a52f6f 100644
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -364,6 +364,15 @@ sub evals_GET {
     );
 }
 
+sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
+
+sub errors_GET {
+    my ($self, $c) = @_;
+
+    $c->stash->{template} = 'eval-error.tt';
+
+    $self->status_ok($c, entity => $c->stash->{jobset});
+}
 
 # Redirect to the latest finished evaluation of this jobset.
 sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm
index 30179d49..aca03d72 100644
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -86,6 +86,15 @@ sub view_GET {
     );
 }
 
+sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
+
+sub errors_GET {
+    my ($self, $c) = @_;
+
+    $c->stash->{template} = 'eval-error.tt';
+
+    $self->status_ok($c, entity => $c->stash->{eval});
+}
 
 sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
     my ($self, $c) = @_;
diff --git a/src/root/eval-error.tt b/src/root/eval-error.tt
new file mode 100644
index 00000000..c2ea28ec
--- /dev/null
+++ b/src/root/eval-error.tt
@@ -0,0 +1,26 @@
+[% PROCESS common.tt %]
+<!DOCTYPE html>
+
+<html lang="en">
+
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
+    [% INCLUDE style.tt %]
+  </head>
+
+  <body>
+
+    <div class="tab-content tab-pane">
+        <div id="tabs-errors" class="">
+          [% IF jobset %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
+          [% ELSIF eval %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+          [% END %]
+        </div>
+    </div>
+  </body>
+</html>
diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt
index fd0d4846..146878f2 100644
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -108,13 +108,6 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
 
 <div class="tab-content">
 
-  [% IF eval.evaluationerror.errormsg %]
-    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
-    </div>
-  [% END %]
-
   <div id="tabs-aborted" class="tab-pane">
     [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
   </div>
@@ -174,8 +167,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
 
   [% IF eval.evaluationerror.errormsg %]
     <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+      <iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
     </div>
   [% END %]
 </div>
diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 5d8345f9..5afcbfde 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -119,8 +119,7 @@
 
   [% IF jobset.errormsg || jobset.fetcherrormsg %]
     <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
+      <iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
     </div>
   [% END %]
 
diff --git a/src/root/layout.tt b/src/root/layout.tt
index 399962b4..b520b455 100644
--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@@ -10,31 +10,7 @@
 
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
     <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
-
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-
-    <link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
-    <script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
-    <link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
-
-    <!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
-
-    <style>
-      .popover { max-width: 40%; }
-    </style>
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
-
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
+    [% INCLUDE style.tt %]
 
     [% IF c.config.enable_google_login %]
       <meta name="google-signin-client_id" content="[% c.config.google_client_id %]">
diff --git a/src/root/static/js/common.js b/src/root/static/js/common.js
index c51f769a..9f31d1e6 100644
--- a/src/root/static/js/common.js
+++ b/src/root/static/js/common.js
@@ -129,6 +129,12 @@ $(document).ready(function() {
             el.addClass("is-local");
         }
     });
+
+    [...document.getElementsByTagName("iframe")].forEach((element) => {
+        element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
+            element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
+        })
+    })
 });
 
 var tabsLoaded = {};
diff --git a/src/root/style.tt b/src/root/style.tt
new file mode 100644
index 00000000..4094b7bc
--- /dev/null
+++ b/src/root/style.tt
@@ -0,0 +1,24 @@
+<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
+
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+
+<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
+<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
+<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
+
+<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
+<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
+<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
+
+<style>
+  .popover { max-width: 40%; }
+</style>
+
+<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
+
+<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
+
+<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
diff --git a/t/Hydra/Controller/Jobset/evals.t b/t/Hydra/Controller/Jobset/evals.t
index 221efb65..25357f8f 100644
--- a/t/Hydra/Controller/Jobset/evals.t
+++ b/t/Hydra/Controller/Jobset/evals.t
@@ -32,4 +32,9 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
     ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
 };
 
+subtest "/jobset/PROJECT/JOBSET/errors" => sub {
+    my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
+    ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
+};
+
 done_testing;
diff --git a/t/Hydra/Controller/JobsetEval/fetch.t b/t/Hydra/Controller/JobsetEval/fetch.t
index 14169c39..609e9224 100644
--- a/t/Hydra/Controller/JobsetEval/fetch.t
+++ b/t/Hydra/Controller/JobsetEval/fetch.t
@@ -35,6 +35,10 @@ subtest "Fetching the eval's overview" => sub {
     is($fetch->code, 200, "channel page is 200");
 };
 
+subtest "Fetching the eval's overview" => sub {
+    my $fetch = request(GET '/eval/' . $eval->id, '/errors');
+    is($fetch->code, 200, "errors page is 200");
+};
 
 
 done_testing;

From abe35881e4d9263babb452bb9169cf6491d418bd Mon Sep 17 00:00:00 2001
From: "git@71rd.net" <git@71rd.net>
Date: Sat, 3 Aug 2024 10:36:10 +0000
Subject: [PATCH 248/285] Stream files from store instead of buffering them

When an artifact is requested from hydra the output is first copied
from the nix store into memory and then sent as a response, delaying
the download and taking up significant amounts of memory.

As reported in https://github.com/NixOS/hydra/issues/1357

Instead of calling a command and blocking while reading in the entire
output, this adds read_into_socket(). the function takes a
command, starting a subprocess with that command, returning a file
descriptor attached to stdout.
This file descriptor is then by responsebuilder of Catalyst to steam
the output directly

(cherry picked from commit 459aa0a5983a0bd546399c08231468d6e9282f54)
---
 src/lib/Hydra/Controller/Build.pm |  2 +-
 src/lib/Hydra/Helper/Nix.pm       | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index de2c204d..5e7b6f24 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -238,7 +238,7 @@ sub serveFile {
         # XSS hole.
         $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
 
-        $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
+        $c->stash->{'plain'} = { data => readIntoSocket(cmd => ["nix", "--experimental-features", "nix-command",
                                                       "store", "cat", "--store", getStoreUri(), "$path"]) };
 
         # Detect MIME type.
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index bff7a5ed..f87946d5 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -36,6 +36,7 @@ our @EXPORT = qw(
     jobsetOverview
     jobsetOverview_
     pathIsInsidePrefix
+    readIntoSocket
     readNixFile
     registerRoot
     restartBuilds
@@ -417,6 +418,17 @@ sub pathIsInsidePrefix {
     return $cur;
 }
 
+sub readIntoSocket{
+    my (%args) = @_;
+    my $sock;
+
+    eval {
+        my $x= join(" ", @{$args{cmd}});
+        open($sock, "-|", $x) or die q(failed to open socket from command:\n $x);
+    };
+
+    return $sock;
+}
 
 
 

From 6133693097c3d24eb67f901eff2a35ded4455ee8 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 18 Aug 2024 10:40:42 +0200
Subject: [PATCH 249/285] readIntoSocket: fix with store URIs containing an `&`

The third argument to `open()` in `-|` mode is passed to a shell if it's
a string. In my case the store URI contains
`?secret-key=${signingKey.directory}/secret&compression=zstd`

For the `nix store cat` case this means that

* until `&` the process will be started in the background. This fails
  immediately because no path to cat is specified.
* `compression=zstd` is a variable assignment
* the `$path` argument to `store cat` is attempted to be executed as
  another command

Passing just the list solves the problem.

(cherry picked from commit 3ee51dbe589458cc54ff753317bbc6db530bddc0)
---
 src/lib/Hydra/Helper/Nix.pm | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index f87946d5..88fbdd6d 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -423,8 +423,7 @@ sub readIntoSocket{
     my $sock;
 
     eval {
-        my $x= join(" ", @{$args{cmd}});
-        open($sock, "-|", $x) or die q(failed to open socket from command:\n $x);
+        open($sock, "-|", @{$args{cmd}}) or die q(failed to open socket from command:\n $x);
     };
 
     return $sock;

From 0d0c4f278bbed3ad31b100cc662933041d34651d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Apr 2025 12:31:02 -0400
Subject: [PATCH 250/285] Fix evaluation of NixOS tests, avoid `with`

---
 nixos-tests.nix | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/nixos-tests.nix b/nixos-tests.nix
index 9efe68c8..77dd8f8a 100644
--- a/nixos-tests.nix
+++ b/nixos-tests.nix
@@ -27,8 +27,7 @@ in
 {
 
   install = forEachSystem (system:
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    simpleTest {
+    (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
       name = "hydra-install";
       nodes.machine = hydraServer;
       testScript =
@@ -43,8 +42,7 @@ in
     });
 
   notifications = forEachSystem (system:
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    simpleTest {
+    (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
       name = "hydra-notifications";
       nodes.machine = {
         imports = [ hydraServer ];
@@ -56,7 +54,7 @@ in
         '';
         services.influxdb.enable = true;
       };
-      testScript = ''
+      testScript = { nodes, ... }: ''
         machine.wait_for_job("hydra-init")
 
         # Create an admin account and some other state.
@@ -87,7 +85,7 @@ in
 
         # Setup the project and jobset
         machine.succeed(
-            "su - hydra -c 'perl -I ${config.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+            "su - hydra -c 'perl -I ${nodes.machine.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
         )
 
         # Wait until hydra has build the job and
@@ -101,9 +99,7 @@ in
     });
 
   gitea = forEachSystem (system:
-    let pkgs = nixpkgs.legacyPackages.${system}; in
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    makeTest {
+    (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest {
       name = "hydra-gitea";
       nodes.machine = { pkgs, ... }: {
         imports = [ hydraServer ];
@@ -129,7 +125,7 @@ in
         networking.firewall.allowedTCPPorts = [ 3000 ];
       };
       skipLint = true;
-      testScript =
+      testScript = { pkgs, ...  }:
         let
           scripts.mktoken = pkgs.writeText "token.sql" ''
             INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');

From 5a9985f96cd00278a137bbbca8ee2789acfae093 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Thu, 11 Apr 2024 17:12:47 +0200
Subject: [PATCH 251/285] web: Skip System on /machines

It is redundant
---
 src/root/machine-status.tt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/root/machine-status.tt b/src/root/machine-status.tt
index 3af5073c..07a2359d 100644
--- a/src/root/machine-status.tt
+++ b/src/root/machine-status.tt
@@ -6,7 +6,6 @@
   <thead>
     <tr>
       <th>Job</th>
-      <th>System</th>
       <th>Build</th>
       <th>Step</th>
       <th>What</th>
@@ -41,7 +40,6 @@
           [% idle = 0 %]
           <tr>
             <td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
-            <td><tt>[% step.system %]</tt></td>
             <td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
             <td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
             <td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>

From 1e2d3211d9fec6dd9e00667bc5c12203ee0bdaa4 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Thu, 11 Apr 2024 15:03:23 +0200
Subject: [PATCH 252/285] queue-runner: limit parallelism of CPU intensive
 operations

My current theory is that running more parallel xz than available CPU
cores is reducing our overall throughput by requiring more scheduling
overhead and more cache thrashing.
---
 src/hydra-queue-runner/build-remote.cc       | 18 ++++++++++++++++++
 src/hydra-queue-runner/hydra-queue-runner.cc |  1 +
 src/hydra-queue-runner/state.hh              |  7 +++++++
 src/root/common.tt                           |  2 ++
 4 files changed, 28 insertions(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 79c32d46..0c8b3f10 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -386,6 +386,16 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
 
 }
 
+/* Utility guard object to auto-release a semaphore on destruction. */
+template <typename T>
+class SemaphoreReleaser {
+public:
+    SemaphoreReleaser(T* s) : sem(s) {}
+    ~SemaphoreReleaser() { sem->release(); }
+
+private:
+    T* sem;
+};
 
 void State::buildRemote(ref<Store> destStore,
     ::Machine::ptr machine, Step::ptr step,
@@ -527,6 +537,14 @@ void State::buildRemote(ref<Store> destStore,
             result.logFile = "";
         }
 
+        /* Throttle CPU-bound work. Opportunistically skip updating the current
+         * step, since this requires a DB roundtrip. */
+        if (!localWorkThrottler.try_acquire()) {
+            updateStep(ssWaitingForLocalSlot);
+            localWorkThrottler.acquire();
+        }
+        SemaphoreReleaser releaser(&localWorkThrottler);
+
         StorePathSet outputs;
         for (auto & [_, realisation] : buildResult.builtOutputs)
             outputs.insert(realisation.outPath);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 64a98797..cf7d4056 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -85,6 +85,7 @@ State::State(std::optional<std::string> metricsAddrOpt)
     : config(std::make_unique<HydraConfig>())
     , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
     , dbPool(config->getIntOption("max_db_connections", 128))
+    , localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
     , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
     , maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
     , uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 8933720d..34b7a676 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -6,6 +6,8 @@
 #include <map>
 #include <memory>
 #include <queue>
+#include <regex>
+#include <semaphore>
 
 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@@ -58,6 +60,7 @@ typedef enum {
     ssConnecting = 10,
     ssSendingInputs = 20,
     ssBuilding = 30,
+    ssWaitingForLocalSlot = 35,
     ssReceivingOutputs = 40,
     ssPostProcessing = 50,
 } StepState;
@@ -353,6 +356,10 @@ private:
     typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
     nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
 
+    /* Throttler for CPU-bound local work. */
+    static constexpr unsigned int maxSupportedLocalWorkers = 1024;
+    std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
+
     /* Various stats. */
     time_t startedAt;
     counter nrBuildsRead{0};
diff --git a/src/root/common.tt b/src/root/common.tt
index 842ad109..6348bee7 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -270,6 +270,8 @@ BLOCK renderBusyStatus;
     <strong>Sending inputs</strong>
   [% ELSIF step.busy == 30 %]
     <strong>Building</strong>
+  [% ELSIF step.busy == 35 %]
+    <strong>Waiting to receive outputs</strong>
   [% ELSIF step.busy == 40 %]
     <strong>Receiving outputs</strong>
   [% ELSIF step.busy == 50 %]

From efcf6815d95134050b0f280668af7655aae9ef72 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sat, 20 Apr 2024 16:48:03 +0200
Subject: [PATCH 253/285] queue-runner: add prom metrics to allow detecting
 internal bottlenecks

By looking at the ratio of running vs. waiting for the dispatcher and
the queue monitor, we should get better visibility into what hydra is
currently bottlenecked on.

There are other side effects we can try to measure to get to the same
result, but having a simple way doesn't cost us much.
---
 src/hydra-queue-runner/dispatcher.cc         | 12 ++++++---
 src/hydra-queue-runner/hydra-queue-runner.cc | 28 ++++++++++++++++++++
 src/hydra-queue-runner/queue-monitor.cc      | 11 ++++++++
 src/hydra-queue-runner/state.hh              |  6 +++++
 4 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index cbf982bf..11db0071 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -40,13 +40,15 @@ void State::dispatcher()
             printMsg(lvlDebug, "dispatcher woken up");
             nrDispatcherWakeups++;
 
-            auto now1 = std::chrono::steady_clock::now();
+            auto t_before_work = std::chrono::steady_clock::now();
 
             auto sleepUntil = doDispatch();
 
-            auto now2 = std::chrono::steady_clock::now();
+            auto t_after_work = std::chrono::steady_clock::now();
 
-            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+            prom.dispatcher_time_spent_running.Increment(
+                std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
+            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
 
             /* Sleep until we're woken up (either because a runnable build
                is added, or because a build finishes). */
@@ -60,6 +62,10 @@ void State::dispatcher()
                 *dispatcherWakeup_ = false;
             }
 
+            auto t_after_sleep = std::chrono::steady_clock::now();
+            prom.dispatcher_time_spent_waiting.Increment(
+                std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
+
         } catch (std::exception & e) {
             printError("dispatcher: %s", e.what());
             sleep(1);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index cf7d4056..8123fd39 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -77,6 +77,34 @@ State::PromMetrics::PromMetrics()
             .Register(*registry)
             .Add({})
     )
+    , dispatcher_time_spent_running(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_dispatcher_time_spent_running")
+            .Help("Time (in micros) spent running the dispatcher")
+            .Register(*registry)
+            .Add({})
+    )
+    , dispatcher_time_spent_waiting(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_dispatcher_time_spent_waiting")
+            .Help("Time (in micros) spent waiting for the dispatcher to obtain work")
+            .Register(*registry)
+            .Add({})
+    )
+    , queue_monitor_time_spent_running(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_queue_monitor_time_spent_running")
+            .Help("Time (in micros) spent running the queue monitor")
+            .Register(*registry)
+            .Add({})
+    )
+    , queue_monitor_time_spent_waiting(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
+            .Help("Time (in micros) spent waiting for the queue monitor to obtain work")
+            .Register(*registry)
+            .Add({})
+    )
 {
 
 }
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 81bda873..3af0752a 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -42,12 +42,19 @@ void State::queueMonitorLoop(Connection & conn)
 
     bool quit = false;
     while (!quit) {
+        auto t_before_work = std::chrono::steady_clock::now();
+
         localStore->clearPathInfoCache();
 
         bool done = getQueuedBuilds(conn, destStore, lastBuildId);
 
         if (buildOne && buildOneDone) quit = true;
 
+        auto t_after_work = std::chrono::steady_clock::now();
+
+        prom.queue_monitor_time_spent_running.Increment(
+            std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
+
         /* Sleep until we get notification from the database about an
            event. */
         if (done && !quit) {
@@ -72,6 +79,10 @@ void State::queueMonitorLoop(Connection & conn)
             printMsg(lvlTalkative, "got notification: jobset shares changed");
             processJobsetSharesChange(conn);
         }
+
+        auto t_after_sleep = std::chrono::steady_clock::now();
+        prom.queue_monitor_time_spent_waiting.Increment(
+            std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
     }
 
     exit(0);
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 34b7a676..5e05157b 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -459,6 +459,12 @@ private:
         prometheus::Counter& queue_checks_finished;
         prometheus::Gauge& queue_max_id;
 
+        prometheus::Counter& dispatcher_time_spent_running;
+        prometheus::Counter& dispatcher_time_spent_waiting;
+
+        prometheus::Counter& queue_monitor_time_spent_running;
+        prometheus::Counter& queue_monitor_time_spent_waiting;
+
         PromMetrics();
     };
     PromMetrics prom;

From d8ffa6b56a49dc6705b1fc3a59f6acfe64dd106a Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sat, 20 Apr 2024 16:53:52 +0200
Subject: [PATCH 254/285] queue-runner: remove id > X from new builds query

Running the query with/without it shows that it makes no difference to
postgres, since there's an index on finished=0 already. This allows a
few simplifications, but also paves the way towards running multiple
parallel monitor threads in the future.
---
 src/hydra-queue-runner/hydra-queue-runner.cc |  7 -------
 src/hydra-queue-runner/queue-monitor.cc      | 21 ++++----------------
 src/hydra-queue-runner/state.hh              |  4 +---
 3 files changed, 5 insertions(+), 27 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 8123fd39..405c44d1 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -70,13 +70,6 @@ State::PromMetrics::PromMetrics()
             .Register(*registry)
             .Add({})
     )
-    , queue_max_id(
-        prometheus::BuildGauge()
-            .Name("hydraqueuerunner_queue_max_build_id_info")
-            .Help("Maximum build record ID in the queue")
-            .Register(*registry)
-            .Add({})
-    )
     , dispatcher_time_spent_running(
         prometheus::BuildCounter()
             .Name("hydraqueuerunner_dispatcher_time_spent_running")
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 3af0752a..a9d386d0 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -38,15 +38,13 @@ void State::queueMonitorLoop(Connection & conn)
 
     auto destStore = getDestStore();
 
-    unsigned int lastBuildId = 0;
-
     bool quit = false;
     while (!quit) {
         auto t_before_work = std::chrono::steady_clock::now();
 
         localStore->clearPathInfoCache();
 
-        bool done = getQueuedBuilds(conn, destStore, lastBuildId);
+        bool done = getQueuedBuilds(conn, destStore);
 
         if (buildOne && buildOneDone) quit = true;
 
@@ -64,12 +62,10 @@ void State::queueMonitorLoop(Connection & conn)
             conn.get_notifs();
 
         if (auto lowestId = buildsAdded.get()) {
-            lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
             printMsg(lvlTalkative, "got notification: new builds added to the queue");
         }
         if (buildsRestarted.get()) {
             printMsg(lvlTalkative, "got notification: builds restarted");
-            lastBuildId = 0; // check all builds
         }
         if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
             printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
@@ -96,11 +92,11 @@ struct PreviousFailure : public std::exception {
 
 
 bool State::getQueuedBuilds(Connection & conn,
-    ref<Store> destStore, unsigned int & lastBuildId)
+    ref<Store> destStore)
 {
     prom.queue_checks_started.Increment();
 
-    printInfo("checking the queue for builds > %d...", lastBuildId);
+    printInfo("checking the queue for builds...");
 
     /* Grab the queued builds from the database, but don't process
        them yet (since we don't want a long-running transaction). */
@@ -108,8 +104,6 @@ bool State::getQueuedBuilds(Connection & conn,
     std::map<BuildID, Build::ptr> newBuildsByID;
     std::multimap<StorePath, BuildID> newBuildsByPath;
 
-    unsigned int newLastBuildId = lastBuildId;
-
     {
         pqxx::work txn(conn);
 
@@ -118,17 +112,12 @@ bool State::getQueuedBuilds(Connection & conn,
              "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
              "globalPriority, priority from Builds "
              "inner join jobsets on builds.jobset_id = jobsets.id "
-             "where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
-            lastBuildId);
+             "where finished = 0 order by globalPriority desc, builds.id");
 
         for (auto const & row : res) {
             auto builds_(builds.lock());
             BuildID id = row["id"].as<BuildID>();
             if (buildOne && id != buildOne) continue;
-            if (id > newLastBuildId) {
-                newLastBuildId = id;
-                prom.queue_max_id.Set(id);
-            }
             if (builds_->count(id)) continue;
 
             auto build = std::make_shared<Build>(
@@ -337,8 +326,6 @@ bool State::getQueuedBuilds(Connection & conn,
     }
 
     prom.queue_checks_finished.Increment();
-
-    lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
     return newBuildsByID.empty();
 }
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 5e05157b..4cb295e7 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -457,7 +457,6 @@ private:
         prometheus::Counter& queue_steps_created;
         prometheus::Counter& queue_checks_early_exits;
         prometheus::Counter& queue_checks_finished;
-        prometheus::Gauge& queue_max_id;
 
         prometheus::Counter& dispatcher_time_spent_running;
         prometheus::Counter& dispatcher_time_spent_waiting;
@@ -507,8 +506,7 @@ private:
     void queueMonitorLoop(Connection & conn);
 
     /* Check the queue for new builds. */
-    bool getQueuedBuilds(Connection & conn,
-        nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
+    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);
 
     /* Handle cancellation, deletion and priority bumps. */
     void processQueueChange(Connection & conn);

From 9265fc5002f6c1073d32c0ced999f334fefd4bc6 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sat, 20 Apr 2024 16:58:10 +0200
Subject: [PATCH 255/285] queue-runner: reduce the time between queue monitor
 restarts

This will induce more DB queries (though these are fairly cheap), but at
the benefit of processing bumps within 1m instead of within 10m.
---
 src/hydra-queue-runner/queue-monitor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index a9d386d0..2049a6c1 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -319,7 +319,7 @@ bool State::getQueuedBuilds(Connection & conn,
 
         /* Stop after a certain time to allow priority bumps to be
            processed. */
-        if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
+        if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
             prom.queue_checks_early_exits.Increment();
             break;
         }

From 52a0199a9bfbb499b564a77d33fe615f39899f6b Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sat, 20 Apr 2024 22:18:13 +0200
Subject: [PATCH 256/285] queue runner: introduce some parallelism for remote
 paths lookup

Each output for a given step being ingested is looked up in parallel,
which should basically multiply the speed of builds ingestion by the
average number of outputs per derivation.
---
 src/hydra-queue-runner/queue-monitor.cc | 40 +++++++++++++++++++++----
 src/hydra-queue-runner/state.hh         |  6 ++++
 2 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 2049a6c1..9eab6e90 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -2,6 +2,7 @@
 #include "hydra-build-result.hh"
 #include <nix/store/globals.hh>
 #include <nix/store/parsed-derivations.hh>
+#include <nix/util/thread-pool.hh>
 
 #include <cstring>
 
@@ -404,6 +405,34 @@ void State::processQueueChange(Connection & conn)
 }
 
 
+std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
+    ref<Store> destStore,
+    const std::map<DrvOutput, std::optional<StorePath>> & paths)
+{
+    Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
+    ThreadPool tp;
+
+    for (auto & [output, maybeOutputPath] : paths) {
+        if (!maybeOutputPath) {
+            auto missing(missing_.lock());
+            missing->insert({output, maybeOutputPath});
+        } else {
+            tp.enqueue([&] {
+                if (!destStore->isValidPath(*maybeOutputPath)) {
+                    auto missing(missing_.lock());
+                    missing->insert({output, maybeOutputPath});
+                }
+            });
+        }
+    }
+
+    tp.process();
+
+    auto missing(missing_.lock());
+    return *missing;
+}
+
+
 Step::ptr State::createStep(ref<Store> destStore,
     Connection & conn, Build::ptr build, const StorePath & drvPath,
     Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
@@ -487,16 +516,15 @@ Step::ptr State::createStep(ref<Store> destStore,
 
     /* Are all outputs valid? */
     auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
-    bool valid = true;
-    std::map<DrvOutput, std::optional<StorePath>> missing;
+    std::map<DrvOutput, std::optional<StorePath>> paths;
     for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
         auto outputHash = outputHashes.at(outputName);
-        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
-            continue;
-        valid = false;
-        missing.insert({{outputHash, outputName}, maybeOutputPath});
+        paths.insert({{outputHash, outputName}, maybeOutputPath});
     }
 
+    auto missing = getMissingRemotePaths(destStore, paths);
+    bool valid = missing.empty();
+
     /* Try to copy the missing paths from the local store or from
        substitutes. */
     if (!missing.empty()) {
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 4cb295e7..18101a0a 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -514,6 +514,12 @@ private:
     BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
         const nix::StorePath & drvPath);
 
+    /* Returns paths missing from the remote store. Paths are processed in
+     * parallel to work around the possible latency of remote stores. */
+    std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
+        nix::ref<nix::Store> destStore,
+        const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
+
     Step::ptr createStep(nix::ref<nix::Store> store,
         Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
         Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,

From 8e02589ac8d6c0c71d55d3a0fa75d321b0af2e50 Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sat, 20 Apr 2024 22:49:24 +0200
Subject: [PATCH 257/285] queue-runner: switch to pseudorandom ordering of
 builds processing

We don't rely on sequential / monotonic build IDs processing anymore, so
randomizing actually has the advantage of mixing builds for different
systems together, to avoid only one chunk of builds for a single system
getting processed while builders for other systems are starved.
---
 src/hydra-queue-runner/queue-monitor.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 9eab6e90..bb15ac04 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -102,7 +102,7 @@ bool State::getQueuedBuilds(Connection & conn,
     /* Grab the queued builds from the database, but don't process
        them yet (since we don't want a long-running transaction). */
     std::vector<BuildID> newIDs;
-    std::map<BuildID, Build::ptr> newBuildsByID;
+    std::unordered_map<BuildID, Build::ptr> newBuildsByID;
     std::multimap<StorePath, BuildID> newBuildsByPath;
 
     {
@@ -113,7 +113,7 @@ bool State::getQueuedBuilds(Connection & conn,
              "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
              "globalPriority, priority from Builds "
              "inner join jobsets on builds.jobset_id = jobsets.id "
-             "where finished = 0 order by globalPriority desc, builds.id");
+             "where finished = 0 order by globalPriority desc, random()");
 
         for (auto const & row : res) {
             auto builds_(builds.lock());

From 78687e23cf1c8d341253689d07d40393f6e5edc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 7 Apr 2025 18:43:12 +0200
Subject: [PATCH 258/285] test/gitea: fix eval

---
 nixos-tests.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/nixos-tests.nix b/nixos-tests.nix
index 77dd8f8a..c70a3cd1 100644
--- a/nixos-tests.nix
+++ b/nixos-tests.nix
@@ -99,6 +99,9 @@ in
     });
 
   gitea = forEachSystem (system:
+    let
+      pkgs = nixpkgs.legacyPackages.${system};
+    in
     (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest {
       name = "hydra-gitea";
       nodes.machine = { pkgs, ... }: {
@@ -125,7 +128,7 @@ in
         networking.firewall.allowedTCPPorts = [ 3000 ];
       };
       skipLint = true;
-      testScript = { pkgs, ...  }:
+      testScript =
         let
           scripts.mktoken = pkgs.writeText "token.sql" ''
             INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all');

From c94ba404fd80538fae0c29788b3dab9deb9f19e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 7 Apr 2025 18:54:39 +0200
Subject: [PATCH 259/285] don't build hydra twice in a pull request + enable
 merge queue

---
 .github/workflows/test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 42cb6843..9c05d752 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,7 +1,10 @@
 name: "Test"
 on:
   pull_request:
+  merge_group:
   push:
+    branches:
+      - master
 jobs:
   tests:
     runs-on: ubuntu-latest

From b3a433336ed4bd2532ff7d71cb2ced90b389700b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 7 Apr 2025 18:43:56 +0200
Subject: [PATCH 260/285] bump nixpkgs

---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 3a7a1672..ddff55ec 100644
--- a/flake.lock
+++ b/flake.lock
@@ -44,11 +44,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1739461644,
-        "narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
+        "lastModified": 1743987495,
+        "narHash": "sha256-46T2vMZ4/AfCK0Y2OjlFzJPxmdpP8GtsuEqSSJv3oe4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
+        "rev": "db8f4fe18ce772a9c8f3adf321416981c8fe9371",
         "type": "github"
       },
       "original": {

From 8a6482bb1c3e3c2361fd3cb0a86c3a86d0420e97 Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Tue, 1 Oct 2024 19:14:24 +0300
Subject: [PATCH 261/285] Add metric for builds waiting for download slot

(cherry picked from commit f23ec71227911891807706b6b978836e4d80edde)
---
 src/hydra-queue-runner/build-remote.cc       | 1 +
 src/hydra-queue-runner/hydra-queue-runner.cc | 1 +
 src/hydra-queue-runner/state.hh              | 1 +
 3 files changed, 3 insertions(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 0c8b3f10..e25da38d 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -540,6 +540,7 @@ void State::buildRemote(ref<Store> destStore,
         /* Throttle CPU-bound work. Opportunistically skip updating the current
          * step, since this requires a DB roundtrip. */
         if (!localWorkThrottler.try_acquire()) {
+            MaintainCount<counter> mc(nrStepsWaitingForDownloadSlot);
             updateStep(ssWaitingForLocalSlot);
             localWorkThrottler.acquire();
         }
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 405c44d1..adc903d8 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -573,6 +573,7 @@ void State::dumpStatus(Connection & conn)
         {"nrActiveSteps", activeSteps_.lock()->size()},
         {"nrStepsBuilding", nrStepsBuilding.load()},
         {"nrStepsCopyingTo", nrStepsCopyingTo.load()},
+        {"nrStepsWaitingForDownloadSlot", nrStepsWaitingForDownloadSlot.load()},
         {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
         {"nrStepsWaiting", nrStepsWaiting.load()},
         {"nrUnsupportedSteps", nrUnsupportedSteps.load()},
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 18101a0a..12aead40 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -369,6 +369,7 @@ private:
     counter nrStepsDone{0};
     counter nrStepsBuilding{0};
     counter nrStepsCopyingTo{0};
+    counter nrStepsWaitingForDownloadSlot{0};
     counter nrStepsCopyingFrom{0};
     counter nrStepsWaiting{0};
     counter nrUnsupportedSteps{0};

From 143a07bff0a4a46f132a404413192e10dcc7150f Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sun, 21 Apr 2024 01:46:41 +0200
Subject: [PATCH 262/285] queue-runner: release machine reservation while
 copying outputs

This allows for better builder usage when the queue runner is busy. To
avoid running into uncontrollable imbalances between builder/queue
runner, we only release the machine reservation after the local
throttler has found a slot to start copying the outputs for that build.

As opposed to asserting uniqueness to understand resource utilization,
we just switch to using `std::unique_ptr`.
---
 src/hydra-queue-runner/build-remote.cc |  9 +++++++++
 src/hydra-queue-runner/builder.cc      | 24 +++++++++++-------------
 src/hydra-queue-runner/dispatcher.cc   |  2 +-
 src/hydra-queue-runner/state.hh        |  6 +++---
 4 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index e25da38d..7e307c75 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -398,6 +398,7 @@ private:
 };
 
 void State::buildRemote(ref<Store> destStore,
+    std::unique_ptr<MachineReservation> reservation,
     ::Machine::ptr machine, Step::ptr step,
     const ServeProto::BuildOptions & buildOptions,
     RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
@@ -546,6 +547,14 @@ void State::buildRemote(ref<Store> destStore,
         }
         SemaphoreReleaser releaser(&localWorkThrottler);
 
+        /* Once we've started copying outputs, release the machine reservation
+         * so further builds can happen. We do not release the machine earlier
+         * to avoid situations where the queue runner is bottlenecked on
+         * copying outputs and we end up building too many things that we
+         * haven't been able to allow copy slots for. */
+        reservation.reset();
+        wakeDispatcher();
+
         StorePathSet outputs;
         for (auto & [_, realisation] : buildResult.builtOutputs)
             outputs.insert(realisation.outPath);
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 018215a5..ff0634b1 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -16,7 +16,7 @@ void setThreadName(const std::string & name)
 }
 
 
-void State::builder(MachineReservation::ptr reservation)
+void State::builder(std::unique_ptr<MachineReservation> reservation)
 {
     setThreadName("bld~" + std::string(reservation->step->drvPath.to_string()));
 
@@ -35,22 +35,20 @@ void State::builder(MachineReservation::ptr reservation)
             activeSteps_.lock()->erase(activeStep);
         });
 
+        std::string machine = reservation->machine->storeUri.render();
+
         try {
             auto destStore = getDestStore();
-            res = doBuildStep(destStore, reservation, activeStep);
+            // Might release the reservation.
+            res = doBuildStep(destStore, std::move(reservation), activeStep);
         } catch (std::exception & e) {
             printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
-                localStore->printStorePath(reservation->step->drvPath),
-                reservation->machine->storeUri.render(),
+                localStore->printStorePath(activeStep->step->drvPath),
+                machine,
                 e.what());
         }
     }
 
-    /* Release the machine and wake up the dispatcher. */
-    assert(reservation.unique());
-    reservation = 0;
-    wakeDispatcher();
-
     /* If there was a temporary failure, retry the step after an
        exponentially increasing interval. */
     Step::ptr step = wstep.lock();
@@ -72,11 +70,11 @@ void State::builder(MachineReservation::ptr reservation)
 
 
 State::StepResult State::doBuildStep(nix::ref<Store> destStore,
-    MachineReservation::ptr reservation,
+    std::unique_ptr<MachineReservation> reservation,
     std::shared_ptr<ActiveStep> activeStep)
 {
-    auto & step(reservation->step);
-    auto & machine(reservation->machine);
+    auto step(reservation->step);
+    auto machine(reservation->machine);
 
     {
         auto step_(step->state.lock());
@@ -211,7 +209,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         try {
             /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, std::move(reservation), machine, step, buildOptions, result, activeStep, updateStep, narMembers);
         } catch (Error & e) {
             if (activeStep->state_.lock()->cancelled) {
                 printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index 11db0071..d3e145de 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -288,7 +288,7 @@ system_time State::doDispatch()
                 /* Make a slot reservation and start a thread to
                    do the build. */
                 auto builderThread = std::thread(&State::builder, this,
-                    std::make_shared<MachineReservation>(*this, step, mi.machine));
+                    std::make_unique<MachineReservation>(*this, step, mi.machine));
                 builderThread.detach(); // FIXME?
 
                 keepGoing = true;
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 12aead40..edfad4fb 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -400,7 +400,6 @@ private:
 
     struct MachineReservation
     {
-        typedef std::shared_ptr<MachineReservation> ptr;
         State & state;
         Step::ptr step;
         Machine::ptr machine;
@@ -550,16 +549,17 @@ private:
 
     void abortUnsupported();
 
-    void builder(MachineReservation::ptr reservation);
+    void builder(std::unique_ptr<MachineReservation> reservation);
 
     /* Perform the given build step. Return true if the step is to be
        retried. */
     enum StepResult { sDone, sRetry, sMaybeCancelled };
     StepResult doBuildStep(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr reservation,
+        std::unique_ptr<MachineReservation> reservation,
         std::shared_ptr<ActiveStep> activeStep);
 
     void buildRemote(nix::ref<nix::Store> destStore,
+        std::unique_ptr<MachineReservation> reservation,
         Machine::ptr machine, Step::ptr step,
         const nix::ServeProto::BuildOptions & buildOptions,
         RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,

From b4322edd05ba509dfe374625b88e5aeb71cd268f Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Fri, 12 Apr 2024 17:33:27 +0200
Subject: [PATCH 263/285] web: replace 'errormsg' with 'errormsg IS NULL' in
 most cases

This is implement in an extremely hacky way due to poor DBIx feature
support. Ideally, what we'd need is a way to tell DBIx to ignore the
errormsg column unless explicitly requested, and to automatically add a
computed 'errormsg IS NULL' column in others. Since it does not support
that, this commit instead hacks some support via method overrides while
taking care to not break anything obvious.
---
 package.nix                                   |  1 +
 src/lib/Hydra/Controller/Jobset.pm            |  6 ++++
 src/lib/Hydra/Controller/JobsetEval.pm        |  2 ++
 src/lib/Hydra/Helper/Nix.pm                   |  3 +-
 .../Hydra/Schema/Result/EvaluationErrors.pm   |  2 ++
 src/lib/Hydra/Schema/Result/Jobsets.pm        |  2 ++
 .../Schema/ResultSet/EvaluationErrors.pm      | 30 +++++++++++++++++++
 src/lib/Hydra/Schema/ResultSet/Jobsets.pm     | 30 +++++++++++++++++++
 src/root/common.tt                            |  4 +--
 src/root/jobset-eval.tt                       |  4 +--
 src/root/jobset.tt                            |  6 ++--
 t/evaluator/evaluate-constituents-globbing.t  |  8 ++---
 t/lib/CliRunners.pm                           |  4 +--
 t/queue-runner/direct-indirect-constituents.t |  2 +-
 14 files changed, 88 insertions(+), 16 deletions(-)
 create mode 100644 src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
 create mode 100644 src/lib/Hydra/Schema/ResultSet/Jobsets.pm

diff --git a/package.nix b/package.nix
index 8bf6a199..4a7840c1 100644
--- a/package.nix
+++ b/package.nix
@@ -89,6 +89,7 @@ let
         DateTime
         DBDPg
         DBDSQLite
+        DBIxClassHelpers
         DigestSHA1
         EmailMIME
         EmailSender
diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm
index 20a52f6f..bc7d7444 100644
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -371,6 +371,12 @@ sub errors_GET {
 
     $c->stash->{template} = 'eval-error.tt';
 
+    my $jobsetName = $c->stash->{params}->{name};
+    $c->stash->{jobset} = $c->stash->{project}->jobsets->find(
+        { name => $jobsetName },
+        { '+columns' => { 'errormsg' => 'errormsg' } }
+    );
+
     $self->status_ok($c, entity => $c->stash->{jobset});
 }
 
diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm
index aca03d72..643a516c 100644
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -93,6 +93,8 @@ sub errors_GET {
 
     $c->stash->{template} = 'eval-error.tt';
 
+    $c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
+
     $self->status_ok($c, entity => $c->stash->{eval});
 }
 
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 88fbdd6d..134b8b7e 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -297,8 +297,7 @@ sub getEvals {
 
     my @evals = $evals_result_set->search(
         { hasnewbuilds => 1 },
-        { order_by => "$me.id DESC", rows => $rows, offset => $offset
-        , prefetch => { evaluationerror => [ ] } });
+        { order_by => "$me.id DESC", rows => $rows, offset => $offset });
     my @res = ();
     my $cache = {};
 
diff --git a/src/lib/Hydra/Schema/Result/EvaluationErrors.pm b/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
index 7033fa5e..f6cc48db 100644
--- a/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
@@ -105,4 +105,6 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
+__PACKAGE__->mk_group_accessors('column' => 'has_error');
+
 1;
diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index cd704ac8..aee87e00 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -386,6 +386,8 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
+__PACKAGE__->mk_group_accessors('column' => 'has_error');
+
 sub supportsDynamicRunCommand {
   my ($self) = @_;
 
diff --git a/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm b/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
new file mode 100644
index 00000000..a4c6d955
--- /dev/null
+++ b/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
@@ -0,0 +1,30 @@
+package Hydra::Schema::ResultSet::EvaluationErrors;
+
+use strict;
+use utf8;
+use warnings;
+
+use parent 'DBIx::Class::ResultSet';
+
+use Storable qw(dclone);
+
+__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
+
+# Exclude expensive error message values unless explicitly requested, and
+# replace them with a summary field describing their presence/absence.
+sub search_rs {
+  my ( $class, $query, $attrs ) = @_;
+
+  if ($attrs) {
+    $attrs = dclone($attrs);
+  }
+
+  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
+    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
+  }
+  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
+    push @{ $attrs->{'remove_columns'} }, 'errormsg';
+  }
+
+  return $class->next::method($query, $attrs);
+}
diff --git a/src/lib/Hydra/Schema/ResultSet/Jobsets.pm b/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
new file mode 100644
index 00000000..1b2a12e3
--- /dev/null
+++ b/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
@@ -0,0 +1,30 @@
+package Hydra::Schema::ResultSet::Jobsets;
+
+use strict;
+use utf8;
+use warnings;
+
+use parent 'DBIx::Class::ResultSet';
+
+use Storable qw(dclone);
+
+__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
+
+# Exclude expensive error message values unless explicitly requested, and
+# replace them with a summary field describing their presence/absence.
+sub search_rs {
+  my ( $class, $query, $attrs ) = @_;
+
+  if ($attrs) {
+    $attrs = dclone($attrs);
+  }
+
+  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
+    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
+  }
+  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
+    push @{ $attrs->{'remove_columns'} }, 'errormsg';
+  }
+
+  return $class->next::method($query, $attrs);
+}
diff --git a/src/root/common.tt b/src/root/common.tt
index 6348bee7..86335a74 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -513,7 +513,7 @@ BLOCK renderEvals %]
             ELSE %]
              -
             [% END %]
-            [% IF eval.evaluationerror.errormsg %]
+            [% IF eval.evaluationerror.has_error %]
               <span class="badge badge-warning">Eval Errors</span>
             [% END %]
           </td>
@@ -639,7 +639,7 @@ BLOCK renderJobsetOverview %]
         <td>[% HTML.escape(j.description) %]</td>
         <td>[% IF j.lastcheckedtime;
                  INCLUDE renderDateTime timestamp = j.lastcheckedtime;
-                 IF j.errormsg || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
+                 IF j.has_error || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
                  ELSE; "-";
                END %]</td>
         [% IF j.get_column('nrtotal') > 0 %]
diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt
index 146878f2..f0b92f97 100644
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
   [% END %]
   <li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>
 
-  [% IF eval.evaluationerror.errormsg %]
+  [% IF eval.evaluationerror.has_error %]
     <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
   [% END %]
 </ul>
@@ -165,7 +165,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
     [% END %]
   </div>
 
-  [% IF eval.evaluationerror.errormsg %]
+  [% IF eval.evaluationerror.has_error %]
     <div id="tabs-errors" class="tab-pane">
       <iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
     </div>
diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 5afcbfde..3e594756 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -61,7 +61,7 @@
   [% END %]
 
   <li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
-  [% IF jobset.errormsg || jobset.fetcherrormsg %]
+  [% IF jobset.has_error || jobset.fetcherrormsg %]
     <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
   [% END %]
   <li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
@@ -79,7 +79,7 @@
         <th>Last checked:</th>
         <td>
           [% IF jobset.lastcheckedtime %]
-            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
+            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
           [% ELSE %]
             <em>never</em>
           [% END %]
@@ -117,7 +117,7 @@
 
   </div>
 
-  [% IF jobset.errormsg || jobset.fetcherrormsg %]
+  [% IF jobset.has_error || jobset.fetcherrormsg %]
     <div id="tabs-errors" class="tab-pane">
       <iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
     </div>
diff --git a/t/evaluator/evaluate-constituents-globbing.t b/t/evaluator/evaluate-constituents-globbing.t
index 49315d58..0ed0ee7c 100644
--- a/t/evaluator/evaluate-constituents-globbing.t
+++ b/t/evaluator/evaluate-constituents-globbing.t
@@ -61,8 +61,8 @@ subtest "* selects all except current aggregate" => sub {
 
         $jobset->discard_changes;  # refresh from DB
         is(
-            $jobset->errormsg,
-            "",
+            $jobset->has_error,
+            0,
             "eval-errors non-empty"
         );
     };
@@ -101,7 +101,7 @@ subtest "trivial cycle check" => sub {
 
     ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
 
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
     like(
         $jobset->errormsg,
         qr/Dependency cycle: indirect_aggregate <-> ok_aggregate/,
@@ -123,7 +123,7 @@ subtest "cycle check with globbing" => sub {
 
     ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
 
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
     like(
         $jobset->errormsg,
         qr/aggregate job ‘indirect_aggregate’ failed with the error: Dependency cycle: indirect_aggregate <-> packages.constituentA/,
diff --git a/t/lib/CliRunners.pm b/t/lib/CliRunners.pm
index 885f2ae4..8c53b551 100644
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -14,7 +14,7 @@ our @EXPORT = qw(
 sub evalSucceeds {
     my ($jobset) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
     if ($res) {
         chomp $stdout; chomp $stderr;
         utf8::decode($stdout) or die "Invalid unicode in stdout.";
@@ -29,7 +29,7 @@ sub evalSucceeds {
 sub evalFails {
     my ($jobset) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
     if (!$res) {
         chomp $stdout; chomp $stderr;
         utf8::decode($stdout) or die "Invalid unicode in stdout.";
diff --git a/t/queue-runner/direct-indirect-constituents.t b/t/queue-runner/direct-indirect-constituents.t
index 35370450..a017c76f 100644
--- a/t/queue-runner/direct-indirect-constituents.t
+++ b/t/queue-runner/direct-indirect-constituents.t
@@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
 my $constituentBuildB = $builds->{"constituentB"};
 
 my $eval = $constituentBuildA->jobsetevals->first();
-is($eval->evaluationerror->errormsg, "");
+is($eval->evaluationerror->has_error, 0);
 
 subtest "Verifying the direct aggregate" => sub {
     my $aggBuild = $builds->{"direct_aggregate"};

From a5b17d0686d9443dab0a552800d8762b8f7422af Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 8 Apr 2025 17:38:19 -0400
Subject: [PATCH 264/285] Queue-runner: Always produce a machines JSON object

Even if there are no machines, there should at least be an empty object.
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index adc903d8..05d7e263 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -615,6 +615,7 @@ void State::dumpStatus(Connection & conn)
         }
 
         {
+            auto machines_json = json::object();
             auto machines_(machines.lock());
             for (auto & i : *machines_) {
                 auto & m(i.second);
@@ -641,8 +642,9 @@ void State::dumpStatus(Connection & conn)
                     machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
                     machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
                 }
-                statusJson["machines"][m->storeUri.render()] = machine;
+                machines_json[m->storeUri.render()] = machine;
             }
+            statusJson["machines"] = machines_json;
         }
 
         {

From 7e0157e387e6bacc83c8a566eba609516415761e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Wed, 9 Apr 2025 17:53:14 +0200
Subject: [PATCH 265/285] Fix compilation with a nix which was compiled withou
 aws sdk

---
 src/hydra-queue-runner/hydra-queue-runner.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 05d7e263..ab146312 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -703,6 +703,7 @@ void State::dumpStatus(Connection & conn)
             : 0.0},
         };
 
+#if NIX_WITH_S3_SUPPORT
         auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
         if (s3Store) {
             auto & s3Stats = s3Store->getS3Stats();
@@ -728,6 +729,7 @@ void State::dumpStatus(Connection & conn)
                         + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09},
             };
         }
+#endif
     }
 
     {

From f1a976d3fdd40e6880cf7d2b6c1b97132f89934e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 3 Apr 2025 10:33:57 +0200
Subject: [PATCH 266/285] Fix displaying eval errors in jobset eval view

Quickfix for something that annoyed me once too often.

Specifically, I'm talking about `/eval/1#tabs-errors`.

To not fetch long errors on each request, this is only done on-demand.
I.e., when the tab is opened, an iframe is requested with the errors.
This iframe uses a template for both the jobset view and the jobset-eval
view. It is differentiated by checking if `jobset` or `eval` is defined.

However, the jobset-eval view also has a `jobset` variable in its stash
which means that in both cases the `if` path was used. Since
`jobset.fetcherrormsg` isn't defined in the eval case though, you always
got an empty error.

The band-aid fix is relatively simple: swap if and else: the `eval`
variable is not defined in the stash of the jobset view, so now this is
a useful condition to decide which view we're in.

(cherry picked from commit https://git.lix.systems/lix-project/hydra/commit/70c3d75f739b184b36908a2c898332444482d1a1)
---
 src/root/eval-error.tt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/root/eval-error.tt b/src/root/eval-error.tt
index c2ea28ec..7b6e702f 100644
--- a/src/root/eval-error.tt
+++ b/src/root/eval-error.tt
@@ -13,12 +13,12 @@
 
     <div class="tab-content tab-pane">
         <div id="tabs-errors" class="">
-          [% IF jobset %]
-              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
-              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
-          [% ELSIF eval %]
+          [% IF eval %]
               <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
               <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+          [% ELSIF jobset %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
           [% END %]
         </div>
     </div>

From 8d750265135b7e203520036a742afdf301b4013f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 7 Apr 2025 18:54:39 +0200
Subject: [PATCH 267/285] re-enable restrict-eval for non-flakes

---
 flake.lock                   | 6 +++---
 src/script/hydra-eval-jobset | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 3a7a1672..49168a96 100644
--- a/flake.lock
+++ b/flake.lock
@@ -29,11 +29,11 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1744018595,
-        "narHash": "sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA=",
+        "lastModified": 1744370057,
+        "narHash": "sha256-n220U5pjzCtTtOJtbga4Xr/PyllowKw9anSevgCqJEw=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "cba718bafe5dc1607c2b6761ecf53c641a6f3b21",
+        "rev": "1260c6599d22dfd8c25fea6893c3d031996b20e1",
         "type": "github"
       },
       "original": {
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index cf3fa294..80f5d79c 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -372,6 +372,7 @@ sub evalJobs {
             or die "cannot find the input containing the job expression\n";
 
         @cmd = ("nix-eval-jobs",
+                "--option", "restrict-eval", "true",
                 "<" . $nixExprInputName . "/" . $nixExprPath . ">",
                 inputsToArgs($inputInfo));
     }

From cf33a9158ab2806f34c4d2bde23ed394417eb555 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Sun, 13 Apr 2025 08:29:01 +0200
Subject: [PATCH 268/285] web: increase colspan for machine row in machine
 status

---
 src/root/machine-status.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/machine-status.tt b/src/root/machine-status.tt
index 07a2359d..725598eb 100644
--- a/src/root/machine-status.tt
+++ b/src/root/machine-status.tt
@@ -17,7 +17,7 @@
     [% name = m.key ? stripSSHUser(m.key) : "localhost" %]
     <thead>
       <tr>
-        <th colspan="6">
+        <th colspan="7">
           <tt [% IF m.value.disabled %]style="text-decoration: line-through;"[% END %]>[% INCLUDE renderMachineName machine=m.key %]</tt>
           [% IF m.value.systemTypes %]
             <span class="muted" style="font-weight: normal;">

From 0ab357e43515b77a6c1deca4cef126cc204236ba Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sat, 20 Jul 2024 13:09:39 +0200
Subject: [PATCH 269/285] jobset-eval: fix actions not showing up sometimes for
 new jobs

New jobs have their "new" status take precedence over them being
"failed" or "queued", which means actions that can act on "failed" or
"queued" jobs weren't shown to the user when they could only act on
"new" jobs.

(cherry picked from commit https://git.lix.systems/lix-project/hydra/commit/9a4a5dd624a1cedc7cdc40687815739b228e5c77)
---
 src/lib/Hydra/Controller/JobsetEval.pm |  4 +++-
 src/lib/Hydra/Helper/BuildDiff.pm      | 18 +++++++++++++++---
 src/root/jobset-eval.tt                |  8 ++++----
 t/Hydra/Helper/BuildDiff.t             | 17 +++++++----------
 4 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm
index 643a516c..77c01a84 100644
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -76,7 +76,9 @@ sub view_GET {
     $c->stash->{removed} = $diff->{removed};
     $c->stash->{unfinished} = $diff->{unfinished};
     $c->stash->{aborted} = $diff->{aborted};
-    $c->stash->{failed} = $diff->{failed};
+    $c->stash->{totalAborted} = $diff->{totalAborted};
+    $c->stash->{totalFailed} = $diff->{totalFailed};
+    $c->stash->{totalQueued} = $diff->{totalQueued};
 
     $c->stash->{full} = ($c->req->params->{full} || "0") eq "1";
 
diff --git a/src/lib/Hydra/Helper/BuildDiff.pm b/src/lib/Hydra/Helper/BuildDiff.pm
index 65dad17c..be8525d6 100644
--- a/src/lib/Hydra/Helper/BuildDiff.pm
+++ b/src/lib/Hydra/Helper/BuildDiff.pm
@@ -32,7 +32,12 @@ sub buildDiff {
         removed => [],
         unfinished => [],
         aborted => [],
-        failed => [],
+
+        # These summary counters cut across the categories to determine whether
+        # actions such as "Restart all failed" or "Bump queue" are available.
+        totalAborted => 0,
+        totalFailed => 0,
+        totalQueued => 0,
     };
 
     my $n = 0;
@@ -80,8 +85,15 @@ sub buildDiff {
         } else {
             push @{$ret->{new}}, $build if !$found;
         }
-        if (defined $build->buildstatus && $build->buildstatus != 0) {
-            push @{$ret->{failed}}, $build;
+
+        if ($build->finished != 0 && $build->buildstatus != 0) {
+            if ($aborted) {
+                ++$ret->{totalAborted};
+            } else {
+                ++$ret->{totalFailed};
+            }
+        } elsif ($build->finished == 0) {
+            ++$ret->{totalQueued};
         }
     }
 
diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt
index f0b92f97..12086d85 100644
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -48,16 +48,16 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
       <a class="nav-link dropdown-toggle" data-toggle="dropdown" href="#">Actions</a>
       <div class="dropdown-menu">
         <a class="dropdown-item" href="[% c.uri_for(c.controller('JobsetEval').action_for('create_jobset'), [eval.id]) %]">Create a jobset from this evaluation</a>
-        [% IF unfinished.size > 0 %]
+        [% IF totalQueued > 0 %]
           <a class="dropdown-item" href="[% c.uri_for(c.controller('JobsetEval').action_for('cancel'), [eval.id]) %]">Cancel all scheduled builds</a>
         [% END %]
-        [% IF aborted.size > 0 || stillFail.size > 0 || nowFail.size > 0 || failed.size > 0 %]
+        [% IF totalFailed > 0 %]
           <a class="dropdown-item" href="[% c.uri_for(c.controller('JobsetEval').action_for('restart_failed'), [eval.id]) %]">Restart all failed builds</a>
         [% END %]
-        [% IF aborted.size > 0 %]
+        [% IF totalAborted > 0 %]
           <a class="dropdown-item" href="[% c.uri_for(c.controller('JobsetEval').action_for('restart_aborted'), [eval.id]) %]">Restart all aborted builds</a>
         [% END %]
-        [% IF unfinished.size > 0 %]
+        [% IF totalQueued > 0 %]
           <a class="dropdown-item" href="[% c.uri_for(c.controller('JobsetEval').action_for('bump'), [eval.id]) %]">Bump builds to front of queue</a>
         [% END %]
       </div>
diff --git a/t/Hydra/Helper/BuildDiff.t b/t/Hydra/Helper/BuildDiff.t
index 243bb596..eef25a0f 100644
--- a/t/Hydra/Helper/BuildDiff.t
+++ b/t/Hydra/Helper/BuildDiff.t
@@ -25,7 +25,10 @@ subtest "empty diff" => sub {
             removed => [],
             unfinished => [],
             aborted => [],
-            failed => [],
+
+            totalAborted => 0,
+            totalFailed => 0,
+            totalQueued => 0,
         },
         "empty list of jobs returns empty diff"
     );
@@ -48,12 +51,7 @@ subtest "2 different jobs" => sub {
         "succeed_with_failed is a new job"
     );
 
-    is(scalar(@{$ret->{failed}}), 1, "list of failed jobs is 1 element long");
-    is(
-        $ret->{failed}[0]->get_column('id'),
-        $builds->{"succeed_with_failed"}->get_column('id'),
-        "succeed_with_failed is a failed job"
-    );
+    is($ret->{totalFailed}, 1, "total failed jobs is 1");
 
     is(
         $ret->{removed},
@@ -70,9 +68,9 @@ subtest "2 different jobs" => sub {
 subtest "failed job with no previous history" => sub {
     my $ret = buildDiff([$builds->{"fails"}], []);
 
-    is(scalar(@{$ret->{failed}}), 1, "list of failed jobs is 1 element long");
+    is($ret->{totalFailed}, 1, "total failed jobs is 1");
     is(
-        $ret->{failed}[0]->get_column('id'),
+        $ret->{new}[0]->get_column('id'),
         $builds->{"fails"}->get_column('id'),
         "fails is a failed job"
     );
@@ -93,7 +91,6 @@ subtest "not-yet-built job with no previous history" => sub {
     is($ret->{removed}, [], "removed");
     is($ret->{unfinished}, [], "unfinished");
     is($ret->{aborted}, [], "aborted");
-    is($ret->{failed}, [], "failed");
 
     is(scalar(@{$ret->{new}}), 1, "list of new jobs is 1 element long");
     is(

From 720db63d52ebcbda617603e7aa5b5c750cc6afec Mon Sep 17 00:00:00 2001
From: Pierre Bourdon <delroth@gmail.com>
Date: Sun, 21 Apr 2024 17:36:16 +0200
Subject: [PATCH 270/285] queue runner: attempt at slightly smarter scheduling
 criteria

Instead of just going for "whatever is the oldest build we know of",
use the following first:

- Is the step more constrained? If so, schedule it first to avoid
  filling up "more desirable" build slots with less constrained builds.

- Does the step have more dependents? If so, schedule it first to try
  and maximize open parallelism and breadth of scheduling options.

(cherry picked from commit https://git.lix.systems/lix-project/hydra/commit/b8d03adaf45105452bf1040deeaaccc8b8b22efb)
---
 src/hydra-queue-runner/dispatcher.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc
index d3e145de..ada25dc6 100644
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@@ -134,6 +134,8 @@ system_time State::doDispatch()
            comparator is a partial ordering (see MachineInfo). */
         int highestGlobalPriority;
         int highestLocalPriority;
+        size_t numRequiredSystemFeatures;
+        size_t numRevDeps;
         BuildID lowestBuildID;
 
         StepInfo(Step::ptr step, Step::State & step_) : step(step)
@@ -142,6 +144,8 @@ system_time State::doDispatch()
                 lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed());
             highestGlobalPriority = step_.highestGlobalPriority;
             highestLocalPriority = step_.highestLocalPriority;
+            numRequiredSystemFeatures = step->requiredSystemFeatures.size();
+            numRevDeps = step_.rdeps.size();
             lowestBuildID = step_.lowestBuildID;
         }
     };
@@ -194,6 +198,8 @@ system_time State::doDispatch()
                 a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority :
                 a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed :
                 a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority :
+                a.numRequiredSystemFeatures != b.numRequiredSystemFeatures ? a.numRequiredSystemFeatures > b.numRequiredSystemFeatures :
+                a.numRevDeps != b.numRevDeps ? a.numRevDeps > b.numRevDeps :
                 a.lowestBuildID < b.lowestBuildID;
         });
 

From 23755bf001592d82c735b49816f66a3bdd96216d Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Wed, 23 Apr 2025 18:27:14 +0200
Subject: [PATCH 271/285] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/a4962f73b5fc874d4b16baef47921daf349addfc' (2025-04-07)
  → 'github:NixOS/nix/70921714cb3b5e6041b7413459541838651079f3' (2025-04-23)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/db8f4fe18ce772a9c8f3adf321416981c8fe9371' (2025-04-07)
  → 'github:NixOS/nixpkgs/eea3403f7ca9f9942098f4f2756adab4ec924b2b' (2025-04-23)
---
 flake.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1a1531ec..53a0526b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -12,11 +12,11 @@
         "nixpkgs-regression": []
       },
       "locked": {
-        "lastModified": 1744030329,
-        "narHash": "sha256-r+psCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I=",
+        "lastModified": 1745420957,
+        "narHash": "sha256-ZbB3IH9OlJvo14GlQZbYHzJojf/HCDT38GzYTod8DaU=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "a4962f73b5fc874d4b16baef47921daf349addfc",
+        "rev": "70921714cb3b5e6041b7413459541838651079f3",
         "type": "github"
       },
       "original": {
@@ -44,11 +44,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1743987495,
-        "narHash": "sha256-46T2vMZ4/AfCK0Y2OjlFzJPxmdpP8GtsuEqSSJv3oe4=",
+        "lastModified": 1745408698,
+        "narHash": "sha256-JT1wMjLIypWJA0N2V27WpUw8feDmTok4Dwkb0oYXDS4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "db8f4fe18ce772a9c8f3adf321416981c8fe9371",
+        "rev": "eea3403f7ca9f9942098f4f2756adab4ec924b2b",
         "type": "github"
       },
       "original": {

From 89fcb931ce7d1cc64b644c4589daecfbf87321f3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 23 Apr 2025 17:51:47 -0400
Subject: [PATCH 272/285] Use Nix without the flake

This is what we do for `nix-eval-jobs` already. It allows for more
fine-grained control over dependencies.
---
 flake.lock | 11 +---------
 flake.nix  | 63 ++++++++++++++++++++++++++++++++----------------------
 2 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/flake.lock b/flake.lock
index 53a0526b..3e403f6f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,16 +1,7 @@
 {
   "nodes": {
     "nix": {
-      "inputs": {
-        "flake-compat": [],
-        "flake-parts": [],
-        "git-hooks-nix": [],
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "nixpkgs-23-11": [],
-        "nixpkgs-regression": []
-      },
+      "flake": false,
       "locked": {
         "lastModified": 1745420957,
         "narHash": "sha256-ZbB3IH9OlJvo14GlQZbYHzJojf/HCDT38GzYTod8DaU=",
diff --git a/flake.nix b/flake.nix
index dc1d1b8a..0e2f32dc 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,14 +5,8 @@
 
   inputs.nix = {
     url = "github:NixOS/nix/2.28-maintenance";
-    inputs.nixpkgs.follows = "nixpkgs";
-
-    # hide nix dev tooling from our lock file
-    inputs.flake-parts.follows = "";
-    inputs.git-hooks-nix.follows = "";
-    inputs.nixpkgs-regression.follows = "";
-    inputs.nixpkgs-23-11.follows = "";
-    inputs.flake-compat.follows = "";
+    # We want to control the deps precisely
+    flake = false;
   };
 
   inputs.nix-eval-jobs = {
@@ -30,10 +24,27 @@
 
       # A Nixpkgs overlay that provides a 'hydra' package.
       overlays.default = final: prev: {
-        nix-eval-jobs = final.callPackage nix-eval-jobs {};
+        nixDependenciesForHydra = final.lib.makeScope final.newScope
+          (import (nix + "/packaging/dependencies.nix") {
+            pkgs = final;
+            inherit (final) stdenv;
+            inputs = {};
+          });
+        nixComponentsForHydra = final.lib.makeScope final.nixDependenciesForHydra.newScope
+          (import (nix + "packaging/components.nix") {
+            officialRelease = true;
+            inherit (final) lib;
+            pkgs = final;
+            src = nix;
+            maintainers = [ ];
+          });
+        nix-eval-jobs = final.callPackage nix-eval-jobs {
+          nixComponents = final.nixComponentsForHydra;
+        };
         hydra = final.callPackage ./package.nix {
-          inherit (nixpkgs.lib) fileset;
+          inherit (final.lib) fileset;
           rawSrc = self;
+          nixComponents = final.nixComponentsForHydra;
         };
       };
 
@@ -73,24 +84,26 @@
       });
 
       packages = forEachSystem (system: let
-        nixComponents = {
-          inherit (nix.packages.${system})
-            nix-util
-            nix-store
-            nix-expr
-            nix-fetchers
-            nix-flake
-            nix-main
-            nix-cmd
-            nix-cli
-            nix-perl-bindings
-            ;
-        };
+        inherit (nixpkgs) lib;
+        pkgs = nixpkgs.legacyPackages.${system};
+        nixDependencies = lib.makeScope pkgs.newScope
+          (import (nix + "/packaging/dependencies.nix") {
+            inherit pkgs;
+            inherit (pkgs) stdenv;
+            inputs = {};
+          });
+        nixComponents = lib.makeScope nixDependencies.newScope
+          (import (nix + "/packaging/components.nix") {
+            officialRelease = true;
+            inherit lib pkgs;
+            src = nix;
+            maintainers = [ ];
+          });
       in {
-        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
+        nix-eval-jobs = pkgs.callPackage nix-eval-jobs {
           inherit nixComponents;
         };
-        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
+        hydra = pkgs.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
           inherit nixComponents;
           inherit (self.packages.${system}) nix-eval-jobs;

From 8218a9ad1b7e61de6449e6efa1bc3e189cc7469a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 29 Apr 2025 20:06:35 +0200
Subject: [PATCH 273/285] hydra: expose nix-cli package

This makes it easier in other packages to get the nix version used to
build Hydra.
---
 package.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index 4a7840c1..5c1a7860 100644
--- a/package.nix
+++ b/package.nix
@@ -277,5 +277,8 @@ stdenv.mkDerivation (finalAttrs: {
   dontStrip = true;
 
   meta.description = "Build of Hydra on ${stdenv.system}";
-  passthru = { inherit perlDeps; };
+  passthru = {
+    inherit perlDeps;
+    nix = nixComponents.nix-cli;
+  };
 })

From 2ae27dd20da6de787a0926e2de38dc537437a215 Mon Sep 17 00:00:00 2001
From: Sandro <sandro.jaeckel@gmail.com>
Date: Mon, 5 May 2025 00:10:59 +0200
Subject: [PATCH 274/285] Add missing slash

error: access to absolute path '/nix/store/sai35xfsrba2a2vasmzxakmn54wdfa13-sourcepackaging' is forbidden in pure evaluation mode (use '--impure' to override)
---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 0e2f32dc..979bfcbd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -31,7 +31,7 @@
             inputs = {};
           });
         nixComponentsForHydra = final.lib.makeScope final.nixDependenciesForHydra.newScope
-          (import (nix + "packaging/components.nix") {
+          (import (nix + "/packaging/components.nix") {
             officialRelease = true;
             inherit (final) lib;
             pkgs = final;

From da1aebe970a96a533e8cff51436bbd160c496bd5 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Thu, 15 May 2025 04:08:57 +0200
Subject: [PATCH 275/285] Migrate from "gc-" prefixed nix options

These have been deprecated, e.g. gc-keep-outputs is now just
keep-outputs.
---
 nixos-modules/hydra.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 79d639e6..283a9b8d 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -228,8 +228,8 @@ in
 
     nix.settings = {
       trusted-users = [ "hydra-queue-runner" ];
-      gc-keep-outputs = true;
-      gc-keep-derivations = true;
+      keep-outputs = true;
+      keep-derivations = true;
     };
 
     services.hydra-dev.extraConfig =

From 8bb7d27588706eb20f68c3103883b2f58b016754 Mon Sep 17 00:00:00 2001
From: Thomas Nixon <tom@tomn.co.uk>
Date: Fri, 23 May 2025 19:53:15 +0100
Subject: [PATCH 276/285] doc/manual: correct nginx reverse proxy example

- hydra does not remove the base URI from the request before processing
  it, so this must be done in the reverse proxy. in nginx this is done
  by giving proxy_pass a URI rather than a protocol/host/port; see:

  https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass

- proxy_redirect is not correct/required: hydra uses proxy headers to
  correctly form redirects in most cases, and where it doesn't it
  produces local redirects which aren't matched by this directive anyway
---
 doc/manual/src/configuration.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index d370312a..856d314c 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -63,8 +63,7 @@ following:
         .. other configuration ..
         location /hydra/ {
 
-            proxy_pass     http://127.0.0.1:3000;
-            proxy_redirect http://127.0.0.1:3000 https://example.com/hydra;
+            proxy_pass http://127.0.0.1:3000/;
 
             proxy_set_header  Host              $host;
             proxy_set_header  X-Real-IP         $remote_addr;
@@ -74,6 +73,9 @@ following:
         }
     }
 
+Note the trailing slash on the `proxy_pass` directive, which causes nginx to
+strip off the `/hydra/` part of the URL before passing it to hydra.
+
 Populating a Cache
 ------------------
 

From 8a50488f6c6be4acdc9ed325f5fd1d88edf5d0d8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 25 May 2025 20:51:05 -0400
Subject: [PATCH 277/285] flake.lock: Update Nixpkgs to 25.05
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/eea3403f7ca9f9942098f4f2756adab4ec924b2b?narHash=sha256-JT1wMjLIypWJA0N2V27WpUw8feDmTok4Dwkb0oYXDS4%3D' (2025-04-23)
  → 'github:NixOS/nixpkgs/db1aed32009f408e4048c1dd0beaf714dd34ed93?narHash=sha256-8A7HjmnvCpDjmETrZY1QwzKunR63LiP7lHu1eA5q6JI%3D' (2025-05-24)
---
 flake.lock | 8 ++++----
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index 3e403f6f..3a8d6870 100644
--- a/flake.lock
+++ b/flake.lock
@@ -35,16 +35,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1745408698,
-        "narHash": "sha256-JT1wMjLIypWJA0N2V27WpUw8feDmTok4Dwkb0oYXDS4=",
+        "lastModified": 1748124805,
+        "narHash": "sha256-8A7HjmnvCpDjmETrZY1QwzKunR63LiP7lHu1eA5q6JI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "eea3403f7ca9f9942098f4f2756adab4ec924b2b",
+        "rev": "db1aed32009f408e4048c1dd0beaf714dd34ed93",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11-small",
+        "ref": "nixos-25.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 979bfcbd..21bd793d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,7 +1,7 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small";
 
   inputs.nix = {
     url = "github:NixOS/nix/2.28-maintenance";

From dafa252d085aa44a3c46e55cfdd3f26908a224d8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 May 2025 00:09:42 -0400
Subject: [PATCH 278/285] flake.lock: Update Nix and nix-eval-jobs to 2.29
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nix':
    'github:NixOS/nix/70921714cb3b5e6041b7413459541838651079f3?narHash=sha256-ZbB3IH9OlJvo14GlQZbYHzJojf/HCDT38GzYTod8DaU%3D' (2025-04-23)
  → 'github:NixOS/nix/d761dad79c79af17aa476a29749bd9d69747548f?narHash=sha256-rCpANMHFIlafta6J/G0ILRd%2BWNSnzv/lzi40Y8f1AR8%3D' (2025-05-25)
• Updated input 'nix-eval-jobs':
    'github:nix-community/nix-eval-jobs/1260c6599d22dfd8c25fea6893c3d031996b20e1?narHash=sha256-n220U5pjzCtTtOJtbga4Xr/PyllowKw9anSevgCqJEw%3D' (2025-04-11)
  → 'github:nix-community/nix-eval-jobs/d9262e535e35454daebcebd434bdb9c1486bb998?narHash=sha256-AJ22q6yWc1hPkqssXMxQqD6QUeJ6hbx52xWHhKsmuP0%3D' (2025-05-25)
---
 flake.lock | 14 +++++++-------
 flake.nix  |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/flake.lock b/flake.lock
index 3a8d6870..2679eecb 100644
--- a/flake.lock
+++ b/flake.lock
@@ -3,16 +3,16 @@
     "nix": {
       "flake": false,
       "locked": {
-        "lastModified": 1745420957,
-        "narHash": "sha256-ZbB3IH9OlJvo14GlQZbYHzJojf/HCDT38GzYTod8DaU=",
+        "lastModified": 1748154947,
+        "narHash": "sha256-rCpANMHFIlafta6J/G0ILRd+WNSnzv/lzi40Y8f1AR8=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "70921714cb3b5e6041b7413459541838651079f3",
+        "rev": "d761dad79c79af17aa476a29749bd9d69747548f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.28-maintenance",
+        "ref": "2.29-maintenance",
         "repo": "nix",
         "type": "github"
       }
@@ -20,11 +20,11 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1744370057,
-        "narHash": "sha256-n220U5pjzCtTtOJtbga4Xr/PyllowKw9anSevgCqJEw=",
+        "lastModified": 1748211873,
+        "narHash": "sha256-AJ22q6yWc1hPkqssXMxQqD6QUeJ6hbx52xWHhKsmuP0=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "1260c6599d22dfd8c25fea6893c3d031996b20e1",
+        "rev": "d9262e535e35454daebcebd434bdb9c1486bb998",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index 21bd793d..e67a3a99 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,7 +4,7 @@
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small";
 
   inputs.nix = {
-    url = "github:NixOS/nix/2.28-maintenance";
+    url = "github:NixOS/nix/2.29-maintenance";
     # We want to control the deps precisely
     flake = false;
   };

From 278a3ebfd5297a976400f93afb1e54c6d1915cca Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 May 2025 00:24:25 -0400
Subject: [PATCH 279/285] Fix build with Nix 2.29

---
 src/hydra-queue-runner/build-remote.cc       |  2 +-
 src/hydra-queue-runner/hydra-queue-runner.cc |  4 ++--
 src/hydra-queue-runner/queue-monitor.cc      | 10 ++++++++--
 src/hydra-queue-runner/state.hh              |  2 +-
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 7e307c75..b372e7dd 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -50,7 +50,7 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
         auto remoteStore = machine->storeUri.params.find("remote-store");
         if (remoteStore != machine->storeUri.params.end()) {
             command.push_back("--store");
-            command.push_back(shellEscape(remoteStore->second));
+            command.push_back(escapeShellArgAlways(remoteStore->second));
         }
     }
 
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index ab146312..a4a7f0a7 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -14,7 +14,7 @@
 #include <nix/util/signals.hh>
 #include "state.hh"
 #include "hydra-build-result.hh"
-#include <nix/store/store-api.hh>
+#include <nix/store/store-open.hh>
 #include <nix/store/remote-store.hh>
 
 #include <nix/store/globals.hh>
@@ -832,7 +832,7 @@ void State::run(BuildID buildOne)
         << metricsAddr << "/metrics (port " << exposerPort << ")"
         << std::endl;
 
-    Store::Params localParams;
+    Store::Config::Params localParams;
     localParams["max-connections"] = "16";
     localParams["max-connection-age"] = "600";
     localStore = openStore(getEnv("NIX_REMOTE").value_or(""), localParams);
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index bb15ac04..0785be6f 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -492,8 +492,14 @@ Step::ptr State::createStep(ref<Store> destStore,
        runnable while step->created == false. */
     step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
     {
-        auto parsedDrv = ParsedDerivation{drvPath, *step->drv};
-        step->drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(parsedDrv));
+        auto parsedOpt = StructuredAttrs::tryParse(step->drv->env);
+        try {
+            step->drvOptions = std::make_unique<DerivationOptions>(
+                DerivationOptions::fromStructuredAttrs(step->drv->env, parsedOpt ? &*parsedOpt : nullptr));
+        } catch (Error & e) {
+            e.addTrace({}, "while parsing derivation '%s'", localStore->printStorePath(drvPath));
+            throw;
+        }
     }
 
     step->preferLocalBuild = step->drvOptions->willBuildLocally(*localStore, *step->drv);
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index edfad4fb..f7ab7de3 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -172,7 +172,7 @@ struct Step
     nix::StorePath drvPath;
     std::unique_ptr<nix::Derivation> drv;
     std::unique_ptr<nix::DerivationOptions> drvOptions;
-    std::set<std::string> requiredSystemFeatures;
+    nix::StringSet requiredSystemFeatures;
     bool preferLocalBuild;
     bool isDeterministic;
     std::string systemType; // concatenation of drv.platform and requiredSystemFeatures

From 635aff50dd14e30baadc278781bad0c8bf7199c3 Mon Sep 17 00:00:00 2001
From: Julien Marquet <j@recursor.wf>
Date: Sat, 23 Apr 2022 23:17:49 +0200
Subject: [PATCH 280/285] docs: refine instructions for proxy setting

---
 doc/manual/src/configuration.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index 856d314c..bd8141a3 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -51,10 +51,12 @@ base_uri example.com
 `base_uri` should be your hydra servers proxied URL. If you are using
 Hydra nixos module then setting `hydraURL` option should be enough.
 
-If you want to serve Hydra with a prefix path, for example
-[http://example.com/hydra]() then you need to configure your reverse
-proxy to pass `X-Request-Base` to hydra, with prefix path as value. For
-example if you are using nginx, then use configuration similar to
+You also need to configure your reverse proxy to pass `X-Request-Base`
+to hydra, with the same value as `base_uri`.
+This also covers the case of serving Hydra with a prefix path,
+as in [http://example.com/hydra]().
+
+For example if you are using nginx, then use configuration similar to
 following:
 
     server {

From c621f274822dbb8f5d67bb3a7e1e1fcae883a1b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 24 Jun 2025 18:45:14 +0200
Subject: [PATCH 281/285] test: bump used nix version

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9c05d752..613e3ef9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,6 +12,6 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v17
+    - uses: cachix/install-nix-action@v31
     #- run: nix flake check
     - run: nix-build -A checks.x86_64-linux.build -A checks.x86_64-linux.validate-openapi

From 9efe38c60b61c225d3333e5bc658d1ea7de5fb7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 24 Jun 2025 18:45:22 +0200
Subject: [PATCH 282/285] add update-flakes action

---
 .github/workflows/update-flakes.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .github/workflows/update-flakes.yml

diff --git a/.github/workflows/update-flakes.yml b/.github/workflows/update-flakes.yml
new file mode 100644
index 00000000..b5c0c2dd
--- /dev/null
+++ b/.github/workflows/update-flakes.yml
@@ -0,0 +1,28 @@
+name: "Update Flakes"
+on:
+  schedule:
+    # Run weekly on Monday at 00:00 UTC
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+jobs:
+  update-flakes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+    - uses: actions/checkout@v3
+    - uses: cachix/install-nix-action@v31
+    - name: Update flake inputs
+      run: nix flake update
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v5
+      with:
+        commit-message: "flake.lock: Update"
+        title: "Update flake inputs"
+        body: |
+          Automated flake input updates.
+          
+          This PR was automatically created by the update-flakes workflow.
+        branch: update-flakes
+        delete-branch: true
\ No newline at end of file

From 605a0e9ce9274073aa994a20916007a79439d6db Mon Sep 17 00:00:00 2001
From: Mic92 <Mic92@users.noreply.github.com>
Date: Tue, 24 Jun 2025 17:03:10 +0000
Subject: [PATCH 283/285] flake.lock: Update

---
 flake.lock | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/flake.lock b/flake.lock
index 2679eecb..0ca074f3 100644
--- a/flake.lock
+++ b/flake.lock
@@ -3,11 +3,11 @@
     "nix": {
       "flake": false,
       "locked": {
-        "lastModified": 1748154947,
-        "narHash": "sha256-rCpANMHFIlafta6J/G0ILRd+WNSnzv/lzi40Y8f1AR8=",
+        "lastModified": 1750777360,
+        "narHash": "sha256-nDWFxwhT+fQNgi4rrr55EKjpxDyVKSl1KaNmSXtYj40=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "d761dad79c79af17aa476a29749bd9d69747548f",
+        "rev": "7bb200199705eddd53cb34660a76567c6f1295d9",
         "type": "github"
       },
       "original": {
@@ -20,11 +20,11 @@
     "nix-eval-jobs": {
       "flake": false,
       "locked": {
-        "lastModified": 1748211873,
-        "narHash": "sha256-AJ22q6yWc1hPkqssXMxQqD6QUeJ6hbx52xWHhKsmuP0=",
+        "lastModified": 1748680938,
+        "narHash": "sha256-TQk6pEMD0mFw7jZXpg7+2qNKGbAluMQgc55OMgEO8bM=",
         "owner": "nix-community",
         "repo": "nix-eval-jobs",
-        "rev": "d9262e535e35454daebcebd434bdb9c1486bb998",
+        "rev": "974a4af3d4a8fd242d8d0e2608da4be87a62b83f",
         "type": "github"
       },
       "original": {
@@ -35,11 +35,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1748124805,
-        "narHash": "sha256-8A7HjmnvCpDjmETrZY1QwzKunR63LiP7lHu1eA5q6JI=",
+        "lastModified": 1750736827,
+        "narHash": "sha256-UcNP7BR41xMTe0sfHBH8R79+HdCw0OwkC/ZKrQEuMeo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "db1aed32009f408e4048c1dd0beaf714dd34ed93",
+        "rev": "b4a30b08433ad7b6e1dfba0833fb0fe69d43dfec",
         "type": "github"
       },
       "original": {

From c544042051269ab0516ce84ca3b259e0a21b59d7 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Fri, 4 Jul 2025 06:44:41 +0200
Subject: [PATCH 284/285] Replace nettools with hostname-debian

As far as I understand we include nettools for its hostname executable
used by the Sys-Hostname-Long perl package. But if we just need that then
the hostname-debian package provides a simpler and better maintained
version.
---
 nixos-modules/hydra.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nixos-modules/hydra.nix b/nixos-modules/hydra.nix
index 283a9b8d..83ffeec4 100644
--- a/nixos-modules/hydra.nix
+++ b/nixos-modules/hydra.nix
@@ -340,7 +340,7 @@ in
         requires = [ "hydra-init.service" ];
         wants = [ "network-online.target" ];
         after = [ "hydra-init.service" "network.target" "network-online.target" ];
-        path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
+        path = [ cfg.package pkgs.hostname-debian pkgs.openssh pkgs.bzip2 config.nix.package ];
         restartTriggers = [ hydraConf ];
         environment = env // {
           PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
@@ -364,7 +364,7 @@ in
         requires = [ "hydra-init.service" ];
         restartTriggers = [ hydraConf ];
         after = [ "hydra-init.service" "network.target" ];
-        path = with pkgs; [ nettools cfg.package jq ];
+        path = with pkgs; [ hostname-debian cfg.package jq ];
         environment = env // {
           HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
         };

From 62fcacb7d2bbafa1fe19087ec93964fe1352b683 Mon Sep 17 00:00:00 2001
From: Dionysis Grigoropoulos <dgrig@erethon.com>
Date: Tue, 15 Jul 2025 19:45:13 +0300
Subject: [PATCH 285/285] fix: Update Nix download url

---
 doc/manual/src/installation.md | 2 +-
 src/root/build.tt              | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/installation.md b/doc/manual/src/installation.md
index cbf3f907..39a86885 100644
--- a/doc/manual/src/installation.md
+++ b/doc/manual/src/installation.md
@@ -48,7 +48,7 @@ Getting Nix
 If your server runs NixOS you are all set to continue with installation
 of Hydra. Otherwise you first need to install Nix. The latest stable
 version can be found one [the Nix web
-site](http://nixos.org/nix/download.html), along with a manual, which
+site](https://nixos.org/download/), along with a manual, which
 includes installation instructions.
 
 Installation
diff --git a/src/root/build.tt b/src/root/build.tt
index 18ff6f01..93629427 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -563,7 +563,7 @@ END;
 
         [% IF eval.flake %]
 
-          <p>If you have <a href='https://nixos.org/nix/download.html'>Nix
+          <p>If you have <a href='https://nixos.org/download/'>Nix
           installed</a>, you can reproduce this build on your own machine by
           running the following command:</p>
 
@@ -573,7 +573,7 @@ END;
 
         [% ELSE %]
 
-          <p>If you have <a href='https://nixos.org/nix/download.html'>Nix
+          <p>If you have <a href='https://nixos.org/download/'>Nix
           installed</a>, you can reproduce this build on your own machine by
           downloading <a [% HTML.attributes(href => url) %]>a script</a>
           that checks out all inputs of the build and then invokes Nix to