Queue-runner: Always produce a machines JSON object

Even if there are no machines, there should at least be an empty object.
web: replace 'errormsg' with 'errormsg IS NULL' in most cases
2025-04-09 11:31:47 -04:00 · 2025-04-09 11:31:47 -04:00 · 2025-04-09 11:31:47 -04:00 · 2025-04-09 11:31:47 -04:00 · 2025-04-09 11:31:47 -04:00 · 2025-04-09 11:31:47 -04:00
48 changed files with 632 additions and 242 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -1,7 +1,10 @@
 name: "Test"
 on:
  pull_request:
+  merge_group:
  push:
+    branches:
+      - master
 jobs:
  tests:
    runs-on: ubuntu-latest
--- a/flake.lock
+++ b/flake.lock
@ -12,16 +12,16 @@
        "nixpkgs-regression": []
      },
      "locked": {
-        "lastModified": 1739899400,
-        "narHash": "sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs=",
+        "lastModified": 1744030329,
+        "narHash": "sha256-r+psCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I=",
        "owner": "NixOS",
        "repo": "nix",
-        "rev": "e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918",
+        "rev": "a4962f73b5fc874d4b16baef47921daf349addfc",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "2.26-maintenance",
+        "ref": "2.28-maintenance",
        "repo": "nix",
        "type": "github"
      }
@ -29,11 +29,11 @@
    "nix-eval-jobs": {
      "flake": false,
      "locked": {
-        "lastModified": 1743008255,
-        "narHash": "sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0=",
+        "lastModified": 1744018595,
+        "narHash": "sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA=",
        "owner": "nix-community",
        "repo": "nix-eval-jobs",
-        "rev": "f7418fc1fa45b96d37baa95ff3c016dd5be3876b",
+        "rev": "cba718bafe5dc1607c2b6761ecf53c641a6f3b21",
        "type": "github"
      },
      "original": {
@ -44,11 +44,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1739461644,
-        "narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
+        "lastModified": 1743987495,
+        "narHash": "sha256-46T2vMZ4/AfCK0Y2OjlFzJPxmdpP8GtsuEqSSJv3oe4=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
+        "rev": "db8f4fe18ce772a9c8f3adf321416981c8fe9371",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -4,7 +4,7 @@
  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";

  inputs.nix = {
-    url = "github:NixOS/nix/2.26-maintenance";
+    url = "github:NixOS/nix/2.28-maintenance";
    inputs.nixpkgs.follows = "nixpkgs";

    # hide nix dev tooling from our lock file
@ -34,7 +34,6 @@
        hydra = final.callPackage ./package.nix {
          inherit (nixpkgs.lib) fileset;
          rawSrc = self;
-          nix-perl-bindings = final.nixComponents.nix-perl-bindings;
        };
      };

@ -73,21 +72,29 @@
        validate-openapi = hydraJobs.tests.validate-openapi.${system};
      });

-      packages = forEachSystem (system: {
-        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
-          nix = nix.packages.${system}.nix;
-        };
-        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
-          inherit (nixpkgs.lib) fileset;
-          inherit (self.packages.${system}) nix-eval-jobs;
-          rawSrc = self;
+      packages = forEachSystem (system: let
+        nixComponents = {
          inherit (nix.packages.${system})
            nix-util
            nix-store
+            nix-expr
+            nix-fetchers
+            nix-flake
            nix-main
+            nix-cmd
            nix-cli
+            nix-perl-bindings
            ;
-          nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
+        };
+      in {
+        nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
+          inherit nixComponents;
+        };
+        hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
+          inherit (nixpkgs.lib) fileset;
+          inherit nixComponents;
+          inherit (self.packages.${system}) nix-eval-jobs;
+          rawSrc = self;
        };
        default = self.packages.${system}.hydra;
      });
--- a/hydra/jobsets.nix
+++ b/hydra/jobsets.nix
@ -0,0 +1,90 @@
+{ pulls, branches, ... }:
+let
+  # create the json spec for the jobset
+  makeSpec =
+    contents:
+    builtins.derivation {
+      name = "spec.json";
+      system = "x86_64-linux";
+      preferLocalBuild = true;
+      allowSubstitutes = false;
+      builder = "/bin/sh";
+      args = [
+        (builtins.toFile "builder.sh" ''
+          echo "$contents" > $out
+        '')
+      ];
+      contents = builtins.toJSON contents;
+    };
+
+  prs = readJSONFile pulls;
+  refs = readJSONFile branches;
+
+  # template for creating a job
+  makeJob =
+    {
+      schedulingshares ? 10,
+      keepnr ? 3,
+      description,
+      flake,
+      enabled ? 1,
+    }:
+    {
+      inherit
+        description
+        flake
+        schedulingshares
+        keepnr
+        enabled
+        ;
+      type = 1;
+      hidden = false;
+      checkinterval = 300; # every 5 minutes
+      enableemail = false;
+      emailoverride = "";
+    };
+
+  giteaHost = "ssh://gitea@nayeonie.com:2222";
+  repo = "ahuston-0/hydra";
+  # # Create a hydra job for a branch
+  jobOfRef =
+    name:
+    { ref, ... }:
+    if ((builtins.match "^refs/heads/(.*)$" ref) == null) then
+      null
+    else
+      {
+        name = builtins.replaceStrings [ "/" ] [ "-" ] "branch-${name}";
+        value = makeJob {
+          description = "Branch ${name}";
+          flake = "git+${giteaHost}/${repo}?ref=${ref}";
+        };
+      };
+
+  # Create a hydra job for a PR
+  jobOfPR = id: info: {
+    name = if info.draft then "draft-${id}" else "pr-${id}";
+    value = makeJob {
+      description = "PR ${id}: ${info.title}";
+      flake = "git+${giteaHost}/${repo}?ref=${info.head.ref}";
+      enabled = info.state == "open";
+    };
+  };
+
+  # some utility functions
+  # converts json to name/value dicts
+  attrsToList = l: builtins.attrValues (builtins.mapAttrs (name: value: { inherit name value; }) l);
+  # wrapper function for reading json from file
+  readJSONFile = f: builtins.fromJSON (builtins.readFile f);
+  # remove null values from a set, in-case of branches that don't exist
+  mapFilter = f: l: builtins.filter (x: (x != null)) (map f l);
+
+  # Create job set from PRs and branches
+  jobs = makeSpec (
+    builtins.listToAttrs (map ({ name, value }: jobOfPR name value) (attrsToList prs))
+    // builtins.listToAttrs (mapFilter ({ name, value }: jobOfRef name value) (attrsToList refs))
+  );
+in
+{
+  jobsets = jobs;
+}
--- a/hydra/spec.json
+++ b/hydra/spec.json
@ -0,0 +1,35 @@
+{
+  "enabled": 1,
+  "hidden": false,
+  "description": "ahuston-0's fork of hydra",
+  "nixexprinput": "nixexpr",
+  "nixexprpath": "hydra/jobsets.nix",
+  "checkinterval": 60,
+  "schedulingshares": 100,
+  "enableemail": false,
+  "emailoverride": "",
+  "keepnr": 3,
+  "type": 0,
+  "inputs": {
+    "nixexpr": {
+      "value": "ssh://gitea@nayeonie.com:2222/ahuston-0/hydra.git add-gitea-pulls",
+      "type": "git",
+      "emailresponsible": false
+    },
+    "nixpkgs": {
+      "value": "https://github.com/NixOS/nixpkgs nixos-unstable",
+      "type": "git",
+      "emailresponsible": false
+    },
+    "pulls": {
+      "type": "giteapulls",
+      "value": "nayeonie.com ahuston-0 hydra https",
+      "emailresponsible": false
+    },
+    "branches": {
+      "type": "gitea_refs",
+      "value": "nayeonie.com ahuston-0 hydra heads https -",
+      "emailresponsible": false
+    }
+  }
+}
--- a/meson.build
+++ b/meson.build
@ -12,20 +12,6 @@ nix_util_dep = dependency('nix-util', required: true)
 nix_store_dep = dependency('nix-store', required: true)
 nix_main_dep = dependency('nix-main', required: true)

-# Nix need extra flags not provided in its pkg-config files.
-nix_dep = declare_dependency(
-  dependencies: [
-    nix_util_dep,
-    nix_store_dep,
-    nix_main_dep,
-  ],
-  compile_args: [
-    '-include', 'nix/config-util.hh',
-    '-include', 'nix/config-store.hh',
-    '-include', 'nix/config-main.hh',
-  ],
-)
-
 pqxx_dep = dependency('libpqxx', required: true)

 prom_cpp_core_dep = dependency('prometheus-cpp-core', required: true)
--- a/nixos-tests.nix
+++ b/nixos-tests.nix
@ -27,8 +27,7 @@ in
 {

  install = forEachSystem (system:
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    simpleTest {
+    (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
      name = "hydra-install";
      nodes.machine = hydraServer;
      testScript =
@ -43,8 +42,7 @@ in
    });

  notifications = forEachSystem (system:
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    simpleTest {
+    (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
      name = "hydra-notifications";
      nodes.machine = {
        imports = [ hydraServer ];
@ -56,7 +54,7 @@ in
        '';
        services.influxdb.enable = true;
      };
-      testScript = ''
+      testScript = { nodes, ... }: ''
        machine.wait_for_job("hydra-init")

        # Create an admin account and some other state.
@ -87,7 +85,7 @@ in

        # Setup the project and jobset
        machine.succeed(
-            "su - hydra -c 'perl -I ${config.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
+            "su - hydra -c 'perl -I ${nodes.machine.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
        )

        # Wait until hydra has build the job and
@ -101,9 +99,10 @@ in
    });

  gitea = forEachSystem (system:
-    let pkgs = nixpkgs.legacyPackages.${system}; in
-    with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
-    makeTest {
+    let
+      pkgs = nixpkgs.legacyPackages.${system};
+    in
+    (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest {
      name = "hydra-gitea";
      nodes.machine = { pkgs, ... }: {
        imports = [ hydraServer ];
--- a/package.nix
+++ b/package.nix
@ -8,11 +8,7 @@

 , perlPackages

-, nix-util
-, nix-store
-, nix-main
-, nix-cli
-, nix-perl-bindings
+, nixComponents
 , git

 , makeWrapper
@ -65,7 +61,7 @@ let
    name = "hydra-perl-deps";
    paths = lib.closePropagation
      ([
-        nix-perl-bindings
+        nixComponents.nix-perl-bindings
        git
      ] ++ (with perlPackages; [
        AuthenSASL
@ -93,6 +89,7 @@ let
        DateTime
        DBDPg
        DBDSQLite
+        DBIxClassHelpers
        DigestSHA1
        EmailMIME
        EmailSender
@ -165,7 +162,7 @@ stdenv.mkDerivation (finalAttrs: {
    nukeReferences
    pkg-config
    mdbook
-    nix-cli
+    nixComponents.nix-cli
    perlDeps
    perl
    unzip
@ -175,9 +172,9 @@ stdenv.mkDerivation (finalAttrs: {
    libpqxx
    openssl
    libxslt
-    nix-util
-    nix-store
-    nix-main
+    nixComponents.nix-util
+    nixComponents.nix-store
+    nixComponents.nix-main
    perlDeps
    perl
    boost
@ -204,14 +201,14 @@ stdenv.mkDerivation (finalAttrs: {
    glibcLocales
    libressl.nc
    python3
-    nix-cli
+    nixComponents.nix-cli
  ];

  hydraPath = lib.makeBinPath (
    [
      subversion
      openssh
-      nix-cli
+      nixComponents.nix-cli
      coreutils
      findutils
      pixz
@ -272,7 +269,7 @@ stdenv.mkDerivation (finalAttrs: {
            --prefix PATH ':' $out/bin:$hydraPath \
            --set HYDRA_RELEASE ${version} \
            --set HYDRA_HOME $out/libexec/hydra \
-            --set NIX_RELEASE ${nix-cli.name or "unknown"} \
+            --set NIX_RELEASE ${nixComponents.nix-cli.name or "unknown"} \
            --set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
    done
  '';
--- a/src/hydra-evaluator/hydra-evaluator.cc
+++ b/src/hydra-evaluator/hydra-evaluator.cc
@ -1,8 +1,8 @@
 #include "db.hh"
 #include "hydra-config.hh"
-#include "pool.hh"
-#include "shared.hh"
-#include "signals.hh"
+#include <nix/util/pool.hh>
+#include <nix/main/shared.hh>
+#include <nix/util/signals.hh>

 #include <algorithm>
 #include <thread>
--- a/src/hydra-evaluator/meson.build
+++ b/src/hydra-evaluator/meson.build
@ -2,7 +2,8 @@ hydra_evaluator = executable('hydra-evaluator',
  'hydra-evaluator.cc',
  dependencies: [
    libhydra_dep,
-    nix_dep,
+    nix_util_dep,
+    nix_main_dep,
    pqxx_dep,
  ],
  install: true,
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@ -5,20 +5,20 @@
 #include <sys/stat.h>
 #include <fcntl.h>

-#include "build-result.hh"
-#include "path.hh"
-#include "legacy-ssh-store.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
+#include <nix/store/build-result.hh>
+#include <nix/store/path.hh>
+#include <nix/store/legacy-ssh-store.hh>
+#include <nix/store/serve-protocol.hh>
+#include <nix/store/serve-protocol-impl.hh>
 #include "state.hh"
-#include "current-process.hh"
-#include "processes.hh"
-#include "util.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
-#include "ssh.hh"
-#include "finally.hh"
-#include "url.hh"
+#include <nix/util/current-process.hh>
+#include <nix/util/processes.hh>
+#include <nix/util/util.hh>
+#include <nix/store/serve-protocol.hh>
+#include <nix/store/serve-protocol-impl.hh>
+#include <nix/store/ssh.hh>
+#include <nix/util/finally.hh>
+#include <nix/util/url.hh>

 using namespace nix;

@ -386,8 +386,19 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)

 }

+/* Utility guard object to auto-release a semaphore on destruction. */
+template <typename T>
+class SemaphoreReleaser {
+public:
+    SemaphoreReleaser(T* s) : sem(s) {}
+    ~SemaphoreReleaser() { sem->release(); }
+
+private:
+    T* sem;
+};

 void State::buildRemote(ref<Store> destStore,
+    std::unique_ptr<MachineReservation> reservation,
    ::Machine::ptr machine, Step::ptr step,
    const ServeProto::BuildOptions & buildOptions,
    RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
@ -527,6 +538,23 @@ void State::buildRemote(ref<Store> destStore,
            result.logFile = "";
        }

+        /* Throttle CPU-bound work. Opportunistically skip updating the current
+         * step, since this requires a DB roundtrip. */
+        if (!localWorkThrottler.try_acquire()) {
+            MaintainCount<counter> mc(nrStepsWaitingForDownloadSlot);
+            updateStep(ssWaitingForLocalSlot);
+            localWorkThrottler.acquire();
+        }
+        SemaphoreReleaser releaser(&localWorkThrottler);
+
+        /* Once we've started copying outputs, release the machine reservation
+         * so further builds can happen. We do not release the machine earlier
+         * to avoid situations where the queue runner is bottlenecked on
+         * copying outputs and we end up building too many things that we
+         * haven't been able to allow copy slots for. */
+        reservation.reset();
+        wakeDispatcher();
+
        StorePathSet outputs;
        for (auto & [_, realisation] : buildResult.builtOutputs)
            outputs.insert(realisation.outPath);
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@ -1,7 +1,7 @@
 #include "hydra-build-result.hh"
-#include "store-api.hh"
-#include "util.hh"
-#include "source-accessor.hh"
+#include <nix/store/store-api.hh>
+#include <nix/util/util.hh>
+#include <nix/util/source-accessor.hh>

 #include <regex>

--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@ -2,8 +2,8 @@

 #include "state.hh"
 #include "hydra-build-result.hh"
-#include "finally.hh"
-#include "binary-cache-store.hh"
+#include <nix/util/finally.hh>
+#include <nix/store/binary-cache-store.hh>

 using namespace nix;

@ -16,7 +16,7 @@ void setThreadName(const std::string & name)
 }


-void State::builder(MachineReservation::ptr reservation)
+void State::builder(std::unique_ptr<MachineReservation> reservation)
 {
    setThreadName("bld~" + std::string(reservation->step->drvPath.to_string()));

@ -35,22 +35,20 @@ void State::builder(MachineReservation::ptr reservation)
            activeSteps_.lock()->erase(activeStep);
        });

+        std::string machine = reservation->machine->storeUri.render();
+
        try {
            auto destStore = getDestStore();
-            res = doBuildStep(destStore, reservation, activeStep);
+            // Might release the reservation.
+            res = doBuildStep(destStore, std::move(reservation), activeStep);
        } catch (std::exception & e) {
            printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
-                localStore->printStorePath(reservation->step->drvPath),
-                reservation->machine->storeUri.render(),
+                localStore->printStorePath(activeStep->step->drvPath),
+                machine,
                e.what());
        }
    }

-    /* Release the machine and wake up the dispatcher. */
-    assert(reservation.unique());
-    reservation = 0;
-    wakeDispatcher();
-
    /* If there was a temporary failure, retry the step after an
       exponentially increasing interval. */
    Step::ptr step = wstep.lock();
@ -72,11 +70,11 @@ void State::builder(MachineReservation::ptr reservation)


 State::StepResult State::doBuildStep(nix::ref<Store> destStore,
-    MachineReservation::ptr reservation,
+    std::unique_ptr<MachineReservation> reservation,
    std::shared_ptr<ActiveStep> activeStep)
 {
-    auto & step(reservation->step);
-    auto & machine(reservation->machine);
+    auto step(reservation->step);
+    auto machine(reservation->machine);

    {
        auto step_(step->state.lock());
@ -211,7 +209,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,

        try {
            /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
+            buildRemote(destStore, std::move(reservation), machine, step, buildOptions, result, activeStep, updateStep, narMembers);
        } catch (Error & e) {
            if (activeStep->state_.lock()->cancelled) {
                printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
--- a/src/hydra-queue-runner/dispatcher.cc
+++ b/src/hydra-queue-runner/dispatcher.cc
@ -40,13 +40,15 @@ void State::dispatcher()
            printMsg(lvlDebug, "dispatcher woken up");
            nrDispatcherWakeups++;

-            auto now1 = std::chrono::steady_clock::now();
+            auto t_before_work = std::chrono::steady_clock::now();

            auto sleepUntil = doDispatch();

-            auto now2 = std::chrono::steady_clock::now();
+            auto t_after_work = std::chrono::steady_clock::now();

-            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+            prom.dispatcher_time_spent_running.Increment(
+                std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
+            dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();

            /* Sleep until we're woken up (either because a runnable build
               is added, or because a build finishes). */
@ -60,6 +62,10 @@ void State::dispatcher()
                *dispatcherWakeup_ = false;
            }

+            auto t_after_sleep = std::chrono::steady_clock::now();
+            prom.dispatcher_time_spent_waiting.Increment(
+                std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
+
        } catch (std::exception & e) {
            printError("dispatcher: %s", e.what());
            sleep(1);
@ -282,7 +288,7 @@ system_time State::doDispatch()
                /* Make a slot reservation and start a thread to
                   do the build. */
                auto builderThread = std::thread(&State::builder, this,
-                    std::make_shared<MachineReservation>(*this, step, mi.machine));
+                    std::make_unique<MachineReservation>(*this, step, mi.machine));
                builderThread.detach(); // FIXME?

                keepGoing = true;
--- a/src/hydra-queue-runner/hydra-build-result.hh
+++ b/src/hydra-queue-runner/hydra-build-result.hh
@ -2,9 +2,9 @@

 #include <memory>

-#include "hash.hh"
-#include "derivations.hh"
-#include "store-api.hh"
+#include <nix/util/hash.hh>
+#include <nix/store/derivations.hh>
+#include <nix/store/store-api.hh>
 #include "nar-extractor.hh"

 struct BuildProduct
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@ -11,16 +11,16 @@

 #include <nlohmann/json.hpp>

-#include "signals.hh"
+#include <nix/util/signals.hh>
 #include "state.hh"
 #include "hydra-build-result.hh"
-#include "store-api.hh"
-#include "remote-store.hh"
+#include <nix/store/store-api.hh>
+#include <nix/store/remote-store.hh>

-#include "globals.hh"
+#include <nix/store/globals.hh>
 #include "hydra-config.hh"
-#include "s3-binary-cache-store.hh"
-#include "shared.hh"
+#include <nix/store/s3-binary-cache-store.hh>
+#include <nix/main/shared.hh>

 using namespace nix;
 using nlohmann::json;
@ -70,10 +70,31 @@ State::PromMetrics::PromMetrics()
            .Register(*registry)
            .Add({})
    )
-    , queue_max_id(
-        prometheus::BuildGauge()
-            .Name("hydraqueuerunner_queue_max_build_id_info")
-            .Help("Maximum build record ID in the queue")
+    , dispatcher_time_spent_running(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_dispatcher_time_spent_running")
+            .Help("Time (in micros) spent running the dispatcher")
+            .Register(*registry)
+            .Add({})
+    )
+    , dispatcher_time_spent_waiting(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_dispatcher_time_spent_waiting")
+            .Help("Time (in micros) spent waiting for the dispatcher to obtain work")
+            .Register(*registry)
+            .Add({})
+    )
+    , queue_monitor_time_spent_running(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_queue_monitor_time_spent_running")
+            .Help("Time (in micros) spent running the queue monitor")
+            .Register(*registry)
+            .Add({})
+    )
+    , queue_monitor_time_spent_waiting(
+        prometheus::BuildCounter()
+            .Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
+            .Help("Time (in micros) spent waiting for the queue monitor to obtain work")
            .Register(*registry)
            .Add({})
    )
@ -85,6 +106,7 @@ State::State(std::optional<std::string> metricsAddrOpt)
    : config(std::make_unique<HydraConfig>())
    , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
    , dbPool(config->getIntOption("max_db_connections", 128))
+    , localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
    , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
    , maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
    , uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
@ -551,6 +573,7 @@ void State::dumpStatus(Connection & conn)
        {"nrActiveSteps", activeSteps_.lock()->size()},
        {"nrStepsBuilding", nrStepsBuilding.load()},
        {"nrStepsCopyingTo", nrStepsCopyingTo.load()},
+        {"nrStepsWaitingForDownloadSlot", nrStepsWaitingForDownloadSlot.load()},
        {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
        {"nrStepsWaiting", nrStepsWaiting.load()},
        {"nrUnsupportedSteps", nrUnsupportedSteps.load()},
@ -592,6 +615,7 @@ void State::dumpStatus(Connection & conn)
        }

        {
+            auto machines_json = json::object();
            auto machines_(machines.lock());
            for (auto & i : *machines_) {
                auto & m(i.second);
@ -618,8 +642,9 @@ void State::dumpStatus(Connection & conn)
                    machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
                    machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
                }
-                statusJson["machines"][m->storeUri.render()] = machine;
+                machines_json[m->storeUri.render()] = machine;
            }
+            statusJson["machines"] = machines_json;
        }

        {
--- a/src/hydra-queue-runner/meson.build
+++ b/src/hydra-queue-runner/meson.build
@ -13,7 +13,9 @@ hydra_queue_runner = executable('hydra-queue-runner',
  srcs,
  dependencies: [
    libhydra_dep,
-    nix_dep,
+    nix_util_dep,
+    nix_store_dep,
+    nix_main_dep,
    pqxx_dep,
    prom_cpp_core_dep,
    prom_cpp_pull_dep,
--- a/src/hydra-queue-runner/nar-extractor.cc
+++ b/src/hydra-queue-runner/nar-extractor.cc
@ -1,6 +1,6 @@
 #include "nar-extractor.hh"

-#include "archive.hh"
+#include <nix/util/archive.hh>

 #include <unordered_set>

--- a/src/hydra-queue-runner/nar-extractor.hh
+++ b/src/hydra-queue-runner/nar-extractor.hh
@ -1,9 +1,9 @@
 #pragma once

-#include "source-accessor.hh"
-#include "types.hh"
-#include "serialise.hh"
-#include "hash.hh"
+#include <nix/util/source-accessor.hh>
+#include <nix/util/types.hh>
+#include <nix/util/serialise.hh>
+#include <nix/util/hash.hh>

 struct NarMemberData
 {
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@ -1,6 +1,8 @@
 #include "state.hh"
 #include "hydra-build-result.hh"
-#include "globals.hh"
+#include <nix/store/globals.hh>
+#include <nix/store/parsed-derivations.hh>
+#include <nix/util/thread-pool.hh>

 #include <cstring>

@ -37,16 +39,21 @@ void State::queueMonitorLoop(Connection & conn)

    auto destStore = getDestStore();

-    unsigned int lastBuildId = 0;
-
    bool quit = false;
    while (!quit) {
+        auto t_before_work = std::chrono::steady_clock::now();
+
        localStore->clearPathInfoCache();

-        bool done = getQueuedBuilds(conn, destStore, lastBuildId);
+        bool done = getQueuedBuilds(conn, destStore);

        if (buildOne && buildOneDone) quit = true;

+        auto t_after_work = std::chrono::steady_clock::now();
+
+        prom.queue_monitor_time_spent_running.Increment(
+            std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
+
        /* Sleep until we get notification from the database about an
           event. */
        if (done && !quit) {
@ -56,12 +63,10 @@ void State::queueMonitorLoop(Connection & conn)
            conn.get_notifs();

        if (auto lowestId = buildsAdded.get()) {
-            lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
            printMsg(lvlTalkative, "got notification: new builds added to the queue");
        }
        if (buildsRestarted.get()) {
            printMsg(lvlTalkative, "got notification: builds restarted");
-            lastBuildId = 0; // check all builds
        }
        if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
            printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
@ -71,6 +76,10 @@ void State::queueMonitorLoop(Connection & conn)
            printMsg(lvlTalkative, "got notification: jobset shares changed");
            processJobsetSharesChange(conn);
        }
+
+        auto t_after_sleep = std::chrono::steady_clock::now();
+        prom.queue_monitor_time_spent_waiting.Increment(
+            std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
    }

    exit(0);
@ -84,20 +93,18 @@ struct PreviousFailure : public std::exception {


 bool State::getQueuedBuilds(Connection & conn,
-    ref<Store> destStore, unsigned int & lastBuildId)
+    ref<Store> destStore)
 {
    prom.queue_checks_started.Increment();

-    printInfo("checking the queue for builds > %d...", lastBuildId);
+    printInfo("checking the queue for builds...");

    /* Grab the queued builds from the database, but don't process
       them yet (since we don't want a long-running transaction). */
    std::vector<BuildID> newIDs;
-    std::map<BuildID, Build::ptr> newBuildsByID;
+    std::unordered_map<BuildID, Build::ptr> newBuildsByID;
    std::multimap<StorePath, BuildID> newBuildsByPath;

-    unsigned int newLastBuildId = lastBuildId;
-
    {
        pqxx::work txn(conn);

@ -106,17 +113,12 @@ bool State::getQueuedBuilds(Connection & conn,
             "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
             "globalPriority, priority from Builds "
             "inner join jobsets on builds.jobset_id = jobsets.id "
-             "where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
-            lastBuildId);
+             "where finished = 0 order by globalPriority desc, random()");

        for (auto const & row : res) {
            auto builds_(builds.lock());
            BuildID id = row["id"].as<BuildID>();
            if (buildOne && id != buildOne) continue;
-            if (id > newLastBuildId) {
-                newLastBuildId = id;
-                prom.queue_max_id.Set(id);
-            }
            if (builds_->count(id)) continue;

            auto build = std::make_shared<Build>(
@ -318,15 +320,13 @@ bool State::getQueuedBuilds(Connection & conn,

        /* Stop after a certain time to allow priority bumps to be
           processed. */
-        if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
+        if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
            prom.queue_checks_early_exits.Increment();
            break;
        }
    }

    prom.queue_checks_finished.Increment();
-
-    lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
    return newBuildsByID.empty();
 }

@ -405,6 +405,34 @@ void State::processQueueChange(Connection & conn)
 }


+std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
+    ref<Store> destStore,
+    const std::map<DrvOutput, std::optional<StorePath>> & paths)
+{
+    Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
+    ThreadPool tp;
+
+    for (auto & [output, maybeOutputPath] : paths) {
+        if (!maybeOutputPath) {
+            auto missing(missing_.lock());
+            missing->insert({output, maybeOutputPath});
+        } else {
+            tp.enqueue([&] {
+                if (!destStore->isValidPath(*maybeOutputPath)) {
+                    auto missing(missing_.lock());
+                    missing->insert({output, maybeOutputPath});
+                }
+            });
+        }
+    }
+
+    tp.process();
+
+    auto missing(missing_.lock());
+    return *missing;
+}
+
+
 Step::ptr State::createStep(ref<Store> destStore,
    Connection & conn, Build::ptr build, const StorePath & drvPath,
    Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
@ -463,14 +491,17 @@ Step::ptr State::createStep(ref<Store> destStore,
       it's not runnable yet, and other threads won't make it
       runnable while step->created == false. */
    step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
-    step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
+    {
+        auto parsedDrv = ParsedDerivation{drvPath, *step->drv};
+        step->drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(parsedDrv));
+    }

-    step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore);
+    step->preferLocalBuild = step->drvOptions->willBuildLocally(*localStore, *step->drv);
    step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1";

    step->systemType = step->drv->platform;
    {
-        StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
+        StringSet features = step->requiredSystemFeatures = step->drvOptions->getRequiredSystemFeatures(*step->drv);
        if (step->preferLocalBuild)
            features.insert("local");
        if (!features.empty()) {
@ -485,16 +516,15 @@ Step::ptr State::createStep(ref<Store> destStore,

    /* Are all outputs valid? */
    auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
-    bool valid = true;
-    std::map<DrvOutput, std::optional<StorePath>> missing;
+    std::map<DrvOutput, std::optional<StorePath>> paths;
    for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
        auto outputHash = outputHashes.at(outputName);
-        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
-            continue;
-        valid = false;
-        missing.insert({{outputHash, outputName}, maybeOutputPath});
+        paths.insert({{outputHash, outputName}, maybeOutputPath});
    }

+    auto missing = getMissingRemotePaths(destStore, paths);
+    bool valid = missing.empty();
+
    /* Try to copy the missing paths from the local store or from
       substitutes. */
    if (!missing.empty()) {
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@ -6,6 +6,8 @@
 #include <map>
 #include <memory>
 #include <queue>
+#include <regex>
+#include <semaphore>

 #include <prometheus/counter.h>
 #include <prometheus/gauge.h>
@ -13,17 +15,18 @@

 #include "db.hh"

-#include "parsed-derivations.hh"
-#include "pathlocks.hh"
-#include "pool.hh"
-#include "build-result.hh"
-#include "store-api.hh"
-#include "sync.hh"
+#include <nix/store/derivations.hh>
+#include <nix/store/derivation-options.hh>
+#include <nix/store/pathlocks.hh>
+#include <nix/util/pool.hh>
+#include <nix/store/build-result.hh>
+#include <nix/store/store-api.hh>
+#include <nix/util/sync.hh>
 #include "nar-extractor.hh"
-#include "serve-protocol.hh"
-#include "serve-protocol-impl.hh"
-#include "serve-protocol-connection.hh"
-#include "machines.hh"
+#include <nix/store/serve-protocol.hh>
+#include <nix/store/serve-protocol-impl.hh>
+#include <nix/store/serve-protocol-connection.hh>
+#include <nix/store/machines.hh>


 typedef unsigned int BuildID;
@ -57,6 +60,7 @@ typedef enum {
    ssConnecting = 10,
    ssSendingInputs = 20,
    ssBuilding = 30,
+    ssWaitingForLocalSlot = 35,
    ssReceivingOutputs = 40,
    ssPostProcessing = 50,
 } StepState;
@ -167,7 +171,7 @@ struct Step

    nix::StorePath drvPath;
    std::unique_ptr<nix::Derivation> drv;
-    std::unique_ptr<nix::ParsedDerivation> parsedDrv;
+    std::unique_ptr<nix::DerivationOptions> drvOptions;
    std::set<std::string> requiredSystemFeatures;
    bool preferLocalBuild;
    bool isDeterministic;
@ -352,6 +356,10 @@ private:
    typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
    nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr

+    /* Throttler for CPU-bound local work. */
+    static constexpr unsigned int maxSupportedLocalWorkers = 1024;
+    std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
+
    /* Various stats. */
    time_t startedAt;
    counter nrBuildsRead{0};
@ -361,6 +369,7 @@ private:
    counter nrStepsDone{0};
    counter nrStepsBuilding{0};
    counter nrStepsCopyingTo{0};
+    counter nrStepsWaitingForDownloadSlot{0};
    counter nrStepsCopyingFrom{0};
    counter nrStepsWaiting{0};
    counter nrUnsupportedSteps{0};
@ -391,7 +400,6 @@ private:

    struct MachineReservation
    {
-        typedef std::shared_ptr<MachineReservation> ptr;
        State & state;
        Step::ptr step;
        Machine::ptr machine;
@ -449,7 +457,12 @@ private:
        prometheus::Counter& queue_steps_created;
        prometheus::Counter& queue_checks_early_exits;
        prometheus::Counter& queue_checks_finished;
-        prometheus::Gauge& queue_max_id;
+
+        prometheus::Counter& dispatcher_time_spent_running;
+        prometheus::Counter& dispatcher_time_spent_waiting;
+
+        prometheus::Counter& queue_monitor_time_spent_running;
+        prometheus::Counter& queue_monitor_time_spent_waiting;

        PromMetrics();
    };
@ -493,8 +506,7 @@ private:
    void queueMonitorLoop(Connection & conn);

    /* Check the queue for new builds. */
-    bool getQueuedBuilds(Connection & conn,
-        nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
+    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);

    /* Handle cancellation, deletion and priority bumps. */
    void processQueueChange(Connection & conn);
@ -502,6 +514,12 @@ private:
    BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
        const nix::StorePath & drvPath);

+    /* Returns paths missing from the remote store. Paths are processed in
+     * parallel to work around the possible latency of remote stores. */
+    std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
+        nix::ref<nix::Store> destStore,
+        const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
+
    Step::ptr createStep(nix::ref<nix::Store> store,
        Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
        Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
@ -531,16 +549,17 @@ private:

    void abortUnsupported();

-    void builder(MachineReservation::ptr reservation);
+    void builder(std::unique_ptr<MachineReservation> reservation);

    /* Perform the given build step. Return true if the step is to be
       retried. */
    enum StepResult { sDone, sRetry, sMaybeCancelled };
    StepResult doBuildStep(nix::ref<nix::Store> destStore,
-        MachineReservation::ptr reservation,
+        std::unique_ptr<MachineReservation> reservation,
        std::shared_ptr<ActiveStep> activeStep);

    void buildRemote(nix::ref<nix::Store> destStore,
+        std::unique_ptr<MachineReservation> reservation,
        Machine::ptr machine, Step::ptr step,
        const nix::ServeProto::BuildOptions & buildOptions,
        RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@ -238,7 +238,7 @@ sub serveFile {
        # XSS hole.
        $c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');

-        $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
+        $c->stash->{'plain'} = { data => readIntoSocket(cmd => ["nix", "--experimental-features", "nix-command",
                                                      "store", "cat", "--store", getStoreUri(), "$path"]) };

        # Detect MIME type.
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@ -364,6 +364,21 @@ sub evals_GET {
    );
 }

+sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
+
+sub errors_GET {
+    my ($self, $c) = @_;
+
+    $c->stash->{template} = 'eval-error.tt';
+
+    my $jobsetName = $c->stash->{params}->{name};
+    $c->stash->{jobset} = $c->stash->{project}->jobsets->find(
+        { name => $jobsetName },
+        { '+columns' => { 'errormsg' => 'errormsg' } }
+    );
+
+    $self->status_ok($c, entity => $c->stash->{jobset});
+}

 # Redirect to the latest finished evaluation of this jobset.
 sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@ -86,6 +86,17 @@ sub view_GET {
    );
 }

+sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
+
+sub errors_GET {
+    my ($self, $c) = @_;
+
+    $c->stash->{template} = 'eval-error.tt';
+
+    $c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
+
+    $self->status_ok($c, entity => $c->stash->{eval});
+}

 sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
    my ($self, $c) = @_;
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@ -162,7 +162,7 @@ sub status_GET {
            { "buildsteps.busy" => { '!=', 0 } },
            { order_by => ["globalpriority DESC", "id"],
              join => "buildsteps",
-              columns => [@buildListColumns]
+              columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
            })]
    );
 }
--- a/src/lib/Hydra/Helper/BuildDiff.pm
+++ b/src/lib/Hydra/Helper/BuildDiff.pm
@ -37,7 +37,16 @@ sub buildDiff {

    my $n = 0;
    foreach my $build (@{$builds}) {
-        my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
+        my $aborted = $build->finished != 0 && (
+            # aborted
+            $build->buildstatus == 3
+            # cancelled
+            || $build->buildstatus == 4
+            # timeout
+            || $build->buildstatus == 7
+            # log limit exceeded
+            || $build->buildstatus == 10
+        );
        my $d;
        my $found = 0;
        while ($n < scalar(@{$builds2})) {
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@ -36,6 +36,7 @@ our @EXPORT = qw(
    jobsetOverview
    jobsetOverview_
    pathIsInsidePrefix
+    readIntoSocket
    readNixFile
    registerRoot
    restartBuilds
@ -296,8 +297,7 @@ sub getEvals {

    my @evals = $evals_result_set->search(
        { hasnewbuilds => 1 },
-        { order_by => "$me.id DESC", rows => $rows, offset => $offset
-        , prefetch => { evaluationerror => [ ] } });
+        { order_by => "$me.id DESC", rows => $rows, offset => $offset });
    my @res = ();
    my $cache = {};

@ -417,6 +417,16 @@ sub pathIsInsidePrefix {
    return $cur;
 }

+sub readIntoSocket{
+    my (%args) = @_;
+    my $sock;
+
+    eval {
+        open($sock, "-|", @{$args{cmd}}) or die q(failed to open socket from command:\n $x);
+    };
+
+    return $sock;
+}



--- a/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/Result/EvaluationErrors.pm
@ -105,4 +105,6 @@ __PACKAGE__->add_column(
    "+id" => { retrieve_on_insert => 1 }
 );

+__PACKAGE__->mk_group_accessors('column' => 'has_error');
+
 1;
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@ -386,6 +386,8 @@ __PACKAGE__->add_column(
    "+id" => { retrieve_on_insert => 1 }
 );

+__PACKAGE__->mk_group_accessors('column' => 'has_error');
+
 sub supportsDynamicRunCommand {
  my ($self) = @_;

--- a/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
+++ b/src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
@ -0,0 +1,30 @@
+package Hydra::Schema::ResultSet::EvaluationErrors;
+
+use strict;
+use utf8;
+use warnings;
+
+use parent 'DBIx::Class::ResultSet';
+
+use Storable qw(dclone);
+
+__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
+
+# Exclude expensive error message values unless explicitly requested, and
+# replace them with a summary field describing their presence/absence.
+sub search_rs {
+  my ( $class, $query, $attrs ) = @_;
+
+  if ($attrs) {
+    $attrs = dclone($attrs);
+  }
+
+  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
+    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
+  }
+  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
+    push @{ $attrs->{'remove_columns'} }, 'errormsg';
+  }
+
+  return $class->next::method($query, $attrs);
+}
--- a/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
+++ b/src/lib/Hydra/Schema/ResultSet/Jobsets.pm
@ -0,0 +1,30 @@
+package Hydra::Schema::ResultSet::Jobsets;
+
+use strict;
+use utf8;
+use warnings;
+
+use parent 'DBIx::Class::ResultSet';
+
+use Storable qw(dclone);
+
+__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
+
+# Exclude expensive error message values unless explicitly requested, and
+# replace them with a summary field describing their presence/absence.
+sub search_rs {
+  my ( $class, $query, $attrs ) = @_;
+
+  if ($attrs) {
+    $attrs = dclone($attrs);
+  }
+
+  unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
+    $attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
+  }
+  unless (exists $attrs->{'+columns'}->{'errormsg'}) {
+    push @{ $attrs->{'remove_columns'} }, 'errormsg';
+  }
+
+  return $class->next::method($query, $attrs);
+}
--- a/src/libhydra/db.hh
+++ b/src/libhydra/db.hh
@ -2,8 +2,8 @@

 #include <pqxx/pqxx>

-#include "environment-variables.hh"
-#include "util.hh"
+#include <nix/util/environment-variables.hh>
+#include <nix/util/util.hh>


 struct Connection : pqxx::connection
--- a/src/libhydra/hydra-config.hh
+++ b/src/libhydra/hydra-config.hh
@ -2,8 +2,8 @@

 #include <map>

-#include "file-system.hh"
-#include "util.hh"
+#include <nix/util/file-system.hh>
+#include <nix/util/util.hh>

 struct HydraConfig
 {
--- a/src/root/build.tt
+++ b/src/root/build.tt
@ -61,21 +61,7 @@ END;
            <td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
            <td class="step-status">
              [% IF step.busy != 0 %]
-                [% IF step.busy == 1 %]
-                  <strong>Preparing</strong>
-                [% ELSIF step.busy == 10 %]
-                  <strong>Connecting</strong>
-                [% ELSIF step.busy == 20 %]
-                  <strong>Sending inputs</strong>
-                [% ELSIF step.busy == 30 %]
-                  <strong>Building</strong>
-                [% ELSIF step.busy == 40 %]
-                  <strong>Receiving outputs</strong>
-                [% ELSIF step.busy == 50 %]
-                  <strong>Post-processing</strong>
-                [% ELSE %]
-                  <strong>Unknown state</strong>
-                [% END %]
+                [% INCLUDE renderBusyStatus %]
              [% ELSIF step.status == 0 %]
                [% IF step.isnondeterministic %]
                  <span class="warn">Succeeded with non-determistic result</span>
--- a/src/root/common.tt
+++ b/src/root/common.tt
@ -91,6 +91,17 @@ BLOCK renderDuration;
  duration % 60 %]s[%
 END;

+BLOCK renderDrvInfo;
+  drvname = step.drvpath
+    .substr(11) # strip `/nix/store/`
+    .split('-').slice(1).join("-") # strip hash part
+    .substr(0, -4); # strip `.drv`
+  IF drvname != releasename;
+    IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
+    IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
+  END;
+END;
+

 BLOCK renderBuildListHeader %]
  <table class="table table-striped table-condensed clickable-rows">
@ -131,7 +142,12 @@ BLOCK renderBuildListBody;
      [% END %]
      <td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
      [% IF !hideJobName %]
-        <td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
+        <td>
+          <a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
+          [% IF showStepName %]
+            [% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
+          [% END %]
+        </td>
      [% END %]
      <td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
      <td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
@ -245,6 +261,27 @@ BLOCK renderBuildStatusIcon;
 END;


+BLOCK renderBusyStatus;
+  IF step.busy == 1 %]
+    <strong>Preparing</strong>
+  [% ELSIF step.busy == 10 %]
+    <strong>Connecting</strong>
+  [% ELSIF step.busy == 20 %]
+    <strong>Sending inputs</strong>
+  [% ELSIF step.busy == 30 %]
+    <strong>Building</strong>
+  [% ELSIF step.busy == 35 %]
+    <strong>Waiting to receive outputs</strong>
+  [% ELSIF step.busy == 40 %]
+    <strong>Receiving outputs</strong>
+  [% ELSIF step.busy == 50 %]
+    <strong>Post-processing</strong>
+  [% ELSE %]
+    <strong>Unknown state</strong>
+  [% END;
+END;
+
+
 BLOCK renderStatus;
  IF build.finished;
    buildstatus = build.buildstatus;
@ -476,7 +513,7 @@ BLOCK renderEvals %]
            ELSE %]
             -
            [% END %]
-            [% IF eval.evaluationerror.errormsg %]
+            [% IF eval.evaluationerror.has_error %]
              <span class="badge badge-warning">Eval Errors</span>
            [% END %]
          </td>
@ -602,7 +639,7 @@ BLOCK renderJobsetOverview %]
        <td>[% HTML.escape(j.description) %]</td>
        <td>[% IF j.lastcheckedtime;
                 INCLUDE renderDateTime timestamp = j.lastcheckedtime;
-                 IF j.errormsg || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
+                 IF j.has_error || j.fetcherrormsg; %]&nbsp;<span class = 'badge badge-warning'>Error</span>[% END;
                 ELSE; "-";
               END %]</td>
        [% IF j.get_column('nrtotal') > 0 %]
--- a/src/root/eval-error.tt
+++ b/src/root/eval-error.tt
@ -0,0 +1,26 @@
+[% PROCESS common.tt %]
+<!DOCTYPE html>
+
+<html lang="en">
+
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
+    [% INCLUDE style.tt %]
+  </head>
+
+  <body>
+
+    <div class="tab-content tab-pane">
+        <div id="tabs-errors" class="">
+          [% IF jobset %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
+          [% ELSIF eval %]
+              <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+              <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+          [% END %]
+        </div>
+    </div>
+  </body>
+</html>
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  [% END %]

  [% IF aborted.size > 0 %]
-    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
+    <li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
  [% END %]
  [% IF nowFail.size > 0 %]
    <li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>
@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  [% END %]
  <li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>

-  [% IF eval.evaluationerror.errormsg %]
+  [% IF eval.evaluationerror.has_error %]
    <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
  [% END %]
 </ul>
@ -108,13 +108,6 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),

 <div class="tab-content">

-  [% IF eval.evaluationerror.errormsg %]
-    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
-    </div>
-  [% END %]
-
  <div id="tabs-aborted" class="tab-pane">
    [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
  </div>
@ -172,10 +165,9 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
    [% END %]
  </div>

-  [% IF eval.evaluationerror.errormsg %]
+  [% IF eval.evaluationerror.has_error %]
    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
+      <iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
    </div>
  [% END %]
 </div>
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@ -61,7 +61,7 @@
  [% END %]

  <li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
-  [% IF jobset.errormsg || jobset.fetcherrormsg %]
+  [% IF jobset.has_error || jobset.fetcherrormsg %]
    <li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
  [% END %]
  <li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
@ -79,7 +79,7 @@
        <th>Last checked:</th>
        <td>
          [% IF jobset.lastcheckedtime %]
-            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
+            [% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
          [% ELSE %]
            <em>never</em>
          [% END %]
@ -117,10 +117,9 @@

  </div>

-  [% IF jobset.errormsg || jobset.fetcherrormsg %]
+  [% IF jobset.has_error || jobset.fetcherrormsg %]
    <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
-      <div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
+      <iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
    </div>
  [% END %]

--- a/src/root/layout.tt
+++ b/src/root/layout.tt
@ -10,31 +10,7 @@

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
-
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-
-    <link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
-    <script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
-    <script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
-    <link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
-
-    <!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
-
-    <style>
-      .popover { max-width: 40%; }
-    </style>
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
-
-    <link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
-
-    <script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
+    [% INCLUDE style.tt %]

    [% IF c.config.enable_google_login %]
      <meta name="google-signin-client_id" content="[% c.config.google_client_id %]">
--- a/src/root/machine-status.tt
+++ b/src/root/machine-status.tt
@ -6,10 +6,10 @@
  <thead>
    <tr>
      <th>Job</th>
-      <th>System</th>
      <th>Build</th>
      <th>Step</th>
      <th>What</th>
+      <th>Status</th>
      <th>Since</th>
    </tr>
  </thead>
@ -40,10 +40,10 @@
          [% idle = 0 %]
          <tr>
            <td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
-            <td><tt>[% step.system %]</tt></td>
            <td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
            <td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
            <td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
+            <td>[% INCLUDE renderBusyStatus %]</td>
            <td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
          </tr>
        [% END %]
--- a/src/root/static/js/common.js
+++ b/src/root/static/js/common.js
@ -129,6 +129,12 @@ $(document).ready(function() {
            el.addClass("is-local");
        }
    });
+
+    [...document.getElementsByTagName("iframe")].forEach((element) => {
+        element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
+            element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
+        })
+    })
 });

 var tabsLoaded = {};
--- a/src/root/status.tt
+++ b/src/root/status.tt
@ -7,7 +7,7 @@

 [% ELSE %]

-  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
+  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]

 [% END %]

--- a/src/root/style.tt
+++ b/src/root/style.tt
@ -0,0 +1,24 @@
+<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
+
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+
+<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
+<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
+<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
+<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
+
+<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
+<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
+<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
+
+<style>
+  .popover { max-width: 40%; }
+</style>
+
+<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
+
+<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
+
+<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
--- a/t/Hydra/Controller/Jobset/evals.t
+++ b/t/Hydra/Controller/Jobset/evals.t
@ -32,4 +32,9 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
    ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
 };

+subtest "/jobset/PROJECT/JOBSET/errors" => sub {
+    my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
+    ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
+};
+
 done_testing;
--- a/t/Hydra/Controller/JobsetEval/fetch.t
+++ b/t/Hydra/Controller/JobsetEval/fetch.t
@ -35,6 +35,10 @@ subtest "Fetching the eval's overview" => sub {
    is($fetch->code, 200, "channel page is 200");
 };

+subtest "Fetching the eval's overview" => sub {
+    my $fetch = request(GET '/eval/' . $eval->id, '/errors');
+    is($fetch->code, 200, "errors page is 200");
+};


 done_testing;
--- a/t/evaluator/evaluate-constituents-globbing.t
+++ b/t/evaluator/evaluate-constituents-globbing.t
@ -61,8 +61,8 @@ subtest "* selects all except current aggregate" => sub {

        $jobset->discard_changes;  # refresh from DB
        is(
-            $jobset->errormsg,
-            "",
+            $jobset->has_error,
+            0,
            "eval-errors non-empty"
        );
    };
@ -101,7 +101,7 @@ subtest "trivial cycle check" => sub {

    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");

-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
    like(
        $jobset->errormsg,
        qr/Dependency cycle: indirect_aggregate <-> ok_aggregate/,
@ -123,7 +123,7 @@ subtest "cycle check with globbing" => sub {

    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");

-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
    like(
        $jobset->errormsg,
        qr/aggregate job ‘indirect_aggregate’ failed with the error: Dependency cycle: indirect_aggregate <-> packages.constituentA/,
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@ -14,7 +14,7 @@ our @EXPORT = qw(
 sub evalSucceeds {
    my ($jobset) = @_;
    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
    if ($res) {
        chomp $stdout; chomp $stderr;
        utf8::decode($stdout) or die "Invalid unicode in stdout.";
@ -29,7 +29,7 @@ sub evalSucceeds {
 sub evalFails {
    my ($jobset) = @_;
    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
-    $jobset->discard_changes;  # refresh from DB
+    $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} });  # refresh from DB
    if (!$res) {
        chomp $stdout; chomp $stderr;
        utf8::decode($stdout) or die "Invalid unicode in stdout.";
--- a/t/queue-runner/direct-indirect-constituents.t
+++ b/t/queue-runner/direct-indirect-constituents.t
@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
 my $constituentBuildB = $builds->{"constituentB"};

 my $eval = $constituentBuildA->jobsetevals->first();
-is($eval->evaluationerror->errormsg, "");
+is($eval->evaluationerror->has_error, 0);

 subtest "Verifying the direct aggregate" => sub {
    my $aggBuild = $builds->{"direct_aggregate"};
Author	SHA1	Message	Date
John Ericson	33a935e8ef	Queue-runner: Always produce a machines JSON object Even if there are no machines, there should at least be an empty object.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	65618fd590	web: replace 'errormsg' with 'errormsg IS NULL' in most cases This is implement in an extremely hacky way due to poor DBIx feature support. Ideally, what we'd need is a way to tell DBIx to ignore the errormsg column unless explicitly requested, and to automatically add a computed 'errormsg IS NULL' column in others. Since it does not support that, this commit instead hacks some support via method overrides while taking care to not break anything obvious.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	06ba54fca7	queue-runner: release machine reservation while copying outputs This allows for better builder usage when the queue runner is busy. To avoid running into uncontrollable imbalances between builder/queue runner, we only release the machine reservation after the local throttler has found a slot to start copying the outputs for that build. As opposed to asserting uniqueness to understand resource utilization, we just switch to using `std::unique_ptr`.	2025-04-09 11:31:47 -04:00
Jörg Thalheim	5b9c22dd18	bump nixpkgs	2025-04-09 11:31:47 -04:00
K900	e15070c6c2	Add metric for builds waiting for download slot (cherry picked from commit f23ec71227911891807706b6b978836e4d80edde)	2025-04-09 11:31:47 -04:00
Jörg Thalheim	37744c7018	don't build hydra twice in a pull request + enable merge queue	2025-04-09 11:31:47 -04:00
Pierre Bourdon	1e3929e75f	queue-runner: switch to pseudorandom ordering of builds processing We don't rely on sequential / monotonic build IDs processing anymore, so randomizing actually has the advantage of mixing builds for different systems together, to avoid only one chunk of builds for a single system getting processed while builders for other systems are starved.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	28da0a705f	queue runner: introduce some parallelism for remote paths lookup Each output for a given step being ingested is looked up in parallel, which should basically multiply the speed of builds ingestion by the average number of outputs per derivation.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	2050b2c324	queue-runner: reduce the time between queue monitor restarts This will induce more DB queries (though these are fairly cheap), but at the benefit of processing bumps within 1m instead of within 10m.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	21d6d805ba	queue-runner: remove id > X from new builds query Running the query with/without it shows that it makes no difference to postgres, since there's an index on finished=0 already. This allows a few simplifications, but also paves the way towards running multiple parallel monitor threads in the future.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	478bb01f7f	queue-runner: add prom metrics to allow detecting internal bottlenecks By looking at the ratio of running vs. waiting for the dispatcher and the queue monitor, we should get better visibility into what hydra is currently bottlenecked on. There are other side effects we can try to measure to get to the same result, but having a simple way doesn't cost us much.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	08bf31b71a	queue-runner: limit parallelism of CPU intensive operations My current theory is that running more parallel xz than available CPU cores is reducing our overall throughput by requiring more scheduling overhead and more cache thrashing.	2025-04-09 11:31:47 -04:00
Pierre Bourdon	641056bd0e	web: Skip System on /machines It is redundant	2025-04-09 11:31:47 -04:00
Jörg Thalheim	29a7ab8009	test/gitea: fix eval	2025-04-09 11:31:47 -04:00
John Ericson	eddc234915	Fix evaluation of NixOS tests, avoid `with`	2025-04-09 11:31:47 -04:00
Maximilian Bosch	80f917d8fa	readIntoSocket: fix with store URIs containing an `&` The third argument to `open()` in `-\|` mode is passed to a shell if it's a string. In my case the store URI contains `?secret-key=${signingKey.directory}/secret&compression=zstd` For the `nix store cat` case this means that * until `&` the process will be started in the background. This fails immediately because no path to cat is specified. * `compression=zstd` is a variable assignment * the `$path` argument to `store cat` is attempted to be executed as another command Passing just the list solves the problem. (cherry picked from commit 3ee51dbe589458cc54ff753317bbc6db530bddc0)	2025-04-09 11:31:47 -04:00
git@71rd.net	5cb82812f2	Stream files from store instead of buffering them When an artifact is requested from hydra the output is first copied from the nix store into memory and then sent as a response, delaying the download and taking up significant amounts of memory. As reported in https://github.com/NixOS/hydra/issues/1357 Instead of calling a command and blocking while reading in the entire output, this adds read_into_socket(). the function takes a command, starting a subprocess with that command, returning a file descriptor attached to stdout. This file descriptor is then by responsebuilder of Catalyst to steam the output directly (cherry picked from commit 459aa0a5983a0bd546399c08231468d6e9282f54)	2025-04-09 11:31:47 -04:00
ajs124	17094c8371	lazy-load evaluation errors Closes #1362	2025-04-09 11:31:47 -04:00
Maximilian Bosch	d5fb163618	Only show stepname if it doesn't equal the name of the drv When building e.g. nixpkgs, the "Running builds" view will mostly look like this hello.x86_64-linux (Build of hello-X.Y) exa.x86_64-linux (Build of exa-X.Y) ... This doesn't provide any useful information. Showing the step name only makes sense if it's not a child of the job's derivation. With this patch, that information will only be shown if the drv name (i.e. w/o `/nix/store/` prefix, .drv ext & hash) is not equal to the drv name of the job itself (build.nixname).	2025-04-09 11:31:47 -04:00
Maximilian Bosch	baec2bbb4c	Running builds view: show build step names When using Hydra to build machine configurations, you'll often see "nixosConfigurations.foo" five times, i.e. for each build step being run. This isn't very helpful I think because in such a case, a single build step can also be compiling the Linux kernel. This change also fetches the `drvpath` and `type` from the `buildsteps` relation. We're already joining it, so this doesn't make much difference (confirmed via query logging that this doesn't cause extra SQL queries). Unfortunately build steps don't have a human readable name, so I'm deriving it from the drvpath by stripping away the hash (assuming that it'll never contain a `-` and that `/nix/store/` is used as prefix). I decided against using the Nix bindings for that to avoid too much overhead due to store operations for each build step.	2025-04-09 11:31:47 -04:00
Maximilian Bosch	b55bd25581	Make "timed out" and "log limit exceeded" builds aborted In 73694087a088ed4481b4ab268a03351b1bcaac3c I gave builds that failed because of a timeout or exceeded log limit a stop sign and I stand by that reasoning: with that it's possible to distinguish between actual build failures and rather transient things such as timeouts. Back then I considered it a feature that these are shown in a different tab, but I don't think that's a good idea anymore. When using a jobset to e.g. track the regressions from a mass rebuild (like a compiler or gcc update), "Newly failed builds" should exclusively display regressions (and flaky builds of course, not much I can do about that). Also, when a bunch of builds fail in such a jobset because of e.g. a broken connection to a builder that results in a timeout, I want to be able to restart them all w/o rebuilding actual regressions. To make it clear that we not only have "Aborted" builds in the tab, I renamed the label to "Aborted / Timed out".	2025-04-09 11:31:47 -04:00
Pierre Bourdon	1ca17faed4	web: include current step status on /machines	2025-04-09 11:31:47 -04:00
John Ericson	9c022848cf	Fix the build	2025-04-09 11:31:47 -04:00
John Ericson	f58a752419	Fix Nix code Can now at least enter dev shell, but build is still broken.	2025-04-09 11:31:47 -04:00
John Ericson	0769853dec	flake.lock: Update to nix and nix-eval-jobs 2.28 Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/d0f98c76f962147610489e84c10033ca92e9c532?narHash=sha256-u6RhBWQ1XohTZ4Ub5ml1PTcaxQgtqFNng6Sohy1rojw%3D' (2025-04-07) → 'github:NixOS/nix/a4962f73b5fc874d4b16baef47921daf349addfc?narHash=sha256-r%2BpsCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I%3D' (2025-04-07) • Updated input 'nix-eval-jobs': 'github:nix-community/nix-eval-jobs/62f9c9e8d00d2ff6ab27a6197ab459a8e0808e59?narHash=sha256-PypQspB7h7EENe4RQQUQj2Ay8J1%2BO49AKNO9JbAU4Ek%3D' (2025-04-07) → 'github:nix-community/nix-eval-jobs/cba718bafe5dc1607c2b6761ecf53c641a6f3b21?narHash=sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA%3D' (2025-04-07)	2025-04-09 11:31:47 -04:00
John Ericson	21c6afa83b	Fix build (due to C++ API changes)	2025-04-09 11:31:47 -04:00
John Ericson	1022514027	flake.lock: Update to nix and nix-eval-jobs 2.27 Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918?narHash=sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs%3D' (2025-02-18) → 'github:NixOS/nix/d0f98c76f962147610489e84c10033ca92e9c532?narHash=sha256-u6RhBWQ1XohTZ4Ub5ml1PTcaxQgtqFNng6Sohy1rojw%3D' (2025-04-07) • Updated input 'nix-eval-jobs': 'github:nix-community/nix-eval-jobs/f7418fc1fa45b96d37baa95ff3c016dd5be3876b?narHash=sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0%3D' (2025-03-26) → 'github:nix-community/nix-eval-jobs/62f9c9e8d00d2ff6ab27a6197ab459a8e0808e59?narHash=sha256-PypQspB7h7EENe4RQQUQj2Ay8J1%2BO49AKNO9JbAU4Ek%3D' (2025-04-07)	2025-04-09 11:31:47 -04:00
Jörg Thalheim	2d4232475c	gitignore hydra-data as created by foreman	2025-04-09 11:31:47 -04:00
Jörg Thalheim	d799742057	fix development workflow after switching to meson-based build	2025-04-09 11:31:47 -04:00
Robin Stumm	485aa93f2d	hydra-eval-jobset: do not wait on n-e-j inside transaction fixes #1429	2025-04-09 11:31:47 -04:00
Josef Kemetmüller	590e8d8511	Fix rendering of metrics with special characters My main motivation here is to get metrics with brackets to work in order to support "pytest" test names: - test_foo.py::test_bar[1] - test_foo.py::test_bar[2] I couldn't find an "HTML escape"-style function that would generate valid html `id` attribute names from random strings, so I went with a hash digest instead.	2025-04-09 11:31:47 -04:00
Maximilian Bosch	90a8a0d94a	Reimplement (named) constituent jobs (+globbing) based on nix-eval-jobs Depends on https://github.com/nix-community/nix-eval-jobs/pull/349 & #1421. Almost equivalent to #1425, but with a small change: when having e.g. an aggregate job with a glob that matches nothing, the jobset evaluation is failed now. This was the intended behavior before (hydra-eval-jobset fails hard if an aggregate is broken), the code-path was never reached however since the aggregate was never marked as broken in this case before.	2025-04-09 11:31:47 -04:00
zowoq	eb17619ee5	flake.lock: Update Flake lock file updates: • Updated input 'nix-eval-jobs': 'github:nix-community/nix-eval-jobs/4b392b284877d203ae262e16af269f702df036bc?narHash=sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY%3D' (2025-02-14) → 'github:nix-community/nix-eval-jobs/f7418fc1fa45b96d37baa95ff3c016dd5be3876b?narHash=sha256-Lo4KFBNcY8tmBuCmEr2XV0IUZtxXHmbXPNLkov/QSU0%3D' (2025-03-26)	2025-04-09 11:31:47 -04:00
zowoq	ebefdb0a3d	hydraTest: remove outdated postgresql version error: postgresql_12 has been removed since it reached its EOL upstream	2025-04-09 11:31:47 -04:00
Martin Weinelt	55349930f1	Fix race condition in hydra-compress-logs	2025-04-09 11:31:47 -04:00
John Ericson	847a8ae6cd	Revert "Use `LegacySSHStore`" There were some hangs caused by this. Need to fix them, ideally reproducing the issue in a test, before trying this again. This reverts commit 4a4a0f901c70676ee47f830d2ff6a72789ba1baf.	2025-04-09 11:31:47 -04:00
ahuston-0	86d0009448	add declaritive hydra spec	2025-04-01 15:02:44 -04:00
ahuston-0	a20f37b97f	add gitea refs Signed-off-by: ahuston-0 <aliceghuston@gmail.com> Reviewed-on: https://<censored>/ahuston-0/hydra/pulls/1	2025-03-31 14:52:51 -04:00
ahuston-0	a94f84118c	add Gitea pulls docs entry Signed-off-by: ahuston-0 <aliceghuston@gmail.com>	2025-03-31 14:52:51 -04:00
Faye Chun	99e3ad325c	Merge branch 'NixOS:master' into add-gitea-pulls	2025-03-01 22:04:13 -05:00
Faye Chun	2f1fa2b069	Add a plugin to poll Gitea pull requests Based off the existing GithubPulls.pm and GitlabPulls.pm plugins. Also adds an integration test for the new 'giteapulls' input type to the existing 'gitea' test.	2024-12-21 08:02:57 -05:00