From d12d4753ee00e9e3640c1c50400e35bc3caef543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Wed, 9 Apr 2025 17:53:14 +0200 Subject: [PATCH 01/10] Fix compilation with a nix which was compiled withou aws sdk --- src/hydra-queue-runner/hydra-queue-runner.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 05d7e263..ab146312 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -703,6 +703,7 @@ void State::dumpStatus(Connection & conn) : 0.0}, }; +#if NIX_WITH_S3_SUPPORT auto s3Store = dynamic_cast(&*store); if (s3Store) { auto & s3Stats = s3Store->getS3Stats(); @@ -728,6 +729,7 @@ void State::dumpStatus(Connection & conn) + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09}, }; } +#endif } { -- 2.49.0 From c7972c368737940cf658011c65433c4cf249fc49 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 3 Apr 2025 10:33:57 +0200 Subject: [PATCH 02/10] Fix displaying eval errors in jobset eval view Quickfix for something that annoyed me once too often. Specifically, I'm talking about `/eval/1#tabs-errors`. To not fetch long errors on each request, this is only done on-demand. I.e., when the tab is opened, an iframe is requested with the errors. This iframe uses a template for both the jobset view and the jobset-eval view. It is differentiated by checking if `jobset` or `eval` is defined. However, the jobset-eval view also has a `jobset` variable in its stash which means that in both cases the `if` path was used. Since `jobset.fetcherrormsg` isn't defined in the eval case though, you always got an empty error. The band-aid fix is relatively simple: swap if and else: the `eval` variable is not defined in the stash of the jobset view, so now this is a useful condition to decide which view we're in. (cherry picked from commit https://git.lix.systems/lix-project/hydra/commit/70c3d75f739b184b36908a2c898332444482d1a1) --- src/root/eval-error.tt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/root/eval-error.tt b/src/root/eval-error.tt index c2ea28ec..7b6e702f 100644 --- a/src/root/eval-error.tt +++ b/src/root/eval-error.tt @@ -13,12 +13,12 @@
- [% IF jobset %] -

Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].

- 
[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]
- [% ELSIF eval %] + [% IF eval %]

Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].

[% HTML.escape(eval.evaluationerror.errormsg) %]
+ [% ELSIF jobset %] +

Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].

+ 
[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]
[% END %]
-- 2.49.0 From 47158cf3600d021d9ac67f1b6740f295f09e49ae Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Sun, 13 Apr 2025 08:29:01 +0200 Subject: [PATCH 03/10] web: increase colspan for machine row in machine status --- src/root/machine-status.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/machine-status.tt b/src/root/machine-status.tt index 07a2359d..725598eb 100644 --- a/src/root/machine-status.tt +++ b/src/root/machine-status.tt @@ -17,7 +17,7 @@ [% name = m.key ? stripSSHUser(m.key) : "localhost" %] - + [% INCLUDE renderMachineName machine=m.key %] [% IF m.value.systemTypes %] -- 2.49.0 From fa1e989b7d7cb3200e875f35ff4d26f9aba7bba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 7 Apr 2025 18:54:39 +0200 Subject: [PATCH 04/10] re-enable restrict-eval for non-flakes --- flake.lock | 6 +++--- src/script/hydra-eval-jobset | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ddff55ec..1a1531ec 100644 --- a/flake.lock +++ b/flake.lock @@ -29,11 +29,11 @@ "nix-eval-jobs": { "flake": false, "locked": { - "lastModified": 1744018595, - "narHash": "sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA=", + "lastModified": 1744370057, + "narHash": "sha256-n220U5pjzCtTtOJtbga4Xr/PyllowKw9anSevgCqJEw=", "owner": "nix-community", "repo": "nix-eval-jobs", - "rev": "cba718bafe5dc1607c2b6761ecf53c641a6f3b21", + "rev": "1260c6599d22dfd8c25fea6893c3d031996b20e1", "type": "github" }, "original": { diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index cf3fa294..80f5d79c 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -372,6 +372,7 @@ sub evalJobs { or die "cannot find the input containing the job expression\n"; @cmd = ("nix-eval-jobs", + "--option", "restrict-eval", "true", "<" . $nixExprInputName . "/" . $nixExprPath . ">", inputsToArgs($inputInfo)); } -- 2.49.0 From 17f9920cf930c622702815089e90b8de9ad7121f Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Sat, 20 Jul 2024 13:09:39 +0200 Subject: [PATCH 05/10] jobset-eval: fix actions not showing up sometimes for new jobs New jobs have their "new" status take precedence over them being "failed" or "queued", which means actions that can act on "failed" or "queued" jobs weren't shown to the user when they could only act on "new" jobs. (cherry picked from commit https://git.lix.systems/lix-project/hydra/commit/9a4a5dd624a1cedc7cdc40687815739b228e5c77) --- src/lib/Hydra/Controller/JobsetEval.pm | 4 +++- src/lib/Hydra/Helper/BuildDiff.pm | 18 +++++++++++++++--- src/root/jobset-eval.tt | 8 ++++---- t/Hydra/Helper/BuildDiff.t | 17 +++++++---------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm index 643a516c..77c01a84 100644 --- a/src/lib/Hydra/Controller/JobsetEval.pm +++ b/src/lib/Hydra/Controller/JobsetEval.pm @@ -76,7 +76,9 @@ sub view_GET { $c->stash->{removed} = $diff->{removed}; $c->stash->{unfinished} = $diff->{unfinished}; $c->stash->{aborted} = $diff->{aborted}; - $c->stash->{failed} = $diff->{failed}; + $c->stash->{totalAborted} = $diff->{totalAborted}; + $c->stash->{totalFailed} = $diff->{totalFailed}; + $c->stash->{totalQueued} = $diff->{totalQueued}; $c->stash->{full} = ($c->req->params->{full} || "0") eq "1"; diff --git a/src/lib/Hydra/Helper/BuildDiff.pm b/src/lib/Hydra/Helper/BuildDiff.pm index 65dad17c..be8525d6 100644 --- a/src/lib/Hydra/Helper/BuildDiff.pm +++ b/src/lib/Hydra/Helper/BuildDiff.pm @@ -32,7 +32,12 @@ sub buildDiff { removed => [], unfinished => [], aborted => [], - failed => [], + + # These summary counters cut across the categories to determine whether + # actions such as "Restart all failed" or "Bump queue" are available. + totalAborted => 0, + totalFailed => 0, + totalQueued => 0, }; my $n = 0; @@ -80,8 +85,15 @@ sub buildDiff { } else { push @{$ret->{new}}, $build if !$found; } - if (defined $build->buildstatus && $build->buildstatus != 0) { - push @{$ret->{failed}}, $build; + + if ($build->finished != 0 && $build->buildstatus != 0) { + if ($aborted) { + ++$ret->{totalAborted}; + } else { + ++$ret->{totalFailed}; + } + } elsif ($build->finished == 0) { + ++$ret->{totalQueued}; } } diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index f0b92f97..12086d85 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -48,16 +48,16 @@ c.uri_for(c.controller('JobsetEval').action_for('view'), Actions
Create a jobset from this evaluation - [% IF unfinished.size > 0 %] + [% IF totalQueued > 0 %] Cancel all scheduled builds [% END %] - [% IF aborted.size > 0 || stillFail.size > 0 || nowFail.size > 0 || failed.size > 0 %] + [% IF totalFailed > 0 %] Restart all failed builds [% END %] - [% IF aborted.size > 0 %] + [% IF totalAborted > 0 %] Restart all aborted builds [% END %] - [% IF unfinished.size > 0 %] + [% IF totalQueued > 0 %] Bump builds to front of queue [% END %]
diff --git a/t/Hydra/Helper/BuildDiff.t b/t/Hydra/Helper/BuildDiff.t index 243bb596..eef25a0f 100644 --- a/t/Hydra/Helper/BuildDiff.t +++ b/t/Hydra/Helper/BuildDiff.t @@ -25,7 +25,10 @@ subtest "empty diff" => sub { removed => [], unfinished => [], aborted => [], - failed => [], + + totalAborted => 0, + totalFailed => 0, + totalQueued => 0, }, "empty list of jobs returns empty diff" ); @@ -48,12 +51,7 @@ subtest "2 different jobs" => sub { "succeed_with_failed is a new job" ); - is(scalar(@{$ret->{failed}}), 1, "list of failed jobs is 1 element long"); - is( - $ret->{failed}[0]->get_column('id'), - $builds->{"succeed_with_failed"}->get_column('id'), - "succeed_with_failed is a failed job" - ); + is($ret->{totalFailed}, 1, "total failed jobs is 1"); is( $ret->{removed}, @@ -70,9 +68,9 @@ subtest "2 different jobs" => sub { subtest "failed job with no previous history" => sub { my $ret = buildDiff([$builds->{"fails"}], []); - is(scalar(@{$ret->{failed}}), 1, "list of failed jobs is 1 element long"); + is($ret->{totalFailed}, 1, "total failed jobs is 1"); is( - $ret->{failed}[0]->get_column('id'), + $ret->{new}[0]->get_column('id'), $builds->{"fails"}->get_column('id'), "fails is a failed job" ); @@ -93,7 +91,6 @@ subtest "not-yet-built job with no previous history" => sub { is($ret->{removed}, [], "removed"); is($ret->{unfinished}, [], "unfinished"); is($ret->{aborted}, [], "aborted"); - is($ret->{failed}, [], "failed"); is(scalar(@{$ret->{new}}), 1, "list of new jobs is 1 element long"); is( -- 2.49.0 From d5d4d19a4c1b981215bd73ef8478a025be57af87 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 23 Apr 2025 18:27:14 +0200 Subject: [PATCH 06/10] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/a4962f73b5fc874d4b16baef47921daf349addfc' (2025-04-07) → 'github:NixOS/nix/70921714cb3b5e6041b7413459541838651079f3' (2025-04-23) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/db8f4fe18ce772a9c8f3adf321416981c8fe9371' (2025-04-07) → 'github:NixOS/nixpkgs/eea3403f7ca9f9942098f4f2756adab4ec924b2b' (2025-04-23) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 1a1531ec..53a0526b 100644 --- a/flake.lock +++ b/flake.lock @@ -12,11 +12,11 @@ "nixpkgs-regression": [] }, "locked": { - "lastModified": 1744030329, - "narHash": "sha256-r+psCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I=", + "lastModified": 1745420957, + "narHash": "sha256-ZbB3IH9OlJvo14GlQZbYHzJojf/HCDT38GzYTod8DaU=", "owner": "NixOS", "repo": "nix", - "rev": "a4962f73b5fc874d4b16baef47921daf349addfc", + "rev": "70921714cb3b5e6041b7413459541838651079f3", "type": "github" }, "original": { @@ -44,11 +44,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1743987495, - "narHash": "sha256-46T2vMZ4/AfCK0Y2OjlFzJPxmdpP8GtsuEqSSJv3oe4=", + "lastModified": 1745408698, + "narHash": "sha256-JT1wMjLIypWJA0N2V27WpUw8feDmTok4Dwkb0oYXDS4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "db8f4fe18ce772a9c8f3adf321416981c8fe9371", + "rev": "eea3403f7ca9f9942098f4f2756adab4ec924b2b", "type": "github" }, "original": { -- 2.49.0 From 1001b6770442f2ce3043a69de473cff0bdda73de Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 23 Apr 2025 17:51:47 -0400 Subject: [PATCH 07/10] Use Nix without the flake This is what we do for `nix-eval-jobs` already. It allows for more fine-grained control over dependencies. --- flake.lock | 11 +--------- flake.nix | 63 ++++++++++++++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/flake.lock b/flake.lock index 53a0526b..3e403f6f 100644 --- a/flake.lock +++ b/flake.lock @@ -1,16 +1,7 @@ { "nodes": { "nix": { - "inputs": { - "flake-compat": [], - "flake-parts": [], - "git-hooks-nix": [], - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-23-11": [], - "nixpkgs-regression": [] - }, + "flake": false, "locked": { "lastModified": 1745420957, "narHash": "sha256-ZbB3IH9OlJvo14GlQZbYHzJojf/HCDT38GzYTod8DaU=", diff --git a/flake.nix b/flake.nix index dc1d1b8a..0e2f32dc 100644 --- a/flake.nix +++ b/flake.nix @@ -5,14 +5,8 @@ inputs.nix = { url = "github:NixOS/nix/2.28-maintenance"; - inputs.nixpkgs.follows = "nixpkgs"; - - # hide nix dev tooling from our lock file - inputs.flake-parts.follows = ""; - inputs.git-hooks-nix.follows = ""; - inputs.nixpkgs-regression.follows = ""; - inputs.nixpkgs-23-11.follows = ""; - inputs.flake-compat.follows = ""; + # We want to control the deps precisely + flake = false; }; inputs.nix-eval-jobs = { @@ -30,10 +24,27 @@ # A Nixpkgs overlay that provides a 'hydra' package. overlays.default = final: prev: { - nix-eval-jobs = final.callPackage nix-eval-jobs {}; + nixDependenciesForHydra = final.lib.makeScope final.newScope + (import (nix + "/packaging/dependencies.nix") { + pkgs = final; + inherit (final) stdenv; + inputs = {}; + }); + nixComponentsForHydra = final.lib.makeScope final.nixDependenciesForHydra.newScope + (import (nix + "packaging/components.nix") { + officialRelease = true; + inherit (final) lib; + pkgs = final; + src = nix; + maintainers = [ ]; + }); + nix-eval-jobs = final.callPackage nix-eval-jobs { + nixComponents = final.nixComponentsForHydra; + }; hydra = final.callPackage ./package.nix { - inherit (nixpkgs.lib) fileset; + inherit (final.lib) fileset; rawSrc = self; + nixComponents = final.nixComponentsForHydra; }; }; @@ -73,24 +84,26 @@ }); packages = forEachSystem (system: let - nixComponents = { - inherit (nix.packages.${system}) - nix-util - nix-store - nix-expr - nix-fetchers - nix-flake - nix-main - nix-cmd - nix-cli - nix-perl-bindings - ; - }; + inherit (nixpkgs) lib; + pkgs = nixpkgs.legacyPackages.${system}; + nixDependencies = lib.makeScope pkgs.newScope + (import (nix + "/packaging/dependencies.nix") { + inherit pkgs; + inherit (pkgs) stdenv; + inputs = {}; + }); + nixComponents = lib.makeScope nixDependencies.newScope + (import (nix + "/packaging/components.nix") { + officialRelease = true; + inherit lib pkgs; + src = nix; + maintainers = [ ]; + }); in { - nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs { + nix-eval-jobs = pkgs.callPackage nix-eval-jobs { inherit nixComponents; }; - hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix { + hydra = pkgs.callPackage ./package.nix { inherit (nixpkgs.lib) fileset; inherit nixComponents; inherit (self.packages.${system}) nix-eval-jobs; -- 2.49.0 From 21f793e21b73f3b43b6d670050380a9e6059b7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 29 Apr 2025 20:06:35 +0200 Subject: [PATCH 08/10] hydra: expose nix-cli package This makes it easier in other packages to get the nix version used to build Hydra. --- package.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 4a7840c1..5c1a7860 100644 --- a/package.nix +++ b/package.nix @@ -277,5 +277,8 @@ stdenv.mkDerivation (finalAttrs: { dontStrip = true; meta.description = "Build of Hydra on ${stdenv.system}"; - passthru = { inherit perlDeps; }; + passthru = { + inherit perlDeps; + nix = nixComponents.nix-cli; + }; }) -- 2.49.0 From 0dd4c0cc8ebec347286fafe600e3eb52a224ee12 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Sun, 21 Apr 2024 17:36:16 +0200 Subject: [PATCH 09/10] queue runner: attempt at slightly smarter scheduling criteria Instead of just going for "whatever is the oldest build we know of", use the following first: - Is the step more constrained? If so, schedule it first to avoid filling up "more desirable" build slots with less constrained builds. - Does the step have more dependents? If so, schedule it first to try and maximize open parallelism and breadth of scheduling options. (cherry picked from commit https://git.lix.systems/lix-project/hydra/commit/b8d03adaf45105452bf1040deeaaccc8b8b22efb) --- src/hydra-queue-runner/dispatcher.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index d3e145de..ada25dc6 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -134,6 +134,8 @@ system_time State::doDispatch() comparator is a partial ordering (see MachineInfo). */ int highestGlobalPriority; int highestLocalPriority; + size_t numRequiredSystemFeatures; + size_t numRevDeps; BuildID lowestBuildID; StepInfo(Step::ptr step, Step::State & step_) : step(step) @@ -142,6 +144,8 @@ system_time State::doDispatch() lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed()); highestGlobalPriority = step_.highestGlobalPriority; highestLocalPriority = step_.highestLocalPriority; + numRequiredSystemFeatures = step->requiredSystemFeatures.size(); + numRevDeps = step_.rdeps.size(); lowestBuildID = step_.lowestBuildID; } }; @@ -194,6 +198,8 @@ system_time State::doDispatch() a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority : a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed : a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority : + a.numRequiredSystemFeatures != b.numRequiredSystemFeatures ? a.numRequiredSystemFeatures > b.numRequiredSystemFeatures : + a.numRevDeps != b.numRevDeps ? a.numRevDeps > b.numRevDeps : a.lowestBuildID < b.lowestBuildID; }); -- 2.49.0 From 84ce142a9d49e2453ce834cf5afa059189a913c9 Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 5 May 2025 00:10:59 +0200 Subject: [PATCH 10/10] Add missing slash error: access to absolute path '/nix/store/sai35xfsrba2a2vasmzxakmn54wdfa13-sourcepackaging' is forbidden in pure evaluation mode (use '--impure' to override) --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 0e2f32dc..979bfcbd 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,7 @@ inputs = {}; }); nixComponentsForHydra = final.lib.makeScope final.nixDependenciesForHydra.newScope - (import (nix + "packaging/components.nix") { + (import (nix + "/packaging/components.nix") { officialRelease = true; inherit (final) lib; pkgs = final; -- 2.49.0