resync hydra with upstream #4
3
.github/workflows/test.yml
vendored
3
.github/workflows/test.yml
vendored
@ -1,7 +1,10 @@
|
||||
name: "Test"
|
||||
on:
|
||||
pull_request:
|
||||
merge_group:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -3,6 +3,7 @@
|
||||
/src/sql/hydra-postgresql.sql
|
||||
/src/sql/hydra-sqlite.sql
|
||||
/src/sql/tmp.sqlite
|
||||
.hydra-data
|
||||
result
|
||||
result-*
|
||||
outputs
|
||||
|
||||
27
README.md
27
README.md
@ -72,17 +72,16 @@ Make sure **State** at the top of the page is set to "_Enabled_" and click on "_
|
||||
You can build Hydra via `nix-build` using the provided [default.nix](./default.nix):
|
||||
|
||||
```
|
||||
$ nix-build
|
||||
$ nix build
|
||||
```
|
||||
|
||||
### Development Environment
|
||||
|
||||
You can use the provided shell.nix to get a working development environment:
|
||||
```
|
||||
$ nix-shell
|
||||
$ autoreconfPhase
|
||||
$ configurePhase # NOTE: not ./configure
|
||||
$ make
|
||||
$ nix develop
|
||||
$ mesonConfigurePhase
|
||||
$ ninja
|
||||
```
|
||||
|
||||
### Executing Hydra During Development
|
||||
@ -91,9 +90,9 @@ When working on new features or bug fixes you need to be able to run Hydra from
|
||||
can be done using [foreman](https://github.com/ddollar/foreman):
|
||||
|
||||
```
|
||||
$ nix-shell
|
||||
$ nix develop
|
||||
$ # hack hack
|
||||
$ make
|
||||
$ ninja -C build
|
||||
$ foreman start
|
||||
```
|
||||
|
||||
@ -115,22 +114,24 @@ Start by following the steps in [Development Environment](#development-environme
|
||||
Then, you can run the tests and the perlcritic linter together with:
|
||||
|
||||
```console
|
||||
$ nix-shell
|
||||
$ make check
|
||||
$ nix develop
|
||||
$ ninja -C build test
|
||||
```
|
||||
|
||||
You can run a single test with:
|
||||
|
||||
```
|
||||
$ nix-shell
|
||||
$ yath test ./t/foo/bar.t
|
||||
$ nix develop
|
||||
$ cd build
|
||||
$ meson test --test-args=../t/Hydra/Event.t testsuite
|
||||
```
|
||||
|
||||
And you can run just perlcritic with:
|
||||
|
||||
```
|
||||
$ nix-shell
|
||||
$ make perlcritic
|
||||
$ nix develop
|
||||
$ cd build
|
||||
$ meson test perlcritic
|
||||
```
|
||||
|
||||
### JSON API
|
||||
|
||||
@ -11,12 +11,6 @@ $ cd hydra
|
||||
To enter a shell in which all environment variables (such as `PERL5LIB`)
|
||||
and dependencies can be found:
|
||||
|
||||
```console
|
||||
$ nix-shell
|
||||
```
|
||||
|
||||
of when flakes are enabled:
|
||||
|
||||
```console
|
||||
$ nix develop
|
||||
```
|
||||
@ -24,15 +18,15 @@ $ nix develop
|
||||
To build Hydra, you should then do:
|
||||
|
||||
```console
|
||||
[nix-shell]$ autoreconfPhase
|
||||
[nix-shell]$ configurePhase
|
||||
[nix-shell]$ make -j$(nproc)
|
||||
$ mesonConfigurePhase
|
||||
$ ninja
|
||||
```
|
||||
|
||||
You start a local database, the webserver, and other components with
|
||||
foreman:
|
||||
|
||||
```console
|
||||
$ ninja -C build
|
||||
$ foreman start
|
||||
```
|
||||
|
||||
@ -47,18 +41,11 @@ $ ./src/script/hydra-server
|
||||
You can run Hydra's test suite with the following:
|
||||
|
||||
```console
|
||||
[nix-shell]$ make check
|
||||
[nix-shell]$ # to run as many tests as you have cores:
|
||||
[nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES
|
||||
[nix-shell]$ # or run yath directly:
|
||||
[nix-shell]$ yath test
|
||||
[nix-shell]$ # to run as many tests as you have cores:
|
||||
[nix-shell]$ yath test -j $NIX_BUILD_CORES
|
||||
$ meson test
|
||||
# to run as many tests as you have cores:
|
||||
$ YATH_JOB_COUNT=$NIX_BUILD_CORES meson test
|
||||
```
|
||||
|
||||
When using `yath` instead of `make check`, ensure you have run `make`
|
||||
in the root of the repository at least once.
|
||||
|
||||
**Warning**: Currently, the tests can fail
|
||||
if run with high parallelism [due to an issue in
|
||||
`Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40)
|
||||
@ -75,7 +62,7 @@ will reload the page every time you save.
|
||||
To build Hydra and its dependencies:
|
||||
|
||||
```console
|
||||
$ nix-build release.nix -A build.x86_64-linux
|
||||
$ nix build .#packages.x86_64-linux.default
|
||||
```
|
||||
|
||||
## Development Tasks
|
||||
|
||||
20
flake.lock
generated
20
flake.lock
generated
@ -12,16 +12,16 @@
|
||||
"nixpkgs-regression": []
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1739899400,
|
||||
"narHash": "sha256-q/RgA4bB7zWai4oPySq9mch7qH14IEeom2P64SXdqHs=",
|
||||
"lastModified": 1744030329,
|
||||
"narHash": "sha256-r+psCOW77vTSTNbxTVrYHeh6OgB0QukbnyUVDwg8s4I=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nix",
|
||||
"rev": "e310c19a1aeb1ce1ed4d41d5ab2d02db596e0918",
|
||||
"rev": "a4962f73b5fc874d4b16baef47921daf349addfc",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "2.26-maintenance",
|
||||
"ref": "2.28-maintenance",
|
||||
"repo": "nix",
|
||||
"type": "github"
|
||||
}
|
||||
@ -29,11 +29,11 @@
|
||||
"nix-eval-jobs": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1739500569,
|
||||
"narHash": "sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY=",
|
||||
"lastModified": 1744018595,
|
||||
"narHash": "sha256-v5n6t49X7MOpqS9j0FtI6TWOXvxuZMmGsp2OfUK5QfA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-eval-jobs",
|
||||
"rev": "4b392b284877d203ae262e16af269f702df036bc",
|
||||
"rev": "cba718bafe5dc1607c2b6761ecf53c641a6f3b21",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -44,11 +44,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1739461644,
|
||||
"narHash": "sha256-1o1qR0KYozYGRrnqytSpAhVBYLNBHX+Lv6I39zGRzKM=",
|
||||
"lastModified": 1743987495,
|
||||
"narHash": "sha256-46T2vMZ4/AfCK0Y2OjlFzJPxmdpP8GtsuEqSSJv3oe4=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "97a719c9f0a07923c957cf51b20b329f9fb9d43f",
|
||||
"rev": "db8f4fe18ce772a9c8f3adf321416981c8fe9371",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
29
flake.nix
29
flake.nix
@ -4,7 +4,7 @@
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
|
||||
|
||||
inputs.nix = {
|
||||
url = "github:NixOS/nix/2.26-maintenance";
|
||||
url = "github:NixOS/nix/2.28-maintenance";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
# hide nix dev tooling from our lock file
|
||||
@ -34,7 +34,6 @@
|
||||
hydra = final.callPackage ./package.nix {
|
||||
inherit (nixpkgs.lib) fileset;
|
||||
rawSrc = self;
|
||||
nix-perl-bindings = final.nixComponents.nix-perl-bindings;
|
||||
};
|
||||
};
|
||||
|
||||
@ -73,21 +72,29 @@
|
||||
validate-openapi = hydraJobs.tests.validate-openapi.${system};
|
||||
});
|
||||
|
||||
packages = forEachSystem (system: {
|
||||
nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
|
||||
nix = nix.packages.${system}.nix;
|
||||
};
|
||||
hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
|
||||
inherit (nixpkgs.lib) fileset;
|
||||
inherit (self.packages.${system}) nix-eval-jobs;
|
||||
rawSrc = self;
|
||||
packages = forEachSystem (system: let
|
||||
nixComponents = {
|
||||
inherit (nix.packages.${system})
|
||||
nix-util
|
||||
nix-store
|
||||
nix-expr
|
||||
nix-fetchers
|
||||
nix-flake
|
||||
nix-main
|
||||
nix-cmd
|
||||
nix-cli
|
||||
nix-perl-bindings
|
||||
;
|
||||
nix-perl-bindings = nix.hydraJobs.perlBindings.${system};
|
||||
};
|
||||
in {
|
||||
nix-eval-jobs = nixpkgs.legacyPackages.${system}.callPackage nix-eval-jobs {
|
||||
inherit nixComponents;
|
||||
};
|
||||
hydra = nixpkgs.legacyPackages.${system}.callPackage ./package.nix {
|
||||
inherit (nixpkgs.lib) fileset;
|
||||
inherit nixComponents;
|
||||
inherit (self.packages.${system}) nix-eval-jobs;
|
||||
rawSrc = self;
|
||||
};
|
||||
default = self.packages.${system}.hydra;
|
||||
});
|
||||
|
||||
14
meson.build
14
meson.build
@ -12,20 +12,6 @@ nix_util_dep = dependency('nix-util', required: true)
|
||||
nix_store_dep = dependency('nix-store', required: true)
|
||||
nix_main_dep = dependency('nix-main', required: true)
|
||||
|
||||
# Nix need extra flags not provided in its pkg-config files.
|
||||
nix_dep = declare_dependency(
|
||||
dependencies: [
|
||||
nix_util_dep,
|
||||
nix_store_dep,
|
||||
nix_main_dep,
|
||||
],
|
||||
compile_args: [
|
||||
'-include', 'nix/config-util.hh',
|
||||
'-include', 'nix/config-store.hh',
|
||||
'-include', 'nix/config-main.hh',
|
||||
],
|
||||
)
|
||||
|
||||
pqxx_dep = dependency('libpqxx', required: true)
|
||||
|
||||
prom_cpp_core_dep = dependency('prometheus-cpp-core', required: true)
|
||||
|
||||
@ -15,7 +15,6 @@
|
||||
systemd.services.hydra-send-stats.enable = false;
|
||||
|
||||
services.postgresql.enable = true;
|
||||
services.postgresql.package = pkgs.postgresql_12;
|
||||
|
||||
# The following is to work around the following error from hydra-server:
|
||||
# [error] Caught exception in engine "Cannot determine local time zone"
|
||||
|
||||
@ -468,7 +468,7 @@ in
|
||||
elif [[ $compression == zstd ]]; then
|
||||
compression="zstd --rm"
|
||||
fi
|
||||
find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
|
||||
find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
|
||||
'';
|
||||
startAt = "Sun 01:45";
|
||||
};
|
||||
|
||||
@ -27,8 +27,7 @@ in
|
||||
{
|
||||
|
||||
install = forEachSystem (system:
|
||||
with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
|
||||
simpleTest {
|
||||
(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
|
||||
name = "hydra-install";
|
||||
nodes.machine = hydraServer;
|
||||
testScript =
|
||||
@ -43,8 +42,7 @@ in
|
||||
});
|
||||
|
||||
notifications = forEachSystem (system:
|
||||
with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
|
||||
simpleTest {
|
||||
(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest {
|
||||
name = "hydra-notifications";
|
||||
nodes.machine = {
|
||||
imports = [ hydraServer ];
|
||||
@ -56,7 +54,7 @@ in
|
||||
'';
|
||||
services.influxdb.enable = true;
|
||||
};
|
||||
testScript = ''
|
||||
testScript = { nodes, ... }: ''
|
||||
machine.wait_for_job("hydra-init")
|
||||
|
||||
# Create an admin account and some other state.
|
||||
@ -87,7 +85,7 @@ in
|
||||
|
||||
# Setup the project and jobset
|
||||
machine.succeed(
|
||||
"su - hydra -c 'perl -I ${config.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
|
||||
"su - hydra -c 'perl -I ${nodes.machine.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./t/setup-notifications-jobset.pl}' >&2"
|
||||
)
|
||||
|
||||
# Wait until hydra has build the job and
|
||||
@ -101,9 +99,10 @@ in
|
||||
});
|
||||
|
||||
gitea = forEachSystem (system:
|
||||
let pkgs = nixpkgs.legacyPackages.${system}; in
|
||||
with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; };
|
||||
makeTest {
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
in
|
||||
(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest {
|
||||
name = "hydra-gitea";
|
||||
nodes.machine = { pkgs, ... }: {
|
||||
imports = [ hydraServer ];
|
||||
|
||||
25
package.nix
25
package.nix
@ -8,11 +8,7 @@
|
||||
|
||||
, perlPackages
|
||||
|
||||
, nix-util
|
||||
, nix-store
|
||||
, nix-main
|
||||
, nix-cli
|
||||
, nix-perl-bindings
|
||||
, nixComponents
|
||||
, git
|
||||
|
||||
, makeWrapper
|
||||
@ -65,7 +61,7 @@ let
|
||||
name = "hydra-perl-deps";
|
||||
paths = lib.closePropagation
|
||||
([
|
||||
nix-perl-bindings
|
||||
nixComponents.nix-perl-bindings
|
||||
git
|
||||
] ++ (with perlPackages; [
|
||||
AuthenSASL
|
||||
@ -93,6 +89,7 @@ let
|
||||
DateTime
|
||||
DBDPg
|
||||
DBDSQLite
|
||||
DBIxClassHelpers
|
||||
DigestSHA1
|
||||
EmailMIME
|
||||
EmailSender
|
||||
@ -165,7 +162,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
nukeReferences
|
||||
pkg-config
|
||||
mdbook
|
||||
nix-cli
|
||||
nixComponents.nix-cli
|
||||
perlDeps
|
||||
perl
|
||||
unzip
|
||||
@ -175,9 +172,9 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
libpqxx
|
||||
openssl
|
||||
libxslt
|
||||
nix-util
|
||||
nix-store
|
||||
nix-main
|
||||
nixComponents.nix-util
|
||||
nixComponents.nix-store
|
||||
nixComponents.nix-main
|
||||
perlDeps
|
||||
perl
|
||||
boost
|
||||
@ -204,14 +201,14 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
glibcLocales
|
||||
libressl.nc
|
||||
python3
|
||||
nix-cli
|
||||
nixComponents.nix-cli
|
||||
];
|
||||
|
||||
hydraPath = lib.makeBinPath (
|
||||
[
|
||||
subversion
|
||||
openssh
|
||||
nix-cli
|
||||
nixComponents.nix-cli
|
||||
coreutils
|
||||
findutils
|
||||
pixz
|
||||
@ -241,7 +238,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
shellHook = ''
|
||||
pushd $(git rev-parse --show-toplevel) >/dev/null
|
||||
|
||||
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH
|
||||
PATH=$(pwd)/build/src/hydra-evaluator:$(pwd)/build/src/script:$(pwd)/build/src/hydra-queue-runner:$PATH
|
||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
|
||||
export HYDRA_HOME="$(pwd)/src/"
|
||||
mkdir -p .hydra-data
|
||||
@ -272,7 +269,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
--prefix PATH ':' $out/bin:$hydraPath \
|
||||
--set HYDRA_RELEASE ${version} \
|
||||
--set HYDRA_HOME $out/libexec/hydra \
|
||||
--set NIX_RELEASE ${nix-cli.name or "unknown"} \
|
||||
--set NIX_RELEASE ${nixComponents.nix-cli.name or "unknown"} \
|
||||
--set NIX_EVAL_JOBS_RELEASE ${nix-eval-jobs.name or "unknown"}
|
||||
done
|
||||
'';
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
#include "db.hh"
|
||||
#include "hydra-config.hh"
|
||||
#include "pool.hh"
|
||||
#include "shared.hh"
|
||||
#include "signals.hh"
|
||||
#include <nix/util/pool.hh>
|
||||
#include <nix/main/shared.hh>
|
||||
#include <nix/util/signals.hh>
|
||||
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
|
||||
@ -2,7 +2,8 @@ hydra_evaluator = executable('hydra-evaluator',
|
||||
'hydra-evaluator.cc',
|
||||
dependencies: [
|
||||
libhydra_dep,
|
||||
nix_dep,
|
||||
nix_util_dep,
|
||||
nix_main_dep,
|
||||
pqxx_dep,
|
||||
],
|
||||
install: true,
|
||||
|
||||
@ -5,17 +5,20 @@
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
#include "build-result.hh"
|
||||
#include "path.hh"
|
||||
#include "legacy-ssh-store.hh"
|
||||
#include "serve-protocol.hh"
|
||||
#include <nix/store/build-result.hh>
|
||||
#include <nix/store/path.hh>
|
||||
#include <nix/store/legacy-ssh-store.hh>
|
||||
#include <nix/store/serve-protocol.hh>
|
||||
#include <nix/store/serve-protocol-impl.hh>
|
||||
#include "state.hh"
|
||||
#include "current-process.hh"
|
||||
#include "processes.hh"
|
||||
#include "util.hh"
|
||||
#include "ssh.hh"
|
||||
#include "finally.hh"
|
||||
#include "url.hh"
|
||||
#include <nix/util/current-process.hh>
|
||||
#include <nix/util/processes.hh>
|
||||
#include <nix/util/util.hh>
|
||||
#include <nix/store/serve-protocol.hh>
|
||||
#include <nix/store/serve-protocol-impl.hh>
|
||||
#include <nix/store/ssh.hh>
|
||||
#include <nix/util/finally.hh>
|
||||
#include <nix/util/url.hh>
|
||||
|
||||
using namespace nix;
|
||||
|
||||
@ -36,6 +39,38 @@ bool ::Machine::isLocalhost() const
|
||||
|
||||
namespace nix::build_remote {
|
||||
|
||||
static std::unique_ptr<SSHMaster::Connection> openConnection(
|
||||
::Machine::ptr machine, SSHMaster & master)
|
||||
{
|
||||
Strings command = {"nix-store", "--serve", "--write"};
|
||||
if (machine->isLocalhost()) {
|
||||
command.push_back("--builders");
|
||||
command.push_back("");
|
||||
} else {
|
||||
auto remoteStore = machine->storeUri.params.find("remote-store");
|
||||
if (remoteStore != machine->storeUri.params.end()) {
|
||||
command.push_back("--store");
|
||||
command.push_back(shellEscape(remoteStore->second));
|
||||
}
|
||||
}
|
||||
|
||||
auto ret = master.startCommand(std::move(command), {
|
||||
"-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
|
||||
});
|
||||
|
||||
// XXX: determine the actual max value we can use from /proc.
|
||||
|
||||
// FIXME: Should this be upstreamed into `startCommand` in Nix?
|
||||
|
||||
int pipesize = 1024 * 1024;
|
||||
|
||||
fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize);
|
||||
fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static void copyClosureTo(
|
||||
::Machine::Connection & conn,
|
||||
Store & destStore,
|
||||
@ -52,8 +87,8 @@ static void copyClosureTo(
|
||||
// FIXME: substitute output pollutes our build log
|
||||
/* Get back the set of paths that are already valid on the remote
|
||||
host. */
|
||||
auto present = conn.store->queryValidPaths(
|
||||
closure, true, useSubstitutes);
|
||||
auto present = conn.queryValidPaths(
|
||||
destStore, true, closure, useSubstitutes);
|
||||
|
||||
if (present.size() == closure.size()) return;
|
||||
|
||||
@ -68,7 +103,12 @@ static void copyClosureTo(
|
||||
std::unique_lock<std::timed_mutex> sendLock(conn.machine->state->sendLock,
|
||||
std::chrono::seconds(600));
|
||||
|
||||
conn.store->addMultipleToStoreLegacy(destStore, missing);
|
||||
conn.to << ServeProto::Command::ImportPaths;
|
||||
destStore.exportPaths(missing, conn.to);
|
||||
conn.to.flush();
|
||||
|
||||
if (readInt(conn.from) != 1)
|
||||
throw Error("remote machine failed to import closure");
|
||||
}
|
||||
|
||||
|
||||
@ -188,7 +228,7 @@ static BuildResult performBuild(
|
||||
counter & nrStepsBuilding
|
||||
)
|
||||
{
|
||||
auto kont = conn.store->buildDerivationAsync(drvPath, drv, options);
|
||||
conn.putBuildDerivationRequest(localStore, drvPath, drv, options);
|
||||
|
||||
BuildResult result;
|
||||
|
||||
@ -197,10 +237,7 @@ static BuildResult performBuild(
|
||||
startTime = time(0);
|
||||
{
|
||||
MaintainCount<counter> mc(nrStepsBuilding);
|
||||
result = kont();
|
||||
// Without proper call-once functions, we need to manually
|
||||
// delete after calling.
|
||||
kont = {};
|
||||
result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
|
||||
}
|
||||
stopTime = time(0);
|
||||
|
||||
@ -216,7 +253,7 @@ static BuildResult performBuild(
|
||||
|
||||
// If the protocol was too old to give us `builtOutputs`, initialize
|
||||
// it manually by introspecting the derivation.
|
||||
if (GET_PROTOCOL_MINOR(conn.store->getProtocol()) < 6)
|
||||
if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 6)
|
||||
{
|
||||
// If the remote is too old to handle CA derivations, we can’t get this
|
||||
// far anyways
|
||||
@ -249,25 +286,26 @@ static void copyPathFromRemote(
|
||||
const ValidPathInfo & info
|
||||
)
|
||||
{
|
||||
/* Receive the NAR from the remote and add it to the
|
||||
destination store. Meanwhile, extract all the info from the
|
||||
NAR that getBuildOutput() needs. */
|
||||
auto source2 = sinkToSource([&](Sink & sink)
|
||||
{
|
||||
/* Note: we should only send the command to dump the store
|
||||
path to the remote if the NAR is actually going to get read
|
||||
by the destination store, which won't happen if this path
|
||||
is already valid on the destination store. Since this
|
||||
lambda function only gets executed if someone tries to read
|
||||
from source2, we will send the command from here rather
|
||||
than outside the lambda. */
|
||||
conn.store->narFromPath(info.path, [&](Source & source) {
|
||||
TeeSource tee{source, sink};
|
||||
extractNarData(tee, conn.store->printStorePath(info.path), narMembers);
|
||||
});
|
||||
});
|
||||
/* Receive the NAR from the remote and add it to the
|
||||
destination store. Meanwhile, extract all the info from the
|
||||
NAR that getBuildOutput() needs. */
|
||||
auto source2 = sinkToSource([&](Sink & sink)
|
||||
{
|
||||
/* Note: we should only send the command to dump the store
|
||||
path to the remote if the NAR is actually going to get read
|
||||
by the destination store, which won't happen if this path
|
||||
is already valid on the destination store. Since this
|
||||
lambda function only gets executed if someone tries to read
|
||||
from source2, we will send the command from here rather
|
||||
than outside the lambda. */
|
||||
conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path);
|
||||
conn.to.flush();
|
||||
|
||||
destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
|
||||
TeeSource tee(conn.from, sink);
|
||||
extractNarData(tee, localStore.printStorePath(info.path), narMembers);
|
||||
});
|
||||
|
||||
destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
|
||||
}
|
||||
|
||||
static void copyPathsFromRemote(
|
||||
@ -348,8 +386,19 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
|
||||
|
||||
}
|
||||
|
||||
/* Utility guard object to auto-release a semaphore on destruction. */
|
||||
template <typename T>
|
||||
class SemaphoreReleaser {
|
||||
public:
|
||||
SemaphoreReleaser(T* s) : sem(s) {}
|
||||
~SemaphoreReleaser() { sem->release(); }
|
||||
|
||||
private:
|
||||
T* sem;
|
||||
};
|
||||
|
||||
void State::buildRemote(ref<Store> destStore,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
::Machine::ptr machine, Step::ptr step,
|
||||
const ServeProto::BuildOptions & buildOptions,
|
||||
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
|
||||
@ -366,39 +415,30 @@ void State::buildRemote(ref<Store> destStore,
|
||||
|
||||
updateStep(ssConnecting);
|
||||
|
||||
// FIXME: rewrite to use Store.
|
||||
::Machine::Connection conn {
|
||||
.machine = machine,
|
||||
.store = [&]{
|
||||
auto * pSpecified = std::get_if<StoreReference::Specified>(&machine->storeUri.variant);
|
||||
if (!pSpecified || pSpecified->scheme != "ssh") {
|
||||
throw Error("Currently, only (legacy-)ssh stores are supported!");
|
||||
}
|
||||
auto storeRef = machine->completeStoreReference();
|
||||
|
||||
auto remoteStore = machine->openStore().dynamic_pointer_cast<LegacySSHStore>();
|
||||
assert(remoteStore);
|
||||
auto * pSpecified = std::get_if<StoreReference::Specified>(&storeRef.variant);
|
||||
if (!pSpecified || pSpecified->scheme != "ssh") {
|
||||
throw Error("Currently, only (legacy-)ssh stores are supported!");
|
||||
}
|
||||
|
||||
remoteStore->connPipeSize = 1024 * 1024;
|
||||
|
||||
if (machine->isLocalhost()) {
|
||||
auto rp_new = remoteStore->remoteProgram.get();
|
||||
rp_new.push_back("--builders");
|
||||
rp_new.push_back("");
|
||||
const_cast<nix::Setting<Strings> &>(remoteStore->remoteProgram).assign(rp_new);
|
||||
}
|
||||
remoteStore->extraSshArgs = {
|
||||
"-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes"
|
||||
};
|
||||
const_cast<nix::Setting<int> &>(remoteStore->logFD).assign(logFD.get());
|
||||
|
||||
return nix::ref{remoteStore};
|
||||
}(),
|
||||
LegacySSHStoreConfig storeConfig {
|
||||
pSpecified->scheme,
|
||||
pSpecified->authority,
|
||||
storeRef.params
|
||||
};
|
||||
|
||||
auto master = storeConfig.createSSHMaster(
|
||||
false, // no SSH master yet
|
||||
logFD.get());
|
||||
|
||||
// FIXME: rewrite to use Store.
|
||||
auto child = build_remote::openConnection(machine, master);
|
||||
|
||||
{
|
||||
auto activeStepState(activeStep->state_.lock());
|
||||
if (activeStepState->cancelled) throw Error("step cancelled");
|
||||
activeStepState->pid = conn.store->getConnectionPid();
|
||||
activeStepState->pid = child->sshPid;
|
||||
}
|
||||
|
||||
Finally clearPid([&]() {
|
||||
@ -413,12 +453,35 @@ void State::buildRemote(ref<Store> destStore,
|
||||
process. Meh. */
|
||||
});
|
||||
|
||||
::Machine::Connection conn {
|
||||
{
|
||||
.to = child->in.get(),
|
||||
.from = child->out.get(),
|
||||
/* Handshake. */
|
||||
.remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize
|
||||
},
|
||||
/*.machine =*/ machine,
|
||||
};
|
||||
|
||||
Finally updateStats([&]() {
|
||||
auto stats = conn.store->getConnectionStats();
|
||||
bytesReceived += stats.bytesReceived;
|
||||
bytesSent += stats.bytesSent;
|
||||
bytesReceived += conn.from.read;
|
||||
bytesSent += conn.to.written;
|
||||
});
|
||||
|
||||
constexpr ServeProto::Version our_version = 0x206;
|
||||
|
||||
try {
|
||||
conn.remoteVersion = decltype(conn)::handshake(
|
||||
conn.to,
|
||||
conn.from,
|
||||
our_version,
|
||||
machine->storeUri.render());
|
||||
} catch (EndOfFile & e) {
|
||||
child->sshPid.wait();
|
||||
std::string s = chomp(readFile(result.logFile));
|
||||
throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s);
|
||||
}
|
||||
|
||||
{
|
||||
auto info(machine->state->connectInfo.lock());
|
||||
info->consecutiveFailures = 0;
|
||||
@ -475,6 +538,23 @@ void State::buildRemote(ref<Store> destStore,
|
||||
result.logFile = "";
|
||||
}
|
||||
|
||||
/* Throttle CPU-bound work. Opportunistically skip updating the current
|
||||
* step, since this requires a DB roundtrip. */
|
||||
if (!localWorkThrottler.try_acquire()) {
|
||||
MaintainCount<counter> mc(nrStepsWaitingForDownloadSlot);
|
||||
updateStep(ssWaitingForLocalSlot);
|
||||
localWorkThrottler.acquire();
|
||||
}
|
||||
SemaphoreReleaser releaser(&localWorkThrottler);
|
||||
|
||||
/* Once we've started copying outputs, release the machine reservation
|
||||
* so further builds can happen. We do not release the machine earlier
|
||||
* to avoid situations where the queue runner is bottlenecked on
|
||||
* copying outputs and we end up building too many things that we
|
||||
* haven't been able to allow copy slots for. */
|
||||
reservation.reset();
|
||||
wakeDispatcher();
|
||||
|
||||
StorePathSet outputs;
|
||||
for (auto & [_, realisation] : buildResult.builtOutputs)
|
||||
outputs.insert(realisation.outPath);
|
||||
@ -487,7 +567,7 @@ void State::buildRemote(ref<Store> destStore,
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
auto infos = conn.store->queryPathInfosUncached(outputs);
|
||||
auto infos = conn.queryPathInfos(*localStore, outputs);
|
||||
|
||||
size_t totalNarSize = 0;
|
||||
for (auto & [_, info] : infos) totalNarSize += info.narSize;
|
||||
@ -522,11 +602,9 @@ void State::buildRemote(ref<Store> destStore,
|
||||
}
|
||||
}
|
||||
|
||||
/* Shut down the connection done by RAII.
|
||||
|
||||
Only difference is kill() instead of wait() (i.e. send signal
|
||||
then wait())
|
||||
*/
|
||||
/* Shut down the connection. */
|
||||
child->in = -1;
|
||||
child->sshPid.wait();
|
||||
|
||||
} catch (Error & e) {
|
||||
/* Disable this machine until a certain period of time has
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#include "hydra-build-result.hh"
|
||||
#include "store-api.hh"
|
||||
#include "util.hh"
|
||||
#include "source-accessor.hh"
|
||||
#include <nix/store/store-api.hh>
|
||||
#include <nix/util/util.hh>
|
||||
#include <nix/util/source-accessor.hh>
|
||||
|
||||
#include <regex>
|
||||
|
||||
|
||||
@ -2,8 +2,8 @@
|
||||
|
||||
#include "state.hh"
|
||||
#include "hydra-build-result.hh"
|
||||
#include "finally.hh"
|
||||
#include "binary-cache-store.hh"
|
||||
#include <nix/util/finally.hh>
|
||||
#include <nix/store/binary-cache-store.hh>
|
||||
|
||||
using namespace nix;
|
||||
|
||||
@ -16,7 +16,7 @@ void setThreadName(const std::string & name)
|
||||
}
|
||||
|
||||
|
||||
void State::builder(MachineReservation::ptr reservation)
|
||||
void State::builder(std::unique_ptr<MachineReservation> reservation)
|
||||
{
|
||||
setThreadName("bld~" + std::string(reservation->step->drvPath.to_string()));
|
||||
|
||||
@ -35,22 +35,20 @@ void State::builder(MachineReservation::ptr reservation)
|
||||
activeSteps_.lock()->erase(activeStep);
|
||||
});
|
||||
|
||||
std::string machine = reservation->machine->storeUri.render();
|
||||
|
||||
try {
|
||||
auto destStore = getDestStore();
|
||||
res = doBuildStep(destStore, reservation, activeStep);
|
||||
// Might release the reservation.
|
||||
res = doBuildStep(destStore, std::move(reservation), activeStep);
|
||||
} catch (std::exception & e) {
|
||||
printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s",
|
||||
localStore->printStorePath(reservation->step->drvPath),
|
||||
reservation->machine->storeUri.render(),
|
||||
localStore->printStorePath(activeStep->step->drvPath),
|
||||
machine,
|
||||
e.what());
|
||||
}
|
||||
}
|
||||
|
||||
/* Release the machine and wake up the dispatcher. */
|
||||
assert(reservation.unique());
|
||||
reservation = 0;
|
||||
wakeDispatcher();
|
||||
|
||||
/* If there was a temporary failure, retry the step after an
|
||||
exponentially increasing interval. */
|
||||
Step::ptr step = wstep.lock();
|
||||
@ -72,11 +70,11 @@ void State::builder(MachineReservation::ptr reservation)
|
||||
|
||||
|
||||
State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
MachineReservation::ptr reservation,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
std::shared_ptr<ActiveStep> activeStep)
|
||||
{
|
||||
auto & step(reservation->step);
|
||||
auto & machine(reservation->machine);
|
||||
auto step(reservation->step);
|
||||
auto machine(reservation->machine);
|
||||
|
||||
{
|
||||
auto step_(step->state.lock());
|
||||
@ -211,7 +209,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||
|
||||
try {
|
||||
/* FIXME: referring builds may have conflicting timeouts. */
|
||||
buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers);
|
||||
buildRemote(destStore, std::move(reservation), machine, step, buildOptions, result, activeStep, updateStep, narMembers);
|
||||
} catch (Error & e) {
|
||||
if (activeStep->state_.lock()->cancelled) {
|
||||
printInfo("marking step %d of build %d as cancelled", stepNr, buildId);
|
||||
|
||||
@ -40,13 +40,15 @@ void State::dispatcher()
|
||||
printMsg(lvlDebug, "dispatcher woken up");
|
||||
nrDispatcherWakeups++;
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
auto t_before_work = std::chrono::steady_clock::now();
|
||||
|
||||
auto sleepUntil = doDispatch();
|
||||
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
auto t_after_work = std::chrono::steady_clock::now();
|
||||
|
||||
dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
||||
prom.dispatcher_time_spent_running.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
|
||||
dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
|
||||
|
||||
/* Sleep until we're woken up (either because a runnable build
|
||||
is added, or because a build finishes). */
|
||||
@ -60,6 +62,10 @@ void State::dispatcher()
|
||||
*dispatcherWakeup_ = false;
|
||||
}
|
||||
|
||||
auto t_after_sleep = std::chrono::steady_clock::now();
|
||||
prom.dispatcher_time_spent_waiting.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
|
||||
|
||||
} catch (std::exception & e) {
|
||||
printError("dispatcher: %s", e.what());
|
||||
sleep(1);
|
||||
@ -282,7 +288,7 @@ system_time State::doDispatch()
|
||||
/* Make a slot reservation and start a thread to
|
||||
do the build. */
|
||||
auto builderThread = std::thread(&State::builder, this,
|
||||
std::make_shared<MachineReservation>(*this, step, mi.machine));
|
||||
std::make_unique<MachineReservation>(*this, step, mi.machine));
|
||||
builderThread.detach(); // FIXME?
|
||||
|
||||
keepGoing = true;
|
||||
|
||||
@ -2,9 +2,9 @@
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "hash.hh"
|
||||
#include "derivations.hh"
|
||||
#include "store-api.hh"
|
||||
#include <nix/util/hash.hh>
|
||||
#include <nix/store/derivations.hh>
|
||||
#include <nix/store/store-api.hh>
|
||||
#include "nar-extractor.hh"
|
||||
|
||||
struct BuildProduct
|
||||
|
||||
@ -11,16 +11,16 @@
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "signals.hh"
|
||||
#include <nix/util/signals.hh>
|
||||
#include "state.hh"
|
||||
#include "hydra-build-result.hh"
|
||||
#include "store-api.hh"
|
||||
#include "remote-store.hh"
|
||||
#include <nix/store/store-api.hh>
|
||||
#include <nix/store/remote-store.hh>
|
||||
|
||||
#include "globals.hh"
|
||||
#include <nix/store/globals.hh>
|
||||
#include "hydra-config.hh"
|
||||
#include "s3-binary-cache-store.hh"
|
||||
#include "shared.hh"
|
||||
#include <nix/store/s3-binary-cache-store.hh>
|
||||
#include <nix/main/shared.hh>
|
||||
|
||||
using namespace nix;
|
||||
using nlohmann::json;
|
||||
@ -70,10 +70,31 @@ State::PromMetrics::PromMetrics()
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_max_id(
|
||||
prometheus::BuildGauge()
|
||||
.Name("hydraqueuerunner_queue_max_build_id_info")
|
||||
.Help("Maximum build record ID in the queue")
|
||||
, dispatcher_time_spent_running(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_dispatcher_time_spent_running")
|
||||
.Help("Time (in micros) spent running the dispatcher")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, dispatcher_time_spent_waiting(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_dispatcher_time_spent_waiting")
|
||||
.Help("Time (in micros) spent waiting for the dispatcher to obtain work")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_monitor_time_spent_running(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_queue_monitor_time_spent_running")
|
||||
.Help("Time (in micros) spent running the queue monitor")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
, queue_monitor_time_spent_waiting(
|
||||
prometheus::BuildCounter()
|
||||
.Name("hydraqueuerunner_queue_monitor_time_spent_waiting")
|
||||
.Help("Time (in micros) spent waiting for the queue monitor to obtain work")
|
||||
.Register(*registry)
|
||||
.Add({})
|
||||
)
|
||||
@ -85,6 +106,7 @@ State::State(std::optional<std::string> metricsAddrOpt)
|
||||
: config(std::make_unique<HydraConfig>())
|
||||
, maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
|
||||
, dbPool(config->getIntOption("max_db_connections", 128))
|
||||
, localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
|
||||
, maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
|
||||
, maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
|
||||
, uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
|
||||
@ -551,6 +573,7 @@ void State::dumpStatus(Connection & conn)
|
||||
{"nrActiveSteps", activeSteps_.lock()->size()},
|
||||
{"nrStepsBuilding", nrStepsBuilding.load()},
|
||||
{"nrStepsCopyingTo", nrStepsCopyingTo.load()},
|
||||
{"nrStepsWaitingForDownloadSlot", nrStepsWaitingForDownloadSlot.load()},
|
||||
{"nrStepsCopyingFrom", nrStepsCopyingFrom.load()},
|
||||
{"nrStepsWaiting", nrStepsWaiting.load()},
|
||||
{"nrUnsupportedSteps", nrUnsupportedSteps.load()},
|
||||
@ -592,6 +615,7 @@ void State::dumpStatus(Connection & conn)
|
||||
}
|
||||
|
||||
{
|
||||
auto machines_json = json::object();
|
||||
auto machines_(machines.lock());
|
||||
for (auto & i : *machines_) {
|
||||
auto & m(i.second);
|
||||
@ -618,8 +642,9 @@ void State::dumpStatus(Connection & conn)
|
||||
machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
|
||||
machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
|
||||
}
|
||||
statusJson["machines"][m->storeUri.render()] = machine;
|
||||
machines_json[m->storeUri.render()] = machine;
|
||||
}
|
||||
statusJson["machines"] = machines_json;
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
@ -13,7 +13,9 @@ hydra_queue_runner = executable('hydra-queue-runner',
|
||||
srcs,
|
||||
dependencies: [
|
||||
libhydra_dep,
|
||||
nix_dep,
|
||||
nix_util_dep,
|
||||
nix_store_dep,
|
||||
nix_main_dep,
|
||||
pqxx_dep,
|
||||
prom_cpp_core_dep,
|
||||
prom_cpp_pull_dep,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#include "nar-extractor.hh"
|
||||
|
||||
#include "archive.hh"
|
||||
#include <nix/util/archive.hh>
|
||||
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include "source-accessor.hh"
|
||||
#include "types.hh"
|
||||
#include "serialise.hh"
|
||||
#include "hash.hh"
|
||||
#include <nix/util/source-accessor.hh>
|
||||
#include <nix/util/types.hh>
|
||||
#include <nix/util/serialise.hh>
|
||||
#include <nix/util/hash.hh>
|
||||
|
||||
struct NarMemberData
|
||||
{
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#include "state.hh"
|
||||
#include "hydra-build-result.hh"
|
||||
#include "globals.hh"
|
||||
#include <nix/store/globals.hh>
|
||||
#include <nix/store/parsed-derivations.hh>
|
||||
#include <nix/util/thread-pool.hh>
|
||||
|
||||
#include <cstring>
|
||||
|
||||
@ -37,16 +39,21 @@ void State::queueMonitorLoop(Connection & conn)
|
||||
|
||||
auto destStore = getDestStore();
|
||||
|
||||
unsigned int lastBuildId = 0;
|
||||
|
||||
bool quit = false;
|
||||
while (!quit) {
|
||||
auto t_before_work = std::chrono::steady_clock::now();
|
||||
|
||||
localStore->clearPathInfoCache();
|
||||
|
||||
bool done = getQueuedBuilds(conn, destStore, lastBuildId);
|
||||
bool done = getQueuedBuilds(conn, destStore);
|
||||
|
||||
if (buildOne && buildOneDone) quit = true;
|
||||
|
||||
auto t_after_work = std::chrono::steady_clock::now();
|
||||
|
||||
prom.queue_monitor_time_spent_running.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
|
||||
|
||||
/* Sleep until we get notification from the database about an
|
||||
event. */
|
||||
if (done && !quit) {
|
||||
@ -56,12 +63,10 @@ void State::queueMonitorLoop(Connection & conn)
|
||||
conn.get_notifs();
|
||||
|
||||
if (auto lowestId = buildsAdded.get()) {
|
||||
lastBuildId = std::min(lastBuildId, static_cast<unsigned>(std::stoul(*lowestId) - 1));
|
||||
printMsg(lvlTalkative, "got notification: new builds added to the queue");
|
||||
}
|
||||
if (buildsRestarted.get()) {
|
||||
printMsg(lvlTalkative, "got notification: builds restarted");
|
||||
lastBuildId = 0; // check all builds
|
||||
}
|
||||
if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
|
||||
printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
|
||||
@ -71,6 +76,10 @@ void State::queueMonitorLoop(Connection & conn)
|
||||
printMsg(lvlTalkative, "got notification: jobset shares changed");
|
||||
processJobsetSharesChange(conn);
|
||||
}
|
||||
|
||||
auto t_after_sleep = std::chrono::steady_clock::now();
|
||||
prom.queue_monitor_time_spent_waiting.Increment(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
|
||||
}
|
||||
|
||||
exit(0);
|
||||
@ -84,20 +93,18 @@ struct PreviousFailure : public std::exception {
|
||||
|
||||
|
||||
bool State::getQueuedBuilds(Connection & conn,
|
||||
ref<Store> destStore, unsigned int & lastBuildId)
|
||||
ref<Store> destStore)
|
||||
{
|
||||
prom.queue_checks_started.Increment();
|
||||
|
||||
printInfo("checking the queue for builds > %d...", lastBuildId);
|
||||
printInfo("checking the queue for builds...");
|
||||
|
||||
/* Grab the queued builds from the database, but don't process
|
||||
them yet (since we don't want a long-running transaction). */
|
||||
std::vector<BuildID> newIDs;
|
||||
std::map<BuildID, Build::ptr> newBuildsByID;
|
||||
std::unordered_map<BuildID, Build::ptr> newBuildsByID;
|
||||
std::multimap<StorePath, BuildID> newBuildsByPath;
|
||||
|
||||
unsigned int newLastBuildId = lastBuildId;
|
||||
|
||||
{
|
||||
pqxx::work txn(conn);
|
||||
|
||||
@ -106,17 +113,12 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||
"jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, "
|
||||
"globalPriority, priority from Builds "
|
||||
"inner join jobsets on builds.jobset_id = jobsets.id "
|
||||
"where builds.id > $1 and finished = 0 order by globalPriority desc, builds.id",
|
||||
lastBuildId);
|
||||
"where finished = 0 order by globalPriority desc, random()");
|
||||
|
||||
for (auto const & row : res) {
|
||||
auto builds_(builds.lock());
|
||||
BuildID id = row["id"].as<BuildID>();
|
||||
if (buildOne && id != buildOne) continue;
|
||||
if (id > newLastBuildId) {
|
||||
newLastBuildId = id;
|
||||
prom.queue_max_id.Set(id);
|
||||
}
|
||||
if (builds_->count(id)) continue;
|
||||
|
||||
auto build = std::make_shared<Build>(
|
||||
@ -318,15 +320,13 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||
|
||||
/* Stop after a certain time to allow priority bumps to be
|
||||
processed. */
|
||||
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
|
||||
if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) {
|
||||
prom.queue_checks_early_exits.Increment();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
prom.queue_checks_finished.Increment();
|
||||
|
||||
lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
|
||||
return newBuildsByID.empty();
|
||||
}
|
||||
|
||||
@ -405,6 +405,34 @@ void State::processQueueChange(Connection & conn)
|
||||
}
|
||||
|
||||
|
||||
std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
|
||||
ref<Store> destStore,
|
||||
const std::map<DrvOutput, std::optional<StorePath>> & paths)
|
||||
{
|
||||
Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
|
||||
ThreadPool tp;
|
||||
|
||||
for (auto & [output, maybeOutputPath] : paths) {
|
||||
if (!maybeOutputPath) {
|
||||
auto missing(missing_.lock());
|
||||
missing->insert({output, maybeOutputPath});
|
||||
} else {
|
||||
tp.enqueue([&] {
|
||||
if (!destStore->isValidPath(*maybeOutputPath)) {
|
||||
auto missing(missing_.lock());
|
||||
missing->insert({output, maybeOutputPath});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
tp.process();
|
||||
|
||||
auto missing(missing_.lock());
|
||||
return *missing;
|
||||
}
|
||||
|
||||
|
||||
Step::ptr State::createStep(ref<Store> destStore,
|
||||
Connection & conn, Build::ptr build, const StorePath & drvPath,
|
||||
Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
|
||||
@ -463,14 +491,17 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||
it's not runnable yet, and other threads won't make it
|
||||
runnable while step->created == false. */
|
||||
step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
|
||||
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
|
||||
{
|
||||
auto parsedDrv = ParsedDerivation{drvPath, *step->drv};
|
||||
step->drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(parsedDrv));
|
||||
}
|
||||
|
||||
step->preferLocalBuild = step->parsedDrv->willBuildLocally(*localStore);
|
||||
step->preferLocalBuild = step->drvOptions->willBuildLocally(*localStore, *step->drv);
|
||||
step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1";
|
||||
|
||||
step->systemType = step->drv->platform;
|
||||
{
|
||||
StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
|
||||
StringSet features = step->requiredSystemFeatures = step->drvOptions->getRequiredSystemFeatures(*step->drv);
|
||||
if (step->preferLocalBuild)
|
||||
features.insert("local");
|
||||
if (!features.empty()) {
|
||||
@ -485,16 +516,15 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||
|
||||
/* Are all outputs valid? */
|
||||
auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
|
||||
bool valid = true;
|
||||
std::map<DrvOutput, std::optional<StorePath>> missing;
|
||||
std::map<DrvOutput, std::optional<StorePath>> paths;
|
||||
for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
|
||||
auto outputHash = outputHashes.at(outputName);
|
||||
if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
|
||||
continue;
|
||||
valid = false;
|
||||
missing.insert({{outputHash, outputName}, maybeOutputPath});
|
||||
paths.insert({{outputHash, outputName}, maybeOutputPath});
|
||||
}
|
||||
|
||||
auto missing = getMissingRemotePaths(destStore, paths);
|
||||
bool valid = missing.empty();
|
||||
|
||||
/* Try to copy the missing paths from the local store or from
|
||||
substitutes. */
|
||||
if (!missing.empty()) {
|
||||
|
||||
@ -6,6 +6,8 @@
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
#include <regex>
|
||||
#include <semaphore>
|
||||
|
||||
#include <prometheus/counter.h>
|
||||
#include <prometheus/gauge.h>
|
||||
@ -13,15 +15,18 @@
|
||||
|
||||
#include "db.hh"
|
||||
|
||||
#include "parsed-derivations.hh"
|
||||
#include "pathlocks.hh"
|
||||
#include "pool.hh"
|
||||
#include "build-result.hh"
|
||||
#include "store-api.hh"
|
||||
#include "sync.hh"
|
||||
#include <nix/store/derivations.hh>
|
||||
#include <nix/store/derivation-options.hh>
|
||||
#include <nix/store/pathlocks.hh>
|
||||
#include <nix/util/pool.hh>
|
||||
#include <nix/store/build-result.hh>
|
||||
#include <nix/store/store-api.hh>
|
||||
#include <nix/util/sync.hh>
|
||||
#include "nar-extractor.hh"
|
||||
#include "legacy-ssh-store.hh"
|
||||
#include "machines.hh"
|
||||
#include <nix/store/serve-protocol.hh>
|
||||
#include <nix/store/serve-protocol-impl.hh>
|
||||
#include <nix/store/serve-protocol-connection.hh>
|
||||
#include <nix/store/machines.hh>
|
||||
|
||||
|
||||
typedef unsigned int BuildID;
|
||||
@ -55,6 +60,7 @@ typedef enum {
|
||||
ssConnecting = 10,
|
||||
ssSendingInputs = 20,
|
||||
ssBuilding = 30,
|
||||
ssWaitingForLocalSlot = 35,
|
||||
ssReceivingOutputs = 40,
|
||||
ssPostProcessing = 50,
|
||||
} StepState;
|
||||
@ -165,7 +171,7 @@ struct Step
|
||||
|
||||
nix::StorePath drvPath;
|
||||
std::unique_ptr<nix::Derivation> drv;
|
||||
std::unique_ptr<nix::ParsedDerivation> parsedDrv;
|
||||
std::unique_ptr<nix::DerivationOptions> drvOptions;
|
||||
std::set<std::string> requiredSystemFeatures;
|
||||
bool preferLocalBuild;
|
||||
bool isDeterministic;
|
||||
@ -290,11 +296,9 @@ struct Machine : nix::Machine
|
||||
bool isLocalhost() const;
|
||||
|
||||
// A connection to a machine
|
||||
struct Connection {
|
||||
struct Connection : nix::ServeProto::BasicClientConnection {
|
||||
// Backpointer to the machine
|
||||
ptr machine;
|
||||
// Opened store
|
||||
nix::ref<nix::LegacySSHStore> store;
|
||||
};
|
||||
};
|
||||
|
||||
@ -352,6 +356,10 @@ private:
|
||||
typedef std::map<nix::StoreReference::Variant, Machine::ptr> Machines;
|
||||
nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
|
||||
|
||||
/* Throttler for CPU-bound local work. */
|
||||
static constexpr unsigned int maxSupportedLocalWorkers = 1024;
|
||||
std::counting_semaphore<maxSupportedLocalWorkers> localWorkThrottler;
|
||||
|
||||
/* Various stats. */
|
||||
time_t startedAt;
|
||||
counter nrBuildsRead{0};
|
||||
@ -361,6 +369,7 @@ private:
|
||||
counter nrStepsDone{0};
|
||||
counter nrStepsBuilding{0};
|
||||
counter nrStepsCopyingTo{0};
|
||||
counter nrStepsWaitingForDownloadSlot{0};
|
||||
counter nrStepsCopyingFrom{0};
|
||||
counter nrStepsWaiting{0};
|
||||
counter nrUnsupportedSteps{0};
|
||||
@ -391,7 +400,6 @@ private:
|
||||
|
||||
struct MachineReservation
|
||||
{
|
||||
typedef std::shared_ptr<MachineReservation> ptr;
|
||||
State & state;
|
||||
Step::ptr step;
|
||||
Machine::ptr machine;
|
||||
@ -449,7 +457,12 @@ private:
|
||||
prometheus::Counter& queue_steps_created;
|
||||
prometheus::Counter& queue_checks_early_exits;
|
||||
prometheus::Counter& queue_checks_finished;
|
||||
prometheus::Gauge& queue_max_id;
|
||||
|
||||
prometheus::Counter& dispatcher_time_spent_running;
|
||||
prometheus::Counter& dispatcher_time_spent_waiting;
|
||||
|
||||
prometheus::Counter& queue_monitor_time_spent_running;
|
||||
prometheus::Counter& queue_monitor_time_spent_waiting;
|
||||
|
||||
PromMetrics();
|
||||
};
|
||||
@ -493,8 +506,7 @@ private:
|
||||
void queueMonitorLoop(Connection & conn);
|
||||
|
||||
/* Check the queue for new builds. */
|
||||
bool getQueuedBuilds(Connection & conn,
|
||||
nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
|
||||
bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> destStore);
|
||||
|
||||
/* Handle cancellation, deletion and priority bumps. */
|
||||
void processQueueChange(Connection & conn);
|
||||
@ -502,6 +514,12 @@ private:
|
||||
BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
|
||||
const nix::StorePath & drvPath);
|
||||
|
||||
/* Returns paths missing from the remote store. Paths are processed in
|
||||
* parallel to work around the possible latency of remote stores. */
|
||||
std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
|
||||
nix::ref<nix::Store> destStore,
|
||||
const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
|
||||
|
||||
Step::ptr createStep(nix::ref<nix::Store> store,
|
||||
Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
|
||||
Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,
|
||||
@ -531,16 +549,17 @@ private:
|
||||
|
||||
void abortUnsupported();
|
||||
|
||||
void builder(MachineReservation::ptr reservation);
|
||||
void builder(std::unique_ptr<MachineReservation> reservation);
|
||||
|
||||
/* Perform the given build step. Return true if the step is to be
|
||||
retried. */
|
||||
enum StepResult { sDone, sRetry, sMaybeCancelled };
|
||||
StepResult doBuildStep(nix::ref<nix::Store> destStore,
|
||||
MachineReservation::ptr reservation,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
std::shared_ptr<ActiveStep> activeStep);
|
||||
|
||||
void buildRemote(nix::ref<nix::Store> destStore,
|
||||
std::unique_ptr<MachineReservation> reservation,
|
||||
Machine::ptr machine, Step::ptr step,
|
||||
const nix::ServeProto::BuildOptions & buildOptions,
|
||||
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
|
||||
|
||||
@ -238,7 +238,7 @@ sub serveFile {
|
||||
# XSS hole.
|
||||
$c->response->header('Content-Security-Policy' => 'sandbox allow-scripts');
|
||||
|
||||
$c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command",
|
||||
$c->stash->{'plain'} = { data => readIntoSocket(cmd => ["nix", "--experimental-features", "nix-command",
|
||||
"store", "cat", "--store", getStoreUri(), "$path"]) };
|
||||
|
||||
# Detect MIME type.
|
||||
|
||||
@ -364,6 +364,21 @@ sub evals_GET {
|
||||
);
|
||||
}
|
||||
|
||||
sub errors :Chained('jobsetChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
|
||||
|
||||
sub errors_GET {
|
||||
my ($self, $c) = @_;
|
||||
|
||||
$c->stash->{template} = 'eval-error.tt';
|
||||
|
||||
my $jobsetName = $c->stash->{params}->{name};
|
||||
$c->stash->{jobset} = $c->stash->{project}->jobsets->find(
|
||||
{ name => $jobsetName },
|
||||
{ '+columns' => { 'errormsg' => 'errormsg' } }
|
||||
);
|
||||
|
||||
$self->status_ok($c, entity => $c->stash->{jobset});
|
||||
}
|
||||
|
||||
# Redirect to the latest finished evaluation of this jobset.
|
||||
sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
|
||||
|
||||
@ -86,6 +86,17 @@ sub view_GET {
|
||||
);
|
||||
}
|
||||
|
||||
sub errors :Chained('evalChain') :PathPart('errors') :Args(0) :ActionClass('REST') { }
|
||||
|
||||
sub errors_GET {
|
||||
my ($self, $c) = @_;
|
||||
|
||||
$c->stash->{template} = 'eval-error.tt';
|
||||
|
||||
$c->stash->{eval} = $c->model('DB::JobsetEvals')->find($c->stash->{eval}->id, { prefetch => 'evaluationerror' });
|
||||
|
||||
$self->status_ok($c, entity => $c->stash->{eval});
|
||||
}
|
||||
|
||||
sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
|
||||
my ($self, $c) = @_;
|
||||
|
||||
@ -162,7 +162,7 @@ sub status_GET {
|
||||
{ "buildsteps.busy" => { '!=', 0 } },
|
||||
{ order_by => ["globalpriority DESC", "id"],
|
||||
join => "buildsteps",
|
||||
columns => [@buildListColumns]
|
||||
columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
|
||||
})]
|
||||
);
|
||||
}
|
||||
|
||||
@ -37,7 +37,16 @@ sub buildDiff {
|
||||
|
||||
my $n = 0;
|
||||
foreach my $build (@{$builds}) {
|
||||
my $aborted = $build->finished != 0 && ($build->buildstatus == 3 || $build->buildstatus == 4);
|
||||
my $aborted = $build->finished != 0 && (
|
||||
# aborted
|
||||
$build->buildstatus == 3
|
||||
# cancelled
|
||||
|| $build->buildstatus == 4
|
||||
# timeout
|
||||
|| $build->buildstatus == 7
|
||||
# log limit exceeded
|
||||
|| $build->buildstatus == 10
|
||||
);
|
||||
my $d;
|
||||
my $found = 0;
|
||||
while ($n < scalar(@{$builds2})) {
|
||||
@ -79,4 +88,4 @@ sub buildDiff {
|
||||
return $ret;
|
||||
}
|
||||
|
||||
1;
|
||||
1;
|
||||
|
||||
@ -36,6 +36,7 @@ our @EXPORT = qw(
|
||||
jobsetOverview
|
||||
jobsetOverview_
|
||||
pathIsInsidePrefix
|
||||
readIntoSocket
|
||||
readNixFile
|
||||
registerRoot
|
||||
restartBuilds
|
||||
@ -296,8 +297,7 @@ sub getEvals {
|
||||
|
||||
my @evals = $evals_result_set->search(
|
||||
{ hasnewbuilds => 1 },
|
||||
{ order_by => "$me.id DESC", rows => $rows, offset => $offset
|
||||
, prefetch => { evaluationerror => [ ] } });
|
||||
{ order_by => "$me.id DESC", rows => $rows, offset => $offset });
|
||||
my @res = ();
|
||||
my $cache = {};
|
||||
|
||||
@ -417,6 +417,16 @@ sub pathIsInsidePrefix {
|
||||
return $cur;
|
||||
}
|
||||
|
||||
sub readIntoSocket{
|
||||
my (%args) = @_;
|
||||
my $sock;
|
||||
|
||||
eval {
|
||||
open($sock, "-|", @{$args{cmd}}) or die q(failed to open socket from command:\n $x);
|
||||
};
|
||||
|
||||
return $sock;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@ -105,4 +105,6 @@ __PACKAGE__->add_column(
|
||||
"+id" => { retrieve_on_insert => 1 }
|
||||
);
|
||||
|
||||
__PACKAGE__->mk_group_accessors('column' => 'has_error');
|
||||
|
||||
1;
|
||||
|
||||
@ -386,6 +386,8 @@ __PACKAGE__->add_column(
|
||||
"+id" => { retrieve_on_insert => 1 }
|
||||
);
|
||||
|
||||
__PACKAGE__->mk_group_accessors('column' => 'has_error');
|
||||
|
||||
sub supportsDynamicRunCommand {
|
||||
my ($self) = @_;
|
||||
|
||||
|
||||
30
src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
Normal file
30
src/lib/Hydra/Schema/ResultSet/EvaluationErrors.pm
Normal file
@ -0,0 +1,30 @@
|
||||
package Hydra::Schema::ResultSet::EvaluationErrors;
|
||||
|
||||
use strict;
|
||||
use utf8;
|
||||
use warnings;
|
||||
|
||||
use parent 'DBIx::Class::ResultSet';
|
||||
|
||||
use Storable qw(dclone);
|
||||
|
||||
__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
|
||||
|
||||
# Exclude expensive error message values unless explicitly requested, and
|
||||
# replace them with a summary field describing their presence/absence.
|
||||
sub search_rs {
|
||||
my ( $class, $query, $attrs ) = @_;
|
||||
|
||||
if ($attrs) {
|
||||
$attrs = dclone($attrs);
|
||||
}
|
||||
|
||||
unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
|
||||
$attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
|
||||
}
|
||||
unless (exists $attrs->{'+columns'}->{'errormsg'}) {
|
||||
push @{ $attrs->{'remove_columns'} }, 'errormsg';
|
||||
}
|
||||
|
||||
return $class->next::method($query, $attrs);
|
||||
}
|
||||
30
src/lib/Hydra/Schema/ResultSet/Jobsets.pm
Normal file
30
src/lib/Hydra/Schema/ResultSet/Jobsets.pm
Normal file
@ -0,0 +1,30 @@
|
||||
package Hydra::Schema::ResultSet::Jobsets;
|
||||
|
||||
use strict;
|
||||
use utf8;
|
||||
use warnings;
|
||||
|
||||
use parent 'DBIx::Class::ResultSet';
|
||||
|
||||
use Storable qw(dclone);
|
||||
|
||||
__PACKAGE__->load_components('Helper::ResultSet::RemoveColumns');
|
||||
|
||||
# Exclude expensive error message values unless explicitly requested, and
|
||||
# replace them with a summary field describing their presence/absence.
|
||||
sub search_rs {
|
||||
my ( $class, $query, $attrs ) = @_;
|
||||
|
||||
if ($attrs) {
|
||||
$attrs = dclone($attrs);
|
||||
}
|
||||
|
||||
unless (exists $attrs->{'select'} || exists $attrs->{'columns'}) {
|
||||
$attrs->{'+columns'}->{'has_error'} = "errormsg != ''";
|
||||
}
|
||||
unless (exists $attrs->{'+columns'}->{'errormsg'}) {
|
||||
push @{ $attrs->{'remove_columns'} }, 'errormsg';
|
||||
}
|
||||
|
||||
return $class->next::method($query, $attrs);
|
||||
}
|
||||
@ -6,6 +6,7 @@ use base 'Catalyst::View::TT';
|
||||
use Template::Plugin::HTML;
|
||||
use Hydra::Helper::Nix;
|
||||
use Time::Seconds;
|
||||
use Digest::SHA qw(sha1_hex);
|
||||
|
||||
__PACKAGE__->config(
|
||||
TEMPLATE_EXTENSION => '.tt',
|
||||
@ -25,8 +26,14 @@ __PACKAGE__->config(
|
||||
makeNameTextForJobset
|
||||
relativeDuration
|
||||
stripSSHUser
|
||||
metricDivId
|
||||
/]);
|
||||
|
||||
sub metricDivId {
|
||||
my ($self, $c, $text) = @_;
|
||||
return "metric-" . sha1_hex($text);
|
||||
}
|
||||
|
||||
sub buildLogExists {
|
||||
my ($self, $c, $build) = @_;
|
||||
return 1 if defined $c->config->{log_prefix};
|
||||
|
||||
@ -2,8 +2,8 @@
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include "environment-variables.hh"
|
||||
#include "util.hh"
|
||||
#include <nix/util/environment-variables.hh>
|
||||
#include <nix/util/util.hh>
|
||||
|
||||
|
||||
struct Connection : pqxx::connection
|
||||
|
||||
@ -2,8 +2,8 @@
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "file-system.hh"
|
||||
#include "util.hh"
|
||||
#include <nix/util/file-system.hh>
|
||||
#include <nix/util/util.hh>
|
||||
|
||||
struct HydraConfig
|
||||
{
|
||||
|
||||
@ -61,21 +61,7 @@ END;
|
||||
<td>[% IF step.busy != 0 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
|
||||
<td class="step-status">
|
||||
[% IF step.busy != 0 %]
|
||||
[% IF step.busy == 1 %]
|
||||
<strong>Preparing</strong>
|
||||
[% ELSIF step.busy == 10 %]
|
||||
<strong>Connecting</strong>
|
||||
[% ELSIF step.busy == 20 %]
|
||||
<strong>Sending inputs</strong>
|
||||
[% ELSIF step.busy == 30 %]
|
||||
<strong>Building</strong>
|
||||
[% ELSIF step.busy == 40 %]
|
||||
<strong>Receiving outputs</strong>
|
||||
[% ELSIF step.busy == 50 %]
|
||||
<strong>Post-processing</strong>
|
||||
[% ELSE %]
|
||||
<strong>Unknown state</strong>
|
||||
[% END %]
|
||||
[% INCLUDE renderBusyStatus %]
|
||||
[% ELSIF step.status == 0 %]
|
||||
[% IF step.isnondeterministic %]
|
||||
<span class="warn">Succeeded with non-determistic result</span>
|
||||
|
||||
@ -91,6 +91,17 @@ BLOCK renderDuration;
|
||||
duration % 60 %]s[%
|
||||
END;
|
||||
|
||||
BLOCK renderDrvInfo;
|
||||
drvname = step.drvpath
|
||||
.substr(11) # strip `/nix/store/`
|
||||
.split('-').slice(1).join("-") # strip hash part
|
||||
.substr(0, -4); # strip `.drv`
|
||||
IF drvname != releasename;
|
||||
IF step.type == 0; action = "Build"; ELSE; action = "Substitution"; END;
|
||||
IF drvname; %]<em> ([% action %] of [% drvname %])</em>[% END;
|
||||
END;
|
||||
END;
|
||||
|
||||
|
||||
BLOCK renderBuildListHeader %]
|
||||
<table class="table table-striped table-condensed clickable-rows">
|
||||
@ -131,7 +142,12 @@ BLOCK renderBuildListBody;
|
||||
[% END %]
|
||||
<td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
|
||||
[% IF !hideJobName %]
|
||||
<td><a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</td>
|
||||
<td>
|
||||
<a href="[%link%]">[% IF !hideJobsetName %][%build.jobset.get_column("project")%]:[%build.jobset.get_column("name")%]:[% END %][%build.get_column("job")%]</a>
|
||||
[% IF showStepName %]
|
||||
[% INCLUDE renderDrvInfo step=build.buildsteps releasename=build.nixname %]
|
||||
[% END %]
|
||||
</td>
|
||||
[% END %]
|
||||
<td class="nowrap">[% t = showSchedulingInfo ? build.timestamp : build.stoptime; IF t; INCLUDE renderRelativeDate timestamp=(showSchedulingInfo ? build.timestamp : build.stoptime); ELSE; "-"; END %]</td>
|
||||
<td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
|
||||
@ -245,6 +261,27 @@ BLOCK renderBuildStatusIcon;
|
||||
END;
|
||||
|
||||
|
||||
BLOCK renderBusyStatus;
|
||||
IF step.busy == 1 %]
|
||||
<strong>Preparing</strong>
|
||||
[% ELSIF step.busy == 10 %]
|
||||
<strong>Connecting</strong>
|
||||
[% ELSIF step.busy == 20 %]
|
||||
<strong>Sending inputs</strong>
|
||||
[% ELSIF step.busy == 30 %]
|
||||
<strong>Building</strong>
|
||||
[% ELSIF step.busy == 35 %]
|
||||
<strong>Waiting to receive outputs</strong>
|
||||
[% ELSIF step.busy == 40 %]
|
||||
<strong>Receiving outputs</strong>
|
||||
[% ELSIF step.busy == 50 %]
|
||||
<strong>Post-processing</strong>
|
||||
[% ELSE %]
|
||||
<strong>Unknown state</strong>
|
||||
[% END;
|
||||
END;
|
||||
|
||||
|
||||
BLOCK renderStatus;
|
||||
IF build.finished;
|
||||
buildstatus = build.buildstatus;
|
||||
@ -476,7 +513,7 @@ BLOCK renderEvals %]
|
||||
ELSE %]
|
||||
-
|
||||
[% END %]
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
[% IF eval.evaluationerror.has_error %]
|
||||
<span class="badge badge-warning">Eval Errors</span>
|
||||
[% END %]
|
||||
</td>
|
||||
@ -602,7 +639,7 @@ BLOCK renderJobsetOverview %]
|
||||
<td>[% HTML.escape(j.description) %]</td>
|
||||
<td>[% IF j.lastcheckedtime;
|
||||
INCLUDE renderDateTime timestamp = j.lastcheckedtime;
|
||||
IF j.errormsg || j.fetcherrormsg; %] <span class = 'badge badge-warning'>Error</span>[% END;
|
||||
IF j.has_error || j.fetcherrormsg; %] <span class = 'badge badge-warning'>Error</span>[% END;
|
||||
ELSE; "-";
|
||||
END %]</td>
|
||||
[% IF j.get_column('nrtotal') > 0 %]
|
||||
|
||||
26
src/root/eval-error.tt
Normal file
26
src/root/eval-error.tt
Normal file
@ -0,0 +1,26 @@
|
||||
[% PROCESS common.tt %]
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
[% INCLUDE style.tt %]
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<div class="tab-content tab-pane">
|
||||
<div id="tabs-errors" class="">
|
||||
[% IF jobset %]
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
|
||||
[% ELSIF eval %]
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
|
||||
[% END %]
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@ -18,8 +18,7 @@
|
||||
|
||||
<h3>Metric: <a [% HTML.attributes(href => c.uri_for('/job' project.name jobset.name job 'metric' metric.name)) %]><tt>[%HTML.escape(metric.name)%]</tt></a></h3>
|
||||
|
||||
[% id = "metric-" _ metric.name;
|
||||
id = id.replace('\.', '_');
|
||||
[% id = metricDivId(metric.name);
|
||||
INCLUDE createChart dataUrl=c.uri_for('/job' project.name jobset.name job 'metric' metric.name); %]
|
||||
|
||||
[% END %]
|
||||
|
||||
@ -65,7 +65,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
[% END %]
|
||||
|
||||
[% IF aborted.size > 0 %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted Jobs ([% aborted.size %])</span></a></li>
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-aborted" data-toggle="tab"><span class="text-warning">Aborted / Timed out Jobs ([% aborted.size %])</span></a></li>
|
||||
[% END %]
|
||||
[% IF nowFail.size > 0 %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-now-fail" data-toggle="tab"><span class="text-warning">Newly Failing Jobs ([% nowFail.size %])</span></a></li>
|
||||
@ -90,7 +90,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
[% END %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-inputs" data-toggle="tab">Inputs</a></li>
|
||||
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
[% IF eval.evaluationerror.has_error %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
|
||||
[% END %]
|
||||
</ul>
|
||||
@ -108,13 +108,6 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
|
||||
<div class="tab-content">
|
||||
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
<div id="tabs-errors" class="tab-pane">
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
|
||||
</div>
|
||||
[% END %]
|
||||
|
||||
<div id="tabs-aborted" class="tab-pane">
|
||||
[% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
|
||||
</div>
|
||||
@ -172,10 +165,9 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
|
||||
[% END %]
|
||||
</div>
|
||||
|
||||
[% IF eval.evaluationerror.errormsg %]
|
||||
[% IF eval.evaluationerror.has_error %]
|
||||
<div id="tabs-errors" class="tab-pane">
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(eval.evaluationerror.errormsg) %]</pre></div></div>
|
||||
<iframe src="[% c.uri_for(c.controller('JobsetEval').action_for('errors'), [eval.id], params) %]" loading="lazy" frameBorder="0" width="100%"></iframe>
|
||||
</div>
|
||||
[% END %]
|
||||
</div>
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
[% END %]
|
||||
|
||||
<li class="nav-item"><a class="nav-link active" href="#tabs-evaluations" data-toggle="tab">Evaluations</a></li>
|
||||
[% IF jobset.errormsg || jobset.fetcherrormsg %]
|
||||
[% IF jobset.has_error || jobset.fetcherrormsg %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation Errors</span></a></li>
|
||||
[% END %]
|
||||
<li class="nav-item"><a class="nav-link" href="#tabs-jobs" data-toggle="tab">Jobs</a></li>
|
||||
@ -79,7 +79,7 @@
|
||||
<th>Last checked:</th>
|
||||
<td>
|
||||
[% IF jobset.lastcheckedtime %]
|
||||
[% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.errormsg || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
|
||||
[% INCLUDE renderDateTime timestamp = jobset.lastcheckedtime %], [% IF jobset.has_error || jobset.fetcherrormsg %]<em class="text-warning">with errors!</em>[% ELSE %]<em>no errors</em>[% END %]
|
||||
[% ELSE %]
|
||||
<em>never</em>
|
||||
[% END %]
|
||||
@ -117,10 +117,9 @@
|
||||
|
||||
</div>
|
||||
|
||||
[% IF jobset.errormsg || jobset.fetcherrormsg %]
|
||||
[% IF jobset.has_error || jobset.fetcherrormsg %]
|
||||
<div id="tabs-errors" class="tab-pane">
|
||||
<p>Errors occurred at [% INCLUDE renderDateTime timestamp=(jobset.errortime || jobset.lastcheckedtime) %].</p>
|
||||
<div class="card bg-light"><div class="card-body"><pre>[% HTML.escape(jobset.fetcherrormsg || jobset.errormsg) %]</pre></div></div>
|
||||
<iframe src="[% c.uri_for('/jobset' project.name jobset.name "errors") %]" loading="lazy" frameBorder="0" width="100%"></iframe>
|
||||
</div>
|
||||
[% END %]
|
||||
|
||||
|
||||
@ -10,31 +10,7 @@
|
||||
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
|
||||
<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
|
||||
<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
|
||||
|
||||
<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
|
||||
|
||||
<style>
|
||||
.popover { max-width: 40%; }
|
||||
</style>
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
|
||||
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
|
||||
[% INCLUDE style.tt %]
|
||||
|
||||
[% IF c.config.enable_google_login %]
|
||||
<meta name="google-signin-client_id" content="[% c.config.google_client_id %]">
|
||||
|
||||
@ -6,10 +6,10 @@
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Job</th>
|
||||
<th>System</th>
|
||||
<th>Build</th>
|
||||
<th>Step</th>
|
||||
<th>What</th>
|
||||
<th>Status</th>
|
||||
<th>Since</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@ -40,10 +40,10 @@
|
||||
[% idle = 0 %]
|
||||
<tr>
|
||||
<td><tt>[% INCLUDE renderFullJobName project=step.project jobset=step.jobset job=step.job %]</tt></td>
|
||||
<td><tt>[% step.system %]</tt></td>
|
||||
<td><a href="[% c.uri_for('/build' step.build) %]">[% step.build %]</a></td>
|
||||
<td>[% IF step.busy >= 30 %]<a class="row-link" href="[% c.uri_for('/build' step.build 'nixlog' step.stepnr 'tail') %]">[% step.stepnr %]</a>[% ELSE; step.stepnr; END %]</td>
|
||||
<td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
|
||||
<td>[% INCLUDE renderBusyStatus %]</td>
|
||||
<td style="width: 10em">[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
|
||||
</tr>
|
||||
[% END %]
|
||||
|
||||
@ -129,6 +129,12 @@ $(document).ready(function() {
|
||||
el.addClass("is-local");
|
||||
}
|
||||
});
|
||||
|
||||
[...document.getElementsByTagName("iframe")].forEach((element) => {
|
||||
element.contentWindow.addEventListener("DOMContentLoaded", (_) => {
|
||||
element.style.height = element.contentWindow.document.body.scrollHeight + 'px';
|
||||
})
|
||||
})
|
||||
});
|
||||
|
||||
var tabsLoaded = {};
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
|
||||
[% ELSE %]
|
||||
|
||||
[% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
|
||||
[% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 showStepName=1 %]
|
||||
|
||||
[% END %]
|
||||
|
||||
|
||||
24
src/root/style.tt
Normal file
24
src/root/style.tt
Normal file
@ -0,0 +1,24 @@
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-3.4.1.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/jquery/jquery-ui-1.10.4.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/moment/moment-2.24.0.min.js") %]"></script>
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
|
||||
<link href="[% c.uri_for("/static/fontawesome/css/all.css") %]" rel="stylesheet" />
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/popper.min.js") %]"></script>
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/bootstrap/js/bootstrap.min.js") %]"></script>
|
||||
<link href="[% c.uri_for("/static/bootstrap/css/bootstrap.min.css") %]" rel="stylesheet" />
|
||||
|
||||
<!-- hydra.css may need to be moved to before boostrap to make the @media rule work. -->
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/hydra.css") %]" type="text/css" />
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/rotated-th.css") %]" type="text/css" />
|
||||
|
||||
<style>
|
||||
.popover { max-width: 40%; }
|
||||
</style>
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/bootbox.min.js") %]"></script>
|
||||
|
||||
<link rel="stylesheet" href="[% c.uri_for("/static/css/tree.css") %]" type="text/css" />
|
||||
|
||||
<script type="text/javascript" src="[% c.uri_for("/static/js/common.js") %]"></script>
|
||||
@ -773,6 +773,9 @@ sub checkJobsetWrapped {
|
||||
my $jobsetChanged = 0;
|
||||
my %buildMap;
|
||||
|
||||
my @jobs;
|
||||
push @jobs, $_ while defined($_ = $jobsIter->());
|
||||
|
||||
$db->txn_do(sub {
|
||||
my $prevEval = getPrevJobsetEval($db, $jobset, 1);
|
||||
|
||||
@ -796,7 +799,7 @@ sub checkJobsetWrapped {
|
||||
|
||||
my @jobsWithConstituents;
|
||||
|
||||
while (defined(my $job = $jobsIter->())) {
|
||||
foreach my $job (@jobs) {
|
||||
if ($jobsetsJobset) {
|
||||
die "The .jobsets jobset must only have a single job named 'jobsets'"
|
||||
unless $job->{attr} eq "jobsets";
|
||||
|
||||
@ -32,4 +32,9 @@ subtest "/jobset/PROJECT/JOBSET/evals" => sub {
|
||||
ok($jobsetevals->is_success, "The page showing the jobset evals returns 200.");
|
||||
};
|
||||
|
||||
subtest "/jobset/PROJECT/JOBSET/errors" => sub {
|
||||
my $jobsetevals = request(GET '/jobset/' . $project->name . '/' . $jobset->name . '/errors');
|
||||
ok($jobsetevals->is_success, "The page showing the jobset eval errors returns 200.");
|
||||
};
|
||||
|
||||
done_testing;
|
||||
|
||||
@ -35,6 +35,10 @@ subtest "Fetching the eval's overview" => sub {
|
||||
is($fetch->code, 200, "channel page is 200");
|
||||
};
|
||||
|
||||
subtest "Fetching the eval's overview" => sub {
|
||||
my $fetch = request(GET '/eval/' . $eval->id, '/errors');
|
||||
is($fetch->code, 200, "errors page is 200");
|
||||
};
|
||||
|
||||
|
||||
done_testing;
|
||||
|
||||
@ -6,27 +6,55 @@ use Hydra::Helper::Exec;
|
||||
|
||||
my $ctx = test_context();
|
||||
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-broken.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
subtest "broken constituents expression" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-broken.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
isnt($res, 0, "hydra-eval-jobset exits non-zero");
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
like(
|
||||
$stderr,
|
||||
qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
|
||||
"The stderr record includes a relevant error message"
|
||||
);
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
isnt($res, 0, "hydra-eval-jobset exits non-zero");
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
like(
|
||||
$stderr,
|
||||
qr/aggregate job 'mixed_aggregate' references non-existent job 'constituentA'/,
|
||||
"The stderr record includes a relevant error message"
|
||||
);
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘mixed_aggregate’ failed with the error: "constituentA": does not exist/,
|
||||
"The jobset records a relevant error message"
|
||||
);
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
|
||||
"The jobset records a relevant error message"
|
||||
);
|
||||
};
|
||||
|
||||
subtest "no matches" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-no-matches.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
isnt($res, 0, "hydra-eval-jobset exits non-zero");
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
like(
|
||||
$stderr,
|
||||
qr/aggregate job 'non_match_aggregate' references constituent glob pattern 'tests\.\*' with no matches/,
|
||||
"The stderr record includes a relevant error message"
|
||||
);
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘non_match_aggregate’ failed with the error: tests\.\*: constituent glob pattern had no matches/,
|
||||
qr/in job ‘non_match_aggregate’:\ntests\.\*: constituent glob pattern had no matches/,
|
||||
"The jobset records a relevant error message"
|
||||
);
|
||||
};
|
||||
|
||||
done_testing;
|
||||
|
||||
138
t/evaluator/evaluate-constituents-globbing.t
Normal file
138
t/evaluator/evaluate-constituents-globbing.t
Normal file
@ -0,0 +1,138 @@
|
||||
use strict;
|
||||
use warnings;
|
||||
use Setup;
|
||||
use Test2::V0;
|
||||
use Hydra::Helper::Exec;
|
||||
use Data::Dumper;
|
||||
|
||||
my $ctx = test_context();
|
||||
|
||||
subtest "general glob testing" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-glob.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
is($res, 0, "hydra-eval-jobset exits zero");
|
||||
|
||||
my $builds = {};
|
||||
for my $build ($jobset->builds) {
|
||||
$builds->{$build->job} = $build;
|
||||
}
|
||||
|
||||
subtest "basic globbing works" => sub {
|
||||
ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
|
||||
my @constituents = $builds->{"ok_aggregate"}->constituents->all;
|
||||
is(2, scalar @constituents, "'ok_aggregate' has two constituents");
|
||||
|
||||
my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
|
||||
|
||||
is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
|
||||
is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
|
||||
};
|
||||
|
||||
subtest "transitivity is OK" => sub {
|
||||
ok(defined $builds->{"indirect_aggregate"}, "'indirect_aggregate' is part of the jobset evaluation");
|
||||
my @constituents = $builds->{"indirect_aggregate"}->constituents->all;
|
||||
is(1, scalar @constituents, "'indirect_aggregate' has one constituent");
|
||||
is($constituents[0]->nixname, "direct_aggregate", "'indirect_aggregate' has 'direct_aggregate' as single constituent");
|
||||
};
|
||||
};
|
||||
|
||||
subtest "* selects all except current aggregate" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-glob-all.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
|
||||
subtest "no eval errors" => sub {
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
ok(
|
||||
$stderr !~ "aggregate job ‘ok_aggregate’ has a constituent .* that doesn't correspond to a Hydra build",
|
||||
"Catchall wildcard must not select itself as constituent"
|
||||
);
|
||||
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
is(
|
||||
$jobset->has_error,
|
||||
0,
|
||||
"eval-errors non-empty"
|
||||
);
|
||||
};
|
||||
|
||||
my $builds = {};
|
||||
for my $build ($jobset->builds) {
|
||||
$builds->{$build->job} = $build;
|
||||
}
|
||||
|
||||
subtest "two constituents" => sub {
|
||||
ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
|
||||
my @constituents = $builds->{"ok_aggregate"}->constituents->all;
|
||||
is(2, scalar @constituents, "'ok_aggregate' has two constituents");
|
||||
|
||||
my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
|
||||
|
||||
is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
|
||||
is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
|
||||
};
|
||||
};
|
||||
|
||||
subtest "trivial cycle check" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-cycle.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
|
||||
ok(
|
||||
$stderr =~ "Found dependency cycle between jobs 'indirect_aggregate' and 'ok_aggregate'",
|
||||
"Dependency cycle error is on stderr"
|
||||
);
|
||||
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/Dependency cycle: indirect_aggregate <-> ok_aggregate/,
|
||||
"eval-errors non-empty"
|
||||
);
|
||||
|
||||
is(0, $jobset->builds->count, "No builds should be scheduled");
|
||||
};
|
||||
|
||||
subtest "cycle check with globbing" => sub {
|
||||
my $jobsetCtx = $ctx->makeJobset(
|
||||
expression => 'constituents-cycle-glob.nix',
|
||||
);
|
||||
my $jobset = $jobsetCtx->{"jobset"};
|
||||
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60,
|
||||
("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
|
||||
);
|
||||
|
||||
ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
|
||||
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/aggregate job ‘indirect_aggregate’ failed with the error: Dependency cycle: indirect_aggregate <-> packages.constituentA/,
|
||||
"packages.constituentA error missing"
|
||||
);
|
||||
|
||||
# on this branch of Hydra, hydra-eval-jobset fails hard if an aggregate
|
||||
# job is broken.
|
||||
is(0, $jobset->builds->count, "Zero jobs are scheduled");
|
||||
};
|
||||
|
||||
done_testing;
|
||||
14
t/jobs/config.nix
Normal file
14
t/jobs/config.nix
Normal file
@ -0,0 +1,14 @@
|
||||
rec {
|
||||
path = "/nix/store/l9mg93sgx50y88p5rr6x1vib6j1rjsds-coreutils-9.1/bin";
|
||||
|
||||
mkDerivation = args:
|
||||
derivation ({
|
||||
system = builtins.currentSystem;
|
||||
PATH = path;
|
||||
} // args);
|
||||
mkContentAddressedDerivation = args: mkDerivation ({
|
||||
__contentAddressed = true;
|
||||
outputHashMode = "recursive";
|
||||
outputHashAlgo = "sha256";
|
||||
} // args);
|
||||
}
|
||||
34
t/jobs/constituents-cycle-glob.nix
Normal file
34
t/jobs/constituents-cycle-glob.nix
Normal file
@ -0,0 +1,34 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
packages.constituentA = mkDerivation {
|
||||
name = "empty-dir-A";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [ "*_aggregate" ];
|
||||
};
|
||||
|
||||
packages.constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"packages.*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
indirect_aggregate = mkDerivation {
|
||||
name = "indirect_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"ok_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
21
t/jobs/constituents-cycle.nix
Normal file
21
t/jobs/constituents-cycle.nix
Normal file
@ -0,0 +1,21 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"indirect_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
indirect_aggregate = mkDerivation {
|
||||
name = "indirect_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"ok_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
22
t/jobs/constituents-glob-all.nix
Normal file
22
t/jobs/constituents-glob-all.nix
Normal file
@ -0,0 +1,22 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
packages.constituentA = mkDerivation {
|
||||
name = "empty-dir-A";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
packages.constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
31
t/jobs/constituents-glob.nix
Normal file
31
t/jobs/constituents-glob.nix
Normal file
@ -0,0 +1,31 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
packages.constituentA = mkDerivation {
|
||||
name = "empty-dir-A";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
packages.constituentB = mkDerivation {
|
||||
name = "empty-dir-B";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
ok_aggregate = mkDerivation {
|
||||
name = "direct_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"packages.*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
indirect_aggregate = mkDerivation {
|
||||
name = "indirect_aggregate";
|
||||
_hydraAggregate = true;
|
||||
constituents = [
|
||||
"ok_aggregate"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
20
t/jobs/constituents-no-matches.nix
Normal file
20
t/jobs/constituents-no-matches.nix
Normal file
@ -0,0 +1,20 @@
|
||||
with import ./config.nix;
|
||||
{
|
||||
non_match_aggregate = mkDerivation {
|
||||
name = "mixed_aggregate";
|
||||
_hydraAggregate = true;
|
||||
_hydraGlobConstituents = true;
|
||||
constituents = [
|
||||
"tests.*"
|
||||
];
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
|
||||
# Without a second job no jobset is attempted to be created
|
||||
# (the only job would be broken)
|
||||
# and thus the constituent validation is never reached.
|
||||
dummy = mkDerivation {
|
||||
name = "dummy";
|
||||
builder = ./empty-dir-builder.sh;
|
||||
};
|
||||
}
|
||||
24
t/jobs/declarative/project.json
Normal file
24
t/jobs/declarative/project.json
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"enabled": 1,
|
||||
"hidden": false,
|
||||
"description": "declarative-jobset-example",
|
||||
"nixexprinput": "src",
|
||||
"nixexprpath": "declarative/generator.nix",
|
||||
"checkinterval": 300,
|
||||
"schedulingshares": 100,
|
||||
"enableemail": false,
|
||||
"emailoverride": "",
|
||||
"keepnr": 3,
|
||||
"inputs": {
|
||||
"src": {
|
||||
"type": "path",
|
||||
"value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
|
||||
"emailresponsible": false
|
||||
},
|
||||
"jobspath": {
|
||||
"type": "string",
|
||||
"value": "/home/ma27/Projects/hydra-cppnix/t/jobs",
|
||||
"emailresponsible": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,7 +14,7 @@ our @EXPORT = qw(
|
||||
sub evalSucceeds {
|
||||
my ($jobset) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
if ($res) {
|
||||
chomp $stdout; chomp $stderr;
|
||||
utf8::decode($stdout) or die "Invalid unicode in stdout.";
|
||||
@ -29,7 +29,7 @@ sub evalSucceeds {
|
||||
sub evalFails {
|
||||
my ($jobset) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
$jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB
|
||||
if (!$res) {
|
||||
chomp $stdout; chomp $stderr;
|
||||
utf8::decode($stdout) or die "Invalid unicode in stdout.";
|
||||
|
||||
@ -22,11 +22,11 @@ is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix
|
||||
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/^"does-not-exist": does not exist$/m,
|
||||
qr/^does-not-exist: does not exist$/m,
|
||||
"Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
|
||||
like(
|
||||
$jobset->errormsg,
|
||||
qr/^"does-not-evaluate": "error: assertion 'false' failed/m,
|
||||
qr/^does-not-evaluate: error: assertion 'false' failed/m,
|
||||
"Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
|
||||
|
||||
done_testing;
|
||||
|
||||
@ -13,7 +13,7 @@ my $constituentBuildA = $builds->{"constituentA"};
|
||||
my $constituentBuildB = $builds->{"constituentB"};
|
||||
|
||||
my $eval = $constituentBuildA->jobsetevals->first();
|
||||
is($eval->evaluationerror->errormsg, "");
|
||||
is($eval->evaluationerror->has_error, 0);
|
||||
|
||||
subtest "Verifying the direct aggregate" => sub {
|
||||
my $aggBuild = $builds->{"direct_aggregate"};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user