d84ff32ce6
incrementally ingest eval results nix-eval-jobs streams output, unlike hydra-eval-jobs. Now that we've migrated, we can use this to: 1. Use less RAM by avoiding buffering a whole eval's worth of metadata into a Perl string and an array of JSON objects. 2. Make evals latency a bit lower by allowing the queue runner to start ingesting builds faster. Also use the newly-restored constituents support in `nix-eval-jobs` Note, we pass --workers and --max-memory-size to n-e-j Lost in the h-e-j -> n-e-j migration, causing evaluation to always be single threaded and limited to 4GiB RAM. Follow the config settings like h-e-j used to do (via C++ code). `nix-eval-jobs` should check `hydraJobs` and then `checks` with flakes (cherry picked from commit 6d4ccff43c41adaf6e4b2b9bced7243bc2f6e97b) (cherry picked from commit b0e9b4b2f99f9d8f5c4e780e89f955c394b5ced4) (cherry picked from commit cdfc5c81e8037d3e4818a3e459d0804b2c157ea9) (cherry picked from commit 4b107e6ff36bd89958fba36e0fe0340903e7cd13) Co-Authored-By: Maximilian Bosch <maximilian@mbosch.me>
278 lines
4.8 KiB
Nix
278 lines
4.8 KiB
Nix
{ stdenv
|
|
, lib
|
|
, fileset
|
|
|
|
, rawSrc
|
|
|
|
, buildEnv
|
|
|
|
, perlPackages
|
|
|
|
, nix
|
|
, nix-perl-bindings
|
|
, git
|
|
|
|
, makeWrapper
|
|
, meson
|
|
, ninja
|
|
, nukeReferences
|
|
, pkg-config
|
|
, mdbook
|
|
|
|
, unzip
|
|
, libpqxx
|
|
, top-git
|
|
, mercurial
|
|
, darcs
|
|
, subversion
|
|
, breezy
|
|
, openssl
|
|
, bzip2
|
|
, libxslt
|
|
, perl
|
|
, pixz
|
|
, boost
|
|
, postgresql_13
|
|
, nlohmann_json
|
|
, prometheus-cpp
|
|
|
|
, cacert
|
|
, foreman
|
|
, glibcLocales
|
|
, libressl
|
|
, openldap
|
|
, python3
|
|
|
|
, openssh
|
|
, coreutils
|
|
, findutils
|
|
, gzip
|
|
, xz
|
|
, gnutar
|
|
, gnused
|
|
, nix-eval-jobs
|
|
|
|
, rpm
|
|
, dpkg
|
|
, cdrkit
|
|
}:
|
|
|
|
let
|
|
perlDeps = buildEnv {
|
|
name = "hydra-perl-deps";
|
|
paths = lib.closePropagation
|
|
([
|
|
nix-perl-bindings
|
|
git
|
|
] ++ (with perlPackages; [
|
|
AuthenSASL
|
|
CatalystActionREST
|
|
CatalystAuthenticationStoreDBIxClass
|
|
CatalystAuthenticationStoreLDAP
|
|
CatalystDevel
|
|
CatalystPluginAccessLog
|
|
CatalystPluginAuthorizationRoles
|
|
CatalystPluginCaptcha
|
|
CatalystPluginPrometheusTiny
|
|
CatalystPluginSessionStateCookie
|
|
CatalystPluginSessionStoreFastMmap
|
|
CatalystPluginStackTrace
|
|
CatalystTraitForRequestProxyBase
|
|
CatalystViewDownload
|
|
CatalystViewJSON
|
|
CatalystViewTT
|
|
CatalystXRoleApplicator
|
|
CatalystXScriptServerStarman
|
|
CryptPassphrase
|
|
CryptPassphraseArgon2
|
|
CryptRandPasswd
|
|
DataDump
|
|
DateTime
|
|
DBDPg
|
|
DBDSQLite
|
|
DigestSHA1
|
|
EmailMIME
|
|
EmailSender
|
|
FileCopyRecursive
|
|
FileLibMagic
|
|
FileSlurper
|
|
FileWhich
|
|
IOCompress
|
|
IPCRun
|
|
IPCRun3
|
|
JSON
|
|
JSONMaybeXS
|
|
JSONXS
|
|
ListSomeUtils
|
|
LWP
|
|
LWPProtocolHttps
|
|
ModulePluggable
|
|
NetAmazonS3
|
|
NetPrometheus
|
|
NetStatsd
|
|
PadWalker
|
|
ParallelForkManager
|
|
PerlCriticCommunity
|
|
PrometheusTinyShared
|
|
ReadonlyX
|
|
SetScalar
|
|
SQLSplitStatement
|
|
Starman
|
|
StringCompareConstantTime
|
|
SysHostnameLong
|
|
TermSizeAny
|
|
TermReadKey
|
|
Test2Harness
|
|
TestPostgreSQL
|
|
TextDiff
|
|
TextTable
|
|
UUID4Tiny
|
|
YAML
|
|
XMLSimple
|
|
]));
|
|
};
|
|
|
|
version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}";
|
|
in
|
|
stdenv.mkDerivation (finalAttrs: {
|
|
pname = "hydra";
|
|
inherit version;
|
|
|
|
src = fileset.toSource {
|
|
root = ./.;
|
|
fileset = fileset.unions ([
|
|
./doc
|
|
./meson.build
|
|
./nixos-modules
|
|
./src
|
|
./t
|
|
./version.txt
|
|
./.perlcriticrc
|
|
]);
|
|
};
|
|
|
|
outputs = [ "out" "doc" ];
|
|
|
|
strictDeps = true;
|
|
|
|
nativeBuildInputs = [
|
|
makeWrapper
|
|
meson
|
|
ninja
|
|
nukeReferences
|
|
pkg-config
|
|
mdbook
|
|
nix
|
|
perlDeps
|
|
perl
|
|
unzip
|
|
];
|
|
|
|
buildInputs = [
|
|
libpqxx
|
|
openssl
|
|
libxslt
|
|
nix
|
|
perlDeps
|
|
perl
|
|
boost
|
|
nlohmann_json
|
|
prometheus-cpp
|
|
];
|
|
|
|
nativeCheckInputs = [
|
|
bzip2
|
|
darcs
|
|
foreman
|
|
top-git
|
|
mercurial
|
|
subversion
|
|
breezy
|
|
openldap
|
|
postgresql_13
|
|
pixz
|
|
nix-eval-jobs
|
|
];
|
|
|
|
checkInputs = [
|
|
cacert
|
|
glibcLocales
|
|
libressl.nc
|
|
python3
|
|
];
|
|
|
|
hydraPath = lib.makeBinPath (
|
|
[
|
|
subversion
|
|
openssh
|
|
nix
|
|
coreutils
|
|
findutils
|
|
pixz
|
|
gzip
|
|
bzip2
|
|
xz
|
|
gnutar
|
|
unzip
|
|
git
|
|
top-git
|
|
mercurial
|
|
darcs
|
|
gnused
|
|
breezy
|
|
nix-eval-jobs
|
|
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
|
|
);
|
|
|
|
OPENLDAP_ROOT = openldap;
|
|
|
|
mesonBuildType = "release";
|
|
|
|
postPatch = ''
|
|
patchShebangs .
|
|
'';
|
|
|
|
shellHook = ''
|
|
pushd $(git rev-parse --show-toplevel) >/dev/null
|
|
|
|
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
|
|
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
|
|
export HYDRA_HOME="$(pwd)/src/"
|
|
mkdir -p .hydra-data
|
|
export HYDRA_DATA="$(pwd)/.hydra-data"
|
|
export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
|
|
|
|
popd >/dev/null
|
|
'';
|
|
|
|
doCheck = true;
|
|
|
|
mesonCheckFlags = [ "--verbose" ];
|
|
|
|
preCheck = ''
|
|
export LOGNAME=''${LOGNAME:-foo}
|
|
# set $HOME for bzr so it can create its trace file
|
|
export HOME=$(mktemp -d)
|
|
'';
|
|
|
|
postInstall = ''
|
|
mkdir -p $out/nix-support
|
|
|
|
for i in $out/bin/*; do
|
|
read -n 4 chars < $i
|
|
if [[ $chars =~ ELF ]]; then continue; fi
|
|
wrapProgram $i \
|
|
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
|
|
--prefix PATH ':' $out/bin:$hydraPath \
|
|
--set HYDRA_RELEASE ${version} \
|
|
--set HYDRA_HOME $out/libexec/hydra \
|
|
--set NIX_RELEASE ${nix.name or "unknown"}
|
|
done
|
|
'';
|
|
|
|
dontStrip = true;
|
|
|
|
meta.description = "Build of Hydra on ${stdenv.system}";
|
|
passthru = { inherit perlDeps nix; };
|
|
})
|