From 5dad72daa450098d364e6a8d81e099fa21308b7c Mon Sep 17 00:00:00 2001 From: ahuston-0 Date: Sun, 3 May 2026 12:34:42 -0400 Subject: [PATCH] add otel and honeycomb --- .vscode/mcp.json | 14 ++- systems/palatine-hill/configuration.nix | 1 + systems/palatine-hill/hydra.nix | 35 ++++++- systems/palatine-hill/otel.nix | 117 ++++++++++++++++++++++++ systems/palatine-hill/secrets.yaml | 6 +- users/alice/secrets.yaml | 8 +- 6 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 systems/palatine-hill/otel.nix diff --git a/.vscode/mcp.json b/.vscode/mcp.json index 0967ef4..5955f97 100644 --- a/.vscode/mcp.json +++ b/.vscode/mcp.json @@ -1 +1,13 @@ -{} +{ + "servers": { + "honeycomb": { + "command": "npx", + "args": [ + "mcp-remote", + "https://mcp.honeycomb.io/mcp" + ], + "type": "stdio" + } + }, + "inputs": [] +} diff --git a/systems/palatine-hill/configuration.nix b/systems/palatine-hill/configuration.nix index 820c23a..c37749d 100644 --- a/systems/palatine-hill/configuration.nix +++ b/systems/palatine-hill/configuration.nix @@ -19,6 +19,7 @@ ./minio.nix ./networking.nix ./nextcloud.nix + ./otel.nix #./plex ./postgresql.nix ./samba.nix diff --git a/systems/palatine-hill/hydra.nix b/systems/palatine-hill/hydra.nix index 6dd66e1..1f45eb5 100644 --- a/systems/palatine-hill/hydra.nix +++ b/systems/palatine-hill/hydra.nix @@ -6,6 +6,8 @@ let hydra_notify_prometheus_port = "9199"; hydra_queue_runner_prometheus_port = "9200"; + postgres_exporter_port = 9187; + zfs_exporter_port = 9134; in { systemd.services.hydra-notify.serviceConfig.EnvironmentFile = @@ -96,10 +98,23 @@ in enable = true; webExternalUrl = "https://prom.alicehuston.xyz"; port = 9001; - exporters.node = { - enable = true; - enabledCollectors = [ "systemd" ]; - port = 9002; + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9002; + }; + postgres = { + enable = true; + listenAddress = "127.0.0.1"; + port = postgres_exporter_port; + runAsLocalSuperUser = true; + }; + zfs = { + enable = true; + listenAddress = "127.0.0.1"; + port = zfs_exporter_port; + }; }; scrapeConfigs = [ { @@ -119,6 +134,18 @@ in } ]; } + { + job_name = "postgres-local"; + static_configs = [ + { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ]; } + ]; + } + { + job_name = "zfs-local"; + static_configs = [ + { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.zfs.port}" ]; } + ]; + } { job_name = "hydra-external"; scheme = "https"; diff --git a/systems/palatine-hill/otel.nix b/systems/palatine-hill/otel.nix new file mode 100644 index 0000000..fe9cd02 --- /dev/null +++ b/systems/palatine-hill/otel.nix @@ -0,0 +1,117 @@ +{ + config, + ... +}: +{ + # node_exporter (port 9002) and Prometheus (port 9001) are already configured + # in hydra.nix — we just scrape the existing exporter here. + + services.opentelemetry-collector = { + enable = true; + settings = { + receivers = { + # Host-level system metrics + hostmetrics = { + collection_interval = "60s"; + scrapers = { + cpu = { }; + memory = { }; + disk = { }; + filesystem = { }; + network = { }; + load = { }; + processes = { }; + }; + }; + + # Scrape node_exporter for per-systemd-unit service state + prometheus = { + config = { + scrape_configs = [ + { + job_name = "node-exporter"; + scrape_interval = "60s"; + static_configs = [ + { + targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; + labels = { + host = "palatine-hill"; + }; + } + ]; + } + { + job_name = "postgres-exporter"; + scrape_interval = "60s"; + static_configs = [ + { + targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ]; + labels = { + host = "palatine-hill"; + }; + } + ]; + } + { + job_name = "zfs-exporter"; + scrape_interval = "60s"; + static_configs = [ + { + targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ]; + labels = { + host = "palatine-hill"; + }; + } + ]; + } + ]; + }; + }; + }; + + processors = { + batch = { }; + + # Attach hostname and other system resource attributes + "resourcedetection/system" = { + detectors = [ "system" ]; + system.hostname_sources = [ "os" ]; + }; + }; + + exporters = { + "otlp/honeycomb" = { + endpoint = "api.honeycomb.io:443"; + headers = { + # Expanded at runtime from the environment file + "x-honeycomb-team" = "\${HONEYCOMB_API_KEY}"; + }; + }; + }; + + service = { + pipelines = { + metrics = { + receivers = [ + "hostmetrics" + "prometheus" + ]; + processors = [ + "resourcedetection/system" + "batch" + ]; + exporters = [ "otlp/honeycomb" ]; + }; + }; + }; + }; + }; + + # Inject the Honeycomb API key at runtime — never stored in the Nix store + systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = + config.sops.secrets."honeycomb/api-key".path; + + sops.secrets = { + "honeycomb/api-key".owner = "root"; + }; +} diff --git a/systems/palatine-hill/secrets.yaml b/systems/palatine-hill/secrets.yaml index 3be1671..8a11354 100644 --- a/systems/palatine-hill/secrets.yaml +++ b/systems/palatine-hill/secrets.yaml @@ -45,6 +45,8 @@ typhon: garage: rpc-secret: ENC[AES256_GCM,data:Q2ZaAXcntD3yK6DynEpxab2TITByMZ7ECVrq1pb0ZU7hXOZnhaBmjdty/Os6len8l+GBl6+WaC0An6cFkhQTlQ==,iv:E8C4bnxMLXK9fky+KC7q8sHpmrEU5un0TEAwxVUBiLk=,tag:PiSiU+9NpyilH2aMs2Qc/Q==,type:str] admin-token: ENC[AES256_GCM,data:Xjm8Xq99aDseR0jN50Uj3gLpeDaq2IGXzJCS0o1H0RgKX9LGdP8w508nWWE=,iv:+L9T3TEUSbIz+jo08ykjGHVhuz5ecmzrlhzD2iv48HE=,tag:7P2rY4F8cWFdG4Lm9n/etQ==,type:str] +honeycomb: + api-key: ENC[AES256_GCM,data:k+Z3tmF8pYwD6RokdZauQ/fMlhD5GbW1ekxzRnj0gEpQFlfGB8gQ6BPwbd7qk12ZhsCA4XdqmvsysAWJldYWrPo=,iv:dNAK/vbQYL7ir4UXhZMTWraZF1E6ps9EOF3skYe5wOk=,tag:5UJfCji/RmB5DIuR4179uw==,type:str] sops: age: - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh @@ -56,8 +58,8 @@ sops: cXNZWmZqd0R0SmhINExscHBKWmxvblUKEFEQvt/zQFARba4S8vHz/1SoKdKg69At LZ58XQGOmlGbBhPr7EzYQ2XSY4flWbnnD174cmCR8DNFm15DsNA5fw== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-05-02T16:02:29Z" - mac: ENC[AES256_GCM,data:dDv33vEGVeEEeTSXZPcIG3BO0GjFOswBGUsOY+/6IJqAC8omHaSQ6hdcVaXKScC56kEn5w/494hfOOEEficJt1nGQBrnfE8u95tdBqcODtSmTWbonXgpfckX68jV7Y9iTSxisih6ciAwFToxovhiI36kLrWoeVlzs5DdfwJp1YU=,iv:jkThy/omE/9SyqfAr6ARrDYLGVhhACmSxm4EgM+Is1s=,tag:Key2xC4btvI8HqQglIcXkg==,type:str] + lastmodified: "2026-05-03T16:28:04Z" + mac: ENC[AES256_GCM,data:dJ00o+Ny6btbOn7Pt5dc4iLx6FHJiTTx7onZDhjxya7Ywg2qAwHTiNP2q2aP6348w5uenlEGrgnV0Dc8xyHfApNCkJwj4G6UnI17jxEGn3lc0ZFNzJJ7jO2CJcwHir0E2G4XdPjmbZUdB3aKmM34dI9EEUWWhNFXdps4X0dNQcM=,iv:vOU3kHq0axRBrkNfVicQ/8H77nF0DGIJlpoDuJmwRGQ=,tag:uvoFkmaaTM8zpr1g2FpCMA==,type:str] pgp: - created_at: "2024-11-28T18:56:39Z" enc: |- diff --git a/users/alice/secrets.yaml b/users/alice/secrets.yaml index 04570de..ec1f868 100644 --- a/users/alice/secrets.yaml +++ b/users/alice/secrets.yaml @@ -10,6 +10,8 @@ alice: attic-nix-cache-admin: ENC[AES256_GCM,data:xHJGeU4EUn1HRy2nIValiJ6iLZnYmmT6Njv/cGMh15Q0hJXKNBSsi8f0mAfLI7EX+GaC299VKh2uTlU25jptrAvogLxNJIc+LZBLsSkyGE/ojqqevHMKmZ/6eciLZRQL5ey9TM3V9HHyDOhGaFgdfawtwg/vyvbV13lZBKpqneAX9T3gPRuKRjV4/Uc/5cUckiOF8bQ50xVFN8Cql9HgGDJEGWgg4XUTPu5eYspof2EN63pYvU7wg6HD2begeLDvqc2/i2DIcsc0wqc5DgkY/dH2YtcssBtU8AR9vKpl+HmH/wvt6dfaEyZ7hF7ITGwWnOO6H2ko3SjYRfHkFK3XDmm1YRRjfkptnw==,iv:BdVgNyZ1azl5tKfH+RTeXuNV/rYY6hPvrareKlIXSeQ=,tag:/ar87eAjMod4TmQXoerNBQ==,type:str] gitea-actions-token: ENC[AES256_GCM,data:QTEPMAh1RWWJ/O3yhkQkEBTdVL8XhIRGCDbiM0lLjfILKF4SpSJ2sA==,iv:mBaaB1JHb2KVc9n2pdeX4pSMvb7q5z3joMT7rR5Whgs=,tag:ef+58SI4AUeqUsk3RVDsRQ==,type:str] gitea-pr-token: ENC[AES256_GCM,data:ybTya4X2wd65pNFSGbQkg73lu66GNtSba4yf8J6tT8XkuOtfvtBS4g==,iv:39mJiAlw4kud4l06jOpxOCRumChE/5q8IBNsPHG1rMc=,tag:MEvHD2b9E3fVHLlz7haNyw==,type:str] + honeycomb-id: ENC[AES256_GCM,data:PndCclCbSMrgmlYdQ5a8//IB+hg9uB0ZwidIZFiKN6w=,iv:mxFcGPnY0eCliugvQT5HR9aGzJIvXZI6FTo8rphVQMM=,tag:3huIaIy7da0gx2G/BVJJLQ==,type:str] + honeycomb-secret: ENC[AES256_GCM,data:x2MZPgAJz3I1m+rjSVpINZdQVbu50XiwEtPGEX/kbJw=,iv:bPNNtXIES0pCLc4Nu/886nwoOVXaZgyTxndSpyWWU0E=,tag:igdd/NPCDpcvzSd9LO4hyw==,type:str] sops: age: - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh @@ -39,8 +41,8 @@ sops: ZERFTlFyNjhOb3VCaW43ZXFHT1Vxc0UK7YV+BU7dCEOZxpqkQA394eDsnthvorj6 7bqrCdeU+6DU7DmFs6++BrNO2tx8vvOa1im+ZGrM/gZAJdv/7R2d6Q== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-04-07T23:43:57Z" - mac: ENC[AES256_GCM,data:ygQzxSpGJqXwkOq7jGDeflA2FTSSxnre/PXm0LxmxzQQW5s7LeIVSI75fMqWir0WU3Pi/xroYGEWjpCG6JvxV5RiJycTONk8VE7c3jtw3AbrHSS0b1K5tJ+Sf+q3rHJFWWk/COrPk8IsRFNb+taqH4jnaH3AAVNo5u0C1CHKMes=,iv:FO2GVDXE8SjjA81/9cDwc+dX8kJ2oHt5kqkhNBuMb54=,tag:hgzRAmsh32SCvJEvKyV+vg==,type:str] + lastmodified: "2026-05-03T16:20:31Z" + mac: ENC[AES256_GCM,data:/Ui0H0wgENYnzVB7V2aAbj4dIbE+sjRpPoso75xyKepmeeMp8nv8CJLKmLmtbp+rtX287teVw4hKxb/z8M6ZHIWmgt9Zgui8/+hw5hwuMmyjPeGqJQjFH8s4zcXkOKLDoFPA65gva5gaPsreqPwOSqgqYwmds5gOcxBI3Tqdh6E=,iv:ugtug8Vb7DRUp1eC2M6ooCoPFjbqZo8htHOV4AXIcSI=,tag:4sMJsh/r57Z4VFoOBmUmXw==,type:str] pgp: - created_at: "2024-09-05T06:10:22Z" enc: |- @@ -55,4 +57,4 @@ sops: -----END PGP MESSAGE----- fp: 5EFFB75F7C9B74EAA5C4637547940175096C1330 unencrypted_suffix: _unencrypted - version: 3.10.1 + version: 3.12.2