add otel and honeycomb

.vscode/mcp.json

@@ -1 +1,13 @@
-{}
+{
+  "servers": {
+    "honeycomb": {
+      "command": "npx",
+      "args": [
+        "mcp-remote",
+        "https://mcp.honeycomb.io/mcp"
+      ],
+      "type": "stdio"
+    }
+  },
+  "inputs": []
+}

systems/palatine-hill/configuration.nix

@@ -19,6 +19,7 @@
     ./minio.nix
     ./networking.nix
     ./nextcloud.nix
+    ./otel.nix
     #./plex
     ./postgresql.nix
     ./samba.nix

systems/palatine-hill/hydra.nix

@@ -6,6 +6,8 @@
 let
   hydra_notify_prometheus_port = "9199";
   hydra_queue_runner_prometheus_port = "9200";
+  postgres_exporter_port = 9187;
+  zfs_exporter_port = 9134;
 in
 {
   systemd.services.hydra-notify.serviceConfig.EnvironmentFile =
@@ -96,11 +98,24 @@ in
       enable = true;
       webExternalUrl = "https://prom.alicehuston.xyz";
       port = 9001;
-      exporters.node = {
+      exporters = {
+        node = {
           enable = true;
           enabledCollectors = [ "systemd" ];
           port = 9002;
         };
+        postgres = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = postgres_exporter_port;
+          runAsLocalSuperUser = true;
+        };
+        zfs = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = zfs_exporter_port;
+        };
+      };
       scrapeConfigs = [
         {
           job_name = "palatine-hill";
@@ -119,6 +134,18 @@ in
             }
           ];
         }
+        {
+          job_name = "postgres-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ]; }
+          ];
+        }
+        {
+          job_name = "zfs-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.zfs.port}" ]; }
+          ];
+        }
         {
           job_name = "hydra-external";
           scheme = "https";

systems/palatine-hill/otel.nix

@@ -0,0 +1,117 @@
+{
+  config,
+  ...
+}:
+{
+  # node_exporter (port 9002) and Prometheus (port 9001) are already configured
+  # in hydra.nix — we just scrape the existing exporter here.
+
+  services.opentelemetry-collector = {
+    enable = true;
+    settings = {
+      receivers = {
+        # Host-level system metrics
+        hostmetrics = {
+          collection_interval = "60s";
+          scrapers = {
+            cpu = { };
+            memory = { };
+            disk = { };
+            filesystem = { };
+            network = { };
+            load = { };
+            processes = { };
+          };
+        };
+
+        # Scrape node_exporter for per-systemd-unit service state
+        prometheus = {
+          config = {
+            scrape_configs = [
+              {
+                job_name = "node-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "postgres-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "zfs-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+            ];
+          };
+        };
+      };
+
+      processors = {
+        batch = { };
+
+        # Attach hostname and other system resource attributes
+        "resourcedetection/system" = {
+          detectors = [ "system" ];
+          system.hostname_sources = [ "os" ];
+        };
+      };
+
+      exporters = {
+        "otlp/honeycomb" = {
+          endpoint = "api.honeycomb.io:443";
+          headers = {
+            # Expanded at runtime from the environment file
+            "x-honeycomb-team" = "\${HONEYCOMB_API_KEY}";
+          };
+        };
+      };
+
+      service = {
+        pipelines = {
+          metrics = {
+            receivers = [
+              "hostmetrics"
+              "prometheus"
+            ];
+            processors = [
+              "resourcedetection/system"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb" ];
+          };
+        };
+      };
+    };
+  };
+
+  # Inject the Honeycomb API key at runtime — never stored in the Nix store
+  systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile =
+    config.sops.secrets."honeycomb/api-key".path;
+
+  sops.secrets = {
+    "honeycomb/api-key".owner = "root";
+  };
+}

systems/palatine-hill/secrets.yaml

@@ -45,6 +45,8 @@ typhon:
 garage:
     rpc-secret: ENC[AES256_GCM,data:Q2ZaAXcntD3yK6DynEpxab2TITByMZ7ECVrq1pb0ZU7hXOZnhaBmjdty/Os6len8l+GBl6+WaC0An6cFkhQTlQ==,iv:E8C4bnxMLXK9fky+KC7q8sHpmrEU5un0TEAwxVUBiLk=,tag:PiSiU+9NpyilH2aMs2Qc/Q==,type:str]
     admin-token: ENC[AES256_GCM,data:Xjm8Xq99aDseR0jN50Uj3gLpeDaq2IGXzJCS0o1H0RgKX9LGdP8w508nWWE=,iv:+L9T3TEUSbIz+jo08ykjGHVhuz5ecmzrlhzD2iv48HE=,tag:7P2rY4F8cWFdG4Lm9n/etQ==,type:str]
+honeycomb:
+    api-key: ENC[AES256_GCM,data:k+Z3tmF8pYwD6RokdZauQ/fMlhD5GbW1ekxzRnj0gEpQFlfGB8gQ6BPwbd7qk12ZhsCA4XdqmvsysAWJldYWrPo=,iv:dNAK/vbQYL7ir4UXhZMTWraZF1E6ps9EOF3skYe5wOk=,tag:5UJfCji/RmB5DIuR4179uw==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -56,8 +58,8 @@ sops:
             cXNZWmZqd0R0SmhINExscHBKWmxvblUKEFEQvt/zQFARba4S8vHz/1SoKdKg69At
             LZ58XQGOmlGbBhPr7EzYQ2XSY4flWbnnD174cmCR8DNFm15DsNA5fw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-02T16:02:29Z"
-    mac: ENC[AES256_GCM,data:dDv33vEGVeEEeTSXZPcIG3BO0GjFOswBGUsOY+/6IJqAC8omHaSQ6hdcVaXKScC56kEn5w/494hfOOEEficJt1nGQBrnfE8u95tdBqcODtSmTWbonXgpfckX68jV7Y9iTSxisih6ciAwFToxovhiI36kLrWoeVlzs5DdfwJp1YU=,iv:jkThy/omE/9SyqfAr6ARrDYLGVhhACmSxm4EgM+Is1s=,tag:Key2xC4btvI8HqQglIcXkg==,type:str]
+    lastmodified: "2026-05-03T16:28:04Z"
+    mac: ENC[AES256_GCM,data:dJ00o+Ny6btbOn7Pt5dc4iLx6FHJiTTx7onZDhjxya7Ywg2qAwHTiNP2q2aP6348w5uenlEGrgnV0Dc8xyHfApNCkJwj4G6UnI17jxEGn3lc0ZFNzJJ7jO2CJcwHir0E2G4XdPjmbZUdB3aKmM34dI9EEUWWhNFXdps4X0dNQcM=,iv:vOU3kHq0axRBrkNfVicQ/8H77nF0DGIJlpoDuJmwRGQ=,tag:uvoFkmaaTM8zpr1g2FpCMA==,type:str]
     pgp:
         - created_at: "2024-11-28T18:56:39Z"
           enc: |-

users/alice/secrets.yaml

@@ -10,6 +10,8 @@ alice:
     attic-nix-cache-admin: ENC[AES256_GCM,data:xHJGeU4EUn1HRy2nIValiJ6iLZnYmmT6Njv/cGMh15Q0hJXKNBSsi8f0mAfLI7EX+GaC299VKh2uTlU25jptrAvogLxNJIc+LZBLsSkyGE/ojqqevHMKmZ/6eciLZRQL5ey9TM3V9HHyDOhGaFgdfawtwg/vyvbV13lZBKpqneAX9T3gPRuKRjV4/Uc/5cUckiOF8bQ50xVFN8Cql9HgGDJEGWgg4XUTPu5eYspof2EN63pYvU7wg6HD2begeLDvqc2/i2DIcsc0wqc5DgkY/dH2YtcssBtU8AR9vKpl+HmH/wvt6dfaEyZ7hF7ITGwWnOO6H2ko3SjYRfHkFK3XDmm1YRRjfkptnw==,iv:BdVgNyZ1azl5tKfH+RTeXuNV/rYY6hPvrareKlIXSeQ=,tag:/ar87eAjMod4TmQXoerNBQ==,type:str]
     gitea-actions-token: ENC[AES256_GCM,data:QTEPMAh1RWWJ/O3yhkQkEBTdVL8XhIRGCDbiM0lLjfILKF4SpSJ2sA==,iv:mBaaB1JHb2KVc9n2pdeX4pSMvb7q5z3joMT7rR5Whgs=,tag:ef+58SI4AUeqUsk3RVDsRQ==,type:str]
     gitea-pr-token: ENC[AES256_GCM,data:ybTya4X2wd65pNFSGbQkg73lu66GNtSba4yf8J6tT8XkuOtfvtBS4g==,iv:39mJiAlw4kud4l06jOpxOCRumChE/5q8IBNsPHG1rMc=,tag:MEvHD2b9E3fVHLlz7haNyw==,type:str]
+    honeycomb-id: ENC[AES256_GCM,data:PndCclCbSMrgmlYdQ5a8//IB+hg9uB0ZwidIZFiKN6w=,iv:mxFcGPnY0eCliugvQT5HR9aGzJIvXZI6FTo8rphVQMM=,tag:3huIaIy7da0gx2G/BVJJLQ==,type:str]
+    honeycomb-secret: ENC[AES256_GCM,data:x2MZPgAJz3I1m+rjSVpINZdQVbu50XiwEtPGEX/kbJw=,iv:bPNNtXIES0pCLc4Nu/886nwoOVXaZgyTxndSpyWWU0E=,tag:igdd/NPCDpcvzSd9LO4hyw==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -39,8 +41,8 @@ sops:
             ZERFTlFyNjhOb3VCaW43ZXFHT1Vxc0UK7YV+BU7dCEOZxpqkQA394eDsnthvorj6
             7bqrCdeU+6DU7DmFs6++BrNO2tx8vvOa1im+ZGrM/gZAJdv/7R2d6Q==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-04-07T23:43:57Z"
-    mac: ENC[AES256_GCM,data:ygQzxSpGJqXwkOq7jGDeflA2FTSSxnre/PXm0LxmxzQQW5s7LeIVSI75fMqWir0WU3Pi/xroYGEWjpCG6JvxV5RiJycTONk8VE7c3jtw3AbrHSS0b1K5tJ+Sf+q3rHJFWWk/COrPk8IsRFNb+taqH4jnaH3AAVNo5u0C1CHKMes=,iv:FO2GVDXE8SjjA81/9cDwc+dX8kJ2oHt5kqkhNBuMb54=,tag:hgzRAmsh32SCvJEvKyV+vg==,type:str]
+    lastmodified: "2026-05-03T16:20:31Z"
+    mac: ENC[AES256_GCM,data:/Ui0H0wgENYnzVB7V2aAbj4dIbE+sjRpPoso75xyKepmeeMp8nv8CJLKmLmtbp+rtX287teVw4hKxb/z8M6ZHIWmgt9Zgui8/+hw5hwuMmyjPeGqJQjFH8s4zcXkOKLDoFPA65gva5gaPsreqPwOSqgqYwmds5gOcxBI3Tqdh6E=,iv:ugtug8Vb7DRUp1eC2M6ooCoPFjbqZo8htHOV4AXIcSI=,tag:4sMJsh/r57Z4VFoOBmUmXw==,type:str]
     pgp:
         - created_at: "2024-09-05T06:10:22Z"
           enc: |-
@@ -55,4 +57,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: 5EFFB75F7C9B74EAA5C4637547940175096C1330
     unencrypted_suffix: _unencrypted
-    version: 3.10.1
+    version: 3.12.2