add otel and honeycomb

systems/palatine-hill/configuration.nix

@@ -19,6 +19,7 @@
     ./minio.nix
     ./networking.nix
     ./nextcloud.nix
+    ./otel.nix
     #./plex
     ./postgresql.nix
     ./samba.nix

systems/palatine-hill/hydra.nix

@@ -6,6 +6,8 @@
 let
   hydra_notify_prometheus_port = "9199";
   hydra_queue_runner_prometheus_port = "9200";
+  postgres_exporter_port = 9187;
+  zfs_exporter_port = 9134;
 in
 {
   systemd.services.hydra-notify.serviceConfig.EnvironmentFile =
@@ -96,10 +98,23 @@ in
       enable = true;
       webExternalUrl = "https://prom.alicehuston.xyz";
       port = 9001;
-      exporters.node = {
-        enable = true;
-        enabledCollectors = [ "systemd" ];
-        port = 9002;
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" ];
+          port = 9002;
+        };
+        postgres = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = postgres_exporter_port;
+          runAsLocalSuperUser = true;
+        };
+        zfs = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = zfs_exporter_port;
+        };
       };
       scrapeConfigs = [
         {
@@ -119,6 +134,18 @@ in
             }
           ];
         }
+        {
+          job_name = "postgres-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ]; }
+          ];
+        }
+        {
+          job_name = "zfs-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.zfs.port}" ]; }
+          ];
+        }
         {
           job_name = "hydra-external";
           scheme = "https";

systems/palatine-hill/otel.nix

@@ -0,0 +1,117 @@
+{
+  config,
+  ...
+}:
+{
+  # node_exporter (port 9002) and Prometheus (port 9001) are already configured
+  # in hydra.nix — we just scrape the existing exporter here.
+
+  services.opentelemetry-collector = {
+    enable = true;
+    settings = {
+      receivers = {
+        # Host-level system metrics
+        hostmetrics = {
+          collection_interval = "60s";
+          scrapers = {
+            cpu = { };
+            memory = { };
+            disk = { };
+            filesystem = { };
+            network = { };
+            load = { };
+            processes = { };
+          };
+        };
+
+        # Scrape node_exporter for per-systemd-unit service state
+        prometheus = {
+          config = {
+            scrape_configs = [
+              {
+                job_name = "node-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "postgres-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "zfs-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+            ];
+          };
+        };
+      };
+
+      processors = {
+        batch = { };
+
+        # Attach hostname and other system resource attributes
+        "resourcedetection/system" = {
+          detectors = [ "system" ];
+          system.hostname_sources = [ "os" ];
+        };
+      };
+
+      exporters = {
+        "otlp/honeycomb" = {
+          endpoint = "api.honeycomb.io:443";
+          headers = {
+            # Expanded at runtime from the environment file
+            "x-honeycomb-team" = "\${HONEYCOMB_API_KEY}";
+          };
+        };
+      };
+
+      service = {
+        pipelines = {
+          metrics = {
+            receivers = [
+              "hostmetrics"
+              "prometheus"
+            ];
+            processors = [
+              "resourcedetection/system"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb" ];
+          };
+        };
+      };
+    };
+  };
+
+  # Inject the Honeycomb API key at runtime — never stored in the Nix store
+  systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile =
+    config.sops.secrets."honeycomb/api-key".path;
+
+  sops.secrets = {
+    "honeycomb/api-key".owner = "root";
+  };
+}

systems/palatine-hill/secrets.yaml

@@ -45,6 +45,8 @@ typhon:
 garage:
     rpc-secret: ENC[AES256_GCM,data:Q2ZaAXcntD3yK6DynEpxab2TITByMZ7ECVrq1pb0ZU7hXOZnhaBmjdty/Os6len8l+GBl6+WaC0An6cFkhQTlQ==,iv:E8C4bnxMLXK9fky+KC7q8sHpmrEU5un0TEAwxVUBiLk=,tag:PiSiU+9NpyilH2aMs2Qc/Q==,type:str]
     admin-token: ENC[AES256_GCM,data:Xjm8Xq99aDseR0jN50Uj3gLpeDaq2IGXzJCS0o1H0RgKX9LGdP8w508nWWE=,iv:+L9T3TEUSbIz+jo08ykjGHVhuz5ecmzrlhzD2iv48HE=,tag:7P2rY4F8cWFdG4Lm9n/etQ==,type:str]
+honeycomb:
+    api-key: ENC[AES256_GCM,data:k+Z3tmF8pYwD6RokdZauQ/fMlhD5GbW1ekxzRnj0gEpQFlfGB8gQ6BPwbd7qk12ZhsCA4XdqmvsysAWJldYWrPo=,iv:dNAK/vbQYL7ir4UXhZMTWraZF1E6ps9EOF3skYe5wOk=,tag:5UJfCji/RmB5DIuR4179uw==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -56,8 +58,8 @@ sops:
             cXNZWmZqd0R0SmhINExscHBKWmxvblUKEFEQvt/zQFARba4S8vHz/1SoKdKg69At
             LZ58XQGOmlGbBhPr7EzYQ2XSY4flWbnnD174cmCR8DNFm15DsNA5fw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-02T16:02:29Z"
-    mac: ENC[AES256_GCM,data:dDv33vEGVeEEeTSXZPcIG3BO0GjFOswBGUsOY+/6IJqAC8omHaSQ6hdcVaXKScC56kEn5w/494hfOOEEficJt1nGQBrnfE8u95tdBqcODtSmTWbonXgpfckX68jV7Y9iTSxisih6ciAwFToxovhiI36kLrWoeVlzs5DdfwJp1YU=,iv:jkThy/omE/9SyqfAr6ARrDYLGVhhACmSxm4EgM+Is1s=,tag:Key2xC4btvI8HqQglIcXkg==,type:str]
+    lastmodified: "2026-05-03T16:28:04Z"
+    mac: ENC[AES256_GCM,data:dJ00o+Ny6btbOn7Pt5dc4iLx6FHJiTTx7onZDhjxya7Ywg2qAwHTiNP2q2aP6348w5uenlEGrgnV0Dc8xyHfApNCkJwj4G6UnI17jxEGn3lc0ZFNzJJ7jO2CJcwHir0E2G4XdPjmbZUdB3aKmM34dI9EEUWWhNFXdps4X0dNQcM=,iv:vOU3kHq0axRBrkNfVicQ/8H77nF0DGIJlpoDuJmwRGQ=,tag:uvoFkmaaTM8zpr1g2FpCMA==,type:str]
     pgp:
         - created_at: "2024-11-28T18:56:39Z"
           enc: |-