otel fixes

systems/palatine-hill/otel.nix

@@ -1,5 +1,6 @@
 {
   config,
+  pkgs,
   ...
 }:
 {
@@ -8,8 +9,17 @@
 
   services.opentelemetry-collector = {
     enable = true;
+    package = pkgs.opentelemetry-collector-contrib;
     settings = {
       receivers = {
+        # Accept OTLP traces/metrics from local services and containers.
+        otlp = {
+          protocols = {
+            grpc.endpoint = "127.0.0.1:4317";
+            http.endpoint = "127.0.0.1:4318";
+          };
+        };
+
         # Host-level system metrics
         hostmetrics = {
           collection_interval = "60s";
@@ -70,7 +80,26 @@
       };
 
       processors = {
-        batch = { };
+        memory_limiter = {
+          check_interval = "1s";
+          limit_percentage = 75;
+          spike_limit_percentage = 15;
+        };
+
+        batch = {
+          send_batch_size = 8192;
+          timeout = "5s";
+        };
+
+        attributes = {
+          actions = [
+            {
+              action = "upsert";
+              key = "deployment.environment";
+              value = "palatine-hill";
+            }
+          ];
+        };
 
         # Attach hostname using the standard resource processor
         resource = {
@@ -85,11 +114,21 @@
       };
 
       exporters = {
-        "otlp/honeycomb" = {
+        "otlp/honeycomb-metrics" = {
           endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
           headers = {
-            # Expanded at runtime from the environment file
-            "x-honeycomb-team" = "\${HONEYCOMB_API_KEY}";
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-metrics";
+          };
+        };
+
+        "otlp/honeycomb-traces" = {
+          endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
+          headers = {
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-traces";
           };
         };
       };
@@ -98,25 +137,38 @@
         pipelines = {
           metrics = {
             receivers = [
+              "otlp"
               "hostmetrics"
               "prometheus"
             ];
             processors = [
+              "memory_limiter"
               "resource"
+              "attributes"
               "batch"
             ];
-            exporters = [ "otlp/honeycomb" ];
-          };
-        };
-      };
-    };
+            exporters = [ "otlp/honeycomb-metrics" ];
           };
 
-  # Inject the Honeycomb API key at runtime — never stored in the Nix store
-  systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile =
-    config.sops.secrets."honeycomb/api-key".path;
+          traces = {
+            receivers = [ "otlp" ];
+            processors = [
+              "memory_limiter"
+              "resource"
+              "attributes"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb-traces" ];
+          };
+        };
+      };
+    };
+  };
 
   sops.secrets = {
-    "honeycomb/api-key".owner = "root";
+    "honeycomb/api-key" = {
+      owner = "root";
+      restartUnits = [ "opentelemetry-collector.service" ];
+    };
   };
 }