diff --git a/systems/palatine-hill/otel.nix b/systems/palatine-hill/otel.nix index 3800213..aa37b90 100644 --- a/systems/palatine-hill/otel.nix +++ b/systems/palatine-hill/otel.nix @@ -1,5 +1,6 @@ { config, + pkgs, ... }: { @@ -8,8 +9,17 @@ services.opentelemetry-collector = { enable = true; + package = pkgs.opentelemetry-collector-contrib; settings = { receivers = { + # Accept OTLP traces/metrics from local services and containers. + otlp = { + protocols = { + grpc.endpoint = "127.0.0.1:4317"; + http.endpoint = "127.0.0.1:4318"; + }; + }; + # Host-level system metrics hostmetrics = { collection_interval = "60s"; @@ -70,7 +80,26 @@ }; processors = { - batch = { }; + memory_limiter = { + check_interval = "1s"; + limit_percentage = 75; + spike_limit_percentage = 15; + }; + + batch = { + send_batch_size = 8192; + timeout = "5s"; + }; + + attributes = { + actions = [ + { + action = "upsert"; + key = "deployment.environment"; + value = "palatine-hill"; + } + ]; + }; # Attach hostname using the standard resource processor resource = { @@ -85,11 +114,21 @@ }; exporters = { - "otlp/honeycomb" = { + "otlp/honeycomb-metrics" = { endpoint = "api.honeycomb.io:443"; + compression = "gzip"; headers = { - # Expanded at runtime from the environment file - "x-honeycomb-team" = "\${HONEYCOMB_API_KEY}"; + "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}"; + "x-honeycomb-dataset" = "palatine-hill-metrics"; + }; + }; + + "otlp/honeycomb-traces" = { + endpoint = "api.honeycomb.io:443"; + compression = "gzip"; + headers = { + "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}"; + "x-honeycomb-dataset" = "palatine-hill-traces"; }; }; }; @@ -98,25 +137,38 @@ pipelines = { metrics = { receivers = [ + "otlp" "hostmetrics" "prometheus" ]; processors = [ + "memory_limiter" "resource" + "attributes" "batch" ]; - exporters = [ "otlp/honeycomb" ]; + exporters = [ "otlp/honeycomb-metrics" ]; + }; + + traces = { + receivers = [ "otlp" ]; + processors = [ + "memory_limiter" + "resource" + "attributes" + "batch" + ]; + exporters = [ "otlp/honeycomb-traces" ]; }; }; }; }; }; - # Inject the Honeycomb API key at runtime — never stored in the Nix store - systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = - config.sops.secrets."honeycomb/api-key".path; - sops.secrets = { - "honeycomb/api-key".owner = "root"; + "honeycomb/api-key" = { + owner = "root"; + restartUnits = [ "opentelemetry-collector.service" ]; + }; }; }