nix-dotfiles/systems/palatine-hill/otel.nix

{
  config,
  pkgs,
  ...
}:
{
  # node_exporter (port 9002) and Prometheus (port 9001) are already configured
  # in hydra.nix — we just scrape the existing exporter here.

  services.opentelemetry-collector = {
    enable = true;
    package = pkgs.opentelemetry-collector-contrib;
    settings = {
      receivers = {
        # Accept OTLP traces/metrics from local services and containers.
        otlp = {
          protocols = {
            grpc.endpoint = "127.0.0.1:4317";
            http.endpoint = "127.0.0.1:4318";
          };
        };

        # Host-level system metrics
        hostmetrics = {
          collection_interval = "60s";
          scrapers = {
            cpu = { };
            memory = { };
            disk = { };
            filesystem = { };
            network = { };
            load = { };
            processes = { };
          };
        };

        # Scrape node_exporter for per-systemd-unit service state
        prometheus = {
          config = {
            scrape_configs = [
              {
                job_name = "node-exporter";
                scrape_interval = "60s";
                static_configs = [
                  {
                    targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
                    labels = {
                      host = "palatine-hill";
                    };
                  }
                ];
              }
              {
                job_name = "postgres-exporter";
                scrape_interval = "60s";
                static_configs = [
                  {
                    targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ];
                    labels = {
                      host = "palatine-hill";
                    };
                  }
                ];
              }
              {
                job_name = "zfs-exporter";
                scrape_interval = "60s";
                static_configs = [
                  {
                    targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ];
                    labels = {
                      host = "palatine-hill";
                    };
                  }
                ];
              }
            ];
          };
        };
      };

      processors = {
        memory_limiter = {
          check_interval = "1s";
          limit_percentage = 75;
          spike_limit_percentage = 15;
        };

        batch = {
          send_batch_size = 8192;
          timeout = "5s";
        };

        attributes = {
          actions = [
            {
              action = "upsert";
              key = "deployment.environment";
              value = "palatine-hill";
            }
          ];
        };

        # Attach hostname using the standard resource processor
        resource = {
          attributes = [
            {
              action = "upsert";
              key = "host.name";
              value = "palatine-hill";
            }
          ];
        };
      };

      exporters = {
        "otlp/honeycomb-metrics" = {
          endpoint = "api.honeycomb.io:443";
          compression = "gzip";
          headers = {
            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
            "x-honeycomb-dataset" = "palatine-hill-metrics";
          };
        };

        "otlp/honeycomb-traces" = {
          endpoint = "api.honeycomb.io:443";
          compression = "gzip";
          headers = {
            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
            "x-honeycomb-dataset" = "palatine-hill-traces";
          };
        };
      };

      service = {
        pipelines = {
          metrics = {
            receivers = [
              "otlp"
              "hostmetrics"
              "prometheus"
            ];
            processors = [
              "memory_limiter"
              "resource"
              "attributes"
              "batch"
            ];
            exporters = [ "otlp/honeycomb-metrics" ];
          };

          traces = {
            receivers = [ "otlp" ];
            processors = [
              "memory_limiter"
              "resource"
              "attributes"
              "batch"
            ];
            exporters = [ "otlp/honeycomb-traces" ];
          };
        };
      };
    };
  };

  sops.secrets = {
    "honeycomb/api-key" = {
      owner = "root";
      restartUnits = [ "opentelemetry-collector.service" ];
    };
  };
}