garage fixes

Reviewed-on: #225

.vscode/mcp.json

@@ -1 +1,13 @@
-{}
+{
+  "servers": {
+    "honeycomb": {
+      "command": "npx",
+      "args": [
+        "mcp-remote",
+        "https://mcp.honeycomb.io/mcp"
+      ],
+      "type": "stdio"
+    }
+  },
+  "inputs": []
+}

systems/palatine-hill/configuration.nix

@@ -19,6 +19,7 @@
     ./minio.nix
     ./networking.nix
     ./nextcloud.nix
+    ./otel.nix
     #./plex
     ./postgresql.nix
     ./samba.nix

systems/palatine-hill/garage.nix

@@ -41,8 +41,43 @@ in
     "d ${basePath}/data 0750 garage garage -"
   ];
 
+  systemd.services.garage = {
+    unitConfig.RequiresMountsFor = [
+      vars.primary_minio
+      basePath
+      "${basePath}/meta"
+      "${basePath}/data"
+    ];
+    preStart = ''
+      mkdir -p ${basePath}/meta ${basePath}/data
+      chown -R garage:garage ${basePath}/meta ${basePath}/data
+    '';
+    serviceConfig = {
+      PermissionsStartOnly = true;
+      DynamicUser = false;
+      User = "garage";
+      Group = "garage";
+    };
+  };
+
+  users.groups.garage = { };
+  users.users.garage = {
+    isSystemUser = true;
+    group = "garage";
+  };
+
   sops.secrets = {
-    "garage/rpc-secret" = { };
-    "garage/admin-token" = { };
+    "garage/rpc-secret" = {
+      owner = "garage";
+      group = "garage";
+      mode = "0400";
+      restartUnits = [ "garage.service" ];
+    };
+    "garage/admin-token" = {
+      owner = "garage";
+      group = "garage";
+      mode = "0400";
+      restartUnits = [ "garage.service" ];
+    };
   };
 }

systems/palatine-hill/hydra.nix

@@ -6,6 +6,8 @@
 let
   hydra_notify_prometheus_port = "9199";
   hydra_queue_runner_prometheus_port = "9200";
+  postgres_exporter_port = 9187;
+  zfs_exporter_port = 9134;
 in
 {
   systemd.services.hydra-notify.serviceConfig.EnvironmentFile =
@@ -96,11 +98,24 @@ in
       enable = true;
       webExternalUrl = "https://prom.alicehuston.xyz";
       port = 9001;
-      exporters.node = {
+      exporters = {
+        node = {
           enable = true;
           enabledCollectors = [ "systemd" ];
           port = 9002;
         };
+        postgres = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = postgres_exporter_port;
+          runAsLocalSuperUser = true;
+        };
+        zfs = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = zfs_exporter_port;
+        };
+      };
       scrapeConfigs = [
         {
           job_name = "palatine-hill";
@@ -119,6 +134,18 @@ in
             }
           ];
         }
+        {
+          job_name = "postgres-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ]; }
+          ];
+        }
+        {
+          job_name = "zfs-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.zfs.port}" ]; }
+          ];
+        }
         {
           job_name = "hydra-external";
           scheme = "https";

systems/palatine-hill/otel.nix

@@ -0,0 +1,174 @@
+{
+  config,
+  pkgs,
+  ...
+}:
+{
+  # node_exporter (port 9002) and Prometheus (port 9001) are already configured
+  # in hydra.nix — we just scrape the existing exporter here.
+
+  services.opentelemetry-collector = {
+    enable = true;
+    package = pkgs.opentelemetry-collector-contrib;
+    settings = {
+      receivers = {
+        # Accept OTLP traces/metrics from local services and containers.
+        otlp = {
+          protocols = {
+            grpc.endpoint = "127.0.0.1:4317";
+            http.endpoint = "127.0.0.1:4318";
+          };
+        };
+
+        # Host-level system metrics
+        hostmetrics = {
+          collection_interval = "60s";
+          scrapers = {
+            cpu = { };
+            memory = { };
+            disk = { };
+            filesystem = { };
+            network = { };
+            load = { };
+            processes = { };
+          };
+        };
+
+        # Scrape node_exporter for per-systemd-unit service state
+        prometheus = {
+          config = {
+            scrape_configs = [
+              {
+                job_name = "node-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "postgres-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "zfs-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+            ];
+          };
+        };
+      };
+
+      processors = {
+        memory_limiter = {
+          check_interval = "1s";
+          limit_percentage = 75;
+          spike_limit_percentage = 15;
+        };
+
+        batch = {
+          send_batch_size = 8192;
+          timeout = "5s";
+        };
+
+        attributes = {
+          actions = [
+            {
+              action = "upsert";
+              key = "deployment.environment";
+              value = "palatine-hill";
+            }
+          ];
+        };
+
+        # Attach hostname using the standard resource processor
+        resource = {
+          attributes = [
+            {
+              action = "upsert";
+              key = "host.name";
+              value = "palatine-hill";
+            }
+          ];
+        };
+      };
+
+      exporters = {
+        "otlp/honeycomb-metrics" = {
+          endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
+          headers = {
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-metrics";
+          };
+        };
+
+        "otlp/honeycomb-traces" = {
+          endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
+          headers = {
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-traces";
+          };
+        };
+      };
+
+      service = {
+        pipelines = {
+          metrics = {
+            receivers = [
+              "otlp"
+              "hostmetrics"
+              "prometheus"
+            ];
+            processors = [
+              "memory_limiter"
+              "resource"
+              "attributes"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb-metrics" ];
+          };
+
+          traces = {
+            receivers = [ "otlp" ];
+            processors = [
+              "memory_limiter"
+              "resource"
+              "attributes"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb-traces" ];
+          };
+        };
+      };
+    };
+  };
+
+  sops.secrets = {
+    "honeycomb/api-key" = {
+      owner = "root";
+      restartUnits = [ "opentelemetry-collector.service" ];
+    };
+  };
+}

systems/palatine-hill/secrets.yaml

@@ -45,6 +45,8 @@ typhon:
 garage:
     rpc-secret: ENC[AES256_GCM,data:Q2ZaAXcntD3yK6DynEpxab2TITByMZ7ECVrq1pb0ZU7hXOZnhaBmjdty/Os6len8l+GBl6+WaC0An6cFkhQTlQ==,iv:E8C4bnxMLXK9fky+KC7q8sHpmrEU5un0TEAwxVUBiLk=,tag:PiSiU+9NpyilH2aMs2Qc/Q==,type:str]
     admin-token: ENC[AES256_GCM,data:Xjm8Xq99aDseR0jN50Uj3gLpeDaq2IGXzJCS0o1H0RgKX9LGdP8w508nWWE=,iv:+L9T3TEUSbIz+jo08ykjGHVhuz5ecmzrlhzD2iv48HE=,tag:7P2rY4F8cWFdG4Lm9n/etQ==,type:str]
+honeycomb:
+    api-key: ENC[AES256_GCM,data:sDhWmpaxLBb+qv/REDEbqpVsTNZBNuuLBGRvv0RYmdAzYBAZUn2OnBTHwgS7Bgv7xRDKgsGW8cOm0gQ8NUdWkmrdwUWvXO8IvDoz3/jzT3y1tw==,iv:mcqnkq3f0FfCnqnN7AdWAE5gDLO7+5PgWyOcK8ZAabs=,tag:+EIMFrp/0LEaf0sFzczK2g==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -56,8 +58,8 @@ sops:
             cXNZWmZqd0R0SmhINExscHBKWmxvblUKEFEQvt/zQFARba4S8vHz/1SoKdKg69At
             LZ58XQGOmlGbBhPr7EzYQ2XSY4flWbnnD174cmCR8DNFm15DsNA5fw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-02T16:02:29Z"
-    mac: ENC[AES256_GCM,data:dDv33vEGVeEEeTSXZPcIG3BO0GjFOswBGUsOY+/6IJqAC8omHaSQ6hdcVaXKScC56kEn5w/494hfOOEEficJt1nGQBrnfE8u95tdBqcODtSmTWbonXgpfckX68jV7Y9iTSxisih6ciAwFToxovhiI36kLrWoeVlzs5DdfwJp1YU=,iv:jkThy/omE/9SyqfAr6ARrDYLGVhhACmSxm4EgM+Is1s=,tag:Key2xC4btvI8HqQglIcXkg==,type:str]
+    lastmodified: "2026-05-05T04:51:42Z"
+    mac: ENC[AES256_GCM,data:6o9xR1B6/AhkHu6E2FlgoPjYY3fUaLY/au0pxLVMweNK3F0/C2FZdnnm0HNwb2cRhEdlWcpbYZ5CyZ6CllVMHlJEGZI/SYXmMzkMx19O5E7AQ85T4GXLWlQl1dksIQ4q3p7fhlZ7uSIy83zRYYXEER9yV/35M1UYjIKC19NLcpY=,iv:meYHkoWnaU4iBLckq0HdpwGcabL0fgSBbtHXUNFoM3Y=,tag:AjLqzIFsCGNk60GO9yQVQw==,type:str]
     pgp:
         - created_at: "2024-11-28T18:56:39Z"
           enc: |-

users/alice/secrets.yaml

@@ -10,6 +10,8 @@ alice:
     attic-nix-cache-admin: ENC[AES256_GCM,data:xHJGeU4EUn1HRy2nIValiJ6iLZnYmmT6Njv/cGMh15Q0hJXKNBSsi8f0mAfLI7EX+GaC299VKh2uTlU25jptrAvogLxNJIc+LZBLsSkyGE/ojqqevHMKmZ/6eciLZRQL5ey9TM3V9HHyDOhGaFgdfawtwg/vyvbV13lZBKpqneAX9T3gPRuKRjV4/Uc/5cUckiOF8bQ50xVFN8Cql9HgGDJEGWgg4XUTPu5eYspof2EN63pYvU7wg6HD2begeLDvqc2/i2DIcsc0wqc5DgkY/dH2YtcssBtU8AR9vKpl+HmH/wvt6dfaEyZ7hF7ITGwWnOO6H2ko3SjYRfHkFK3XDmm1YRRjfkptnw==,iv:BdVgNyZ1azl5tKfH+RTeXuNV/rYY6hPvrareKlIXSeQ=,tag:/ar87eAjMod4TmQXoerNBQ==,type:str]
     gitea-actions-token: ENC[AES256_GCM,data:QTEPMAh1RWWJ/O3yhkQkEBTdVL8XhIRGCDbiM0lLjfILKF4SpSJ2sA==,iv:mBaaB1JHb2KVc9n2pdeX4pSMvb7q5z3joMT7rR5Whgs=,tag:ef+58SI4AUeqUsk3RVDsRQ==,type:str]
     gitea-pr-token: ENC[AES256_GCM,data:ybTya4X2wd65pNFSGbQkg73lu66GNtSba4yf8J6tT8XkuOtfvtBS4g==,iv:39mJiAlw4kud4l06jOpxOCRumChE/5q8IBNsPHG1rMc=,tag:MEvHD2b9E3fVHLlz7haNyw==,type:str]
+    honeycomb-id: ENC[AES256_GCM,data:PndCclCbSMrgmlYdQ5a8//IB+hg9uB0ZwidIZFiKN6w=,iv:mxFcGPnY0eCliugvQT5HR9aGzJIvXZI6FTo8rphVQMM=,tag:3huIaIy7da0gx2G/BVJJLQ==,type:str]
+    honeycomb-secret: ENC[AES256_GCM,data:x2MZPgAJz3I1m+rjSVpINZdQVbu50XiwEtPGEX/kbJw=,iv:bPNNtXIES0pCLc4Nu/886nwoOVXaZgyTxndSpyWWU0E=,tag:igdd/NPCDpcvzSd9LO4hyw==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -39,8 +41,8 @@ sops:
             ZERFTlFyNjhOb3VCaW43ZXFHT1Vxc0UK7YV+BU7dCEOZxpqkQA394eDsnthvorj6
             7bqrCdeU+6DU7DmFs6++BrNO2tx8vvOa1im+ZGrM/gZAJdv/7R2d6Q==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-04-07T23:43:57Z"
-    mac: ENC[AES256_GCM,data:ygQzxSpGJqXwkOq7jGDeflA2FTSSxnre/PXm0LxmxzQQW5s7LeIVSI75fMqWir0WU3Pi/xroYGEWjpCG6JvxV5RiJycTONk8VE7c3jtw3AbrHSS0b1K5tJ+Sf+q3rHJFWWk/COrPk8IsRFNb+taqH4jnaH3AAVNo5u0C1CHKMes=,iv:FO2GVDXE8SjjA81/9cDwc+dX8kJ2oHt5kqkhNBuMb54=,tag:hgzRAmsh32SCvJEvKyV+vg==,type:str]
+    lastmodified: "2026-05-03T16:20:31Z"
+    mac: ENC[AES256_GCM,data:/Ui0H0wgENYnzVB7V2aAbj4dIbE+sjRpPoso75xyKepmeeMp8nv8CJLKmLmtbp+rtX287teVw4hKxb/z8M6ZHIWmgt9Zgui8/+hw5hwuMmyjPeGqJQjFH8s4zcXkOKLDoFPA65gva5gaPsreqPwOSqgqYwmds5gOcxBI3Tqdh6E=,iv:ugtug8Vb7DRUp1eC2M6ooCoPFjbqZo8htHOV4AXIcSI=,tag:4sMJsh/r57Z4VFoOBmUmXw==,type:str]
     pgp:
         - created_at: "2024-09-05T06:10:22Z"
           enc: |-
@@ -55,4 +57,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: 5EFFB75F7C9B74EAA5C4637547940175096C1330
     unencrypted_suffix: _unencrypted
-    version: 3.10.1
+    version: 3.12.2