garage fixes

flake.nix

@@ -148,7 +148,7 @@
       systems = [
         "x86_64-linux"
         # disable arm for now as hydra isn't set up for it
-        "aarch64-linux"
+        # "aarch64-linuxa
       ];
 
       forEachSystem = lib.genAttrs systems;

systems/palatine-hill/garage.nix

@@ -41,8 +41,43 @@ in
     "d ${basePath}/data 0750 garage garage -"
   ];
 
+  systemd.services.garage = {
+    unitConfig.RequiresMountsFor = [
+      vars.primary_minio
+      basePath
+      "${basePath}/meta"
+      "${basePath}/data"
+    ];
+    preStart = ''
+      mkdir -p ${basePath}/meta ${basePath}/data
+      chown -R garage:garage ${basePath}/meta ${basePath}/data
+    '';
+    serviceConfig = {
+      PermissionsStartOnly = true;
+      DynamicUser = false;
+      User = "garage";
+      Group = "garage";
+    };
+  };
+
+  users.groups.garage = { };
+  users.users.garage = {
+    isSystemUser = true;
+    group = "garage";
+  };
+
   sops.secrets = {
-    "garage/rpc-secret" = { };
-    "garage/admin-token" = { };
+    "garage/rpc-secret" = {
+      owner = "garage";
+      group = "garage";
+      mode = "0400";
+      restartUnits = [ "garage.service" ];
+    };
+    "garage/admin-token" = {
+      owner = "garage";
+      group = "garage";
+      mode = "0400";
+      restartUnits = [ "garage.service" ];
+    };
   };
 }

systems/palatine-hill/otel.nix

@@ -1,5 +1,6 @@
 {
   config,
+  pkgs,
   ...
 }:
 {
@@ -8,8 +9,17 @@
 
   services.opentelemetry-collector = {
     enable = true;
+    package = pkgs.opentelemetry-collector-contrib;
     settings = {
       receivers = {
+        # Accept OTLP traces/metrics from local services and containers.
+        otlp = {
+          protocols = {
+            grpc.endpoint = "127.0.0.1:4317";
+            http.endpoint = "127.0.0.1:4318";
+          };
+        };
+
         # Host-level system metrics
         hostmetrics = {
           collection_interval = "60s";
@@ -70,21 +80,55 @@
       };
 
       processors = {
-        batch = { };
+        memory_limiter = {
+          check_interval = "1s";
+          limit_percentage = 75;
+          spike_limit_percentage = 15;
+        };
 
-        # Attach hostname and other system resource attributes
-        "resourcedetection/system" = {
-          detectors = [ "system" ];
-          system.hostname_sources = [ "os" ];
+        batch = {
+          send_batch_size = 8192;
+          timeout = "5s";
+        };
+
+        attributes = {
+          actions = [
+            {
+              action = "upsert";
+              key = "deployment.environment";
+              value = "palatine-hill";
+            }
+          ];
+        };
+
+        # Attach hostname using the standard resource processor
+        resource = {
+          attributes = [
+            {
+              action = "upsert";
+              key = "host.name";
+              value = "palatine-hill";
+            }
+          ];
         };
       };
 
       exporters = {
-        "otlp/honeycomb" = {
+        "otlp/honeycomb-metrics" = {
           endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
           headers = {
-            # Expanded at runtime from the environment file
-            "x-honeycomb-team" = "\${HONEYCOMB_API_KEY}";
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-metrics";
+          };
+        };
+
+        "otlp/honeycomb-traces" = {
+          endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
+          headers = {
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-traces";
           };
         };
       };
@@ -93,25 +137,38 @@
         pipelines = {
           metrics = {
             receivers = [
+              "otlp"
               "hostmetrics"
               "prometheus"
             ];
             processors = [
-              "resourcedetection/system"
+              "memory_limiter"
+              "resource"
+              "attributes"
               "batch"
             ];
-            exporters = [ "otlp/honeycomb" ];
-          };
-        };
-      };
-    };
+            exporters = [ "otlp/honeycomb-metrics" ];
           };
 
-  # Inject the Honeycomb API key at runtime — never stored in the Nix store
-  systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile =
-    config.sops.secrets."honeycomb/api-key".path;
+          traces = {
+            receivers = [ "otlp" ];
+            processors = [
+              "memory_limiter"
+              "resource"
+              "attributes"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb-traces" ];
+          };
+        };
+      };
+    };
+  };
 
   sops.secrets = {
-    "honeycomb/api-key".owner = "root";
+    "honeycomb/api-key" = {
+      owner = "root";
+      restartUnits = [ "opentelemetry-collector.service" ];
+    };
   };
 }

systems/palatine-hill/secrets.yaml

@@ -46,7 +46,7 @@ garage:
     rpc-secret: ENC[AES256_GCM,data:Q2ZaAXcntD3yK6DynEpxab2TITByMZ7ECVrq1pb0ZU7hXOZnhaBmjdty/Os6len8l+GBl6+WaC0An6cFkhQTlQ==,iv:E8C4bnxMLXK9fky+KC7q8sHpmrEU5un0TEAwxVUBiLk=,tag:PiSiU+9NpyilH2aMs2Qc/Q==,type:str]
     admin-token: ENC[AES256_GCM,data:Xjm8Xq99aDseR0jN50Uj3gLpeDaq2IGXzJCS0o1H0RgKX9LGdP8w508nWWE=,iv:+L9T3TEUSbIz+jo08ykjGHVhuz5ecmzrlhzD2iv48HE=,tag:7P2rY4F8cWFdG4Lm9n/etQ==,type:str]
 honeycomb:
-    api-key: ENC[AES256_GCM,data:k+Z3tmF8pYwD6RokdZauQ/fMlhD5GbW1ekxzRnj0gEpQFlfGB8gQ6BPwbd7qk12ZhsCA4XdqmvsysAWJldYWrPo=,iv:dNAK/vbQYL7ir4UXhZMTWraZF1E6ps9EOF3skYe5wOk=,tag:5UJfCji/RmB5DIuR4179uw==,type:str]
+    api-key: ENC[AES256_GCM,data:sDhWmpaxLBb+qv/REDEbqpVsTNZBNuuLBGRvv0RYmdAzYBAZUn2OnBTHwgS7Bgv7xRDKgsGW8cOm0gQ8NUdWkmrdwUWvXO8IvDoz3/jzT3y1tw==,iv:mcqnkq3f0FfCnqnN7AdWAE5gDLO7+5PgWyOcK8ZAabs=,tag:+EIMFrp/0LEaf0sFzczK2g==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -58,8 +58,8 @@ sops:
             cXNZWmZqd0R0SmhINExscHBKWmxvblUKEFEQvt/zQFARba4S8vHz/1SoKdKg69At
             LZ58XQGOmlGbBhPr7EzYQ2XSY4flWbnnD174cmCR8DNFm15DsNA5fw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-05-03T16:28:04Z"
-    mac: ENC[AES256_GCM,data:dJ00o+Ny6btbOn7Pt5dc4iLx6FHJiTTx7onZDhjxya7Ywg2qAwHTiNP2q2aP6348w5uenlEGrgnV0Dc8xyHfApNCkJwj4G6UnI17jxEGn3lc0ZFNzJJ7jO2CJcwHir0E2G4XdPjmbZUdB3aKmM34dI9EEUWWhNFXdps4X0dNQcM=,iv:vOU3kHq0axRBrkNfVicQ/8H77nF0DGIJlpoDuJmwRGQ=,tag:uvoFkmaaTM8zpr1g2FpCMA==,type:str]
+    lastmodified: "2026-05-05T04:51:42Z"
+    mac: ENC[AES256_GCM,data:6o9xR1B6/AhkHu6E2FlgoPjYY3fUaLY/au0pxLVMweNK3F0/C2FZdnnm0HNwb2cRhEdlWcpbYZ5CyZ6CllVMHlJEGZI/SYXmMzkMx19O5E7AQ85T4GXLWlQl1dksIQ4q3p7fhlZ7uSIy83zRYYXEER9yV/35M1UYjIKC19NLcpY=,iv:meYHkoWnaU4iBLckq0HdpwGcabL0fgSBbtHXUNFoM3Y=,tag:AjLqzIFsCGNk60GO9yQVQw==,type:str]
     pgp:
         - created_at: "2024-11-28T18:56:39Z"
           enc: |-