garage fixes

Reviewed-on: #225

.github/workflows/update-claurst.yml

@@ -0,0 +1,169 @@
+name: "Update claurst"
+on:
+    repository_dispatch:
+    workflow_dispatch:
+    schedule:
+        - cron: "00 14 * * 1" # Every Monday at 14:00 UTC
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
+jobs:
+    update_claurst:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            - name: Install nix
+              uses: https://github.com/DeterminateSystems/nix-installer-action@main
+            - name: Setup Attic cache
+              uses: ryanccn/attic-action@v0
+              with:
+                endpoint: ${{ secrets.ATTIC_ENDPOINT }}
+                cache: ${{ secrets.ATTIC_CACHE }}
+                token: ${{ secrets.ATTIC_TOKEN }}
+                skip-push: "true"
+            - name: Get current claurst version
+              id: current
+              run: |
+                VERSION=$(grep 'version = ' pkgs/claurst/default.nix | head -1 | sed 's/.*version = "\(.*\)".*/\1/')
+                echo "version=$VERSION" >> $GITHUB_OUTPUT
+                echo "Current version: $VERSION"
+            - name: Get latest claurst release
+              id: latest
+              uses: actions/github-script@v7
+              with:
+                github-token: ${{ secrets.GITHUB_TOKEN }}
+                script: |
+                    const release = await github.rest.repos.getLatestRelease({
+                      owner: 'Kuberwastaken',
+                      repo: 'claurst',
+                    });
+                    const tag = release.data.tag_name.replace(/^v/, '');
+                    core.setOutput('version', tag);
+                    core.info(`Latest release: ${tag}`);
+            - name: Check if update needed
+              id: check_update
+              run: |
+                CURRENT="${{ steps.current.outputs.version }}"
+                LATEST="${{ steps.latest.outputs.version }}"
+                if [ "$CURRENT" = "$LATEST" ]; then
+                  echo "No update needed (current: $CURRENT, latest: $LATEST)"
+                  echo "update_needed=false" >> $GITHUB_OUTPUT
+                else
+                  echo "Update needed (current: $CURRENT, latest: $LATEST)"
+                  echo "update_needed=true" >> $GITHUB_OUTPUT
+                fi
+            - name: Update claurst if new version available
+              if: steps.check_update.outputs.update_needed == 'true'
+              id: update
+              run: |
+                NEW_VERSION="${{ steps.latest.outputs.version }}"
+
+                # Backup original file
+                cp pkgs/claurst/default.nix pkgs/claurst/default.nix.bak
+
+                # Update version placeholder with empty hash to compute it
+                sed -i "s/version = \"[^\"]*\"/version = \"$NEW_VERSION\"/" pkgs/claurst/default.nix
+
+                # Try to fetch the new src hash
+                echo "Computing src hash for v$NEW_VERSION..."
+                SRC_HASH=$(nix-prefetch-url --unpack "https://github.com/Kuberwastaken/claurst/archive/refs/tags/v$NEW_VERSION.tar.gz" 2>/dev/null | tail -1 || echo "")
+
+                if [ -z "$SRC_HASH" ]; then
+                  echo "Failed to compute src hash, reverting"
+                  mv pkgs/claurst/default.nix.bak pkgs/claurst/default.nix
+                  exit 1
+                fi
+
+                SRC_HASH="sha256-$SRC_HASH"
+                echo "New src hash: $SRC_HASH"
+
+                # Update src hash
+                sed -i "s|hash = \"sha256-[^\"]*\"|hash = \"$SRC_HASH\"|" pkgs/claurst/default.nix
+
+                # Compute cargoHash - this requires building
+                echo "Computing cargo hash..."
+                CARGO_HASH=$(nix build \
+                  --no-eval-cache \
+                  --expr "(import ./pkgs/default.nix { nixpkgs = import <nixpkgs> { }; }).mkPkgs \"x86_64-linux\" | .claurst" \
+                  2>&1 | grep -oP 'got:\s*\K[^"]+' | head -1 || echo "")
+
+                if [ -z "$CARGO_HASH" ]; then
+                  echo "Failed to compute cargo hash, trying with attribute substitution..."
+                  CARGO_HASH=$(nix eval \
+                    --impure \
+                    --expr "
+                      let
+                        pkgs = import <nixpkgs> { config.allowUnsupportedSystem = true; };
+                        claurst = import pkgs/claurst { inherit pkgs; };
+                      in claurst.cargoHash
+                    " 2>&1 | tail -1)
+                fi
+
+                if [ ! -z "$CARGO_HASH" ]; then
+                  echo "New cargo hash: $CARGO_HASH"
+                  sed -i "s|cargoHash = \"[^\"]*\"|cargoHash = \"$CARGO_HASH\"|" pkgs/claurst/default.nix
+                fi
+
+                rm -f pkgs/claurst/default.nix.bak
+                echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT
+            - name: Validate nix flake
+              if: steps.check_update.outputs.update_needed == 'true'
+              run: |
+                echo "Running nix flake check..."
+                nix flake check --show-trace || true
+            - name: Build claurst to verify changes
+              if: steps.check_update.outputs.update_needed == 'true'
+              run: |
+                echo "Building updated claurst package..."
+                nix build ".#artemision.config.environment.systemPackages" --no-eval-cache 2>&1 | tail -20 || true
+            - name: Generate PR body
+              if: steps.check_update.outputs.update_needed == 'true'
+              id: pr_body
+              run: |
+                cat > pr_body.md << 'EOF'
+                # Claurst Update
+
+                Automated claurst package update.
+
+                **Changes:**
+                - Version: `${{ steps.current.outputs.version }}` → `${{ steps.update.outputs.version }}`
+                - Source hash updated
+                - Cargo hash updated
+
+                Auto-generated by [update-claurst.yml][1].
+
+                [1]: https://nayeonie.com/ahuston-0/nix-dotfiles/src/branch/main/.github/workflows/update-claurst.yml
+                EOF
+                cat pr_body.md
+            - name: Create Pull Request
+              if: steps.check_update.outputs.update_needed == 'true'
+              uses: https://nayeonie.com/ahuston-0/create-pull-request@main
+              with:
+                token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+                add-paths: pkgs/claurst/default.nix
+                body-path: pr_body.md
+                author: '"github-actions[bot]" <github-actions[bot]@users.noreply.github.com>'
+                title: "automated: Update claurst to ${{ steps.update.outputs.version }}"
+                commit-message: |
+                    automated: Update claurst to ${{ steps.update.outputs.version }}
+
+                    - Bumped version from ${{ steps.current.outputs.version }} to ${{ steps.update.outputs.version }}
+                    - Updated src and cargo hashes
+
+                    Auto-generated by [update-claurst.yml][1].
+
+                    [1]: https://nayeonie.com/ahuston-0/nix-dotfiles/src/branch/main/.github/workflows/update-claurst.yml
+                branch: update-claurst
+                delete-branch: true
+                pr-labels: |
+                    dependencies
+                    automated
+            - name: Print PR result
+              if: steps.check_update.outputs.update_needed == 'true'
+              run: |
+                echo "Pull request created successfully"
+                echo "Version updated: ${{ steps.current.outputs.version }} → ${{ steps.update.outputs.version }}"
+permissions:
+    pull-requests: write
+    contents: write

.vscode/extensions.json

@@ -1,5 +1,5 @@
 {
-    "recommendations": [
+  "recommendations": [
-        "davidanson.vscode-markdownlint"
+    "davidanson.vscode-markdownlint"
-    ]
+  ]
-}
+}

.vscode/mcp.json

@@ -1 +1,13 @@
-{}
+{
+  "servers": {
+    "honeycomb": {
+      "command": "npx",
+      "args": [
+        "mcp-remote",
+        "https://mcp.honeycomb.io/mcp"
+      ],
+      "type": "stdio"
+    }
+  },
+  "inputs": []
+}

AGENTS.md

@@ -0,0 +1,105 @@
+> Note: This document was AI-generated and reviewed by a maintainer.
+
+# AGENTS Guide for nix-dotfiles
+
+This file is the quick-start map for coding agents working in this repository.
+Use this first, then follow the linked source files for full detail.
+
+## Purpose and Scope
+
+- Repository type: flake-based NixOS + Home Manager dotfiles/infrastructure.
+- Primary goals: safe system/user config edits, reproducible builds, and clean secrets handling.
+- Default assumption: preserve existing module patterns and avoid broad refactors unless requested.
+
+## Source of Truth
+
+Read these files before substantial changes:
+
+- `.github/copilot-instructions.md`: Full repository guide for structure, workflows, dynamic system generation, module patterns, and SOPS handling.
+- `.github/instructions/ai-doc-attribution.instructions.md`: Markdown rule for top-of-document attribution when docs are fully AI-generated.
+- `flake.nix`: Flake inputs/outputs entrypoint; system generation begins here.
+- `lib/systems.nix`: Core dynamic config assembly (`genSystems`, `constructSystem`, and wrapper generators).
+- `systems/<hostname>/default.nix`: Per-host parameters (users, home, sops, server role, extra modules).
+- `systems/<hostname>/configuration.nix`: Main host config.
+- `modules/*.nix`: Global modules automatically imported into all systems.
+- `users/<username>/home.nix` and `users/<username>/default.nix`: Home Manager and user account configuration.
+- `hydra/jobs.nix` and `hydra/jobsets.nix`: CI/build orchestration details.
+
+## Repo Mental Model
+
+- `systems/` contains host-specific configs.
+- `modules/` contains global modules applied across hosts.
+- `users/` contains user and home-manager configs.
+- `lib/systems.nix` auto-discovers hosts and composes final configs.
+- SOPS secrets are colocated with hosts/users via `secrets.yaml` files.
+
+## Dynamic Configuration Rules
+
+- Hosts are auto-discovered from subdirectories in `systems/`.
+- Each host's `default.nix` feeds `constructSystem` parameters.
+- Effective module merge order matters. High-level order is: 1) base external
+  modules, 2) host essentials (`hardware.nix`, `configuration.nix`), 3)
+  host-specific modules from `systems/<host>/default.nix`, 4) global
+  `modules/*.nix`, 5) optional SOPS and Home Manager/user layers.
+- Global modules load after host config, so explicit overrides may require `lib.mkForce` depending on target option.
+
+## Editing Conventions
+
+- Keep changes minimal and scoped to the requested behavior.
+- Preserve existing Nix style and option naming patterns.
+- Prefer module options + `lib.mkIf` toggles over hard-coded behavior.
+- Use `lib.mkDefault` for soft defaults and `lib.mkForce` only when necessary.
+- Do not commit plaintext secrets.
+- Update docs when behavior/workflow changes.
+
+## Validation and Workflow
+
+Typical local sequence:
+
+1. Make targeted edits.
+2. Evaluate and build with `nix flake check` and `nix build .#<hostname>`.
+3. Optionally deploy/apply with `nh os switch` or `nh home switch`.
+4. For secrets-related changes, edit with `sops .../secrets.yaml` and validate expected `config.sops.secrets` evaluation paths.
+
+## Secrets and Safety
+
+- Secrets live in `systems/<hostname>/secrets.yaml` and `users/<username>/secrets.yaml`.
+- Use SOPS for create/edit/rekey operations.
+- During merge conflicts in encrypted files, prefer repository SOPS merge tooling (`utils/sops-mergetool.sh`, `utils/sops-mergetool-new.sh`).
+
+## Agent and Tool Routing
+
+When a specialized agent is available, route work by intent:
+
+- `Explore`: Fast read-only repository exploration and Q&A.
+- `dependency-auditor`: Flake/module dependency security and CVE-oriented audits.
+- `security-researcher`: Read-only server security configuration audits.
+- `server-architect`: Server integration/review planning for `palatine-hill` style infra changes.
+
+Use Nix lookup tooling for package/options discovery; prefer `unstable` channel when channel selection is available.
+
+## Where To Look Next (By Task)
+
+- Add a new host: see `.github/copilot-instructions.md` sections on "Adding a New NixOS System", plus `systems/<new-host>/default.nix`, `hardware.nix`, and `configuration.nix`.
+- Add/modify a global capability: see `modules/*.nix` and the `.github/copilot-instructions.md` section "Adding a Global Module to modules/".
+- Change user/home-manager behavior: see `users/<username>/home.nix` and `users/<username>/default.nix`.
+- Modify build/release automation: see `hydra/jobs.nix` and `hydra/jobsets.nix`.
+- Work with secrets: see `.sops.yaml`, `systems/*/secrets.yaml`, `users/*/secrets.yaml`, and the `.github/copilot-instructions.md` section "Secrets Management".
+- Validate module composition/debug evaluation: see `lib/systems.nix` and `nix eval .#nixosConfigurations.<host>...`.
+
+## Documentation Attribution Rule
+
+For Markdown docs (`**/*.md`):
+
+- If a document is fully AI-generated, include explicit attribution near the top.
+- Accepted label includes "AI-generated documentation" wording.
+- Do not imply fully human authorship for fully AI-authored content.
+
+## Quick Command Reference
+
+- `nh os build`
+- `nh os switch`
+- `nh home switch`
+- `nix build .#<hostname>`
+- `nix flake check`
+- `nix eval .#nixosConfigurations.<hostname>.config.<path>`

flake.nix

@@ -164,19 +164,23 @@
           lib = self;
         }
       );
+      packageSetup = import ./pkgs/default.nix { inherit nixpkgs; };
+      inherit (packageSetup) localPackagesOverlay;
       inherit (lib.adev.systems) genSystems getImages;
       inherit (self) outputs; # for hydra
     in
     rec {
       inherit lib; # for allowing use of custom functions in nix repl
 
+      overlays.default = localPackagesOverlay;
+
       hydraJobs = import ./hydra/jobs.nix { inherit inputs outputs systems; };
       formatter = forEachSystem (system: nixpkgs.legacyPackages.${system}.nixfmt);
 
       nixosConfigurations = genSystems inputs outputs src (src + "/systems");
       homeConfigurations = {
         "alice" = inputs.home-manager.lib.homeManagerConfiguration {
-          pkgs = import nixpkgs { system = "x86_64-linux"; };
+          pkgs = packageSetup.mkPkgs "x86_64-linux";
           modules = [
             inputs.stylix.homeModules.stylix
             inputs.sops-nix.homeManagerModules.sops
@@ -203,9 +207,7 @@
         qcow = getImages nixosConfigurations "qcow";
       };
 
-      packages.x86_64-linux.lego-latest =
+      packages = forEachSystem packageSetup.mkPackages;
-        nixpkgs.legacyPackages.x86_64-linux.callPackage ./pkgs/lego-latest/default.nix
-          { };
 
       checks = import ./checks.nix { inherit inputs forEachSystem formatter; };
       devShells = import ./shell.nix { inherit inputs forEachSystem checks; };

lib/systems.nix

@@ -172,6 +172,7 @@ rec {
       modules = [
         inputs.nixos-modules.nixosModule
         inputs.nix-index-database.nixosModules.nix-index
+        { nixpkgs.overlays = [ outputs.overlays.default ]; }
         (genHostName hostname)
         (configPath + "/hardware.nix")
         (configPath + "/configuration.nix")

pkgs/bitwarden-rofi/default.nix

@@ -19,6 +19,7 @@
   libnotify,
 }:
 let
+  maintainers = import ../maintainers.nix;
   bins = [
     jq
     bitwarden-cli
@@ -64,6 +65,7 @@ stdenv.mkDerivation {
     description = "Wrapper for Bitwarden and Rofi";
     homepage = "https://github.com/mattydebie/bitwarden-rofi";
     license = licenses.gpl3;
+    maintainers = [ maintainers.alice ];
     platforms = platforms.linux;
   };
 

pkgs/claurst/default.nix

@@ -0,0 +1,52 @@
+{
+  lib,
+  fetchFromGitHub,
+  rustPlatform,
+  pkg-config,
+  openssl,
+  alsa-lib,
+  dbus,
+  libxkbcommon,
+  libxcb,
+}:
+
+let
+  maintainers = import ../maintainers.nix;
+in
+rustPlatform.buildRustPackage rec {
+  pname = "claurst";
+  version = "0.0.9";
+
+  src = fetchFromGitHub {
+    owner = "Kuberwastaken";
+    repo = "claurst";
+    rev = "v${version}";
+    hash = "sha256-bTQHtZGZxhEAki0JxSC8smAC3w+otm8ubHvZ9MvwDaE=";
+  };
+
+  cargoRoot = "src-rust";
+  cargoHash = "sha256-6+B43spqmUZ983YMl5UBH5647DcUOS2ngw5ChMIPFFo=";
+  buildAndTestSubdir = "src-rust";
+  doCheck = false;
+
+  nativeBuildInputs = [
+    pkg-config
+  ];
+
+  buildInputs = [
+    openssl
+    alsa-lib
+    dbus
+    libxkbcommon
+    libxcb
+  ];
+
+  meta = with lib; {
+    description = "Terminal coding agent written in Rust";
+    homepage = "https://github.com/Kuberwastaken/claurst";
+    license = licenses.gpl3Only;
+    mainProgram = "claurst";
+    maintainers = [ maintainers.alice ];
+    platforms = platforms.linux;
+  };
+}

pkgs/default.nix

@@ -0,0 +1,33 @@
+{ nixpkgs }:
+let
+  localPackagesOverlay = final: _prev: {
+    lego-latest = final.callPackage ./lego-latest/default.nix { };
+    claurst = final.callPackage ./claurst/default.nix { };
+  };
+
+  mkPkgs =
+    system:
+    import nixpkgs {
+      inherit system;
+      overlays = [ localPackagesOverlay ];
+    };
+
+  mkPackages =
+    system:
+    let
+      pkgs = mkPkgs system;
+    in
+    {
+      inherit (pkgs)
+        lego-latest
+        claurst
+        ;
+    };
+in
+{
+  inherit
+    localPackagesOverlay
+    mkPkgs
+    mkPackages
+    ;
+}

pkgs/maintainers.nix

@@ -0,0 +1,8 @@
+{
+  alice = {
+    name = "Alice Huston";
+    email = "aliceghuston@gmail.com";
+    github = "ahuston-0";
+    githubId = 43225907;
+  };
+}

systems/palatine-hill/configuration.nix

@@ -9,6 +9,7 @@
     ./acme.nix
     ./attic
     ./docker
+    ./garage.nix
     ./gitea.nix
     ./firewall.nix
     ./haproxy
@@ -18,6 +19,7 @@
     ./minio.nix
     ./networking.nix
     ./nextcloud.nix
+    ./otel.nix
     #./plex
     ./postgresql.nix
     ./samba.nix

systems/palatine-hill/docker/haproxy.cfg

@@ -50,6 +50,7 @@ frontend ContentSwitching
   acl host_minio hdr(host) -i minio.alicehuston.xyz
   acl host_minio_console hdr(host) -i minio-console.alicehuston.xyz
   acl host_attic hdr(host) -i attic.nayeonie.com
+  acl host_s3 hdr(host) -i s3.nayeonie.com
   acl host_minio hdr(host) -i minio.nayeonie.com
   acl host_minio_console hdr(host) -i minio-console.nayeonie.com
   #acl host_nextcloud_vol hdr(host) -i nextcloud-vol.alicehuston.xyz
@@ -67,6 +68,7 @@ frontend ContentSwitching
   use_backend nextcloud_nodes if host_nextcloud
   use_backend hydra_nodes if host_hydra
   use_backend attic_nodes if host_attic
+  use_backend garage_nodes if host_s3
   #use_backend nextcloud_vol_nodes if host_nextcloud_vol
 #  use_backend collabora_nodes if host_collabora
   use_backend prometheus_nodes if host_prometheus
@@ -142,6 +144,10 @@ backend minio_console_nodes
   mode http
   server server 192.168.76.2:8501
 
+backend garage_nodes
+  mode http
+  server server 192.168.76.2:8502
+
 # backend foundry_nodes
 #   timeout tunnel 50s
 #   mode http

systems/palatine-hill/garage.nix

@@ -0,0 +1,83 @@
+{
+  config,
+  pkgs,
+  ...
+}:
+
+let
+  vars = import ./vars.nix;
+  basePath = "${vars.primary_minio}/garage";
+in
+{
+  services.garage = {
+    enable = true;
+    package = pkgs.garage;
+    logLevel = "info";
+    settings = {
+      metadata_dir = "${basePath}/meta";
+      data_dir = "${basePath}/data";
+      db_engine = "sqlite";
+      replication_factor = 1;
+
+      rpc_bind_addr = "127.0.0.1:8504";
+      rpc_public_addr = "127.0.0.1:8504";
+      rpc_secret_file = config.sops.secrets."garage/rpc-secret".path;
+
+      s3_api = {
+        api_bind_addr = "127.0.0.1:8502";
+        s3_region = "us-east-1";
+        root_domain = ".s3.nayeonie.com";
+      };
+
+      admin = {
+        api_bind_addr = "127.0.0.1:8503";
+        admin_token_file = config.sops.secrets."garage/admin-token".path;
+      };
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d ${basePath}/meta 0750 garage garage -"
+    "d ${basePath}/data 0750 garage garage -"
+  ];
+
+  systemd.services.garage = {
+    unitConfig.RequiresMountsFor = [
+      vars.primary_minio
+      basePath
+      "${basePath}/meta"
+      "${basePath}/data"
+    ];
+    preStart = ''
+      mkdir -p ${basePath}/meta ${basePath}/data
+      chown -R garage:garage ${basePath}/meta ${basePath}/data
+    '';
+    serviceConfig = {
+      PermissionsStartOnly = true;
+      DynamicUser = false;
+      User = "garage";
+      Group = "garage";
+    };
+  };
+
+  users.groups.garage = { };
+  users.users.garage = {
+    isSystemUser = true;
+    group = "garage";
+  };
+
+  sops.secrets = {
+    "garage/rpc-secret" = {
+      owner = "garage";
+      group = "garage";
+      mode = "0400";
+      restartUnits = [ "garage.service" ];
+    };
+    "garage/admin-token" = {
+      owner = "garage";
+      group = "garage";
+      mode = "0400";
+      restartUnits = [ "garage.service" ];
+    };
+  };
+}

systems/palatine-hill/hydra.nix

@@ -6,6 +6,8 @@
 let
   hydra_notify_prometheus_port = "9199";
   hydra_queue_runner_prometheus_port = "9200";
+  postgres_exporter_port = 9187;
+  zfs_exporter_port = 9134;
 in
 {
   systemd.services.hydra-notify.serviceConfig.EnvironmentFile =
@@ -96,10 +98,23 @@ in
       enable = true;
       webExternalUrl = "https://prom.alicehuston.xyz";
       port = 9001;
-      exporters.node = {
+      exporters = {
-        enable = true;
+        node = {
-        enabledCollectors = [ "systemd" ];
+          enable = true;
-        port = 9002;
+          enabledCollectors = [ "systemd" ];
+          port = 9002;
+        };
+        postgres = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = postgres_exporter_port;
+          runAsLocalSuperUser = true;
+        };
+        zfs = {
+          enable = true;
+          listenAddress = "127.0.0.1";
+          port = zfs_exporter_port;
+        };
       };
       scrapeConfigs = [
         {
@@ -119,6 +134,18 @@ in
             }
           ];
         }
+        {
+          job_name = "postgres-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ]; }
+          ];
+        }
+        {
+          job_name = "zfs-local";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.zfs.port}" ]; }
+          ];
+        }
         {
           job_name = "hydra-external";
           scheme = "https";

systems/palatine-hill/otel.nix

@@ -0,0 +1,174 @@
+{
+  config,
+  pkgs,
+  ...
+}:
+{
+  # node_exporter (port 9002) and Prometheus (port 9001) are already configured
+  # in hydra.nix — we just scrape the existing exporter here.
+
+  services.opentelemetry-collector = {
+    enable = true;
+    package = pkgs.opentelemetry-collector-contrib;
+    settings = {
+      receivers = {
+        # Accept OTLP traces/metrics from local services and containers.
+        otlp = {
+          protocols = {
+            grpc.endpoint = "127.0.0.1:4317";
+            http.endpoint = "127.0.0.1:4318";
+          };
+        };
+
+        # Host-level system metrics
+        hostmetrics = {
+          collection_interval = "60s";
+          scrapers = {
+            cpu = { };
+            memory = { };
+            disk = { };
+            filesystem = { };
+            network = { };
+            load = { };
+            processes = { };
+          };
+        };
+
+        # Scrape node_exporter for per-systemd-unit service state
+        prometheus = {
+          config = {
+            scrape_configs = [
+              {
+                job_name = "node-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "postgres-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+              {
+                job_name = "zfs-exporter";
+                scrape_interval = "60s";
+                static_configs = [
+                  {
+                    targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ];
+                    labels = {
+                      host = "palatine-hill";
+                    };
+                  }
+                ];
+              }
+            ];
+          };
+        };
+      };
+
+      processors = {
+        memory_limiter = {
+          check_interval = "1s";
+          limit_percentage = 75;
+          spike_limit_percentage = 15;
+        };
+
+        batch = {
+          send_batch_size = 8192;
+          timeout = "5s";
+        };
+
+        attributes = {
+          actions = [
+            {
+              action = "upsert";
+              key = "deployment.environment";
+              value = "palatine-hill";
+            }
+          ];
+        };
+
+        # Attach hostname using the standard resource processor
+        resource = {
+          attributes = [
+            {
+              action = "upsert";
+              key = "host.name";
+              value = "palatine-hill";
+            }
+          ];
+        };
+      };
+
+      exporters = {
+        "otlp/honeycomb-metrics" = {
+          endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
+          headers = {
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-metrics";
+          };
+        };
+
+        "otlp/honeycomb-traces" = {
+          endpoint = "api.honeycomb.io:443";
+          compression = "gzip";
+          headers = {
+            "x-honeycomb-team" = "\${file:" + config.sops.secrets."honeycomb/api-key".path + "}";
+            "x-honeycomb-dataset" = "palatine-hill-traces";
+          };
+        };
+      };
+
+      service = {
+        pipelines = {
+          metrics = {
+            receivers = [
+              "otlp"
+              "hostmetrics"
+              "prometheus"
+            ];
+            processors = [
+              "memory_limiter"
+              "resource"
+              "attributes"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb-metrics" ];
+          };
+
+          traces = {
+            receivers = [ "otlp" ];
+            processors = [
+              "memory_limiter"
+              "resource"
+              "attributes"
+              "batch"
+            ];
+            exporters = [ "otlp/honeycomb-traces" ];
+          };
+        };
+      };
+    };
+  };
+
+  sops.secrets = {
+    "honeycomb/api-key" = {
+      owner = "root";
+      restartUnits = [ "opentelemetry-collector.service" ];
+    };
+  };
+}

systems/palatine-hill/secrets.yaml

@@ -42,6 +42,11 @@ server-validation:
     webhook: ENC[AES256_GCM,data:Lwqy4UhyFutpXjai7EJPKp8MDlI+ayDna4T8jluvC6qkeJ7o1UaaDCOsgLy4Fw7LC77tXhJtkcmep9w37JaiHp2CoDOfy2iAaq8o9CCSi/a0zqMJx+HdZYZNemvmpc6E/be0K+JDrFZLbjr3unSpCidQ3whccC6XyY013R12swN3bFZIu1gtzXCgUZ4U,iv:pVbrRwH3ziu4+R5BfimPV7N71QmyerJEc9M5K4eofOc=,tag:zNrCXrIioQWPEPVz/wMDpQ==,type:str]
 typhon:
     hashedPassword: ENC[AES256_GCM,data:gMyY8gxUn3HzycQRu2cminqRFWghqWcjzZzTxAQZ5PJqn604iSwDiVdr7icHB7drJfCAfsE7L4oKRJgxaIAE32043oOkb2T7DDH8y2jxMzqmZCfbvrfMI4wdfRTHGqzxb6X/aZ5ai2rr1Q==,iv:4EsTo/lQld0o9iktDX9gobMlPUCitx1i9wn8EL16sIs=,tag:FgVDRHk2glDwpC/mprrPqQ==,type:str]
+garage:
+    rpc-secret: ENC[AES256_GCM,data:Q2ZaAXcntD3yK6DynEpxab2TITByMZ7ECVrq1pb0ZU7hXOZnhaBmjdty/Os6len8l+GBl6+WaC0An6cFkhQTlQ==,iv:E8C4bnxMLXK9fky+KC7q8sHpmrEU5un0TEAwxVUBiLk=,tag:PiSiU+9NpyilH2aMs2Qc/Q==,type:str]
+    admin-token: ENC[AES256_GCM,data:Xjm8Xq99aDseR0jN50Uj3gLpeDaq2IGXzJCS0o1H0RgKX9LGdP8w508nWWE=,iv:+L9T3TEUSbIz+jo08ykjGHVhuz5ecmzrlhzD2iv48HE=,tag:7P2rY4F8cWFdG4Lm9n/etQ==,type:str]
+honeycomb:
+    api-key: ENC[AES256_GCM,data:sDhWmpaxLBb+qv/REDEbqpVsTNZBNuuLBGRvv0RYmdAzYBAZUn2OnBTHwgS7Bgv7xRDKgsGW8cOm0gQ8NUdWkmrdwUWvXO8IvDoz3/jzT3y1tw==,iv:mcqnkq3f0FfCnqnN7AdWAE5gDLO7+5PgWyOcK8ZAabs=,tag:+EIMFrp/0LEaf0sFzczK2g==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -53,8 +58,8 @@ sops:
             cXNZWmZqd0R0SmhINExscHBKWmxvblUKEFEQvt/zQFARba4S8vHz/1SoKdKg69At
             LZ58XQGOmlGbBhPr7EzYQ2XSY4flWbnnD174cmCR8DNFm15DsNA5fw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-01-17T01:50:50Z"
+    lastmodified: "2026-05-05T04:51:42Z"
-    mac: ENC[AES256_GCM,data:8TGSqwEcfmrW1PjuzTVNyDTNs6s3oWbT0tI+rg7u2w5Dcw1EEU+SjJ6VpNY06AZHTjSD6E0O7NzUxybtMpslHUGitOGWwQCk+sbqRJuUseFe7bWFboEVoJpEoYGN5pnn52opMT+NeHGkXumaxjhDjCxfwn1RBHR7TgD4ZHEH6pE=,iv:szBUnn3HL/osWhmTwYmHrUghobWdBR60Lc6uUD/eGMY=,tag:6vgdJeJjL4ZYKc8WjixClg==,type:str]
+    mac: ENC[AES256_GCM,data:6o9xR1B6/AhkHu6E2FlgoPjYY3fUaLY/au0pxLVMweNK3F0/C2FZdnnm0HNwb2cRhEdlWcpbYZ5CyZ6CllVMHlJEGZI/SYXmMzkMx19O5E7AQ85T4GXLWlQl1dksIQ4q3p7fhlZ7uSIy83zRYYXEER9yV/35M1UYjIKC19NLcpY=,iv:meYHkoWnaU4iBLckq0HdpwGcabL0fgSBbtHXUNFoM3Y=,tag:AjLqzIFsCGNk60GO9yQVQw==,type:str]
     pgp:
         - created_at: "2024-11-28T18:56:39Z"
           enc: |-
@@ -69,4 +74,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: 5EFFB75F7C9B74EAA5C4637547940175096C1330
     unencrypted_suffix: _unencrypted
-    version: 3.11.0
+    version: 3.12.2

users/alice/home.nix

@@ -90,6 +90,7 @@
 
       gocryptfs
       awscli2
+      claurst
     ];
   };
 

users/alice/secrets.yaml

@@ -10,6 +10,8 @@ alice:
     attic-nix-cache-admin: ENC[AES256_GCM,data:xHJGeU4EUn1HRy2nIValiJ6iLZnYmmT6Njv/cGMh15Q0hJXKNBSsi8f0mAfLI7EX+GaC299VKh2uTlU25jptrAvogLxNJIc+LZBLsSkyGE/ojqqevHMKmZ/6eciLZRQL5ey9TM3V9HHyDOhGaFgdfawtwg/vyvbV13lZBKpqneAX9T3gPRuKRjV4/Uc/5cUckiOF8bQ50xVFN8Cql9HgGDJEGWgg4XUTPu5eYspof2EN63pYvU7wg6HD2begeLDvqc2/i2DIcsc0wqc5DgkY/dH2YtcssBtU8AR9vKpl+HmH/wvt6dfaEyZ7hF7ITGwWnOO6H2ko3SjYRfHkFK3XDmm1YRRjfkptnw==,iv:BdVgNyZ1azl5tKfH+RTeXuNV/rYY6hPvrareKlIXSeQ=,tag:/ar87eAjMod4TmQXoerNBQ==,type:str]
     gitea-actions-token: ENC[AES256_GCM,data:QTEPMAh1RWWJ/O3yhkQkEBTdVL8XhIRGCDbiM0lLjfILKF4SpSJ2sA==,iv:mBaaB1JHb2KVc9n2pdeX4pSMvb7q5z3joMT7rR5Whgs=,tag:ef+58SI4AUeqUsk3RVDsRQ==,type:str]
     gitea-pr-token: ENC[AES256_GCM,data:ybTya4X2wd65pNFSGbQkg73lu66GNtSba4yf8J6tT8XkuOtfvtBS4g==,iv:39mJiAlw4kud4l06jOpxOCRumChE/5q8IBNsPHG1rMc=,tag:MEvHD2b9E3fVHLlz7haNyw==,type:str]
+    honeycomb-id: ENC[AES256_GCM,data:PndCclCbSMrgmlYdQ5a8//IB+hg9uB0ZwidIZFiKN6w=,iv:mxFcGPnY0eCliugvQT5HR9aGzJIvXZI6FTo8rphVQMM=,tag:3huIaIy7da0gx2G/BVJJLQ==,type:str]
+    honeycomb-secret: ENC[AES256_GCM,data:x2MZPgAJz3I1m+rjSVpINZdQVbu50XiwEtPGEX/kbJw=,iv:bPNNtXIES0pCLc4Nu/886nwoOVXaZgyTxndSpyWWU0E=,tag:igdd/NPCDpcvzSd9LO4hyw==,type:str]
 sops:
     age:
         - recipient: age1qw5k8h72k3fjg5gmlxx8q8gwlc2k6n6u08d8hdzpm2pk9r0fnfxsmw33nh
@@ -39,8 +41,8 @@ sops:
             ZERFTlFyNjhOb3VCaW43ZXFHT1Vxc0UK7YV+BU7dCEOZxpqkQA394eDsnthvorj6
             7bqrCdeU+6DU7DmFs6++BrNO2tx8vvOa1im+ZGrM/gZAJdv/7R2d6Q==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-04-07T23:43:57Z"
+    lastmodified: "2026-05-03T16:20:31Z"
-    mac: ENC[AES256_GCM,data:ygQzxSpGJqXwkOq7jGDeflA2FTSSxnre/PXm0LxmxzQQW5s7LeIVSI75fMqWir0WU3Pi/xroYGEWjpCG6JvxV5RiJycTONk8VE7c3jtw3AbrHSS0b1K5tJ+Sf+q3rHJFWWk/COrPk8IsRFNb+taqH4jnaH3AAVNo5u0C1CHKMes=,iv:FO2GVDXE8SjjA81/9cDwc+dX8kJ2oHt5kqkhNBuMb54=,tag:hgzRAmsh32SCvJEvKyV+vg==,type:str]
+    mac: ENC[AES256_GCM,data:/Ui0H0wgENYnzVB7V2aAbj4dIbE+sjRpPoso75xyKepmeeMp8nv8CJLKmLmtbp+rtX287teVw4hKxb/z8M6ZHIWmgt9Zgui8/+hw5hwuMmyjPeGqJQjFH8s4zcXkOKLDoFPA65gva5gaPsreqPwOSqgqYwmds5gOcxBI3Tqdh6E=,iv:ugtug8Vb7DRUp1eC2M6ooCoPFjbqZo8htHOV4AXIcSI=,tag:4sMJsh/r57Z4VFoOBmUmXw==,type:str]
     pgp:
         - created_at: "2024-09-05T06:10:22Z"
           enc: |-
@@ -55,4 +57,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: 5EFFB75F7C9B74EAA5C4637547940175096C1330
     unencrypted_suffix: _unencrypted
-    version: 3.10.1
+    version: 3.12.2