nix-dotfiles/systems/palatine-hill/zfs.nix

{
  config,
  lib,
  pkgs,
  ...
}:
{
  boot = {
    zfs.extraPools = [ "ZFS-primary" ];
    filesystem = "zfs";
    extraModprobeConfig = ''
      options zfs zfs_arc_min=82463372083
      options zfs zfs_arc_max=192414534860
    '';

    initrd.systemd.services = {
      zfs-import-zfs-primary = {
        description = "Import ZFS-primary pool in initrd";
        wantedBy = [ "initrd-root-fs.target" ];
        wants = [ "systemd-udev-settle.service" ];
        after = [ "systemd-udev-settle.service" ];
        before = [
          "sysroot.mount"
          "initrd-root-fs.target"
        ];
        unitConfig.DefaultDependencies = "no";
        serviceConfig = {
          Type = "oneshot";
          RemainAfterExit = true;
        };
        path = with pkgs; [
          coreutils
          gawk
          zfs
        ];
        script = ''
          ZFS_FORCE="-f"
          msg=""

          for o in $(cat /proc/cmdline); do
            case "$o" in
              zfs_force|zfs_force=1|zfs_force=y)
                ZFS_FORCE="-f"
                ;;
            esac
          done

          pool_ready() {
            pool="$1"
            state="$(zpool import -d /dev/disk/by-id/ 2>/dev/null | awk '/pool: '"$pool"'/ { found = 1 }; /state:/ { if (found == 1) { print $2; exit } }; END { if (found == 0) { print "MISSING" } }')"
            if [ "$state" = "ONLINE" ]; then
              return 0
            fi
            echo "Pool $pool in state $state, waiting"
            return 1
          }

          pool_imported() {
            pool="$1"
            zpool list "$pool" >/dev/null 2>/dev/null
          }

          pool_import() {
            pool="$1"
            zpool import -d /dev/disk/by-id/ -N $ZFS_FORCE "$pool"
          }

          echo -n 'importing root ZFS pool "ZFS-primary"...'
          # Loop until import succeeds, because by-id devices may not be discovered yet.
          if ! pool_imported "ZFS-primary"; then
            trial=1
            while [ "$trial" -le 60 ]; do
              if pool_ready "ZFS-primary" >/dev/null && msg="$(pool_import "ZFS-primary" 2>&1)"; then
                break
              fi
              sleep 1
              echo -n .
              trial=$((trial + 1))
            done
            echo
            if [ -n "$msg" ]; then
              echo "$msg"
            fi
            pool_imported "ZFS-primary" || pool_import "ZFS-primary"  # Try one last time, e.g. to import a degraded pool.
          fi
        '';
      };

      zfs-load-nix-key = {
        description = "Load ZFS key for ZFS-primary/nix in initrd";
        wantedBy = [ "initrd-fs.target" ];
        requires = [
          "sysroot.mount"
          "zfs-import-zfs-primary.service"
        ];
        after = [
          "sysroot.mount"
          "zfs-import-zfs-primary.service"
        ];
        before = [
          "initrd-fs.target"
          "sysroot-nix.mount"
        ];
        unitConfig.DefaultDependencies = "no";
        serviceConfig = {
          Type = "oneshot";
          RemainAfterExit = true;
        };
        path = with pkgs; [ zfs ];
        script = ''
          key_file="/sysroot/crypto/keys/zfs-nix-store-key"
          zfs load-key -L "file://$key_file" "ZFS-primary/nix"
        '';
      };
    };
  };

  services = {
    zfs = {
      trim.enable = true;
      autoScrub.enable = true;
    };

    sanoid = {
      enable = true;

      datasets = {
        "ZFS-primary/attic".useTemplate = [ "nix-prod" ];
        "ZFS-primary/backups".useTemplate = [ "production" ];
        "ZFS-primary/calibre".useTemplate = [ "production" ];
        "ZFS-primary/db".useTemplate = [ "production" ];
        "ZFS-primary/docker".useTemplate = [ "production" ];
        "ZFS-primary/hydra".useTemplate = [ "nix-prod" ];
        "ZFS-primary/nextcloud".useTemplate = [ "production" ];
        # all docker containers should have a bind mount if they expect lasting zfs snapshots
        "ZFS-primary/vardocker".useTemplate = [ "nix-prod" ];
        "ZFS-primary/minio".useTemplate = [ "nix-prod" ];
        "ZFS-primary/games" = {
          useTemplate = [ "games" ];
          recursive = true;
          processChildrenOnly = true;
        };
      };

      templates = {
        # full resiliency
        production = {
          frequently = 0;
          hourly = 36;
          daily = 30;
          weekly = 0;
          monthly = 6;
          yearly = 2;
          autosnap = true;
          autoprune = true;
        };
        # some resiliency, but not much
        # common option for things like nix store and attic where there is
        # already a lot of resiliency built in
        nix-prod = {
          frequently = 4;
          hourly = 24;
          daily = 7;
          weekly = 0;
          monthly = 0;
          yearly = 0;
          autosnap = true;
          autoprune = true;
        };
        # much shorter lived than others
        games = {
          frequently = 6;
          hourly = 36;
          daily = 3;
          weekly = 0;
          monthly = 0;
          yearly = 0;
          autosnap = true;
          autoprune = true;
        };
      };
    };
  };

}