nix-dotfiles/systems/palatine-hill/zfs.nix

{
  config,
  lib,
  pkgs,
  ...
}:
{
  boot = {
    zfs.extraPools = [ "ZFS-primary" ];
    filesystem = "zfs";
    extraModprobeConfig = ''
      options zfs zfs_arc_min=82463372083
      options zfs zfs_arc_max=192414534860
    '';
  };

  services = {
    zfs = {
      trim.enable = true;
      autoScrub.enable = true;
    };

    sanoid = {
      enable = true;

      datasets = {
        "ZFS-primary/attic".useTemplate = [ "nix-prod" ];
        "ZFS-primary/backups".useTemplate = [ "production" ];
        "ZFS-primary/calibre".useTemplate = [ "production" ];
        "ZFS-primary/db".useTemplate = [ "production" ];
        "ZFS-primary/docker".useTemplate = [ "production" ];
        "ZFS-primary/hydra".useTemplate = [ "nix-prod" ];
        "ZFS-primary/nextcloud".useTemplate = [ "production" ];
        # all docker containers should have a bind mount if they expect lasting zfs snapshots
        "ZFS-primary/vardocker".useTemplate = [ "nix-prod" ];
        "ZFS-primary/minio".useTemplate = [ "nix-prod" ];
        "ZFS-primary/games" = {
          useTemplate = [ "games" ];
          recursive = true;
          processChildrenOnly = true;
        };
      };

      templates = {
        # full resiliency
        production = {
          frequently = 0;
          hourly = 36;
          daily = 30;
          weekly = 0;
          monthly = 6;
          yearly = 3;
          autosnap = true;
          autoprune = true;
        };
        # some resiliency, but not much
        # common option for things like nix store and attic where there is
        # already a lot of resiliency built in
        nix-prod = {
          frequently = 4;
          hourly = 24;
          daily = 7;
          weekly = 0;
          monthly = 0;
          yearly = 0;
          autosnap = true;
          autoprune = true;
        };
        # much shorter lived than others
        games = {
          frequently = 6;
          hourly = 36;
          daily = 3;
          weekly = 0;
          monthly = 0;
          yearly = 0;
          autosnap = true;
          autoprune = true;
        };
      };
    };
  };

  # hack to make sure pool is imported before keys are loaded,
  # and also keys are imported before things get mounted
  # note to self: move zfs encryption over to luks lol
  boot.initrd.postResumeCommands = ''
    ZFS_FORCE="-f"

    for o in $(cat /proc/cmdline); do
      case $o in
        zfs_force|zfs_force=1|zfs_force=y)
          ZFS_FORCE="-f"
          ;;
      esac
    done
    poolReady() {
      pool="$1"
      state="$("zpool" import -d "/dev/disk/by-id/" 2>/dev/null | "awk" "/pool: $pool/ { found = 1 }; /state:/ { if (found == 1) { print \$2; exit } }; END { if (found == 0) { print \"MISSING\" } }")"
      if [[ "$state" = "ONLINE" ]]; then
        return 0
      else
        echo "Pool $pool in state $state, waiting"
        return 1
      fi
    }
    poolImported() {
      pool="$1"
      "zpool" list "$pool" >/dev/null 2>/dev/null
    }
    poolImport() {
      pool="$1"
      "zpool" import -d "/dev/disk/by-id/" -N $ZFS_FORCE "$pool"
    }

    echo -n "importing root ZFS pool \"ZFS-primary\"..."
    # Loop across the import until it succeeds, because the devices needed may not be discovered yet.
    if ! poolImported "ZFS-primary"; then
      for trial in `seq 1 60`; do
        poolReady "ZFS-primary" > /dev/null && msg="$(poolImport "ZFS-primary" 2>&1)" && break
        sleep 1
        echo -n .
      done
      echo
      if [[ -n "$msg" ]]; then
        echo "$msg";
      fi
      poolImported "ZFS-primary" || poolImport "ZFS-primary"  # Try one last time, e.g. to import a degraded pool.
    fi

    # let root mount and everything, then manually unlock stuff
    load_zfs_nix() {
      local device="/dev/disk/by-uuid/8bfaa32b-09dd-45c8-831e-05e80be82f9e"
      local mountPoint="/"
      local options="x-initrd.mount,noatime,nodiratime"
      local fsType="ext4"

      echo "manually mounting key location, then unmounting"
      udevadm settle

      mountFS "$device" "$(escapeFstab "$mountPoint")" "$(escapeFstab "$options")" "$fsType"

      zfs load-key -L "file://$targetRoot/crypto/keys/zfs-nix-store-key" "ZFS-primary/nix"
      umount "$targetRoot/"
    }

    load_zfs_nix
  '';
}