From b77a34415defe764193cecf7991c74e4efe12112 Mon Sep 17 00:00:00 2001 From: Alice Huston Date: Sun, 24 Mar 2024 13:08:42 -0400 Subject: [PATCH] Add attic cache (#117) * add attic * change formatter to nixfmt-rfc-style Signed-off-by: ahuston-0 * update lock Signed-off-by: ahuston-0 * initial format Signed-off-by: ahuston-0 * move back to postgres Signed-off-by: ahuston-0 * fix attic * fix signing key * formatting Signed-off-by: ahuston-0 --------- Signed-off-by: ahuston-0 --- flake.lock | 82 +++++++++++++++++++++++- flake.nix | 11 ++++ modules/flake-update-service.nix | 4 +- systems/configuration.nix | 1 - systems/palatine-hill/configuration.nix | 83 ++++++++++++++++++++++++- systems/palatine-hill/default.nix | 3 +- systems/palatine-hill/secrets.yaml | 9 ++- 7 files changed, 185 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 831608d..97b1965 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,52 @@ { "nodes": { + "attic": { + "inputs": { + "crane": "crane", + "flake-compat": "flake-compat", + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1707922053, + "narHash": "sha256-wSZjK+rOXn+UQiP1NbdNn5/UW6UcBxjvlqr2wh++MbM=", + "owner": "zhaofengli", + "repo": "attic", + "rev": "6eabc3f02fae3683bffab483e614bebfcd476b21", + "type": "github" + }, + "original": { + "owner": "zhaofengli", + "repo": "attic", + "type": "github" + } + }, + "crane": { + "inputs": { + "nixpkgs": [ + "attic", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1702918879, + "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=", + "owner": "ipetkov", + "repo": "crane", + "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "fenix": { "inputs": { "nixpkgs": [ @@ -37,6 +84,22 @@ "type": "github" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": [ @@ -95,7 +158,7 @@ }, "nix": { "inputs": { - "flake-compat": "flake-compat", + "flake-compat": "flake-compat_2", "libgit2": "libgit2", "nixpkgs": [ "nixpkgs" @@ -215,8 +278,25 @@ "type": "github" } }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1702780907, + "narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { + "attic": "attic", "fenix": "fenix", "flake-utils": "flake-utils", "home-manager": "home-manager", diff --git a/flake.nix b/flake.nix index fe46612..f6345d6 100644 --- a/flake.nix +++ b/flake.nix @@ -5,12 +5,14 @@ trusted-users = [ "root" ]; substituters = [ "https://cache.nixos.org/?priority=1&want-mass-query=true" + "https://attic.alicehuston.xyz/cache-nix-dot?priority=4&want-mass-query=true" "https://cache.alicehuston.xyz/?priority=5&want-mass-query=true" "https://nix-community.cachix.org/?priority=10&want-mass-query=true" ]; trusted-substituters = [ "https://cache.nixos.org" + "https://attic.alicehuston.xyz/cache-nix-dot" "https://cache.alicehuston.xyz" "https://nix-community.cachix.org" ]; @@ -19,6 +21,7 @@ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" "cache.alicehuston.xyz:SJAm8HJVTWUjwcTTLAoi/5E1gUOJ0GWum2suPPv7CUo=%" "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + "cache-nix-dot:0hp/F6mUJXNyZeLBPNBjmyEh8gWsNVH+zkuwlWMmwXg=" ]; }; @@ -73,6 +76,14 @@ flake-utils.follows = "flake-utils"; }; }; + + attic = { + url = "github:zhaofengli/attic"; + inputs = { + nixpkgs.follows = "nixpkgs"; + flake-utils.follows = "flake-utils"; + }; + }; }; outputs = diff --git a/modules/flake-update-service.nix b/modules/flake-update-service.nix index 92b9c95..870debd 100644 --- a/modules/flake-update-service.nix +++ b/modules/flake-update-service.nix @@ -50,8 +50,8 @@ in pkgs.git ]; systemd.services."autopull@${cfg.name}" = { - after = [ "multi-user.target" ]; - requires = [ "multi-user.target" ]; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; description = "Pull the latest data for ${cfg.name}"; environment = lib.mkIf (cfg.ssh-key != "") { GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.ssh-key} -o IdentitiesOnly=yes"; diff --git a/systems/configuration.nix b/systems/configuration.nix index 316db0a..dd86329 100644 --- a/systems/configuration.nix +++ b/systems/configuration.nix @@ -144,7 +144,6 @@ enableCompletion = true; enableBashCompletion = true; ohMyZsh.enable = true; - shellAliases.update = "sudo nixos-rebuild switch --fast --accept-flake-config --flake /root/dotfiles#${config.networking.hostName} -L |& nom"; autosuggestions = { enable = true; strategy = [ "completion" ]; diff --git a/systems/palatine-hill/configuration.nix b/systems/palatine-hill/configuration.nix index 687232f..2492369 100644 --- a/systems/palatine-hill/configuration.nix +++ b/systems/palatine-hill/configuration.nix @@ -92,6 +92,8 @@ }; environment.systemPackages = with pkgs; [ + attic-client + attic docker-compose jellyfin-ffmpeg ]; @@ -117,9 +119,27 @@ superuser_map /^(.*)$ \1 ''; + ensureDatabases = [ "atticd" ]; + ensureUsers = [ + { + name = "atticd"; + ensureDBOwnership = true; + } + ]; + + # initialScript = config.sops.secrets."postgres/init".path; + upgrade = { enable = true; - stopServices = [ "hydra" ]; + stopServices = [ + "hydra-evaluator" + "hydra-init" + "hydra-notify" + "hydra-queue-runner" + "hydra-send-stats" + "hydra-server" + "atticd" + ]; }; }; @@ -138,6 +158,64 @@ enable = true; secretKeyFile = config.sops.secrets."nix-serve/secret-key".path; }; + atticd = { + enable = true; + + credentialsFile = config.sops.secrets."attic/secret-key".path; + + settings = { + listen = "[::]:8183"; + allowed-hosts = [ "attic.alicehuston.xyz" ]; + api-endpoint = "https://attic.alicehuston.xyz"; + compression.type = "none"; # let ZFS do the compressing + database = { + url = "postgres://atticd?host=/run/postgresql"; + # disable postgres, using SOPS fails at below :( + # https://github.com/zhaofengli/attic/blob/main/nixos/atticd.nix#L57 + # url = "sqlite:///ZFS/ZFS-primary/attic/server.db?mode=rwc"; + heartbeat = true; + }; + storage = { + type = "local"; + path = "/ZFS/ZFS-primary/attic/storage"; + }; + + # Warning: If you change any of the values here, it will be + # difficult to reuse existing chunks for newly-uploaded NARs + # since the cutpoints will be different. As a result, the + # deduplication ratio will suffer for a while after the change. + chunking = { + # The minimum NAR size to trigger chunking + # + # If 0, chunking is disabled entirely for newly-uploaded NARs. + # If 1, all NARs are chunked. + nar-size-threshold = 64 * 1024; # 64 KiB + + # The preferred minimum size of a chunk, in bytes + min-size = 16 * 1024; # 16 KiB + + # The preferred average size of a chunk, in bytes + avg-size = 64 * 1024; # 64 KiB + + # The preferred maximum size of a chunk, in bytes + max-size = 256 * 1024; # 256 KiB + }; + }; + }; + }; + + # borrowing from https://github.com/Shawn8901/nix-configuration/blob/4b8d1d44f47aec60feb58ca7b7ab5ed000506e90/modules/nixos/private/hydra.nix + # configured default webstore for this on root user separately + systemd.services.attic-watch-store = { + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + requires = [ "network-online.target" ]; + description = "Upload all store content to binary catch"; + serviceConfig = { + User = "root"; + Restart = "always"; + ExecStart = "${pkgs.attic}/bin/attic watch-store cache-nix-dot"; + }; }; sops = { @@ -145,6 +223,9 @@ secrets = { "hydra/environment".owner = "hydra"; "nix-serve/secret-key".owner = "root"; + "attic/secret-key".owner = "root"; + "attic/database-url".owner = "root"; + "postgres/init".owner = "postgres"; }; }; diff --git a/systems/palatine-hill/default.nix b/systems/palatine-hill/default.nix index 01f0304..94d5707 100644 --- a/systems/palatine-hill/default.nix +++ b/systems/palatine-hill/default.nix @@ -1,7 +1,8 @@ -{ ... }: +{ inputs, ... }: { users = [ "alice" "richie" ]; + modules = [ inputs.attic.nixosModules.atticd ]; } diff --git a/systems/palatine-hill/secrets.yaml b/systems/palatine-hill/secrets.yaml index 63d6bb3..df5d607 100644 --- a/systems/palatine-hill/secrets.yaml +++ b/systems/palatine-hill/secrets.yaml @@ -2,6 +2,11 @@ hydra: environment: ENC[AES256_GCM,data:k6t0jVLgsCbOwAnj71ogmsdoLsMaMjeScYRblE72FNEk8cgWc2Q5kw5LVShIC5Kgl2XhSJIoi1+pDS1X5huyWs+cz4T9oUtOJhtSlL9+UCLmaqoR0SCI1eCZT1fkRZ3QtitrRmtvm77Sld7Ckz/apG7cQsfpKhymkEz+Y8WdC3mc5Kjt05eAn66IbQYO8y1HQc9bkCAWYD+NSwOqC80W5RIfkKActWz1DFoeTESwMcpA9MKHlGMKP82Uo/qlRhXq+riY5e5voFGQw0O3CKRTy1Q=,iv:Fbl/9XkNTe5qmn7wvPtQ1Hpfzp7+3WLeuipkme9a29A=,tag:+git1pCZzSirfFsxj91WUQ==,type:str] nix-serve: secret-key: ENC[AES256_GCM,data:a+N7udOUnls35wCyO/icqtMWEVMorg3mSlZKih8LHQM4wgemZXuXYdhvw65CTPHvzcS0mr6QEMNzkqXios4kvlNDUvbG0OuaVhtqWqtuutz4J9VsGf8PdIvXNkLSHfm2fEY4n84nYM5tUidzwfA=,iv:045gOacG0t9rbzaszQ/5quZkRvfHLF8cETG2tABUrvk=,tag:sLs/yFdUlwf+YZf/Ja8YbA==,type:str] +attic: + secret-key: ENC[AES256_GCM,data:h6DQhTgEhcFnjwUojPEleZh8vkBiCCCwLM+dECRpqGURiRJ4mDa3Edb1Ja42GWyAYy8X5B0UmsVmc+UxzVkbsDs4G/HMRM+KMNUjhC4J0vePWU87T7AMJa0rgNF22bCfJMhpYzVtjZQZ2UlvjDoKf20do+rsC25E8b02x+tgvfiC,iv:bY5VnXfIGD/4I4Bj7+oSLdBQinY+Tuq2dGnJmzfaVQY=,tag:DpZ4DlAY7svMPk/e3tI5wg==,type:str] + database-url: ENC[AES256_GCM,data:tLmfslMFP3TtFSna3zT6UNeotGn0GcvQDmGGNgxUKtGQVBtKc7ph/hTeMhFvLOibPUJuU4xs00Cd,iv:BFr6HDYQHUCLJhL8TTqBPr8OhxYhdVZ2OxlxdEEht80=,tag:nolIYbAdadKC9FU9mS8R1A==,type:str] +postgres: + init: ENC[AES256_GCM,data:iKgzmEq/3zBaDMLFdH/DZtfhZuqdLFhndyILwwDr5MwHiR3tQ+wT2+DQ8dBFwvAK12btrp07T7k=,iv:jEfFVS9YyGCohaORKLA8YQr2HUyCBwaYWrVYUe1UPDU=,tag:IJslOL6/ajDPEtXc7ggc5g==,type:str] sops: kms: [] gcp_kms: [] @@ -17,8 +22,8 @@ sops: Qkt2U1hWWGo4VzB2bEFYTWUwL0tyYzgKE1H8Wx5VH8D5cBHrniAAVQXD8yyR1eWY wUjeAOgiTEe8gjulqGDKxjMqcz7w/wuHBTICXEUEi6fBSdDE4RJkkA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-02-01T05:36:42Z" - mac: ENC[AES256_GCM,data:mUpprU3khFg6ioJlv6dD0SfD6vmLsKKWqX/nHMkUooyc3SbLTEh7u40bmIhpQLMTvxryRB1+oV+K87NTUYBlD34SglH4a3/FyCzdeP0cgpc+pkswa5LQsJrPcB2IN2MJe4cWGGDkzVS80747HSdAqHw6fv2lNjQBFfvsp3Jo8ck=,iv:ltDI4nOBYRPVTTbSfEYfLFee3H7b0G9tjOu0eNnpvgw=,tag:+l3NsxJ+HSy8RI2ZAUn0Jw==,type:str] + lastmodified: "2024-02-18T17:10:38Z" + mac: ENC[AES256_GCM,data:ImYBdEk+DqoG9J5fmj2aPqxFuko5AIWzVk0/v2YlMPHwBQ0dUGnYrNMXpZ4KyYlulsQ1R78agjF4Xk6jumvNbAwGZXshSSOx4A6CCAK/Xl7WbS7ilHYl9+H6K4wzTV0f8v1ShGH1INkFF+jWEpeQSSHvhHMs5lOu/N5+ZSLdC9k=,iv:17H07sayQNQmAv4hxtXYimQJX/FibannQn/7rojSrC8=,tag:15+OQlcAVitB/OYmfm+Y9Q==,type:str] pgp: - created_at: "2024-02-01T04:49:29Z" enc: |-