creating kubnet-accuse

Signed-off-by: ahuston-0 <aliceghuston@gmail.com>

.vscode/settings.json

@@ -115,6 +115,7 @@
     "keyserver",
     "keyservers",
     "KRNL",
+    "kubnet",
     "kuma",
     "libexec",
     "libglvnd",
@@ -132,6 +133,7 @@
     "markdownlint",
     "mechatroner",
     "mediainfo",
+    "microvm",
     "modesetting",
     "mousewheel",
     "mpris",

flake.lock

@@ -267,6 +267,30 @@
         "type": "github"
       }
     },
+    "microvm": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "spectrum": "spectrum"
+      },
+      "locked": {
+        "lastModified": 1716382614,
+        "narHash": "sha256-dwUYl8jyMNKidPEM9gTafcRe+3pUh2rH3ZlBrUpsvnw=",
+        "owner": "astro",
+        "repo": "microvm.nix",
+        "rev": "a59c3167f673ce29b65c674deb2bee73d151a96c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "astro",
+        "repo": "microvm.nix",
+        "type": "github"
+      }
+    },
     "nix": {
       "inputs": {
         "flake-compat": [
@@ -609,6 +633,7 @@
         "flake-utils": "flake-utils_2",
         "home-manager": "home-manager",
         "hyprland-contrib": "hyprland-contrib",
+        "microvm": "microvm",
         "nix": "nix",
         "nix-index-database": "nix-index-database",
         "nixos-generators": "nixos-generators",
@@ -733,6 +758,22 @@
         "type": "github"
       }
     },
+    "spectrum": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1708358594,
+        "narHash": "sha256-e71YOotu2FYA67HoC/voJDTFsiPpZNRwmiQb4f94OxQ=",
+        "ref": "refs/heads/main",
+        "rev": "6d0e73864d28794cdbd26ab7b37259ab0e1e044c",
+        "revCount": 614,
+        "type": "git",
+        "url": "https://spectrum-os.org/git/spectrum"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://spectrum-os.org/git/spectrum"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,

flake.nix

@@ -72,6 +72,14 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    microvm = {
+      url = "github:astro/microvm.nix";
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        flake-utils.follows = "flake-utils";
+      };
+    };
+
     nix = {
       url = "github:NixOS/nix/latest-release";
       inputs = {

lib/default.nix

@@ -3,6 +3,7 @@
   # create rad-dev namespace for lib
   rad-dev = rec {
     systems = import ./systems.nix { inherit lib; };
+    microvm = import ./microvms.nix { inherit lib; };
 
     # any(), but checks if any value in the list is true
     #
@@ -56,5 +57,30 @@
     # type:
     # fileList :: Path -> String -> [Path]
     fileList = dir: map (file: dir + "/${file}") (ls dir);
+
+    # constructs a mac address from a string's hash
+    #
+    # args:
+    # hashable: the string to hash
+    #
+    # type:
+    # strToMac :: String -> String
+    strToMac =
+      hashable:
+      let
+        # computes sha512 hash of input
+        hashStr = builtins.hashString "sha512" hashable;
+        # grabs first 12 letters of hash
+        hashSub = start: builtins.substring start 2 (builtins.substring 0 12 hashStr);
+        # joins list of strings with a delimiter between
+        joiner =
+          delim: arr:
+          builtins.foldl' (
+            a: b: lib.concatStrings ([ a ] ++ (lib.optionals (a != "") [ delim ]) ++ [ b ])
+          ) "" arr;
+        # generates a list of indexes for the hash
+        starts = builtins.genList (x: x * 2) 6;
+      in
+      joiner ":" (map hashSub starts);
   };
 }

lib/microvms.nix

@@ -0,0 +1,116 @@
+{ lib, ... }:
+rec {
+  genK3SVM =
+    server-config: agent-config: vms:
+    lib.mapAttrs (
+      host:
+      {
+        address,
+        gateway,
+        machine-id,
+        server ? false,
+      }:
+      genMicroVM host address gateway "x86_64-linux" machine-id (
+        if server then server-config else agent-config
+      )
+    ) vms;
+
+  genMicroVM =
+    hostName: address: gateway: _system: machine-id: vm-config:
+    # microvm refers to microvm.nixosModules
+
+    # {
+    #   config,
+    #   pkgs,
+    #   lib,
+    #   ...
+    # }:
+    {
+      # The package set to use for the microvm. This also determines the microvm's architecture.
+      # Defaults to the host system's package set if not given.
+      # pkgs = import pkgs { inherit system; };
+
+      # (Optional) A set of special arguments to be passed to the MicroVM's NixOS modules.
+      #specialArgs = {};
+
+      # The configuration for the MicroVM.
+      # Multiple definitions will be merged as expected.
+      config = {
+        imports = [ vm-config ];
+        # It is highly recommended to share the host's nix-store
+        # with the VMs to prevent building huge images.
+
+        system.stateVersion = "24.05";
+
+        environment.etc."machine-id" = {
+          mode = "0644";
+          text = machine-id + "\n";
+        };
+
+        networking.hostName = hostName;
+
+        microvm = {
+          interfaces = [
+            {
+              type = "tap";
+              # bridge = "ztkubnet";
+              id = "vm-${hostName}";
+              mac = lib.rad-dev.strToMac hostName;
+            }
+          ];
+          shares = [
+            {
+              source = "/nix/store";
+              mountPoint = "/nix/.ro-store";
+              tag = "ro-store";
+              proto = "virtiofs";
+            }
+            {
+              # On the host
+              source = "/var/lib/microvms/${hostName}/journal";
+              # In the MicroVM
+              mountPoint = "/var/log/journal";
+              tag = "journal";
+              proto = "virtiofs";
+              socket = "journal.sock";
+            }
+          ];
+        };
+
+        systemd.network.enable = true;
+
+        systemd.network.networks."20-lan" = {
+          matchConfig.Type = "ether";
+          networkConfig = {
+            Address = address;
+            Gateway = gateway;
+            DNS = [ "9.9.9.9" ];
+            IPv6AcceptRA = true;
+            DHCP = "no";
+          };
+        };
+
+        services.openssh = {
+          enable = true;
+          openFirewall = true;
+        };
+        users.users.alice = {
+          openssh.authorizedKeys.keys = [
+            # photon
+            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOGcqhLaKsjwAnb6plDavAhEyQHNvFS9Uh5lMTuwMhGF alice@parthenon-7588"
+            # gh
+            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGoaEmzaS9vANckvBmqrYSHdFR0sPL4Xgeonbh9KcgFe gitlab keypair"
+            # janus
+            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfcO9p5opG8Tym6tcLkat6YGCcE6vwg0+V4MTC5WKop alice@parthenon-7588"
+            # palatine
+            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP59pDsx34k2ikrKa0eVacj0APSGivaij3lP9L0Zd9au alice@parthenon-7588"
+            # jeeves
+            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJDgkUndkfns6f779T5ckHOVhyOKP8GttQ9RfaO9uJdx alice@parthenon-7588"
+          ];
+          isNormalUser = true;
+        };
+        # Any other configuration for your MicroVM
+        # [...]
+      };
+    };
+}

lib/systems.nix

@@ -160,7 +160,12 @@ rec {
     lib.nixosSystem {
       inherit system;
       specialArgs = {
-        inherit inputs server system;
+        inherit
+          inputs
+          server
+          system
+          src
+          ;
       };
       modules =
         [
@@ -208,7 +213,7 @@ rec {
               inherit inputs src configPath;
               hostname = name;
             }
-            // import configPath { inherit inputs; }
+            // import configPath { inherit inputs src; }
           );
         }
       ) (lib.rad-dev.lsdir path)

modules/kub_net.nix

@@ -20,13 +20,13 @@ in
       echo "ebe7fbd44565ba9d=ztkubnet" > /var/lib/zerotier-one/devicemap 
     '';
 
-    services.zerotierone = lib.mkDefault {
+    services.zerotierone = {
-      enable = true;
+      enable = lib.mkDefault true;
       joinNetworks = [ "ebe7fbd44565ba9d" ];
     };
 
-    systemd.network = lib.mkDefault {
+    systemd.network = {
-      enable = true;
+      enable = lib.mkDefault true;
       wait-online.anyInterface = true;
       netdevs = {
         "20-brkubnet" = {
@@ -38,15 +38,27 @@ in
       };
       networks = {
         "30-ztkubnet" = {
-          matchConfig.Name = "ztkubnet";
+          matchConfig.Name = [ "ztkubnet" ];
           networkConfig.Bridge = "brkubnet";
           linkConfig.RequiredForOnline = "enslaved";
         };
         "40-brkubnet" = {
           matchConfig.Name = "brkubnet";
           bridgeConfig = { };
+          networkConfig.LinkLocalAddressing = "no";
           linkConfig.RequiredForOnline = "no";
         };
+        "41-vms" = {
+          matchConfig.Name = [ "vm-*" ];
+          networkConfig.Bridge = "brkubnet";
+          linkConfig.RequiredForOnline = "enslaved";
+        };
+        "42-kubnet-accuse" = {
+          matchConfig.Name = "kubnet-accuse";
+          networkConfig.Bridge = "brkubnet";
+          linkConfig.RequiredForOnline = "enslaved";
+          address = [ "192.168.69.20/24" ];
+        };
       };
     };
 

modules/opt/default.nix

@@ -0,0 +1,5 @@
+{ ... }:
+
+{
+
+}

modules/opt/k3s-agent.nix

@@ -0,0 +1,5 @@
+{ ... }:
+{
+  imports = [ ./k3s-common.nix ];
+  services.k3s.role = "agent";
+}

modules/opt/k3s-common.nix

@@ -0,0 +1,15 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  services.k3s = {
+    enable = true;
+    extraFlags = "--cluster-cidr 192.168.69.0/24";
+    # tokenFile = #TODO: set this up after building the first node lol
+    # serverAddr =
+  };
+}

modules/opt/k3s-server.nix

@@ -0,0 +1,6 @@
+{ ... }:
+{
+  # imports = [ ./k3s-common.nix ];
+
+  services.k3s.role = "server";
+}

modules/opt/microvm-host.nix

@@ -0,0 +1,34 @@
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+let
+  cfg = config.rad-dev.microvm-host;
+  microvm = inputs.microvm.nixosModules;
+in
+{
+  imports = [ microvm.host ];
+  options.rad-dev.microvm-host = {
+    vms = lib.mkOption {
+      type = lib.types.attrs;
+      default = { };
+      description = "A list of VMs to construct on the host";
+    };
+  };
+  config = {
+    networking.useNetworkd = true;
+    microvm.vms = cfg.vms;
+
+    # TODO: deprecate this once we have syslog forwarders
+    systemd.tmpfiles.rules = map (
+      vmHost:
+      let
+        machineId = cfg.vms.${vmHost}.config.environment.etc."machine-id".text;
+      in
+      # creates a symlink of each MicroVM's journal under the host's /var/log/journal
+      "L+ /var/log/journal/${machineId} - - - - /var/lib/microvms/${vmHost}/journal/${machineId}"
+    ) (builtins.attrNames cfg.vms);
+  };
+}

systems/palatine-hill/configuration.nix

@@ -9,11 +9,13 @@
     ./attic.nix
     ./docker.nix
     ./hydra.nix
+    ./microvms.nix
     ./minio.nix
     ./networking.nix
     ./nextcloud.nix
     ./services.nix
     ./zfs.nix
+    ./networking.nix
   ];
 
   programs.git.lfs.enable = false;

systems/palatine-hill/default.nix

@@ -1,8 +1,11 @@
-{ inputs, ... }:
+{ inputs, src, ... }:
 {
   users = [
     "alice"
     "richie"
   ];
-  modules = [ inputs.attic.nixosModules.atticd ];
+  modules = [
+    inputs.attic.nixosModules.atticd
+    (src + "/modules/opt/microvm-host.nix")
+  ];
 }

systems/palatine-hill/microvms.nix

@@ -0,0 +1,27 @@
+{
+  config,
+  lib,
+  pkgs,
+  src,
+  ...
+}:
+let
+  inherit (lib.rad-dev.microvm) genK3SVM;
+in
+
+{
+  # rad-dev.microvm-host.enable = true;
+  rad-dev.microvm-host.vms =
+    genK3SVM (src + "/modules/opt/k3s-server.nix") (src + "/modules/opt/k3s-agent.nix")
+      {
+        "ph-server-1" = {
+          address = [ "192.168.69.10/24" ];
+          gateway = "192.168.69.1";
+          machine-id = "d694ad1e88b356887bb204ac665263f7";
+          server = true;
+        };
+        # "ph-agent-1" = {
+        #   ipv4 = "192.168.69.30";
+        # };
+      };
+}