feat: add riscv64-linux image

Flake lock file updates:

• Updated input 'nixos-stable':
    'github:NixOS/nixpkgs/c0b1da36f7c34a7146501f684e9ebdf15d2bebf8?narHash=sha256-WLxED18lodtQiayIPDE5zwAfkPJSjHJ35UhZ8h3cJUg%3D' (2024-10-14)
  → 'github:NixOS/nixpkgs/4eb33fe664af7b41a4c446f87d20c9a0a6321fa3?narHash=sha256-LDiPhQ3l%2BfBjRATNtnuDZsBS7hqoBtPkKBkhpoBHv3I%3D' (2024-10-17)
• Updated input 'nixos-unstable':
    'github:NixOS/nixpkgs/8f1d45587bd9af3dbf5146aa8a1347e20421597b?narHash=sha256-6sIuRVqVMHq9ZwcEVdpf2BuZeuLIUgvFznhIfsc75Jo%3D' (2024-10-16)
  → 'github:NixOS/nixpkgs/e3f55158e7587c5a5fdb0e86eb7ca4f455f0928f?narHash=sha256-3GNZr0V4b19RZ5mlyiY/4F8N2pzitvjDU6aHMWjAqLI%3D' (2024-10-20)

.github/workflows/build.yml

@@ -11,15 +11,18 @@ jobs:
       fail-fast: false
       matrix:
         tag:
-          - nixos-23.05
+          - nixos-24.05
           - nixos-unstable
-    runs-on: ubuntu-latest
+        os:
+          - nscloud-ubuntu-22.04-arm64-4x16
+          - ubuntu-latest
+    runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: cachix/install-nix-action@v21
+    - uses: actions/checkout@v4
+    - uses: cachix/install-nix-action@v30
       with:
         nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixpkgs-unstable.tar.gz
     - name: Build image
-      run: ./build-images.sh "${{matrix.tag}}"
+      run: ./build-images.sh "${{ matrix.tag }}" "$(nix eval --raw --impure --expr builtins.currentSystem)"
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update-flake-lock.yml

@@ -9,16 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Nix
-        uses: cachix/install-nix-action@v21
+        uses: cachix/install-nix-action@v30
       - name: Update flake.lock
-        uses: DeterminateSystems/update-flake-lock@v19
+        uses: DeterminateSystems/update-flake-lock@v24
         with:
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
-          pr-body: |
-            Automated changes by the update-flake-lock
-            ```
-            {{ env.GIT_COMMIT_MESSAGE }}
-            ```
-            bors merge
+          pr-labels: |
+            merge-queue

.mergify.yml

@@ -0,0 +1,15 @@
+queue_rules:
+  - name: default
+    merge_conditions:
+      - check-success=buildbot/nix-build
+defaults:
+  actions:
+    queue:
+      merge_method: rebase
+pull_request_rules:
+  - name: merge using the merge queue
+    conditions:
+      - base=main
+      - label~=merge-queue|dependencies
+    actions:
+      queue: {}

README.md

@@ -3,11 +3,6 @@
 Automatically weekly updated images for NixOS. This project is intended to extend the images created by hydra.nixos.org.
 We are currently creating the images listed below:
 
-## Netboot images
-
-You can boot the netboot image using this [ipxe script](https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/netboot-x86_64-linux.ipxe).
-It consists of the [kernel image](https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/bzImage-x86_64-linux) and [initrd](https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/bzImage-x86_64-linux).
-
 ## Kexec tarballs
 
 These images are used for unattended remote installation in [nixos-anywhere](https://github.com/numtide/nixos-anywhere).
@@ -16,10 +11,10 @@ Kexec is a mechanism in Linux to load a new kernel from a running Linux to
 replace the current kernel. This is useful for booting the Nixos installer from
 existing Linux distributions, such as server provider that do not offer a NixOS
 option. After running kexec, the NixOS installer exists only in memory. At the
-time of writing, this requires at least 1.5GB of physical RAM (swap does not
-count) in the system. If not enough RAM is available, the initrd cannot be
-loaded. Because the NixOS runs only in RAM, users can reformat all the system's
-discs to prepare for a new NixOS installation.
+time of writing, this requires secure boot off in BIOS settings and at least 1GB
+of physical RAM (swap does not count) in the system. If not enough RAM is available,
+the initrd cannot be loaded. Because the NixOS runs only in RAM, users can reformat
+all the system's discs to prepare for a new NixOS installation.
 
 It can be booted as follows by running these commands as root:
 
@@ -32,12 +27,32 @@ The kexec installer comes with the following features:
 
 - Re-uses ssh host keys from the sshd to not break `.ssh/known_hosts`
 - Authorized ssh keys are read from `/root/.ssh/authorized_keys`, `/root/.ssh/authorized_keys2` and `/etc/ssh/authorized_keys.d/root`
-- (experimental, only tested for nixos-unstable) Static ip addresses and routes
-  are restored after reboot. Interface that had dynamic addresses before are
-  configured with DHCP and to accept prefixes from ipv6 router advertisment
+- Static ip addresses and routes are restored after reboot.
+  Interface that had dynamic addresses before are configured with DHCP and
+  to accept prefixes from ipv6 router advertisement
 
-
-The actual kexec happens with a slight delay (6s).  This allows for easier
+The actual kexec happens with a slight delay (6s). This allows for easier
 integration into automated nixos installation scripts, since you can cleanly
 disconnect from the running machine before the kexec takes place. The tarball
 is also designed to be run from NixOS, which can be useful for new installations
+
+## Iso installer images
+
+This image allows to boot a NixOS installer off a USB-Stick.
+This installer has been optimized for remote installation i.e.
+with [nixos-anywhere](https://github.com/numtide/nixos-anywhere) and [clan](https://docs.clan.lol/getting-started/installer/) notably:
+
+* Enables openssh by default
+* Generates a random root password on each login
+* Enables a Tor hidden SSH service so that by using the `torify ssh <hash>.onion`,
+  one can log in from remote machines.
+* Prints a QR-Code that contains local addresses, the root password
+* Includes the [IWD](https://wiki.archlinux.org/title/iwd) deamon for easier wifi setups:
+  * Run `iwctl` in the terminal for an interactive wifi setup interface.
+
+![Screenshot of the installer](https://github.com/nix-community/nixos-images/releases/download/assets/image-installer-screenshot.jpg)
+
+## Netboot images
+
+You can boot the netboot image using this [ipxe script](https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/netboot-x86_64-linux.ipxe).
+It consists of the [kernel image](https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/bzImage-x86_64-linux) and [initrd](https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/initrd-x86_64-linux).

bors.toml

@@ -1,15 +0,0 @@
-cut_body_after = "" # don't include text from the PR body in the merge commit message
-status = [
-  "Evaluate flake.nix",
-  "check kexec-installer-2305 [x86_64-linux]",
-  "check kexec-installer-unstable [x86_64-linux]",
-  "check shellcheck [x86_64-linux]",
-  "package kexec-installer-nixos-2305 [x86_64-linux]",
-  "package kexec-installer-nixos-2305-noninteractive [x86_64-linux]",
-  "package kexec-installer-nixos-unstable [x86_64-linux]",
-  "package kexec-installer-nixos-unstable-noninteractive [x86_64-linux]",
-  "package netboot-installer-nixos-2305 [x86_64-linux]",
-  "package netboot-installer-nixos-unstable [x86_64-linux]",
-  "package netboot-nixos-2305 [x86_64-linux]",
-  "package netboot-nixos-unstable [x86_64-linux]"
-]

build-images.sh

@@ -5,45 +5,54 @@ set -xeuo pipefail
 shopt -s lastpipe
 
 build_netboot_image() {
-  declare -r tag=$1 arch=$2 tmp=$3
-  img=$(nix build --print-out-paths --option accept-flake-config true -L ".#packages.${arch}.netboot-${tag//.}")
-  ln -s "$img/bzImage" "$tmp/bzImage-$arch"
-  echo "$tmp/bzImage-$arch"
+  declare -r tag=$1 channel=$2 arch=$3 tmp=$4
+  img=$(nix build --print-out-paths --option accept-flake-config true -L ".#packages.${arch}.netboot-nixos-${channel//./}")
+  kernel=$(echo "$img"/*Image)
+  kernelName=$(basename "$kernel")
+  ln -s "$kernel" "$tmp/$kernelName-$arch"
+  echo "$tmp/$kernelName-$arch"
   ln -s "$img/initrd" "$tmp/initrd-$arch"
   echo "$tmp/initrd-$arch"
-  sed -e "s!^kernel bzImage!kernel https://github.com/nix-community/nixos-images/releases/download/${tag}/bzImage-${arch}!" \
+  sed -e "s!^kernel $kernelName!kernel https://github.com/nix-community/nixos-images/releases/download/${tag}/$kernelName-${arch}!" \
     -e "s!^initrd initrd!initrd https://github.com/nix-community/nixos-images/releases/download/${tag}/initrd-${arch}!" \
     -e "s!initrd=initrd!initrd=initrd-${arch}!" \
-    < "$img/netboot.ipxe" \
-    > "$tmp/netboot-$arch.ipxe"
+    <"$img/netboot.ipxe" \
+    >"$tmp/netboot-$arch.ipxe"
   echo "$tmp/netboot-$arch.ipxe"
 }
 
 build_kexec_installer() {
-  declare -r tag=$1 arch=$2 tmp=$3 variant=$4
-  out=$(nix build --print-out-paths --option accept-flake-config true -L ".#packages.${arch}.kexec-installer-${tag//.}${variant}")
+  declare -r channel=$1 arch=$2 tmp=$3 variant=$4
+  out=$(nix build --print-out-paths --option accept-flake-config true -L ".#packages.${arch}.kexec-installer-nixos-${channel}${variant}")
   echo "$out/nixos-kexec-installer${variant}-$arch.tar.gz"
 }
 
+build_image_installer() {
+  declare -r channel=$1 arch=$2 tmp=$3
+  out=$(nix build --print-out-paths --option accept-flake-config true -L ".#packages.${arch}.image-installer-nixos-${channel//./}")
+  echo "$out/iso/nixos-installer-${arch}.iso"
+}
+
 main() {
   declare -r tag=${1:-nixos-unstable} arch=${2:-x86_64-linux}
   tmp="$(mktemp -d)"
   trap 'rm -rf -- "$tmp"' EXIT
   (
-    build_kexec_installer "$tag" "$arch" "$tmp" ""
-    build_kexec_installer "$tag" "$arch" "$tmp" "-noninteractive"
-    build_netboot_image "$tag" "$arch" "$tmp"
+    channel=$(if [[ "$tag" == nixos-unstable ]]; then echo "unstable"; else echo "stable"; fi)
+    build_kexec_installer "$channel" "$arch" "$tmp" ""
+    build_kexec_installer "$channel" "$arch" "$tmp" "-noninteractive"
+    build_netboot_image "$tag" "$channel" "$arch" "$tmp"
+    build_image_installer "$channel" "$arch" "$tmp"
   ) | readarray -t assets
   for asset in "${assets[@]}"; do
     pushd "$(dirname "$asset")"
-    sha256sum "$(basename "$asset")" >> "$TMP/sha256sums"
     popd
   done
-  assets+=("$TMP/sha256sums")
 
-  # Since we cannot atomically update a release, we delete the old one before
-  gh release delete "$tag" </dev/null || true
-  gh release create --title "$tag (build $(date +"%Y-%m-%d"))" "$tag" "${assets[@]}" </dev/null
+  if ! gh release view "$tag"; then
+    gh release create --title "$tag (build $(date +"%Y-%m-%d"))" "$tag"
+  fi
+  gh release upload --clobber "$tag" "${assets[@]}"
 }
 
 main "$@"

flake.lock

@@ -1,28 +1,28 @@
 {
   "nodes": {
-    "nixos-2305": {
+    "nixos-stable": {
       "locked": {
-        "lastModified": 1686780367,
-        "narHash": "sha256-hEn8ebT32APeDSoT80VUGktqQ4zqI4gCFLzbIZdSuoo=",
+        "lastModified": 1729181673,
+        "narHash": "sha256-LDiPhQ3l+fBjRATNtnuDZsBS7hqoBtPkKBkhpoBHv3I=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "75eb7c2d47fdc01a0d477e9a89eac7ed366fe898",
+        "rev": "4eb33fe664af7b41a4c446f87d20c9a0a6321fa3",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "release-23.05",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixos-unstable": {
       "locked": {
-        "lastModified": 1686718773,
-        "narHash": "sha256-x+4xs6+jWhFaYwt6REH7e91rm5vt2GCPEfmRdNcHyi4=",
+        "lastModified": 1729450260,
+        "narHash": "sha256-3GNZr0V4b19RZ5mlyiY/4F8N2pzitvjDU6aHMWjAqLI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ba1a6ec548000d4a50719d14e6f73f63016674d5",
+        "rev": "e3f55158e7587c5a5fdb0e86eb7ca4f455f0928f",
         "type": "github"
       },
       "original": {
@@ -34,7 +34,7 @@
     },
     "root": {
       "inputs": {
-        "nixos-2305": "nixos-2305",
+        "nixos-stable": "nixos-stable",
         "nixos-unstable": "nixos-unstable"
       }
     }

flake.nix

@@ -2,62 +2,84 @@
   description = "NixOS images";
 
   inputs.nixos-unstable.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-  inputs.nixos-2305.url = "github:NixOS/nixpkgs/release-23.05";
+  inputs.nixos-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
 
-  nixConfig.extra-substituters = [
-    "https://cache.garnix.io"
-  ];
-  nixConfig.extra-trusted-public-keys = [
-    "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g="
-  ];
+  nixConfig.extra-substituters = [ "https://nix-community.cachix.org" ];
+  nixConfig.extra-trusted-public-keys = [ "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" ];
 
-  outputs = { self, nixos-unstable, nixos-2305 }: let
-    supportedSystems = [ "aarch64-linux" "x86_64-linux" ];
-    forAllSystems = nixos-unstable.lib.genAttrs supportedSystems;
-  in {
-    packages = forAllSystems (system: let
-      netboot = nixpkgs: (import (nixpkgs + "/nixos/release.nix") {}).netboot.${system};
-      kexec-installer = nixpkgs: modules: (nixpkgs.legacyPackages.${system}.nixos (modules ++ [self.nixosModules.kexec-installer])).config.system.build.kexecTarball;
-      netboot-installer = nixpkgs: (nixpkgs.legacyPackages.${system}.nixos [self.nixosModules.netboot-installer]).config.system.build.netboot;
-    in {
-      netboot-nixos-unstable = netboot nixos-unstable;
-      netboot-nixos-2305 = netboot nixos-2305;
-      kexec-installer-nixos-unstable = kexec-installer nixos-unstable [];
-      kexec-installer-nixos-2305 = kexec-installer nixos-2305 [];
+  outputs = { self, nixos-unstable, nixos-stable }:
+    let
+      supportedSystems = [ "riscv64-linux" ];
+      forAllSystems = nixos-unstable.lib.genAttrs supportedSystems;
+    in
+    {
+      hydraJobs = { inherit (self) checks; };
+      packages = forAllSystems (system:
+        let
+          netboot = nixpkgs: (import (nixpkgs + "/nixos/release.nix") { }).netboot.${system};
+          kexec-installer = nixpkgs: module: (nixpkgs.legacyPackages.${system}.nixos [ module self.nixosModules.kexec-installer ]).config.system.build.kexecTarball;
+          netboot-installer = nixpkgs: (nixpkgs.legacyPackages.${system}.nixos [ self.nixosModules.netboot-installer ]).config.system.build.netboot;
+          image-installer = nixpkgs: (nixpkgs.legacyPackages.${system}.nixos [ self.nixosModules.image-installer ]).config.system.build.isoImage;
+        in
+        {
+          netboot-nixos-unstable = netboot nixos-unstable;
+          netboot-nixos-stable = netboot nixos-stable;
+          kexec-installer-nixos-unstable = kexec-installer nixos-unstable {};
+          kexec-installer-nixos-stable = kexec-installer nixos-stable {};
 
-      kexec-installer-nixos-unstable-noninteractive = kexec-installer nixos-unstable [ 
-        { system.kexec-installer.name = "nixos-kexec-installer-noninteractive"; }
-        self.nixosModules.noninteractive 
-      ];
-      kexec-installer-nixos-2305-noninteractive = kexec-installer nixos-2305 [ 
-        { system.kexec-installer.name = "nixos-kexec-installer-noninteractive"; }
-        self.nixosModules.noninteractive 
-      ];
+          image-installer-nixos-unstable = image-installer nixos-unstable;
+          image-installer-nixos-stable = image-installer nixos-stable;
 
-      netboot-installer-nixos-unstable = netboot-installer nixos-unstable;
-      netboot-installer-nixos-2305 = netboot-installer nixos-2305;
-    });
-    nixosModules = {
-      kexec-installer = ./nix/kexec-installer/module.nix;
-      noninteractive = ./nix/noninteractive.nix;
-      # TODO: also add a test here once we have https://github.com/NixOS/nixpkgs/pull/228346 merged
-      netboot-installer = ./nix/netboot-installer/module.nix;
-    };
-    checks.x86_64-linux = let
-      pkgs = nixos-unstable.legacyPackages.x86_64-linux;
-    in {
-      kexec-installer-unstable = pkgs.callPackage ./nix/kexec-installer/test.nix {
-        kexecTarball = self.packages.x86_64-linux.kexec-installer-nixos-unstable-noninteractive;
-      };
-      shellcheck = pkgs.runCommand "shellcheck" {
-        nativeBuildInputs = [ pkgs.shellcheck ];
-      } ''
-        shellcheck ${(pkgs.nixos [self.nixosModules.kexec-installer]).config.system.build.kexecRun}
-        touch $out
-      '';
-      kexec-installer-2305 = nixos-2305.legacyPackages.x86_64-linux.callPackage ./nix/kexec-installer/test.nix {
-        kexecTarball = self.packages.x86_64-linux.kexec-installer-nixos-2305-noninteractive;
+          kexec-installer-nixos-unstable-noninteractive = kexec-installer nixos-unstable {
+            _file = __curPos.file;
+            system.kexec-installer.name = "nixos-kexec-installer-noninteractive";
+            imports = [
+              self.nixosModules.noninteractive
+            ];
+          };
+          kexec-installer-nixos-stable-noninteractive = kexec-installer nixos-stable {
+            _file = __curPos.file;
+            system.kexec-installer.name = "nixos-kexec-installer-noninteractive";
+            imports = [
+              self.nixosModules.noninteractive
+            ];
+          };
+
+          netboot-installer-nixos-unstable = netboot-installer nixos-unstable;
+          netboot-installer-nixos-stable = netboot-installer nixos-stable;
+        });
+      nixosModules = {
+        kexec-installer = ./nix/kexec-installer/module.nix;
+        noninteractive = ./nix/noninteractive.nix;
+        # TODO: also add a test here once we have https://github.com/NixOS/nixpkgs/pull/228346 merged
+        netboot-installer = ./nix/netboot-installer/module.nix;
+        image-installer = ./nix/image-installer/module.nix;
       };
+      checks =
+        let
+          # re-export the packages as checks
+          packages = forAllSystems (system: nixos-unstable.lib.mapAttrs' (n: nixos-unstable.lib.nameValuePair "package-${n}") self.packages.${system});
+          checks =
+            let
+              pkgsUnstable = nixos-unstable.legacyPackages.riscv64-linux;
+              pkgsStable = nixos-stable.legacyPackages.riscv64-linux;
+
+              bootTests = pkgs: channel: suffix: pkgs.lib.mapAttrs' (name: pkgs.lib.nameValuePair "${name}${suffix}") (pkgs.callPackages ./nix/image-installer/tests.nix {
+                nixpkgs = channel;
+                nixosModules = self.nixosModules;
+              });
+            in
+            {
+              kexec-installer-unstable = pkgsUnstable.callPackage ./nix/kexec-installer/test.nix {
+                kexecTarball = self.packages.riscv64-linux.kexec-installer-nixos-unstable-noninteractive;
+              };
+
+              kexec-installer-stable = nixos-stable.legacyPackages.riscv64-linux.callPackage ./nix/kexec-installer/test.nix {
+                kexecTarball = self.packages.riscv64-linux.kexec-installer-nixos-stable-noninteractive;
+              };
+            } // (bootTests pkgsUnstable nixos-unstable "-nixos-unstable")
+              // (bootTests pkgsStable nixos-stable "-nixos-stable");
+        in
+        nixos-unstable.lib.recursiveUpdate packages { riscv64-linux = checks; };
     };
-  };
 }

garnix.yaml

@@ -1,5 +0,0 @@
-builds:
-  include:
-  - '*.x86_64-linux.*'
-  - '*.aarch64-linux.*'
-  - nixosConfigurations.*

nix/image-installer/hidden-ssh-announcement.nix

@@ -0,0 +1,63 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.hidden-ssh-announce = {
+    enable = lib.mkEnableOption "hidden-ssh-announce";
+    script = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.writers.writeDash "test-output" "echo $1";
+      description = ''
+        script to run when the hidden tor service was started and they hostname is known.
+        takes the hostname as $1
+      '';
+    };
+  };
+
+  config = lib.mkIf config.hidden-ssh-announce.enable {
+    services.openssh.enable = true;
+    services.tor = {
+      enable = true;
+      relay.onionServices.hidden-ssh = {
+        version = 3;
+        map = [
+          {
+            port = 22;
+            target.port = 22;
+          }
+        ];
+      };
+      client.enable = true;
+    };
+    systemd.services.hidden-ssh-announce = {
+      description = "announce hidden ssh";
+      after = [
+        "tor.service"
+        "network-online.target"
+      ];
+      wants = [
+        "tor.service"
+        "network-online.target"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        # ${pkgs.tor}/bin/torify
+        ExecStart = pkgs.writeShellScript "announce-hidden-service" ''
+          set -efu
+          until test -e ${config.services.tor.settings.DataDirectory}/onion/hidden-ssh/hostname; do
+            echo "still waiting for ${config.services.tor.settings.DataDirectory}/onion/hidden-ssh/hostname"
+            sleep 1
+          done
+
+          ${config.hidden-ssh-announce.script} "$(cat ${config.services.tor.settings.DataDirectory}/onion/hidden-ssh/hostname)"
+        '';
+        PrivateTmp = "true";
+        User = "tor";
+        Type = "oneshot";
+      };
+    };
+  };
+}

nix/image-installer/module.nix

@@ -0,0 +1,125 @@
+{
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+let
+  network-status = pkgs.writeShellScriptBin "network-status" ''
+    export PATH=${
+      lib.makeBinPath (
+        with pkgs;
+        [
+          iproute2
+          coreutils
+          gnugrep
+          nettools
+          gum
+        ]
+      )
+    }
+    set -efu -o pipefail
+    msgs=()
+    if [[ -e /var/shared/qrcode.utf8 ]]; then
+      qrcode=$(gum style --border-foreground 240 --border normal "$(< /var/shared/qrcode.utf8)")
+      msgs+=("$qrcode")
+    fi
+    network_status="Root password: $(cat /var/shared/root-password)
+    Local network addresses:
+    $(ip -brief -color addr | grep -v 127.0.0.1)
+    $([[ -e /var/shared/onion-hostname ]] && echo "Onion address: $(cat /var/shared/onion-hostname)" || echo "Onion address: Waiting for tor network to be ready...")
+    Multicast DNS: $(hostname).local"
+    network_status=$(gum style --border-foreground 240 --border normal "$network_status")
+    msgs+=("$network_status")
+    msgs+=("Press 'Ctrl-C' for console access")
+
+    gum join --vertical "''${msgs[@]}"
+  '';
+in
+{
+  imports = [
+    (modulesPath + "/installer/cd-dvd/installation-cd-base.nix")
+    ../installer.nix
+    ../noveau-workaround.nix
+    ./hidden-ssh-announcement.nix
+    ./wifi.nix
+  ];
+  systemd.tmpfiles.rules = [ "d /var/shared 0777 root root - -" ];
+  services.openssh.settings.PermitRootLogin = "yes";
+  system.activationScripts.root-password = ''
+    mkdir -p /var/shared
+    ${pkgs.xkcdpass}/bin/xkcdpass --numwords 3 --delimiter - --count 1 > /var/shared/root-password
+    echo "root:$(cat /var/shared/root-password)" | chpasswd
+  '';
+  hidden-ssh-announce = {
+    enable = true;
+    script = pkgs.writeShellScript "write-hostname" ''
+      set -efu
+      export PATH=${
+        lib.makeBinPath (
+          with pkgs;
+          [
+            iproute2
+            coreutils
+            jq
+            qrencode
+          ]
+        )
+      }
+
+      mkdir -p /var/shared
+      echo "$1" > /var/shared/onion-hostname
+      local_addrs=$(ip -json addr | jq '[map(.addr_info) | flatten | .[] | select(.scope == "global") | .local]')
+      jq -nc \
+        --arg password "$(cat /var/shared/root-password)" \
+        --arg onion_address "$(cat /var/shared/onion-hostname)" \
+        --argjson local_addrs "$local_addrs" \
+        '{ pass: $password, tor: $onion_address, addrs: $local_addrs }' \
+        > /var/shared/login.json
+      cat /var/shared/login.json | qrencode -s 2 -m 2 -t utf8 -o /var/shared/qrcode.utf8
+    '';
+  };
+
+  services.getty.autologinUser = lib.mkForce "root";
+
+  console.earlySetup = true;
+  console.font = lib.mkDefault "${pkgs.terminus_font}/share/consolefonts/ter-u22n.psf.gz";
+
+  environment.systemPackages = [ network-status ];
+
+  # Less ipv6 addresses to reduce the noise
+  networking.tempAddresses = "disabled";
+
+  # Tango theme: https://yayachiken.net/en/posts/tango-colors-in-terminal/
+  console.colors = lib.mkDefault [
+    "000000"
+    "CC0000"
+    "4E9A06"
+    "C4A000"
+    "3465A4"
+    "75507B"
+    "06989A"
+    "D3D7CF"
+    "555753"
+    "EF2929"
+    "8AE234"
+    "FCE94F"
+    "739FCF"
+    "AD7FA8"
+    "34E2E2"
+    "EEEEEC"
+  ];
+
+  programs.bash.interactiveShellInit = ''
+    if [[ "$(tty)" =~ /dev/(tty1|hvc0|ttyS0)$ ]]; then
+      # workaround for https://github.com/NixOS/nixpkgs/issues/219239
+      systemctl restart systemd-vconsole-setup.service
+
+      watch --no-title --color ${network-status}/bin/network-status
+    fi
+  '';
+
+  # No one got time for xz compression.
+  isoImage.squashfsCompression = "zstd";
+  isoImage.isoName = lib.mkForce "nixos-installer-${pkgs.system}.iso";
+}

nix/image-installer/tests.nix

@@ -0,0 +1,105 @@
+{
+  pkgs,
+  lib,
+  nixpkgs,
+  nixos,
+  nixosModules,
+}:
+
+let
+  testConfig = (
+    nixos [
+      (
+        { modulesPath, ... }:
+        {
+          imports = [
+            nixosModules.image-installer
+            "${modulesPath}/testing/test-instrumentation.nix"
+          ];
+        }
+      )
+    ]
+  );
+  iso = testConfig.config.system.build.isoImage;
+  mkStartCommand =
+    {
+      memory ? 2048,
+      cdrom ? null,
+      usb ? null,
+      uefi ? false,
+      extraFlags ? [ ],
+    }:
+    let
+      qemu-common = import (nixpkgs + "/nixos/lib/qemu-common.nix") { inherit lib pkgs; };
+      qemu = qemu-common.qemuBinary pkgs.qemu_test;
+
+      flags =
+        [
+          "-m"
+          (toString memory)
+          "-netdev"
+          "user,id=net0"
+          "-device"
+          "virtio-net-pci,netdev=net0"
+        ]
+        ++ lib.optionals (cdrom != null) [
+          "-cdrom"
+          cdrom
+        ]
+        ++ lib.optionals (usb != null) [
+          "-device"
+          "usb-ehci"
+          "-drive"
+          "id=usbdisk,file=${usb},if=none,readonly"
+          "-device"
+          "usb-storage,drive=usbdisk"
+        ]
+        ++ lib.optionals uefi [
+          "-drive"
+          "if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
+          "-drive"
+          "if=pflash,format=raw,unit=1,readonly=on,file=${pkgs.OVMF.variables}"
+        ]
+        ++ extraFlags;
+
+      flagsStr = lib.concatStringsSep " " flags;
+    in
+    "${qemu} ${flagsStr}";
+
+  makeBootTest =
+    name: config:
+    let
+      startCommand = mkStartCommand config;
+    in
+    pkgs.testers.runNixOSTest {
+      name = "boot-${name}";
+      nodes = { };
+      testScript = ''
+        machine = create_machine("${startCommand}")
+        machine.start()
+        machine.wait_for_unit("multi-user.target")
+        machine.succeed("nix store verify --no-trust -r --option experimental-features nix-command /run/current-system")
+
+        machine.shutdown()
+      '';
+    };
+in
+{
+  uefi-cdrom = makeBootTest "uefi-cdrom" {
+    uefi = true;
+    cdrom = "${iso}/iso/nixos-installer-${pkgs.hostPlatform.system}.iso";
+  };
+
+  uefi-usb = makeBootTest "uefi-usb" {
+    uefi = true;
+    usb = "${iso}/iso/nixos-installer-${pkgs.hostPlatform.system}.iso";
+  };
+
+  bios-cdrom = makeBootTest "bios-cdrom" {
+    cdrom = "${iso}/iso/nixos-installer-${pkgs.hostPlatform.system}.iso";
+  };
+
+  bios-usb = makeBootTest "bios-usb" {
+    usb = "${iso}/iso/nixos-installer-${pkgs.hostPlatform.system}.iso";
+  };
+}

nix/image-installer/wifi.nix

@@ -0,0 +1,17 @@
+{
+  imports = [ ../networkd.nix ];
+  # use iwd instead of wpa_supplicant
+  networking.wireless.enable = false;
+
+  # Use iwd instead of wpa_supplicant. It has a user friendly CLI
+  networking.wireless.iwd = {
+    enable = true;
+    settings = {
+      Network = {
+        EnableIPv6 = true;
+        RoutePriorityOffset = 300;
+      };
+      Settings.AutoConnect = true;
+    };
+  };
+}

nix/installer.nix

@@ -1,56 +1,45 @@
-{ config, lib, pkgs, ... }: {
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+#    ./latest-zfs-kernel.nix
+    ./nix-settings.nix
+  ];
+  # more descriptive hostname than just "nixos"
+  networking.hostName = lib.mkDefault "nixos-installer";
+
   # We are stateless, so just default to latest.
   system.stateVersion = config.system.nixos.version;
 
-  # use latest kernel we can support to get more hardware support
-  boot.kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
+  # Enable bcachefs support
+  boot.supportedFilesystems.bcachefs = lib.mkDefault true;
 
-  # IPMI SOL console redirection stuff
-  boot.kernelParams =
-    [ "console=tty0" ] ++
-    (lib.optional (pkgs.stdenv.hostPlatform.isAarch32 || pkgs.stdenv.hostPlatform.isAarch64) "console=ttyAMA0,115200") ++
-    (lib.optional (pkgs.stdenv.hostPlatform.isRiscV) "console=ttySIF0,115200") ++
-    [ "console=ttyS0,115200" ];
+  # use latest kernel we can support to get more hardware support
+#  boot.zfs.package = pkgs.zfsUnstable;
 
   documentation.enable = false;
-  # Not really needed. Saves a few bytes and the only service we are running is sshd, which we want to be reachable.
-  networking.firewall.enable = false;
+  documentation.man.man-db.enable = false;
 
-  systemd.network.enable = true;
-  networking.dhcpcd.enable = false;
+  # make it easier to debug boot failures
+  boot.initrd.systemd.emergencyAccess = true;
 
-  # for zapping of disko
-  environment.systemPackages = [ pkgs.jq ];
+  environment.systemPackages = [
+    pkgs.nixos-install-tools
+    # for zapping of disko
+    pkgs.jq
+    # for copying extra files of nixos-anywhere
+    pkgs.rsync
+    # alternative to nixos-generate-config
+    # TODO: use nixpkgs again after next nixos release
+    (pkgs.callPackage ./nixos-facter.nix {})
 
-  systemd.services.log-network-status = {
-    wantedBy = [ "multi-user.target" ];
-    # No point in restarting this. We just need this after boot
-    restartIfChanged = false;
+    pkgs.disko
+  ];
 
-    serviceConfig = {
-      Type = "oneshot";
-      StandardOutput = "journal+console";
-      ExecStart = [
-        # Allow failures, so it still prints what interfaces we have even if we
-        # not get online
-        "-${pkgs.systemd}/lib/systemd/systemd-networkd-wait-online"
-        "${pkgs.iproute2}/bin/ip -c addr"
-        "${pkgs.iproute2}/bin/ip -c -6 route"
-        "${pkgs.iproute2}/bin/ip -c -4 route"
-      ];
-    };
-  };
-
-  # Restore ssh host and user keys if they are available.
-  # This avoids warnings of unknown ssh keys.
-  boot.initrd.postMountCommands = ''
-    mkdir -m 700 -p /mnt-root/root/.ssh
-    mkdir -m 755 -p /mnt-root/etc/ssh
-    mkdir -m 755 -p /mnt-root/root/network
-    if [[ -f ssh/authorized_keys ]]; then
-      install -m 400 ssh/authorized_keys /mnt-root/root/.ssh
-    fi
-    install -m 400 ssh/ssh_host_* /mnt-root/etc/ssh
-    cp *.json /mnt-root/root/network/
-  '';
+  # Don't add nixpkgs to the image to save space, for our intended use case we don't need it
+  system.installer.channel.enable = false;
 }

nix/kexec-installer/kexec-run.sh

@@ -1,6 +1,19 @@
 #!/bin/sh
 
 set -ex
+
+kexec_extra_flags=""
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+  --kexec-extra-flags)
+    kexec_extra_flags="$2"
+    shift
+    ;;
+  esac
+  shift
+done
+
 # provided by nix
 init="@init@"
 kernelParams="@kernelParams@"
@@ -21,12 +34,16 @@ extractPubKeys() {
     key="$home/$file"
     if test -e "$key"; then
       # workaround for debian shenanigans
-      grep -o '\(ssh-[^ ]* .*\)' "$key" >> ssh/authorized_keys || true
+      grep -o '\(\(ssh\|ecdsa\|sk\)-[^ ]* .*\)' "$key" >> ssh/authorized_keys || true
     fi
   done
 }
 extractPubKeys /root
 
+if test -n "${DOAS_USER-}"; then
+  SUDO_USER="$DOAS_USER"
+fi
+
 if test -n "${SUDO_USER-}"; then
   sudo_home=$(sh -c "echo ~$SUDO_USER")
   extractPubKeys "$sudo_home"
@@ -50,25 +67,36 @@ done
 "$SCRIPT_DIR/ip" -4 --json route > routes-v4.json
 "$SCRIPT_DIR/ip" -6 --json route > routes-v6.json
 
+[ -f /etc/machine-id ] && cp /etc/machine-id machine-id
+
 find . | cpio -o -H newc | gzip -9 >> "$SCRIPT_DIR/initrd"
 
-# Dropped --kexec-syscall-auto because it broke on GCP...
-if ! "$SCRIPT_DIR/kexec" --load "$SCRIPT_DIR/bzImage" \
-  --initrd="$SCRIPT_DIR/initrd" --no-checks \
-  --command-line "init=$init $kernelParams"; then
+kexecSyscallFlags=""
+# only do kexec-syscall-auto on kernels newer than 6.0.
+# On older kernel we often get errors like: https://github.com/nix-community/nixos-anywhere/issues/264
+if printf "%s\n" "6.1" "$(uname -r)" | sort -c -V 2>&1; then
+  kexecSyscallFlags="--kexec-syscall-auto"
+fi
+
+if ! sh -c "'$SCRIPT_DIR/kexec' --load '$SCRIPT_DIR/bzImage' \
+  $kexecSyscallFlags \
+  $kexec_extra_flags \
+  --initrd='$SCRIPT_DIR/initrd' --no-checks \
+  --command-line 'init=$init $kernelParams'"
+then
   echo "kexec failed, dumping dmesg"
   dmesg | tail -n 100
   exit 1
 fi
 
 # Disconnect our background kexec from the terminal
-echo "machine will boot into nixos in in 6s..."
+echo "machine will boot into nixos in 6s..."
 if test -e /dev/kmsg; then
-  # this makes logging visible in `dmesg`, or the system consol or tools like journald
+  # this makes logging visible in `dmesg`, or the system console or tools like journald
   exec > /dev/kmsg 2>&1
 else
   exec > /dev/null 2>&1
 fi
 # We will kexec in background so we can cleanly finish the script before the hosts go down.
 # This makes integration with tools like terraform easier.
-nohup sh -c "sleep 6 && '$SCRIPT_DIR/kexec' -e" &
+nohup sh -c "sleep 6 && '$SCRIPT_DIR/kexec' -e ${kexec_extra_flags}" &

nix/kexec-installer/local-network-restore-test.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env -S nix shell --inputs-from .# nixos-unstable#bash nixos-unstable#iproute2 nixos-unstable#findutils nixos-unstable#coreutils nixos-unstable#python3 nixos-unstable#jq --command bash
+
+set -eu
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+
+# This script can be used to see what network configuration would be restored by the restore_routes.py script for the current system.
+
+tmp=$(mktemp -d)
+trap "rm -rf $tmp" EXIT
+ip --json address >"$tmp/addrs.json"
+ip -6 --json route >"$tmp/routes-v6.json"
+ip -4 --json route >"$tmp/routes-v4.json"
+python3 "$SCRIPT_DIR/restore_routes.py" "$tmp/addrs.json" "$tmp/routes-v4.json" "$tmp/routes-v6.json" "$tmp"
+ls -la "$tmp"
+
+find "$tmp" -type f -name "*.json" -print0 | while IFS= read -r -d '' file; do
+  echo -e "\033[0;31m$(basename "$file")\033[0m"
+  jq . "$file"
+  echo ""
+done
+
+find "$tmp" -type f -name "*.network" -print0 | while IFS= read -r -d '' file; do
+  echo -e "\033[0;31m$(basename "$file")\033[0m"
+  cat "$file"
+  echo ""
+done

nix/kexec-installer/module.nix

@@ -1,7 +1,14 @@
 { config, lib, modulesPath, pkgs, ... }:
 let
-  restore-network = pkgs.writers.writePython3 "restore-network" { flakeIgnore = [ "E501" ]; }
-    ./restore_routes.py;
+  writePython3 = pkgs.writers.makePythonWriter
+    pkgs.python3Minimal pkgs.python3Packages pkgs.buildPackages.python3Packages;
+
+  # writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin)
+  writePython3Bin = name: writePython3 "/bin/${name}";
+
+  restore-network = writePython3Bin "restore-network" {
+     flakeIgnore = [ "E501" ];
+  } ./restore_routes.py;
 
   # does not link with iptables enabled
   iprouteStatic = pkgs.pkgsStatic.iproute2.override { iptables = null; };
@@ -10,7 +17,11 @@ in
   imports = [
     (modulesPath + "/installer/netboot/netboot-minimal.nix")
     ../installer.nix
+    ../networkd.nix
+    ../serial.nix
+    ../restore-remote-access.nix
   ];
+
   options = {
     system.kexec-installer.name = lib.mkOption {
       type = lib.types.str;
@@ -22,6 +33,7 @@ in
   };
 
   config = {
+    boot.initrd.compressor = "xz";
     # This is a variant of the upstream kexecScript that also allows embedding
     # a ssh key.
     system.build.kexecRun = pkgs.runCommand "kexec-run" { } ''
@@ -61,7 +73,7 @@ in
         Type = "oneshot";
         RemainAfterExit = true;
         ExecStart = [
-          "${restore-network} /root/network/addrs.json /root/network/routes-v4.json /root/network/routes-v6.json /etc/systemd/network"
+          "${restore-network}/bin/restore-network /root/network/addrs.json /root/network/routes-v4.json /root/network/routes-v6.json /etc/systemd/network"
         ];
       };
 

nix/kexec-installer/restore_routes.py

@@ -1,40 +1,86 @@
 import json
 import sys
 from pathlib import Path
-from typing import Any
+from typing import Any, Iterator
+from dataclasses import dataclass
 
 
-def filter_interfaces(network: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    output = []
+@dataclass
+class Address:
+    address: str
+    family: str
+    prefixlen: int
+    preferred_life_time: int = 0
+    valid_life_time: int = 0
+
+
+@dataclass
+class Interface:
+    name: str
+    ifname: str | None
+    mac_address: str
+    dynamic_addresses: list[Address]
+    static_addresses: list[Address]
+    static_routes: list[dict[str, Any]]
+
+
+def filter_interfaces(network: list[dict[str, Any]]) -> list[Interface]:
+    interfaces = []
     for net in network:
         if net.get("link_type") == "loopback":
             continue
-        if not net.get("address"):
+        if not (mac_address := net.get("address")):
             # We need a mac address to match devices reliable
             continue
-        addr_info = []
-        has_dynamic_address = False
-        for addr in net.get("addr_info", []):
+        static_addresses = []
+        dynamic_addresses = []
+        for info in net.get("addr_info", []):
             # no link-local ipv4/ipv6
-            if addr.get("scope") == "link":
+            if info.get("scope") == "link":
                 continue
-            # do not explicitly configure addresses from dhcp or router advertisment
-            if addr.get("dynamic", False):
-                has_dynamic_address = True
+            if (preferred_life_time := info.get("preferred_life_time")) is None:
+                continue
+            if (valid_life_time := info.get("valid_life_time")) is None:
+                continue
+            if (prefixlen := info.get("prefixlen")) is None:
+                continue
+            if (family := info.get("family")) not in ["inet", "inet6"]:
+                continue
+            if (local := info.get("local")) is None:
+                continue
+            if (dynamic := info.get("dynamic", False)) is None:
                 continue
-            else:
-                addr_info.append(addr)
-        if addr_info != [] or has_dynamic_address:
-            net["addr_info"] = addr_info
-            output.append(net)
 
-    return output
+            address = Address(
+                address=local,
+                family=family,
+                prefixlen=prefixlen,
+                preferred_life_time=preferred_life_time,
+                valid_life_time=valid_life_time,
+            )
+
+            if dynamic:
+                dynamic_addresses.append(address)
+            else:
+                static_addresses.append(address)
+        interfaces.append(
+            Interface(
+                name=net.get("ifname", mac_address.replace(":", "-")),
+                ifname=net.get("ifname"),
+                mac_address=mac_address,
+                dynamic_addresses=dynamic_addresses,
+                static_addresses=static_addresses,
+                static_routes=[],
+            )
+        )
+
+    return interfaces
 
 
 def filter_routes(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
     filtered = []
     for route in routes:
-        # Filter out routes set by addresses with subnets, dhcp and router advertisment
+        # Filter out routes set by addresses with subnets, dhcp and router advertisement
         if route.get("protocol") in ["dhcp", "kernel", "ra"]:
             continue
         filtered.append(route)
@@ -42,46 +88,81 @@ def filter_routes(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
     return filtered
 
 
+def find_most_recent_v4_lease(addresses: list[Address]) -> Address | None:
+    most_recent_address = None
+    most_recent_lifetime = -1
+    for addr in addresses:
+        if addr.family == "inet6":
+            continue
+        lifetime = max(addr.preferred_life_time, addr.valid_life_time)
+        if lifetime > most_recent_lifetime:
+            most_recent_lifetime = lifetime
+            most_recent_address = addr
+    return most_recent_address
+
+
+def generate_routes(
+    interface: Interface, routes: list[dict[str, Any]]
+) -> Iterator[str]:
+    for route in routes:
+        if interface.ifname is None or route.get("dev") != interface.ifname:
+            continue
+
+        # we may ignore on-link default routes here, but I don't see how
+        # they would be useful for internet connectivity anyway
+
+        yield "[Route]"
+        if route.get("dst") != "default":
+            # can be skipped for default routes
+            yield f"Destination = {route['dst']}"
+        gateway = route.get("gateway")
+        # route v4 via v6
+        route_via = route.get("via")
+        if route_via and route_via.get("family") == "inet6":
+            gateway = route_via.get("host")
+            if route.get("dst") == "default":
+                yield "Destination = 0.0.0.0/0"
+        if gateway:
+            yield f"Gateway = {gateway}"
+
+
 def generate_networkd_units(
-    interfaces: list[dict[str, Any]], routes: list[dict[str, Any]], directory: Path
+    interfaces: list[Interface], routes: list[dict[str, Any]], directory: Path
 ) -> None:
     directory.mkdir(exist_ok=True)
     for interface in interfaces:
-        name = f"{interface['ifname']}.network"
-        addresses = [
-            f"Address = {addr['local']}/{addr['prefixlen']}"
-            for addr in interface.get("addr_info", [])
-        ]
-
-        route_sections = []
-        for route in routes:
-            if route.get("dev", "nodev") != interface.get("ifname", "noif"):
-                continue
-
-            route_section = "[Route]\n"
-            if route.get("dst") != "default":
-                # can be skipped for default routes
-                route_section += f"Destination = {route['dst']}\n"
-            gateway = route.get("gateway")
-            if gateway:
-                route_section += f"Gateway = {gateway}\n"
-
-            # we may ignore on-link default routes here, but I don't see how
-            # they would be useful for internet connectivity anyway
-            route_sections.append(route_section)
-
-        # FIXME in some networks we might not want to trust dhcp or router advertisments
-        unit = f"""
+        # FIXME in some networks we might not want to trust dhcp or router advertisements
+        unit_sections = [
+            f"""
 [Match]
-MACAddress = {interface["address"]}
+MACAddress = {interface.mac_address}
 
 [Network]
+# both ipv4 and ipv6
 DHCP = yes
+# lets us discover the switch port we're connected to
+LLDP = yes
+# ipv6 router advertisements
 IPv6AcceptRA = yes
-"""
-        unit += "\n".join(addresses)
-        unit += "\n" + "\n".join(route_sections)
-        (directory / name).write_text(unit)
+# allows us to ping "nixos.local"
+MulticastDNS = yes"""
+        ]
+        unit_sections.extend(
+            f"Address = {addr.address}/{addr.prefixlen}"
+            for addr in interface.static_addresses
+        )
+        unit_sections.extend(generate_routes(interface, routes))
+        most_recent_v4_lease = find_most_recent_v4_lease(interface.dynamic_addresses)
+        if most_recent_v4_lease:
+            unit_sections.append("[DHCPv4]")
+            unit_sections.append(f"RequestAddress = {most_recent_v4_lease.address}")
+
+        # trailing newline at the end
+        unit_sections.append("")
+
+        (directory / f"00-{interface.name}.network").write_text(
+            "\n".join(unit_sections)
+        )
 
 
 def main() -> None:

nix/kexec-installer/test.nix

@@ -1,30 +1,24 @@
 { pkgs
+, lib
 , kexecTarball
+, nixos-facter ? null
 }:
 
-let
-  makeTest = import (pkgs.path + "/nixos/tests/make-test-python.nix");
-  makeTest' = args: makeTest args {
-    inherit pkgs;
-    inherit (pkgs) system;
-  };
-in
-makeTest' {
+pkgs.testers.runNixOSTest {
   name = "kexec-installer";
   meta = with pkgs.lib.maintainers; {
     maintainers = [ mic92 ];
   };
 
   nodes = {
-    node1 = { modulesPath, ... }: {
+    node1 = { modulesPath, pkgs, ... }: {
       virtualisation.vlans = [ ];
-      environment.noXlibs = false; # avoid recompilation
       imports = [
         (modulesPath + "/profiles/minimal.nix")
       ];
 
       system.extraDependencies = [ kexecTarball ];
-      virtualisation.memorySize = 1 * 1024 + 512;
+      virtualisation.memorySize = 1 * 1024;
       virtualisation.diskSize = 4 * 1024;
       virtualisation.forwardPorts = [{
         host.port = 2222;
@@ -33,10 +27,8 @@ makeTest' {
 
       services.openssh.enable = true;
 
-      networking = {
-        useNetworkd = true;
-        useDHCP = false;
-      };
+      networking.useNetworkd = true;
+      networking.useDHCP = false;
 
       users.users.root.openssh.authorizedKeys.keyFiles = [ ./ssh-keys/id_ed25519.pub ];
 
@@ -52,12 +44,20 @@ makeTest' {
             name = "eth0";
             address = [
               # Some static addresses that we want to see in the kexeced image
-              "192.168.42.1/24" "42::1/64"
+              "192.168.42.1/24"
+              "42::1/64"
             ];
-            routes = [
+            routes = if pkgs.lib.versionAtLeast lib.version "24.11" then [
+              { Destination = "192.168.43.0/24"; }
+              { Destination = "192.168.44.0/24"; Gateway = "192.168.43.1"; }
+              { Destination = "192.168.45.0/24"; Gateway = "43::1"; }
+              { Destination = "43::0/64"; }
+              { Destination = "44::1/64"; Gateway = "43::1"; }
+            ] else [
               # Some static routes that we want to see in the kexeced image
               { routeConfig = { Destination = "192.168.43.0/24"; }; }
               { routeConfig = { Destination = "192.168.44.0/24"; Gateway = "192.168.43.1"; }; }
+              { routeConfig = { Destination = "192.168.45.0/24"; Gateway = "43::1"; }; }
               { routeConfig = { Destination = "43::0/64"; }; }
               { routeConfig = { Destination = "44::1/64"; Gateway = "43::1"; }; }
             ];
@@ -65,10 +65,14 @@ makeTest' {
           };
         };
       };
+    } // lib.optionalAttrs (lib.versionOlder lib.version "24.11pre") {
+      # avoid second overlay
+      environment.noXlibs = false;
     };
   };
 
-  testScript = ''
+  testScript = /*python*/ ''
+    import json
     import time
     import subprocess
     import socket
@@ -102,19 +106,19 @@ makeTest' {
     node1.succeed("ip -6 route >&2")
     node1.succeed("networkctl status eth0 >&2")
 
-    def ssh(cmd: list[str], check: bool = True, stdout: Optional[int] = None) -> subprocess.CompletedProcess:
+    def ssh(cmd: list[str], check: bool = True, stdout: Optional[int] = None) -> subprocess.CompletedProcess[str]:
         ssh_cmd = [
-          "${pkgs.openssh}/bin/ssh", 
-          "-o", "StrictHostKeyChecking=no", 
+          "${pkgs.openssh}/bin/ssh",
+          "-o", "StrictHostKeyChecking=no",
           "-o", "ConnectTimeout=1",
-          "-i", "${./ssh-keys/id_ed25519}", 
+          "-i", "${./ssh-keys/id_ed25519}",
           "-p", "2222",
           "root@127.0.0.1",
           "--"
         ] + cmd
         print(" ".join(ssh_cmd))
-        return subprocess.run(ssh_cmd, 
-                              text=True, 
+        return subprocess.run(ssh_cmd,
+                              text=True,
                               check=check,
                               stdout=stdout)
 
@@ -129,6 +133,7 @@ makeTest' {
 
     # Kexec node1 to the toplevel of node2 via the kexec-boot script
     node1.succeed('touch /run/foo')
+    old_machine_id = node1.succeed("cat /etc/machine-id").strip()
     node1.fail('parted --version >&2')
     node1.succeed('tar -xf ${kexecTarball}/nixos-kexec-installer-noninteractive-${pkgs.system}.tar.gz -C /root')
     node1.succeed('/root/kexec/ip -V >&2')
@@ -136,23 +141,36 @@ makeTest' {
     node1.succeed('/root/kexec/run >&2')
 
     # wait for kexec to finish
-    while ssh(["true"], check=False).returncode == 0: 
+    while ssh(["true"], check=False).returncode == 0:
         print("Waiting for kexec to finish...")
         time.sleep(1)
 
-    while ssh(["true"], check=False).returncode != 0: 
+    while ssh(["true"], check=False).returncode != 0:
         print("Waiting for node2 to come up...")
         time.sleep(1)
 
+    while ssh(["systemctl is-active restore-network"], check=False).returncode != 0:
+        print("Waiting for network to be restored...")
+        time.sleep(1)
+    ssh(["systemctl", "status", "restore-network"])
+
     print(ssh(["ip", "addr"]))
     print(ssh(["ip", "route"]))
     print(ssh(["ip", "-6", "route"]))
     print(ssh(["networkctl", "status"]))
 
+    new_machine_id = ssh(["cat", "/etc/machine-id"], stdout=subprocess.PIPE).stdout.strip()
+    assert old_machine_id == new_machine_id, f"{old_machine_id} != {new_machine_id}, machine-id changed"
+
     assert ssh(["ls", "-la", "/run/foo"], check=False).returncode != 0, "kexeced node1 still has /run/foo"
     print(ssh(["parted", "--version"]))
     host = ssh(["hostname"], stdout=subprocess.PIPE).stdout.strip()
-    assert host == "nixos", f"hostname is {host}, not nixos"
+    assert host == "nixos-installer", f"hostname is {host}, not nixos-installer"
+
+    has_nixos_facter=${if nixos-facter != null then "True" else "False"}
+    if has_nixos_facter == True:
+        data = json.loads(ssh(["nixos-facter"], stdout=subprocess.PIPE).stdout)
+        assert data["virtualisation"] == "kvm", f"virtualisation is {data['virtualisation']}, not kvm"
 
     host_ed25519_after = ssh(["cat", "/etc/ssh/ssh_host_ed25519_key.pub"], stdout=subprocess.PIPE).stdout.strip()
     assert host_ed25519_before == host_ed25519_after, f"'{host_ed25519_before}' != '{host_ed25519_after}'"
@@ -160,7 +178,7 @@ makeTest' {
     root_ed25519_after = ssh(["cat", "/root/.ssh/authorized_keys"], stdout=subprocess.PIPE).stdout.strip()
     assert root_ed25519_before in root_ed25519_after, f"'{root_ed25519_before}' not included in '{root_ed25519_after}'"
 
-    print(ssh(["cat", "/etc/systemd/network/eth0.network"]))
+    print(ssh(["cat", "/etc/systemd/network/00-eth0.network"]))
     ssh(["curl", "-v", "-I", f"http://10.0.2.2:{port}"])
     ssh(["curl", "-v", "-I", f"http://[fec0::2]:{port}"])
 
@@ -176,6 +194,10 @@ makeTest' {
     print(out)
     assert "192.168.44.2 via 192.168.43.1" in out, f"route to `192.168.44.2 via 192.168.43.1` not found: {out}"
 
+    out = ssh(["ip", "route", "get", "192.168.45.2"], stdout=subprocess.PIPE).stdout
+    print(out)
+    assert "192.168.45.2 via inet6 43::1" in out, f"route to `192.168.45.2 via inet6 43::1` not found: {out}"
+
     out = ssh(["ip", "route", "get", "43::2"], stdout=subprocess.PIPE).stdout
     print(out)
     assert "43::2 from :: dev" in out, f"route `43::2 from dev` not found: {out}"

nix/latest-zfs-kernel.nix

@@ -0,0 +1,26 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+
+let
+  isUnstable = config.boot.zfs.package == pkgs.zfsUnstable;
+  zfsCompatibleKernelPackages = lib.filterAttrs (
+    name: kernelPackages:
+    (builtins.match "linux_[0-9]+_[0-9]+" name) != null
+    && (builtins.tryEval kernelPackages).success
+    && (
+      (!isUnstable && !kernelPackages.zfs.meta.broken)
+      || (isUnstable && !kernelPackages.zfs_unstable.meta.broken)
+    )
+  ) pkgs.linuxKernel.packages;
+  latestKernelPackage = lib.last (
+    lib.sort (a: b: (lib.versionOlder a.kernel.version b.kernel.version)) (builtins.attrValues zfsCompatibleKernelPackages)
+  );
+in
+{
+  # Note this might jump back and worth as kernel get added or removed.
+  boot.kernelPackages = latestKernelPackage;
+}

nix/log-network-status.nix

@@ -0,0 +1,22 @@
+{ pkgs, ... }:
+{
+  systemd.services.log-network-status = {
+    wantedBy = [ "multi-user.target" ];
+    # No point in restarting this. We just need this after boot
+    restartIfChanged = false;
+
+    serviceConfig = {
+      Type = "oneshot";
+      StandardOutput = "journal+console";
+      ExecStart = [
+        # Allow failures, so it still prints what interfaces we have even if we
+        # not get online
+        "-${pkgs.systemd}/lib/systemd/systemd-networkd-wait-online"
+        "${pkgs.iproute2}/bin/ip -c addr"
+        "${pkgs.iproute2}/bin/ip -c -6 route"
+        "${pkgs.iproute2}/bin/ip -c -4 route"
+        "${pkgs.systemd}/bin/networkctl status"
+      ];
+    };
+  };
+}

nix/netboot-installer/module.nix

@@ -3,6 +3,9 @@
   imports = [
     (modulesPath + "/installer/netboot/netboot-minimal.nix")
     ../installer.nix
+    ../networkd.nix
+    ../serial.nix
+    ../restore-remote-access.nix
   ];
 
   # We are stateless, so just default to latest.
@@ -13,7 +16,7 @@
     paths = with config.system.build; [
       netbootRamdisk
       kernel
-      (pkgs.runCommand "kernel-params" {} ''
+      (pkgs.runCommand "kernel-params" { } ''
         mkdir -p $out
         ln -s "${config.system.build.toplevel}/kernel-params" $out/kernel-params
         ln -s "${config.system.build.toplevel}/init" $out/init
@@ -25,9 +28,8 @@
     matchConfig.Type = "ether";
     networkConfig = {
       DHCP = "yes";
-      LLMNR = "yes";
       EmitLLDP = "yes";
-      IPv6AcceptRA = "no";
+      IPv6AcceptRA = "yes";
       MulticastDNS = "yes";
       LinkLocalAddressing = "yes";
       LLDP = "yes";

nix/networkd.nix

@@ -0,0 +1,13 @@
+{ lib, ... }:
+{
+  # Not really needed. Saves a few bytes and the only service we are running is sshd, which we want to be reachable.
+  networking.firewall.enable = false;
+
+  networking.useNetworkd = true;
+  systemd.network.enable = true;
+
+  # mdns
+  networking.firewall.allowedUDPPorts = [ 5353 ];
+  systemd.network.networks."99-ethernet-default-dhcp".networkConfig.MulticastDNS = lib.mkDefault "yes";
+  systemd.network.networks."99-wireless-client-dhcp".networkConfig.MulticastDNS = lib.mkDefault "yes";
+}

nix/nix-settings.nix

@@ -0,0 +1,24 @@
+# take from srvos
+{ lib, ... }:
+{
+  # Fallback quickly if substituters are not available.
+  nix.settings.connect-timeout = 5;
+
+  # Enable flakes
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+
+  # The default at 10 is rarely enough.
+  nix.settings.log-lines = lib.mkDefault 25;
+
+  # Avoid disk full issues
+  nix.settings.max-free = lib.mkDefault (3000 * 1024 * 1024);
+  nix.settings.min-free = lib.mkDefault (512 * 1024 * 1024);
+
+  # TODO: cargo culted.
+  nix.daemonCPUSchedPolicy = lib.mkDefault "batch";
+  nix.daemonIOSchedClass = lib.mkDefault "idle";
+  nix.daemonIOSchedPriority = lib.mkDefault 7;
+
+  # Avoid copying unnecessary stuff over SSH
+  nix.settings.builders-use-substitutes = true;
+}

nix/nixos-facter.nix

@@ -0,0 +1,71 @@
+{
+  lib,
+  buildGoModule,
+  fetchFromGitHub,
+  hwinfo,
+  libusb1,
+  gcc,
+  pkg-config,
+  util-linux,
+  pciutils,
+  stdenv,
+}:
+let
+  # We are waiting on some changes to be merged upstream: https://github.com/openSUSE/hwinfo/pulls
+  hwinfoOverride = hwinfo.overrideAttrs {
+    src = fetchFromGitHub {
+      owner = "numtide";
+      repo = "hwinfo";
+      rev = "a559f34934098d54096ed2078e750a8245ae4044";
+      hash = "sha256-3abkWPr98qXXQ17r1Z43gh2M5hl/DHjW2hfeWl+GSAs=";
+    };
+  };
+in
+buildGoModule rec {
+  pname = "nixos-facter";
+  version = "0.1.1";
+
+  src = fetchFromGitHub {
+    owner = "numtide";
+    repo = "nixos-facter";
+    rev = "v${version}";
+    hash = "sha256-vlPmvCrgX64dcf//BPtQszBt7dkq35JpgQg+/LW0AqM=";
+  };
+
+  vendorHash = "sha256-5leiTNp3FJmgFd0SKhu18hxYZ2G9SuQPhZJjki2SDVs=";
+
+  CGO_ENABLED = 1;
+
+  buildInputs = [
+    libusb1
+    hwinfoOverride
+  ];
+
+  nativeBuildInputs = [
+    gcc
+    pkg-config
+  ];
+
+  runtimeInputs = [
+    libusb1
+    util-linux
+    pciutils
+  ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X git.numtide.com/numtide/nixos-facter/build.Name=nixos-facter"
+    "-X git.numtide.com/numtide/nixos-facter/build.Version=v${version}"
+    "-X github.com/numtide/nixos-facter/pkg/build.System=${stdenv.hostPlatform.system}"
+  ];
+
+  meta = {
+    description = "Declarative hardware configuration for NixOS";
+    homepage = "https://github.com/numtide/nixos-facter";
+    license = lib.licenses.gpl3Plus;
+    maintainers = [ lib.maintainers.brianmcgee ];
+    mainProgram = "nixos-facter";
+    platforms = lib.platforms.linux;
+  };
+}

nix/no-bootloaders.nix

@@ -0,0 +1,12 @@
+{ lib, ... }: {
+  # HACK: Drop this, once we have 24.11 everywhere
+  nixpkgs.overlays = lib.optionals (lib.versionOlder lib.version "24.11pre") [
+    # Both syslinux and grub also reference perl
+    (final: prev: {
+      # we don't need grub: save ~ 60MB
+      grub2 = prev.coreutils;
+      grub2_efi = prev.coreutils;
+      syslinux = prev.coreutils;
+    })
+  ];
+}

nix/noninteractive.nix

@@ -1,13 +1,25 @@
 # This module optimizes for non-interactive deployments by remove some store paths
 # which are primarily useful for interactive installations.
 
-{ config, lib, pkgs, ... }: {
+{ lib, pkgs, modulesPath, ... }:
+{
   disabledModules = [
     # This module adds values to multiple lists (systemPackages, supportedFilesystems)
     # which are impossible/unpractical to remove, so we disable the entire module.
     "profiles/base.nix"
   ];
 
+  imports = [
+#    ./zfs-minimal.nix
+    ./no-bootloaders.nix
+    ./python-minimal.nix
+    ./noveau-workaround.nix
+    # reduce closure size by removing perl
+    "${modulesPath}/profiles/perlless.nix"
+    # FIXME: we still are left with nixos-generate-config due to nixos-install-tools
+    { system.forbiddenDependenciesRegexes = lib.mkForce []; }
+  ];
+
   # among others, this prevents carrying a stdenv with gcc in the image
   system.extraDependencies = lib.mkForce [ ];
 
@@ -15,20 +27,32 @@
   nix.registry = lib.mkForce { };
 
   # would pull in nano
-  programs.nano.syntaxHighlight = lib.mkForce false;
+  programs.nano.enable = false;
 
-  # prevents nano, strace
+  # prevents strace
   environment.defaultPackages = lib.mkForce [
     pkgs.rsync
     pkgs.parted
-    (pkgs.zfs.override {
-      # this overrides saves 10MB
-      samba = pkgs.coreutils;
-    })
+    pkgs.gptfdisk
+    pkgs.e2fsprogs
   ];
 
+  # normal users are not allowed with sys-users
+  # see https://github.com/NixOS/nixpkgs/pull/328926
+  users.users.nixos = {
+    isSystemUser = true;
+    isNormalUser = lib.mkForce false;
+    shell = "/run/current-system/sw/bin/bash";
+    group = "nixos";
+  };
+  users.groups.nixos = {};
+
+  # we prefer root as this is also what we use in nixos-anywhere
+  services.getty.autologinUser = lib.mkForce "root";
+
   # we are missing this from base.nix
   boot.supportedFilesystems = [
+    "ext4"
     "btrfs"
     # probably not needed but does not seem to increase closure size
     "cifs"
@@ -41,15 +65,9 @@
     "vfat"
     "xfs"
   ];
-  boot = {
-    kernelModules = [ "zfs" ];
-    extraModulePackages = [
-      config.boot.kernelPackages.zfs
-    ];
-  };
-
-  networking.hostId = lib.mkDefault "8425e349";
-
-  # we can drop this after 23.05 has been released, which has this set by default
-  hardware.enableRedistributableFirmware = lib.mkForce false;
+  boot.kernelModules = [
+    # we have to explicitly enable this, otherwise it is not loaded even when creating a raid:
+    # https://github.com/nix-community/nixos-anywhere/issues/249
+    "dm-raid"
+  ];
 }

nix/noveau-workaround.nix

@@ -0,0 +1,4 @@
+{
+  # fixes blank screen on boot for some cards
+  boot.kernelParams = [ "nouveau.modeset=0" ];
+}

nix/python-minimal.nix

@@ -0,0 +1,10 @@
+{
+  nixpkgs.overlays = [
+    (final: prev: {
+      bcachefs-tools = prev.bcachefs-tools.override { python3 = final.python3Minimal; };
+      cifs-utils = prev.cifs-utils.override { python3 = final.python3Minimal; };
+      nfs-utils = prev.nfs-utils.override { python3 = final.python3Minimal; };
+      talloc = prev.talloc.override { python3 = final.python3Minimal; };
+    })
+  ];
+}

nix/restore-remote-access.nix

@@ -0,0 +1,29 @@
+{
+  # We have a bug in 23.11 in combination with netboot.
+  boot.initrd.systemd.enable = true;
+  boot.initrd.systemd.services.restore-state-from-initrd = {
+    unitConfig = {
+      DefaultDependencies = false;
+      RequiresMountsFor = "/sysroot /dev";
+    };
+    wantedBy = [ "initrd.target" ];
+    requiredBy = [ "rw-etc.service" ];
+    before = [ "rw-etc.service" ];
+    serviceConfig.Type = "oneshot";
+    # Restore ssh host and user keys if they are available.
+    # This avoids warnings of unknown ssh keys.
+    script = ''
+      mkdir -m 700 -p /sysroot/root/.ssh
+      mkdir -m 755 -p /sysroot/etc/ssh
+      mkdir -m 755 -p /sysroot/root/network
+      if [[ -f ssh/authorized_keys ]]; then
+        install -m 400 ssh/authorized_keys /sysroot/root/.ssh
+      fi
+      install -m 400 ssh/ssh_host_* /sysroot/etc/ssh
+      cp *.json /sysroot/root/network/
+      if [[ -f machine-id ]]; then
+        cp machine-id /sysroot/etc/machine-id
+      fi
+    '';
+  };
+}

nix/serial.nix

@@ -0,0 +1,11 @@
+{ pkgs, lib, ... }:
+{
+  # IPMI SOL console redirection stuff
+  boot.kernelParams =
+    [ "console=tty0" ]
+    ++ (lib.optional (
+      pkgs.stdenv.hostPlatform.isAarch32 || pkgs.stdenv.hostPlatform.isAarch64
+    ) "console=ttyAMA0,115200")
+    ++ (lib.optional (pkgs.stdenv.hostPlatform.isRiscV) "console=ttySIF0,115200")
+    ++ [ "console=ttyS0,115200" ];
+}

nix/zfs-minimal.nix

@@ -0,0 +1,21 @@
+{ config, lib, pkgs, ... }:
+# incorperate a space-optimized version of zfs
+let
+  zfs = pkgs.zfsUnstable.override {
+    # this overrides saves 10MB
+    samba = pkgs.coreutils;
+
+    python3 = pkgs.python3Minimal;
+  };
+in
+{
+  services.udev.packages = [ zfs ]; # to hook zvol naming, etc.
+  # unsure if need this, but in future udev rules could potentially point to systemd services.
+  systemd.packages = [ zfs ];
+  environment.defaultPackages = lib.mkForce [ zfs ]; # this merges with outer noninteractive module.
+
+  boot.kernelModules = [ "zfs" ];
+  boot.extraModulePackages = [ config.boot.kernelPackages.zfs_unstable ];
+
+  networking.hostId = lib.mkDefault "8425e349";
+}