hydra: add even more power

config/hosts/crime/configuration.nix

@@ -1,4 +1,4 @@
-{ ... }: {
+{ inputs, ... }: {
   imports = [
     ../../common
     ../../users/lucy
@@ -8,6 +8,7 @@
     ../../services/arrs
     ../../services/jellyfin.nix
     ../../services/nginx.nix
+    inputs.oth.nixosModules.default
   ];
   networking.hostName = "crime";
   systemd.network.networks."98-eth-default" = {
@@ -24,6 +25,7 @@
       "/var/lib/radarr"
       "/var/lib/sonarr"
       "/var/lib/private/prowlarr"
+      "/home"
     ];
   };
 }

config/profiles/rpi.nix

@@ -9,10 +9,6 @@
       options snd_bcm2835 enable_headphones=1
     '';
     kernelParams = [ "snd_bcm2835.enable_hdmi=1" ];
-    loader.raspberryPi.firmwareConfig = ''
-      dtparam=audio=on
-      dtparam=krnbt=on
-    '';
   };
   console.enable = false;
 

config/services/hydra/default.nix

@@ -30,8 +30,8 @@
     extraConfig = ''
       server_store_uri = https://cache.kyouma.net
       binary_cache_public_uri = https://cache.kyouma.net
-      evaluator_workers = 6
+      evaluator_workers = 1
-      evaluator_max_memory_size = 12288
+      evaluator_max_memory_size = 4096
       max_output_size = ${builtins.toString (24 * 1024 * 1024 * 1024)}
     '';
   };
@@ -44,8 +44,8 @@
   };
 
   systemd.services.hydra-evaluator.serviceConfig = {
-    MemoryHigh = "250G";
+    MemoryHigh = "150G";
-    MemoryMax = "254G";
+    MemoryMax = "155G";
   };
 
   kyouma.nginx.defaultForbidden = "hydra.kyouma.net";

config/services/hydra/nix-config.nix

@@ -28,8 +28,8 @@
       supportedFeatures = base ++ [ "gccarch-rv64imac" "gccarch-rv64imacfd" "gccarch-rv64gc" ];
       sshKey = config.sops.secrets."services/hydra/id_ed25519_hydra".path;
     }
-  ] ++ lib.forEach (lib.genList (i: i + 1) 8) (num: {
+  ] ++ lib.forEach (lib.range 0 11) (num: {
-      hostName = "build-worker-0${toString num}";
+      hostName = "build-worker-${lib.fixedWidthNumber 2 num}";
       sshUser = "root";
       maxJobs = 2;
       speedFactor = 20;
@@ -56,6 +56,7 @@
       "schrodinger.kyouma.net".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKo7vZ6lS1wx76YsbAdhOsGcc20YMAW52ep8SZ/FCHDp";
       "lab.nyantec.com".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIUePtVPtBK+CYosufbaGiMT4EVanti4V5t2Wg0g/Fy4";
       "localhost".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPNVavo3YHVsrYwXRVISu7kDoknn+5inFGySn4azlB8P";
+      "[build-worker-kyoumanet.fly.dev]:2200".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJUGzlilikAUfUGKXVCoTeDvPRoWUgDDkNU5WaRUBzls";
       "[build-worker-kyoumanet.fly.dev]:2201".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDL2M97UBHg9aUfjDUxzmzg1r0ga0m3/stummBVwuEAB";
       "[build-worker-kyoumanet.fly.dev]:2202".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOTwVKL0P0chPM2Gz23rbT94844+w1CGJdCaZdzfjThz";
       "[build-worker-kyoumanet.fly.dev]:2203".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAjy2eZGJQeAYy0+fLgW9jiS0jVY2LInY0NDMnzCvvKp";
@@ -64,15 +65,18 @@
       "[build-worker-kyoumanet.fly.dev]:2206".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGf0kxGgwOG9KhUhvxxTSiQC5YikrzZXKDgSpBw33qN4";
       "[build-worker-kyoumanet.fly.dev]:2207".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL9z95a6Fn/dB+iNigEYpuJdBnBwCkIZYaKHcFbGP+RY";
       "[build-worker-kyoumanet.fly.dev]:2208".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAk+FNMhTfAVqk3MfLp4QiG/i5ti53DlpnC0q+sOvU9O";
+      "[build-worker-kyoumanet-cdg.fly.dev]:2209".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJGlQD/3fLn/Kyb7v0RIycHRcArGi75jURj803EMpW0S";
+      "[build-worker-kyoumanet-cdg.fly.dev]:2210".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMQm1FSGBGdCR5f8MvBvdKM0M4yIQVnH1po7hHO5T1qz";
+      "[build-worker-kyoumanet-cdg.fly.dev]:2211".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINlH+v2ZlcDQY3itw4b7aRbwRTqDsTE0R5Ua3vF0VaGr";
     }; 
     extraConfig = ''
       Host machine-0008.cloud-v.co
         Hostname machine.cloud-v.co
         Port 20008
-    '' + lib.concatLines (lib.genList (i: ''
+    '' + lib.concatLines (lib.forEach (lib.range 0 11) (num: ''
-      Host build-worker-0${toString (i + 1)}
+      Host build-worker-${lib.fixedWidthNumber 2 num}
-        Hostname build-worker-kyoumanet.fly.dev
+        Hostname build-worker-kyoumanet${lib.optionalString (num > 8) "-cdg"}.fly.dev
-        Port 220${toString (i + 1)}
+        Port 22${lib.fixedWidthNumber 2 num}
-    '') 8);
+    ''));
   };
 }

flake.lock

@@ -402,11 +402,11 @@
         "systems": "systems_3"
       },
       "locked": {
-        "lastModified": 1710146030,
+        "lastModified": 1726560853,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -775,15 +775,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1723510904,
+        "lastModified": 1729360442,
-        "narHash": "sha256-zNW/rqNJwhq2lYmQf19wJerRuNimjhxHKmzrWWFJYts=",
+        "narHash": "sha256-6U0CyPycIBc04hbYy2hBINnVso58n/ZyywY2BD3hu+s=",
-        "rev": "622a2253a071a1fb97a4d3c8103a91114acc1140",
+        "rev": "9098ac95768f7006d7e070b88bae76939f6034e6",
         "type": "tarball",
-        "url": "https://git.lix.systems/api/v1/repos/lix-project/nixos-module/archive/622a2253a071a1fb97a4d3c8103a91114acc1140.tar.gz?rev=622a2253a071a1fb97a4d3c8103a91114acc1140"
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/nixos-module/archive/9098ac95768f7006d7e070b88bae76939f6034e6.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.lix.systems/lix-project/nixos-module/archive/2.91.0.tar.gz"
+        "url": "https://git.lix.systems/lix-project/nixos-module/archive/2.91.1-1.tar.gz"
       }
     },
     "lix_2": {
@@ -795,29 +795,29 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1723503926,
+        "lastModified": 1729298361,
-        "narHash": "sha256-Rosl9iA9MybF5Bud4BTAQ9adbY81aGmPfV8dDBGl34s=",
+        "narHash": "sha256-hiGtfzxFkDc9TSYsb96Whg0vnqBVV7CUxyscZNhed0U=",
-        "rev": "bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2",
+        "rev": "ad9d06f7838a25beec425ff406fe68721fef73be",
         "type": "tarball",
-        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2.tar.gz?rev=bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2"
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ad9d06f7838a25beec425ff406fe68721fef73be.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.lix.systems/lix-project/lix/archive/2.91.0.tar.gz"
+        "url": "https://git.lix.systems/lix-project/lix/archive/2.91.1.tar.gz"
       }
     },
     "lix_3": {
       "flake": false,
       "locked": {
-        "lastModified": 1723503926,
+        "lastModified": 1729298361,
-        "narHash": "sha256-Rosl9iA9MybF5Bud4BTAQ9adbY81aGmPfV8dDBGl34s=",
+        "narHash": "sha256-hiGtfzxFkDc9TSYsb96Whg0vnqBVV7CUxyscZNhed0U=",
-        "rev": "bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2",
+        "rev": "ad9d06f7838a25beec425ff406fe68721fef73be",
         "type": "tarball",
-        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2.tar.gz?rev=bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2"
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ad9d06f7838a25beec425ff406fe68721fef73be.tar.gz?rev=ad9d06f7838a25beec425ff406fe68721fef73be"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.lix.systems/lix-project/lix/archive/2.91.0.tar.gz"
+        "url": "https://git.lix.systems/lix-project/lix/archive/2.91.1.tar.gz"
       }
     },
     "neoidiosyn": {
@@ -1151,6 +1151,29 @@
         "type": "github"
       }
     },
+    "oth": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732215904,
+        "narHash": "sha256-bXXoyQcVMULYOj1KlzbquUyMTyByzmLatIgm0ra/7sk=",
+        "ref": "refs/heads/main",
+        "rev": "847cc8493f7f44bd5ada0283d6b96457f4ee5a9b",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://forgejo@woof.rip/emily/oth.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://forgejo@woof.rip/emily/oth.git"
+      }
+    },
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
@@ -1187,6 +1210,7 @@
         "nixos-needsreboot": "nixos-needsreboot",
         "nixpkgs": "nixpkgs_4",
         "nixvim": "nixvim",
+        "oth": "oth",
         "sops-nix": "sops-nix",
         "stylix": "stylix"
       }

flake.nix

@@ -50,9 +50,9 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.flake-utils.follows = "flake-utils";
     };
-    lix.url = "https://git.lix.systems/lix-project/lix/archive/2.91.0.tar.gz";
+    lix.url = "https://git.lix.systems/lix-project/lix/archive/2.91.1.tar.gz";
     lix-module = {
-      url = "https://git.lix.systems/lix-project/nixos-module/archive/2.91.0.tar.gz";
+      url = "https://git.lix.systems/lix-project/nixos-module/archive/2.91.1-1.tar.gz";
       inputs.nixpkgs.follows = "nixpkgs";
     };
     nixos-needsreboot = {
@@ -64,6 +64,11 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.home-manager.follows = "home-manager";
     };
+    oth = {
+      url = "git+ssh://forgejo@woof.rip/emily/oth.git";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
     sops-nix = {
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";

pkgs/build-worker-oci/ams/fly.toml

@@ -5,6 +5,7 @@ primary_region = 'ams'
   image = 'registry.fly.io/build-worker-kyoumanet:latest'
 
 [processes]
+  bw-00 = '/entrypoint.sh'
   bw-01 = '/entrypoint.sh'
   bw-02 = '/entrypoint.sh'
   bw-03 = '/entrypoint.sh'
@@ -14,28 +15,34 @@ primary_region = 'ams'
   bw-07 = '/entrypoint.sh'
   bw-08 = '/entrypoint.sh'
 
+[[mounts]]
+  source = 'bw00'
+  destination = '/mnt/data'
+  initial_size = '256GB'
+  processes = ['bw-00']
+
 [[mounts]]
   source = 'bw01'
   destination = '/mnt/data'
-  initial_size = '128GB'
+  initial_size = '256GB'
   processes = ['bw-01']
 
 [[mounts]]
   source = 'bw02'
   destination = '/mnt/data'
-  initial_size = '128GB'
+  initial_size = '256GB'
   processes = ['bw-02']
 
 [[mounts]]
   source = 'bw03'
   destination = '/mnt/data'
-  initial_size = '128GB'
+  initial_size = '256GB'
   processes = ['bw-03']
 
 [[mounts]]
   source = 'bw04'
   destination = '/mnt/data'
-  initial_size = '128GB'
+  initial_size = '256GB'
   processes = ['bw-04']
 
 [[mounts]]
@@ -62,6 +69,15 @@ primary_region = 'ams'
   initial_size = '256GB'
   processes = ['bw-08']
 
+[[services]]
+  protocol = 'tcp'
+  internal_port = 2222
+  auto_stop_machines = 'off'
+  processes = ['bw-00']
+
+  [[services.ports]]
+    port = 2200
+
 [[services]]
   protocol = 'tcp'
   internal_port = 2222

pkgs/build-worker-oci/cdg/fly.toml

@@ -0,0 +1,67 @@
+# fly.toml app configuration file generated for build-worker-kyoumanet-cdg on 2024-11-21T00:31:54+01:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = 'build-worker-kyoumanet-cdg'
+primary_region = 'cdg'
+
+[build]
+  image = 'registry.fly.io/build-worker-kyoumanet:latest'
+
+[processes]
+  bw-09 = '/entrypoint.sh'
+  bw-10 = '/entrypoint.sh'
+  bw-11 = '/entrypoint.sh'
+
+[[mounts]]
+  source = 'bw09'
+  destination = '/mnt/data'
+  initial_size = '256GB'
+  processes = ['bw-09']
+
+[[mounts]]
+  source = 'bw10'
+  destination = '/mnt/data'
+  initial_size = '256GB'
+  processes = ['bw-10']
+
+[[mounts]]
+  source = 'bw11'
+  destination = '/mnt/data'
+  initial_size = '256GB'
+  processes = ['bw-11']
+
+[[services]]
+  protocol = 'tcp'
+  internal_port = 2222
+  auto_stop_machines = 'off'
+  processes = ['bw-09']
+
+  [[services.ports]]
+    port = 2209
+
+[[services]]
+  protocol = 'tcp'
+  internal_port = 2222
+  auto_stop_machines = 'off'
+  processes = ['bw-10']
+
+  [[services.ports]]
+    port = 2210
+
+[[services]]
+  protocol = 'tcp'
+  internal_port = 2222
+  auto_stop_machines = 'off'
+  processes = ['bw-11']
+
+  [[services.ports]]
+    port = 2211
+
+[[restart]]
+  policy = 'never'
+
+[[vm]]
+  size = 'performance-16x'
+  memory = '96GB'

pkgs/update-nixfiles/update-nixfiles.sh

@@ -23,7 +23,9 @@ merge_theirs () {
 test_build () {
   local build_jobs
 
-  build_jobs="$(curl --fail -s -L -H "Accept: application/json" "${JOBSET_URL}/latest-eval" | jq -r ".builds | .[]")"
+  sleep 30
+
+  build_jobs="$(curl --fail -s -L -H "Accept: application/json" "${JOBSET_URL}/evals" | jq -r ".evals | max_by(.id) | .builds | .[]")"
   for build in ${build_jobs}; do 
     local build_status