From af4e5a712799e730180567a83c15c6a08a878854 Mon Sep 17 00:00:00 2001 From: emily Date: Thu, 21 Nov 2024 00:51:39 +0100 Subject: [PATCH] hydra: add even more power --- config/hosts/crime/configuration.nix | 4 +- config/profiles/rpi.nix | 4 -- config/services/hydra/default.nix | 8 +-- config/services/hydra/nix-config.nix | 18 +++-- flake.lock | 60 ++++++++++++----- flake.nix | 9 ++- .../build-worker-oci/ams/fly.toml | 24 +++++-- pkgs/build-worker-oci/cdg/fly.toml | 67 +++++++++++++++++++ pkgs/update-nixfiles/update-nixfiles.sh | 4 +- 9 files changed, 157 insertions(+), 41 deletions(-) rename fly.toml => pkgs/build-worker-oci/ams/fly.toml (86%) create mode 100644 pkgs/build-worker-oci/cdg/fly.toml diff --git a/config/hosts/crime/configuration.nix b/config/hosts/crime/configuration.nix index 5ab41bc..75bc6d5 100644 --- a/config/hosts/crime/configuration.nix +++ b/config/hosts/crime/configuration.nix @@ -1,4 +1,4 @@ -{ ... }: { +{ inputs, ... }: { imports = [ ../../common ../../users/lucy @@ -8,6 +8,7 @@ ../../services/arrs ../../services/jellyfin.nix ../../services/nginx.nix + inputs.oth.nixosModules.default ]; networking.hostName = "crime"; systemd.network.networks."98-eth-default" = { @@ -24,6 +25,7 @@ "/var/lib/radarr" "/var/lib/sonarr" "/var/lib/private/prowlarr" + "/home" ]; }; } diff --git a/config/profiles/rpi.nix b/config/profiles/rpi.nix index 0644af9..22e27a1 100644 --- a/config/profiles/rpi.nix +++ b/config/profiles/rpi.nix @@ -9,10 +9,6 @@ options snd_bcm2835 enable_headphones=1 ''; kernelParams = [ "snd_bcm2835.enable_hdmi=1" ]; - loader.raspberryPi.firmwareConfig = '' - dtparam=audio=on - dtparam=krnbt=on - ''; }; console.enable = false; diff --git a/config/services/hydra/default.nix b/config/services/hydra/default.nix index 2bd0cc2..c723a7b 100644 --- a/config/services/hydra/default.nix +++ b/config/services/hydra/default.nix @@ -30,8 +30,8 @@ extraConfig = '' server_store_uri = https://cache.kyouma.net binary_cache_public_uri = https://cache.kyouma.net - evaluator_workers = 6 - evaluator_max_memory_size = 12288 + evaluator_workers = 1 + evaluator_max_memory_size = 4096 max_output_size = ${builtins.toString (24 * 1024 * 1024 * 1024)} ''; }; @@ -44,8 +44,8 @@ }; systemd.services.hydra-evaluator.serviceConfig = { - MemoryHigh = "250G"; - MemoryMax = "254G"; + MemoryHigh = "150G"; + MemoryMax = "155G"; }; kyouma.nginx.defaultForbidden = "hydra.kyouma.net"; diff --git a/config/services/hydra/nix-config.nix b/config/services/hydra/nix-config.nix index 9933888..949a114 100644 --- a/config/services/hydra/nix-config.nix +++ b/config/services/hydra/nix-config.nix @@ -28,8 +28,8 @@ supportedFeatures = base ++ [ "gccarch-rv64imac" "gccarch-rv64imacfd" "gccarch-rv64gc" ]; sshKey = config.sops.secrets."services/hydra/id_ed25519_hydra".path; } - ] ++ lib.forEach (lib.genList (i: i + 1) 8) (num: { - hostName = "build-worker-0${toString num}"; + ] ++ lib.forEach (lib.range 0 11) (num: { + hostName = "build-worker-${lib.fixedWidthNumber 2 num}"; sshUser = "root"; maxJobs = 2; speedFactor = 20; @@ -56,6 +56,7 @@ "schrodinger.kyouma.net".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKo7vZ6lS1wx76YsbAdhOsGcc20YMAW52ep8SZ/FCHDp"; "lab.nyantec.com".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIUePtVPtBK+CYosufbaGiMT4EVanti4V5t2Wg0g/Fy4"; "localhost".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPNVavo3YHVsrYwXRVISu7kDoknn+5inFGySn4azlB8P"; + "[build-worker-kyoumanet.fly.dev]:2200".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJUGzlilikAUfUGKXVCoTeDvPRoWUgDDkNU5WaRUBzls"; "[build-worker-kyoumanet.fly.dev]:2201".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDL2M97UBHg9aUfjDUxzmzg1r0ga0m3/stummBVwuEAB"; "[build-worker-kyoumanet.fly.dev]:2202".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOTwVKL0P0chPM2Gz23rbT94844+w1CGJdCaZdzfjThz"; "[build-worker-kyoumanet.fly.dev]:2203".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAjy2eZGJQeAYy0+fLgW9jiS0jVY2LInY0NDMnzCvvKp"; @@ -64,15 +65,18 @@ "[build-worker-kyoumanet.fly.dev]:2206".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGf0kxGgwOG9KhUhvxxTSiQC5YikrzZXKDgSpBw33qN4"; "[build-worker-kyoumanet.fly.dev]:2207".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL9z95a6Fn/dB+iNigEYpuJdBnBwCkIZYaKHcFbGP+RY"; "[build-worker-kyoumanet.fly.dev]:2208".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAk+FNMhTfAVqk3MfLp4QiG/i5ti53DlpnC0q+sOvU9O"; + "[build-worker-kyoumanet-cdg.fly.dev]:2209".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJGlQD/3fLn/Kyb7v0RIycHRcArGi75jURj803EMpW0S"; + "[build-worker-kyoumanet-cdg.fly.dev]:2210".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMQm1FSGBGdCR5f8MvBvdKM0M4yIQVnH1po7hHO5T1qz"; + "[build-worker-kyoumanet-cdg.fly.dev]:2211".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINlH+v2ZlcDQY3itw4b7aRbwRTqDsTE0R5Ua3vF0VaGr"; }; extraConfig = '' Host machine-0008.cloud-v.co Hostname machine.cloud-v.co Port 20008 - '' + lib.concatLines (lib.genList (i: '' - Host build-worker-0${toString (i + 1)} - Hostname build-worker-kyoumanet.fly.dev - Port 220${toString (i + 1)} - '') 8); + '' + lib.concatLines (lib.forEach (lib.range 0 11) (num: '' + Host build-worker-${lib.fixedWidthNumber 2 num} + Hostname build-worker-kyoumanet${lib.optionalString (num > 8) "-cdg"}.fly.dev + Port 22${lib.fixedWidthNumber 2 num} + '')); }; } diff --git a/flake.lock b/flake.lock index dc457d7..0504454 100644 --- a/flake.lock +++ b/flake.lock @@ -402,11 +402,11 @@ "systems": "systems_3" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", "type": "github" }, "original": { @@ -775,15 +775,15 @@ ] }, "locked": { - "lastModified": 1723510904, - "narHash": "sha256-zNW/rqNJwhq2lYmQf19wJerRuNimjhxHKmzrWWFJYts=", - "rev": "622a2253a071a1fb97a4d3c8103a91114acc1140", + "lastModified": 1729360442, + "narHash": "sha256-6U0CyPycIBc04hbYy2hBINnVso58n/ZyywY2BD3hu+s=", + "rev": "9098ac95768f7006d7e070b88bae76939f6034e6", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/nixos-module/archive/622a2253a071a1fb97a4d3c8103a91114acc1140.tar.gz?rev=622a2253a071a1fb97a4d3c8103a91114acc1140" + "url": "https://git.lix.systems/api/v1/repos/lix-project/nixos-module/archive/9098ac95768f7006d7e070b88bae76939f6034e6.tar.gz" }, "original": { "type": "tarball", - "url": "https://git.lix.systems/lix-project/nixos-module/archive/2.91.0.tar.gz" + "url": "https://git.lix.systems/lix-project/nixos-module/archive/2.91.1-1.tar.gz" } }, "lix_2": { @@ -795,29 +795,29 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1723503926, - "narHash": "sha256-Rosl9iA9MybF5Bud4BTAQ9adbY81aGmPfV8dDBGl34s=", - "rev": "bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2", + "lastModified": 1729298361, + "narHash": "sha256-hiGtfzxFkDc9TSYsb96Whg0vnqBVV7CUxyscZNhed0U=", + "rev": "ad9d06f7838a25beec425ff406fe68721fef73be", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2.tar.gz?rev=bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ad9d06f7838a25beec425ff406fe68721fef73be.tar.gz" }, "original": { "type": "tarball", - "url": "https://git.lix.systems/lix-project/lix/archive/2.91.0.tar.gz" + "url": "https://git.lix.systems/lix-project/lix/archive/2.91.1.tar.gz" } }, "lix_3": { "flake": false, "locked": { - "lastModified": 1723503926, - "narHash": "sha256-Rosl9iA9MybF5Bud4BTAQ9adbY81aGmPfV8dDBGl34s=", - "rev": "bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2", + "lastModified": 1729298361, + "narHash": "sha256-hiGtfzxFkDc9TSYsb96Whg0vnqBVV7CUxyscZNhed0U=", + "rev": "ad9d06f7838a25beec425ff406fe68721fef73be", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2.tar.gz?rev=bcaeb6388b8916ac6d1736e3aa2b13313e6a6bd2" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/ad9d06f7838a25beec425ff406fe68721fef73be.tar.gz?rev=ad9d06f7838a25beec425ff406fe68721fef73be" }, "original": { "type": "tarball", - "url": "https://git.lix.systems/lix-project/lix/archive/2.91.0.tar.gz" + "url": "https://git.lix.systems/lix-project/lix/archive/2.91.1.tar.gz" } }, "neoidiosyn": { @@ -1151,6 +1151,29 @@ "type": "github" } }, + "oth": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1732215904, + "narHash": "sha256-bXXoyQcVMULYOj1KlzbquUyMTyByzmLatIgm0ra/7sk=", + "ref": "refs/heads/main", + "rev": "847cc8493f7f44bd5ada0283d6b96457f4ee5a9b", + "revCount": 1, + "type": "git", + "url": "ssh://forgejo@woof.rip/emily/oth.git" + }, + "original": { + "type": "git", + "url": "ssh://forgejo@woof.rip/emily/oth.git" + } + }, "pre-commit-hooks": { "flake": false, "locked": { @@ -1187,6 +1210,7 @@ "nixos-needsreboot": "nixos-needsreboot", "nixpkgs": "nixpkgs_4", "nixvim": "nixvim", + "oth": "oth", "sops-nix": "sops-nix", "stylix": "stylix" } diff --git a/flake.nix b/flake.nix index 583df84..4cc9544 100644 --- a/flake.nix +++ b/flake.nix @@ -50,9 +50,9 @@ inputs.nixpkgs.follows = "nixpkgs"; inputs.flake-utils.follows = "flake-utils"; }; - lix.url = "https://git.lix.systems/lix-project/lix/archive/2.91.0.tar.gz"; + lix.url = "https://git.lix.systems/lix-project/lix/archive/2.91.1.tar.gz"; lix-module = { - url = "https://git.lix.systems/lix-project/nixos-module/archive/2.91.0.tar.gz"; + url = "https://git.lix.systems/lix-project/nixos-module/archive/2.91.1-1.tar.gz"; inputs.nixpkgs.follows = "nixpkgs"; }; nixos-needsreboot = { @@ -64,6 +64,11 @@ inputs.nixpkgs.follows = "nixpkgs"; inputs.home-manager.follows = "home-manager"; }; + oth = { + url = "git+ssh://forgejo@woof.rip/emily/oth.git"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; sops-nix = { url = "github:Mic92/sops-nix"; inputs.nixpkgs.follows = "nixpkgs"; diff --git a/fly.toml b/pkgs/build-worker-oci/ams/fly.toml similarity index 86% rename from fly.toml rename to pkgs/build-worker-oci/ams/fly.toml index b796763..635a20f 100644 --- a/fly.toml +++ b/pkgs/build-worker-oci/ams/fly.toml @@ -5,6 +5,7 @@ primary_region = 'ams' image = 'registry.fly.io/build-worker-kyoumanet:latest' [processes] + bw-00 = '/entrypoint.sh' bw-01 = '/entrypoint.sh' bw-02 = '/entrypoint.sh' bw-03 = '/entrypoint.sh' @@ -14,28 +15,34 @@ primary_region = 'ams' bw-07 = '/entrypoint.sh' bw-08 = '/entrypoint.sh' +[[mounts]] + source = 'bw00' + destination = '/mnt/data' + initial_size = '256GB' + processes = ['bw-00'] + [[mounts]] source = 'bw01' destination = '/mnt/data' - initial_size = '128GB' + initial_size = '256GB' processes = ['bw-01'] [[mounts]] source = 'bw02' destination = '/mnt/data' - initial_size = '128GB' + initial_size = '256GB' processes = ['bw-02'] [[mounts]] source = 'bw03' destination = '/mnt/data' - initial_size = '128GB' + initial_size = '256GB' processes = ['bw-03'] [[mounts]] source = 'bw04' destination = '/mnt/data' - initial_size = '128GB' + initial_size = '256GB' processes = ['bw-04'] [[mounts]] @@ -62,6 +69,15 @@ primary_region = 'ams' initial_size = '256GB' processes = ['bw-08'] +[[services]] + protocol = 'tcp' + internal_port = 2222 + auto_stop_machines = 'off' + processes = ['bw-00'] + + [[services.ports]] + port = 2200 + [[services]] protocol = 'tcp' internal_port = 2222 diff --git a/pkgs/build-worker-oci/cdg/fly.toml b/pkgs/build-worker-oci/cdg/fly.toml new file mode 100644 index 0000000..47a9062 --- /dev/null +++ b/pkgs/build-worker-oci/cdg/fly.toml @@ -0,0 +1,67 @@ +# fly.toml app configuration file generated for build-worker-kyoumanet-cdg on 2024-11-21T00:31:54+01:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'build-worker-kyoumanet-cdg' +primary_region = 'cdg' + +[build] + image = 'registry.fly.io/build-worker-kyoumanet:latest' + +[processes] + bw-09 = '/entrypoint.sh' + bw-10 = '/entrypoint.sh' + bw-11 = '/entrypoint.sh' + +[[mounts]] + source = 'bw09' + destination = '/mnt/data' + initial_size = '256GB' + processes = ['bw-09'] + +[[mounts]] + source = 'bw10' + destination = '/mnt/data' + initial_size = '256GB' + processes = ['bw-10'] + +[[mounts]] + source = 'bw11' + destination = '/mnt/data' + initial_size = '256GB' + processes = ['bw-11'] + +[[services]] + protocol = 'tcp' + internal_port = 2222 + auto_stop_machines = 'off' + processes = ['bw-09'] + + [[services.ports]] + port = 2209 + +[[services]] + protocol = 'tcp' + internal_port = 2222 + auto_stop_machines = 'off' + processes = ['bw-10'] + + [[services.ports]] + port = 2210 + +[[services]] + protocol = 'tcp' + internal_port = 2222 + auto_stop_machines = 'off' + processes = ['bw-11'] + + [[services.ports]] + port = 2211 + +[[restart]] + policy = 'never' + +[[vm]] + size = 'performance-16x' + memory = '96GB' diff --git a/pkgs/update-nixfiles/update-nixfiles.sh b/pkgs/update-nixfiles/update-nixfiles.sh index 02e6fbb..b1cb7ec 100644 --- a/pkgs/update-nixfiles/update-nixfiles.sh +++ b/pkgs/update-nixfiles/update-nixfiles.sh @@ -23,7 +23,9 @@ merge_theirs () { test_build () { local build_jobs - build_jobs="$(curl --fail -s -L -H "Accept: application/json" "${JOBSET_URL}/latest-eval" | jq -r ".builds | .[]")" + sleep 30 + + build_jobs="$(curl --fail -s -L -H "Accept: application/json" "${JOBSET_URL}/evals" | jq -r ".evals | max_by(.id) | .builds | .[]")" for build in ${build_jobs}; do local build_status