diff --git a/home/gabriel/features/ai/pi/default.nix b/home/gabriel/features/ai/pi/default.nix index 10fa5982..c49685bc 100644 --- a/home/gabriel/features/ai/pi/default.nix +++ b/home/gabriel/features/ai/pi/default.nix @@ -42,8 +42,6 @@ in { "claude-bridge/claude-opus-5" "claude-bridge/claude-sonnet-5" "claude-bridge/claude-haiku-4-5" - "llama.cpp/qwen3.6-35b-a3b" - "llama.cpp/gemma-4-26b-a4b" ]; skills = [./skills]; @@ -64,7 +62,6 @@ in { }; }; home.sessionVariables = { - LLAMA_BASE_URL = "http://llm.m7.rs"; PI_SKIP_VERSION_CHECK = true; PI_TELEMETRY = false; }; diff --git a/hosts/nixos/atlas/default.nix b/hosts/nixos/atlas/default.nix index 0c33ab1f..8b51d23a 100644 --- a/hosts/nixos/atlas/default.nix +++ b/hosts/nixos/atlas/default.nix @@ -9,7 +9,6 @@ inputs.hardware.nixosModules.common-pc-ssd ./hardware-configuration.nix - ./llm-server.nix ../common/global ../common/users/gabriel diff --git a/hosts/nixos/atlas/llm-server.nix b/hosts/nixos/atlas/llm-server.nix deleted file mode 100644 index 9d30a672..00000000 --- a/hosts/nixos/atlas/llm-server.nix +++ /dev/null @@ -1,57 +0,0 @@ -{outputs, ...}: let - port = 18080; - models = { - # flash-attn + q8_0 KV cache halves the KV footprint to stay under 8GB VRAM. - # Qwen3.6 35B-A3B (MoE, vision-capable - served text-only here). Q5_K_M - # unsloth UD quant; ~16GB+ stays in RAM. ~16/48 layers on GPU as a start. - # 64k of the model's native 256k ctx; full attention so KV grows linearly. - # RAM holds the CPU-layer KV fine; drop ngl if the GPU-side KV won't fit. - "qwen3.6-35b-a3b" = { - hf = "unsloth/Qwen3.6-35B-A3B-GGUF:Q5_K_M"; - ctx-size = 65536; - n-gpu-layers = 15; - flash-attn = "on"; - cache-type-k = "q8_0"; - cache-type-v = "q8_0"; - threads = 8; - parallel = 1; - cont-batching = 1; - sleep-idle-seconds = 300; - }; - # Gemma-4-26B-A4B (MoE, 4B active, vision-capable - text-only here). - # QAT q4_0: trained for q4 robustness, ~bf16 quality at only 13.5GB. Good - # for chatty/creative work. Official Google GGUF, ungated. Light enough to - # push ngl high - nudge up while watching VRAM. Sliding-window attention - # bounds the KV cache, so its full native 256k ctx is cheap. - gemma-4-26b-a4b = { - hf = "google/gemma-4-26B-A4B-it-qat-q4_0-gguf:Q4_0"; - ctx-size = 262144; - n-gpu-layers = 20; - flash-attn = "on"; - cache-type-k = "q8_0"; - cache-type-v = "q8_0"; - threads = 8; - parallel = 1; - cont-batching = 1; - sleep-idle-seconds = 300; - }; - }; -in { - services.llama-router = { - enable = true; - inherit port models; - }; - - services.nginx.virtualHosts."llm.m7.rs" = { - locations."/" = { - proxyPass = "http://localhost:${toString port}"; - extraConfig = '' - allow 127.0.0.1; - allow ::1; - allow ${outputs.nixosConfigurations.alcyone.config.services.headscale.settings.prefixes.v4}; - allow ${outputs.nixosConfigurations.alcyone.config.services.headscale.settings.prefixes.v6}; - deny all; - ''; - }; - }; -} diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index 7f58b84e..cfb322af 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -4,5 +4,4 @@ opencode = import ./opencode.nix; openrgb = import ./openrgb.nix; nix-registry-prometheus-exporter = import ./nix-registry-prometheus-exporter.nix; - llama-router = import ./llama-router.nix; } diff --git a/modules/nixos/llama-router.nix b/modules/nixos/llama-router.nix deleted file mode 100644 index bf5dbe48..00000000 --- a/modules/nixos/llama-router.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ - pkgs, - lib, - config, - ... -}: let - cfg = config.services.llama-router; -in { - options.services.llama-router = { - enable = lib.mkEnableOption "the local llama.cpp router API"; - - package = lib.mkPackageOption pkgs "llama-cpp-vulkan" {}; - - models = lib.mkOption { - type = (pkgs.formats.ini {}).type; - default = {}; - description = '' - Model presets to serve, as an attrset mapping each model name to its - llama-server settings (e.g. `hf`, `ctx-size`, `n-gpu-layers`). - ''; - }; - - host = lib.mkOption { - type = lib.types.str; - default = "127.0.0.1"; - description = "Address llama-server binds to."; - }; - - port = lib.mkOption { - type = lib.types.port; - default = 18080; - description = "Port llama-server listens on."; - }; - - user = lib.mkOption { - type = lib.types.str; - default = "llama"; - description = "User to run the router service as."; - }; - - group = lib.mkOption { - type = lib.types.str; - default = "llama"; - description = "Group to run the router service as."; - }; - }; - - config = lib.mkIf cfg.enable { - users.users = lib.mkIf (cfg.user == "llama") { - llama = { - isSystemUser = true; - group = cfg.group; - home = "/var/lib/llama"; - }; - }; - users.groups = lib.mkIf (cfg.group == "llama") { - llama = {}; - }; - - systemd.services.llama-cpp-router = { - description = "Local llama.cpp router API"; - after = ["network-online.target"]; - wants = ["network-online.target"]; - wantedBy = ["multi-user.target"]; - environment = { - XDG_CACHE_HOME = "/var/lib/llama"; - XDG_DATA_HOME = "/var/lib/llama"; - }; - path = [cfg.package]; - script = lib.concatStringsSep " " [ - "llama-server" - # No --models-dir: all models come from --models-preset below. - "--models-preset ${(pkgs.formats.ini {}).generate "llama-cpp-models.ini" cfg.models}" - "--models-max 1" - "--host ${cfg.host}" - "--port ${toString cfg.port}" - ]; - serviceConfig = { - User = cfg.user; - Group = cfg.group; - StateDirectory = "llama"; - RuntimeDirectory = "llama"; - SupplementaryGroups = ["render" "video"]; - Restart = "on-failure"; - RestartSec = 2; - # llama-server can dawdle on shutdown; SIGKILL it after 15s. - TimeoutStopSec = 15; - }; - }; - }; -} diff --git a/overlays/default.nix b/overlays/default.nix index 6b4b94d9..5bab71fe 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -113,12 +113,6 @@ in { }) ]; - # Make the llama.cpp router's HF-cache scan opt-in (LLAMA_ROUTER_SCAN_CACHE) - # so --models-preset is the single source of truth and models can be named - # freely without untuned repo:tag twins. Patch the base so the -vulkan and - # -rocm variants (llama-cpp.override) inherit it. - llama-cpp = addPatches prev.llama-cpp [./llama-cpp-optional-cache-scan.patch]; - # https://gitlab.freedesktop.org/mstoeckl/waypipe/-/releases#v0.11.1 # Raise when it's time to remove waypipe = assert final.lib.versionOlder prev.waypipe.version "0.11.1";