From 0eb78dcf9b34d658509b3c99fba0ed6219cc1c10 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:16:47 +0200 Subject: [PATCH] feat(dev): pass the host NVIDIA GPU through to the KVM dev VM Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../InstallDevelopmentGpuDriver.php | 47 ++++++++ .../Development/ManageDevelopmentQemuVm.php | 33 ++++-- .../Development/StartDevelopmentQemuVm.php | 100 ++++++++++++++-- .../ManageDevelopmentQemuVmCommand.php | 14 ++- .../SeedDevelopmentQemuServerCommand.php | 9 +- config/development-qemu.php | 2 + scripts/dev | 5 +- tests/Feature/DevScriptTest.php | 20 ++++ tests/Feature/DevelopmentQemuVmTest.php | 112 ++++++++++++++++++ 10 files changed, 318 insertions(+), 26 deletions(-) create mode 100644 app/Actions/Development/InstallDevelopmentGpuDriver.php diff --git a/AGENTS.md b/AGENTS.md index 661a920c56..bfa80f40cf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ Docker Compose-based dev setup with services: coolify (app, which also runs Reve docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d ``` -The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-` (`10.221..0/24`) and VMs `coolify-dev---`; VMs are reused, use `php artisan dev:qemu --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev--` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f `. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=` to run another instance from the same checkout. +The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-` (`10.221..0/24`) and VMs `coolify-dev---`; VMs are reused, use `php artisan dev:qemu --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev--` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f `. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. To pass the host NVIDIA GPU through to the KVM localhost VM (`ubuntu-root` only), start with `COOLIFY_DEV_GPU=1 ./scripts/dev start` (or `php artisan dev:qemu ubuntu-root --as-localhost --gpu`). This attaches every PCI function of the GPU, gives the VM 4 GiB, stops `nvidia-persistenced` on the host, and installs the NVIDIA driver and container toolkit in the VM. The host cannot use the GPU while the VM runs. Remove it with `--fresh` without `--gpu`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=` to run another instance from the same checkout. ## Testing the Self-Hosted Upgrade Process diff --git a/app/Actions/Development/InstallDevelopmentGpuDriver.php b/app/Actions/Development/InstallDevelopmentGpuDriver.php new file mode 100644 index 0000000000..ffe62f497c --- /dev/null +++ b/app/Actions/Development/InstallDevelopmentGpuDriver.php @@ -0,0 +1,47 @@ +/dev/null 2>&1; then', + 'apt-get update -qq', + 'apt-get install -y -qq ubuntu-drivers-common', + 'ubuntu-drivers install --gpgpu', + // For example 580-server-open. The utilities package has no -open suffix. + 'flavor=$(dpkg-query -W -f=\'${Package}\n\' \'nvidia-headless-no-dkms-*\' | sed -n \'s/^nvidia-headless-no-dkms-//p\' | head -n1)', + // ubuntu-drivers installs the modules for the newest kernel, which can differ from the running one. + 'apt-get install -y -qq "nvidia-utils-${flavor%-open}" "linux-modules-nvidia-${flavor}-$(uname -r)"', + 'modprobe nvidia', + 'nvidia-smi', + 'fi', + 'if ! command -v nvidia-ctk >/dev/null 2>&1; then', + 'curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --batch --yes --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg', + 'curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed \'s#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g\' > /etc/apt/sources.list.d/nvidia-container-toolkit.list', + 'apt-get update -qq', + 'apt-get install -y -qq nvidia-container-toolkit', + 'nvidia-ctk runtime configure --runtime=docker', + 'systemctl restart docker', + 'fi', + ], $server, timeout: 1800, disableMultiplexing: true, maxAttempts: 1); + + $server->gatherServerMetadata(); + } +} diff --git a/app/Actions/Development/ManageDevelopmentQemuVm.php b/app/Actions/Development/ManageDevelopmentQemuVm.php index 9b1a1c791f..9fab0050a4 100644 --- a/app/Actions/Development/ManageDevelopmentQemuVm.php +++ b/app/Actions/Development/ManageDevelopmentQemuVm.php @@ -2,6 +2,7 @@ namespace App\Actions\Development; +use Illuminate\Contracts\Process\ProcessResult; use Illuminate\Database\QueryException; use Illuminate\Support\Facades\Process; use Lorisleiva\Actions\Concerns\AsAction; @@ -12,28 +13,44 @@ class ManageDevelopmentQemuVm /** * Start (or create) the selected VMs and seed their servers. Existing VMs are reused unless $fresh is set. + * With $gpu, the host GPU is passed through and the container seeds the server, because it installs the + * driver over Coolify's SSH connection. * * @param string|array $profileNames */ - public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false): void + public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false, bool $gpu = false): void { $profileNames = is_array($profileNames) ? array_values(array_unique($profileNames)) : [$profileNames]; foreach ($profileNames as $index => $profileName) { - StartDevelopmentQemuVm::run($profileName, $fresh); + StartDevelopmentQemuVm::run($profileName, $fresh, $gpu); + + if ($gpu) { + $this->seedInContainer($profileName, $index > 0, $asLocalhost, true)->throw(); + + continue; + } try { SeedDevelopmentQemuServer::run($profileName, $index === 0, $asLocalhost); } catch (QueryException $exception) { - $keepOthers = $index === 0 ? '' : ' --keep-others'; - $localhostOption = $asLocalhost ? ' --as-localhost' : ''; - $container = escapeshellarg(config('development-qemu.coolify_container')); - $result = Process::run("docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName).$keepOthers.$localhostOption); - - if ($result->failed()) { + if ($this->seedInContainer($profileName, $index > 0, $asLocalhost, false)->failed()) { throw $exception; } } } } + + private function seedInContainer(string $profileName, bool $keepOthers, bool $asLocalhost, bool $gpu): ProcessResult + { + $container = escapeshellarg(config('development-qemu.coolify_container')); + $command = "docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName) + .($keepOthers ? ' --keep-others' : '') + .($asLocalhost ? ' --as-localhost' : '') + .($gpu ? ' --gpu' : ''); + + return $gpu + ? Process::forever()->run($command, fn (string $type, string $output) => fwrite(STDOUT, $output)) + : Process::run($command); + } } diff --git a/app/Actions/Development/StartDevelopmentQemuVm.php b/app/Actions/Development/StartDevelopmentQemuVm.php index 5815185508..8e52a6dcc4 100644 --- a/app/Actions/Development/StartDevelopmentQemuVm.php +++ b/app/Actions/Development/StartDevelopmentQemuVm.php @@ -15,8 +15,9 @@ class StartDevelopmentQemuVm /** * Start the VM of a profile, creating it only when its libvirt domain does not exist yet. * With $fresh, only this profile's VM is destroyed, undefined, and deleted before it is created again. + * With $gpu, the NVIDIA GPU of the host is passed through to the VM. An existing VM is stopped to add it. */ - public function handle(string $profileName, bool $fresh = false): void + public function handle(string $profileName, bool $fresh = false, bool $gpu = false): void { $this->ensureDevelopmentEnvironment(); $profile = config("development-qemu.profiles.{$profileName}"); @@ -33,9 +34,12 @@ class StartDevelopmentQemuVm } if (! $fresh && $this->domainExists($profile['domain'])) { + if ($gpu) { + $this->attachGpu($profile['domain']); + } $this->startExistingVm($profile['domain']); } else { - $this->createPreparedVm($profile); + $this->createPreparedVm($profile, $gpu); } ConfigureDevelopmentQemuHost::run(); @@ -43,13 +47,13 @@ class StartDevelopmentQemuVm } /** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */ - private function createPreparedVm(array $profile): void + private function createPreparedVm(array $profile, bool $gpu): void { $preparedImage = $this->preparedImage($profile['template']); if (! File::exists($preparedImage)) { $this->createVm($profile, false); - $this->waitForPreparation($profile['domain']); + $this->waitForShutOff($profile['domain'], 900); $this->runOrFail('virsh undefine '.escapeshellarg($profile['domain'])); $this->runOrFail('mv '.escapeshellarg($this->vmDisk($profile['domain'])).' '.escapeshellarg($preparedImage)); if (File::exists($preparedImage)) { @@ -57,7 +61,7 @@ class StartDevelopmentQemuVm } } - $this->createVm($profile, true); + $this->createVm($profile, true, $gpu); } private function domainExists(string $domain): bool @@ -79,20 +83,84 @@ class StartDevelopmentQemuVm return; } - $this->applyMemory($domain); + $hasGpu = $this->hasGpu($domain); + $this->applyMemory($domain, $hasGpu); + + if ($hasGpu) { + $this->releaseHostGpu(); + } + $this->runOrFail('virsh start '.escapeshellarg($domain)); } /** * Sets the configured memory on a stopped VM, so VMs created with a larger size shrink on their next start. */ - private function applyMemory(string $domain): void + private function applyMemory(string $domain, bool $hasGpu): void { - $memory = (int) config('development-qemu.memory').'M'; + $memory = (int) config($hasGpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory').'M'; $this->runOrFail(sprintf('virsh setmaxmem %s %s --config', escapeshellarg($domain), $memory)); $this->runOrFail(sprintf('virsh setmem %s %s --config', escapeshellarg($domain), $memory)); } + private function hasGpu(string $domain): bool + { + return str_contains(Process::run('virsh dumpxml --inactive '.escapeshellarg($domain))->output(), 'hasGpu($domain)) { + return; + } + + if (trim(Process::run('virsh domstate '.escapeshellarg($domain))->output()) !== 'shut off') { + $this->runOrFail('virsh shutdown '.escapeshellarg($domain)); + $this->waitForShutOff($domain, 120); + } + + foreach ($this->hostGpuDevices() as $device) { + $this->runOrFail(sprintf('virt-xml %s --add-device --hostdev %s', escapeshellarg($domain), escapeshellarg($device))); + } + } + + /** + * Returns the libvirt node device names of every PCI function of the first NVIDIA display controller, + * for example pci_0000_01_00_0 (GPU) and pci_0000_01_00_1 (its audio device). All functions must move together. + * + * @return array + */ + private function hostGpuDevices(): array + { + $devices = collect(explode("\n", trim(Process::run('lspci -Dnmm -d 10de:')->output()))) + ->filter() + ->map(fn (string $line): array => str_getcsv($line, ' ')); + $gpu = $devices->first(fn (array $device): bool => in_array($device[1] ?? null, ['0300', '0302'], true)); + + if ($gpu === null) { + throw new RuntimeException('No NVIDIA GPU found on this host.'); + } + + $slot = substr($gpu[0], 0, strrpos($gpu[0], '.')); + + return $devices + ->filter(fn (array $device): bool => str_starts_with($device[0], "{$slot}.")) + ->map(fn (array $device): string => 'pci_'.str_replace([':', '.'], '_', $device[0])) + ->values() + ->all(); + } + + /** + * nvidia-persistenced keeps the GPU open, which blocks libvirt from moving it to vfio-pci. + */ + private function releaseHostGpu(): void + { + Process::run('systemctl stop nvidia-persistenced'); + } + private function destroyVm(string $domain): void { Process::run('virsh destroy '.escapeshellarg($domain)); @@ -101,7 +169,7 @@ class StartDevelopmentQemuVm } /** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */ - private function createVm(array $profile, bool $prepared): void + private function createVm(array $profile, bool $prepared, bool $gpu = false): void { $directory = config('development-qemu.storage_path'); File::ensureDirectoryExists($directory); @@ -155,11 +223,17 @@ class StartDevelopmentQemuVm $seedDisk = $prepared ? '' : ' --disk path='.escapeshellarg($seedImage).',format=raw,bus=virtio,readonly=on'; // Instance networks are isolated, so VMs of different instances may share the profile MAC. $macCheck = config('development-qemu.instance') === null ? '' : ' --check mac_in_use=off'; + $hostDevices = ''; + + if ($gpu) { + $hostDevices = collect($this->hostGpuDevices())->map(fn (string $device): string => ' --hostdev '.escapeshellarg($device))->implode(''); + $this->releaseHostGpu(); + } $this->runOrFail(sprintf( - 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck, + 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck.$hostDevices, escapeshellarg($profile['domain']), - config('development-qemu.memory'), + config($gpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory'), config('development-qemu.vcpus'), escapeshellarg($profile['os_variant']), escapeshellarg($disk), @@ -182,10 +256,10 @@ class StartDevelopmentQemuVm return config('development-qemu.storage_path')."/{$template}-prepared.qcow2"; } - private function waitForPreparation(string $domain): void + private function waitForShutOff(string $domain, int $timeoutSeconds): void { $check = 'while [ "$(virsh domstate '.escapeshellarg($domain).')" != "shut off" ]; do sleep 2; done'; - $this->runOrFail('timeout 900 bash -c '.escapeshellarg($check)); + $this->runOrFail("timeout {$timeoutSeconds} bash -c ".escapeshellarg($check)); } private function moveLegacyFiles(string $directory): void diff --git a/app/Console/Commands/ManageDevelopmentQemuVmCommand.php b/app/Console/Commands/ManageDevelopmentQemuVmCommand.php index 1dc235d0d0..bf27d1747e 100644 --- a/app/Console/Commands/ManageDevelopmentQemuVmCommand.php +++ b/app/Console/Commands/ManageDevelopmentQemuVmCommand.php @@ -9,7 +9,7 @@ use function Laravel\Prompts\multiselect; class ManageDevelopmentQemuVmCommand extends Command { - protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them}'; + protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them} {--gpu : Pass the NVIDIA GPU of the host through to the VM and install its driver}'; protected $description = 'Start (or create) selected development QEMU VMs and seed their Coolify servers'; @@ -34,7 +34,17 @@ class ManageDevelopmentQemuVmCommand extends Command return self::FAILURE; } - ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh')); + if ($this->option('gpu')) { + $profile = $profiles[$profileNames[0]] ?? null; + + if (count($profileNames) !== 1 || $profile === null || $profile['user'] !== 'root' || ! str_starts_with($profile['os_variant'], 'ubuntu')) { + $this->error('GPU passthrough needs exactly one Ubuntu root profile (ubuntu-root).'); + + return self::FAILURE; + } + } + + ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'), (bool) $this->option('gpu')); foreach ($profileNames as $profileName) { $profile = $profiles[$profileName]; diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php index 26fc463ff7..aee9925163 100644 --- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php +++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php @@ -2,6 +2,7 @@ namespace App\Console\Commands; +use App\Actions\Development\InstallDevelopmentGpuDriver; use App\Actions\Development\SeedDevelopmentQemuServer; use App\Actions\Development\StartDevelopmentInstanceDatabase; use Illuminate\Console\Command; @@ -9,7 +10,7 @@ use Throwable; class SeedDevelopmentQemuServerCommand extends Command { - protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port}'; + protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port} {--gpu : Install the NVIDIA driver and container toolkit in the VM}'; protected $description = 'Seed one development QEMU server in the Coolify database'; @@ -33,6 +34,12 @@ class SeedDevelopmentQemuServerCommand extends Command } } + if ($this->option('gpu')) { + $this->info('Installing the NVIDIA driver and container toolkit. This can take some minutes.'); + InstallDevelopmentGpuDriver::run($server); + $this->info('The GPU is ready.'); + } + return self::SUCCESS; } } diff --git a/config/development-qemu.php b/config/development-qemu.php index 983cd587aa..c9b30d049a 100644 --- a/config/development-qemu.php +++ b/config/development-qemu.php @@ -147,6 +147,8 @@ return [ 'dns' => '1.1.1.1', // MiB per VM. Many dev instances share one host, so keep this small; existing VMs get it on their next start. 'memory' => (int) env('DEVELOPMENT_QEMU_MEMORY', 1024), + // MiB for a VM with a passed-through GPU (dev:qemu --gpu). The NVIDIA driver needs more than the default. + 'gpu_memory' => (int) env('DEVELOPMENT_QEMU_GPU_MEMORY', 4096), 'vcpus' => 2, 'disk_size' => '20G', 'libvirt_network' => $developmentQemuInstance === null ? 'default' : "coolify-dev-{$developmentQemuSlot}", diff --git a/scripts/dev b/scripts/dev index 3102f0b9d7..952190b699 100755 --- a/scripts/dev +++ b/scripts/dev @@ -23,6 +23,7 @@ # redis +4, vite +5, mailpit +6/+7, minio +8/+9. # # Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE= +# COOLIFY_DEV_GPU=1 passes the host NVIDIA GPU through to the KVM VM (ubuntu-root only) # # Side by side: COOLIFY_DEV_INSTANCE= COOLIFY_DEV_SIDE_BY_SIDE=1 starts a second instance of # this checkout without stopping the other ones (for example a transfer target, see @@ -402,6 +403,8 @@ start_backend() { local host="${DEV_BIND_ADDRESS:-127.0.0.1}" [[ "$host" == 0.0.0.0 ]] && host=127.0.0.1 echo "Using KVM profile: $profile" + local gpu=() + [[ "${COOLIFY_DEV_GPU:-}" == 1 ]] && gpu=(--gpu) # Use this instance database and encryption key, without a stale environment or config cache. # Send logs to stderr to avoid root-owned files in storage/logs. APP_ENV=local APP_CONFIG_CACHE="$ROOT/bootstrap/cache/dev-qemu-$$.php" \ @@ -412,7 +415,7 @@ start_backend() { DEVELOPMENT_QEMU_SLOT="$(env_value "$instance" DEVELOPMENT_QEMU_SLOT)" \ DEVELOPMENT_QEMU_DOCKER_NETWORK="$(project_name "$instance")" \ DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$(project_name "$instance")" \ - "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost + "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost "${gpu[@]}" elif [[ "$backend" == "lima" ]]; then echo "Using Lima profile: $profile" start_lima_vm diff --git a/tests/Feature/DevScriptTest.php b/tests/Feature/DevScriptTest.php index 232bde15de..79c5821c6d 100644 --- a/tests/Feature/DevScriptTest.php +++ b/tests/Feature/DevScriptTest.php @@ -269,6 +269,26 @@ BASH); ->and(devScriptLog())->toContain('php artisan dev:qemu ubuntu-root --as-localhost ENV=local CACHE='); }); +it('passes the gpu option to the host qemu command when COOLIFY_DEV_GPU is 1', function () { + if (! function_exists('posix_geteuid') || posix_geteuid() !== 0 || ! file_exists('/dev/kvm') || filetype('/dev/kvm') !== 'char' || ! is_readable('/dev/kvm') || ! is_writable('/dev/kvm')) { + $this->markTestSkipped('KVM and root access are required for this launcher branch.'); + } + + file_put_contents($this->devRoot.'/bin/php', <<<'BASH' +#!/usr/bin/env bash +printf 'php %s|\n' "$*" >> "$DEV_TEST_LOG" +BASH); + + $withGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '1']); + $withoutGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '0']); + + expect($withGpu->isSuccessful())->toBeTrue($withGpu->getErrorOutput()) + ->and($withoutGpu->isSuccessful())->toBeTrue($withoutGpu->getErrorOutput()) + ->and(devScriptLog()) + ->toContain('php artisan dev:qemu ubuntu-root --as-localhost --gpu|') + ->toContain('php artisan dev:qemu ubuntu-root --as-localhost|'); +}); + it('starts a lima vm as localhost on macos and seeds its forwarded ssh port', function () { ($this->fakeLima)(); runDevScript('main', ['start']); diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.php index 60c273a36a..3512a20fd2 100644 --- a/tests/Feature/DevelopmentQemuVmTest.php +++ b/tests/Feature/DevelopmentQemuVmTest.php @@ -690,3 +690,115 @@ function useDevelopmentQemuInstance(string $instance, int $slot, string $storage 'storage_path' => $storagePath, ]]); } + +function fakeQemuHostWithGpu(string $domainState = 'running', string $domainXml = ''): void +{ + Process::fake([ + '* net-dumpxml *' => Process::result(output: ''), + '* network inspect *' => Process::result(output: "172.18.0.0/16\n"), + '* dominfo *' => Process::result(output: 'exists'), + '* domstate *' => Process::result(output: "{$domainState}\n"), + '* dumpxml *' => Process::result(output: $domainXml), + 'lspci *' => Process::result(output: implode("\n", [ + '0000:01:00.0 "0300" "10de" "2584" -ra1 -p00 "1043" "8976"', + '0000:01:00.1 "0403" "10de" "2291" -ra1 -p00 "1043" "8976"', + ])."\n"), + '*' => Process::result(), + ]); +} + +it('creates a vm with every function of the host nvidia gpu and more memory', function () { + $storagePath = sys_get_temp_dir().'/coolify-qemu-gpu-create-test-'.uniqid(); + config(['development-qemu.storage_path' => $storagePath, 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]); + File::ensureDirectoryExists($storagePath); + File::put("{$storagePath}/coolify-dev-ubuntu-root-prepared.qcow2", 'prepared image'); + fakeQemuHostWithGpu(); + + StartDevelopmentQemuVm::run('ubuntu-root', true, true); + + Process::assertRan(fn ($process) => $process->command === 'systemctl stop nvidia-persistenced'); + Process::assertRan(fn ($process) => str_starts_with($process->command, 'virt-install ') + && str_contains($process->command, '--memory 4096 ') + && str_contains($process->command, "--hostdev 'pci_0000_01_00_0' --hostdev 'pci_0000_01_00_1'")); +}); + +it('stops an existing vm to attach the host gpu and starts it with the gpu memory', function () { + config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-attach-test-'.uniqid(), 'development-qemu.gpu_memory' => 4096]); + $hostdevAttached = false; + $ran = []; + Process::fake(function ($process) use (&$hostdevAttached, &$ran) { + $ran[] = $process->command; + $hostdevAttached = $hostdevAttached || str_starts_with($process->command, 'virt-xml '); + + return match (true) { + str_contains($process->command, ' net-dumpxml ') => Process::result(output: ''), + str_contains($process->command, ' network inspect ') => Process::result(output: "172.18.0.0/16\n"), + str_contains($process->command, ' dominfo ') => Process::result(output: 'exists'), + str_contains($process->command, ' domstate ') => Process::result(output: $hostdevAttached ? "shut off\n" : "running\n"), + str_contains($process->command, ' dumpxml ') => Process::result(output: $hostdevAttached ? "" : ''), + str_starts_with($process->command, 'lspci ') => Process::result(output: "0000:01:00.0 \"0300\" \"10de\" \"2584\"\n0000:01:00.1 \"0403\" \"10de\" \"2291\"\n"), + default => Process::result(), + }; + }); + + StartDevelopmentQemuVm::run('ubuntu-root', false, true); + + $commands = collect($ran); + $shutdown = $commands->search("virsh shutdown 'coolify-dev-ubuntu-root'"); + $attach = $commands->search("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_0'"); + expect($shutdown)->not->toBeFalse() + ->and($attach)->toBeGreaterThan($shutdown) + ->and($commands)->toContain("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_1'") + ->and($commands->search("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config"))->toBeGreaterThan($attach) + ->and($commands->search("virsh start 'coolify-dev-ubuntu-root'"))->toBeGreaterThan($commands->search('systemctl stop nvidia-persistenced')); + Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-install ')); +}); + +it('keeps the gpu memory of a gpu vm that starts without the gpu option', function () { + config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-restart-test-'.uniqid(), 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]); + fakeQemuHostWithGpu('shut off', ""); + + StartDevelopmentQemuVm::run('ubuntu-root'); + + Process::assertRan("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config"); + Process::assertRan('systemctl stop nvidia-persistenced'); + Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml ')); + Process::assertNotRan(fn ($process) => str_contains($process->command, '1024M')); +}); + +it('fails before it changes the vm when the host has no nvidia gpu', function () { + config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-no-gpu-test-'.uniqid()]); + Process::fake([ + '* net-dumpxml *' => Process::result(output: ''), + '* network inspect *' => Process::result(output: "172.18.0.0/16\n"), + '* dominfo *' => Process::result(output: 'exists'), + '* domstate *' => Process::result(output: "shut off\n"), + '*' => Process::result(), + ]); + + expect(fn () => StartDevelopmentQemuVm::run('ubuntu-root', false, true)) + ->toThrow(RuntimeException::class, 'No NVIDIA GPU found'); + Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml ') || str_contains($process->command, 'virsh start')); +}); + +it('allows gpu passthrough only for one ubuntu root profile', function (array $profiles) { + Process::fake(); + + expect(Artisan::call('dev:qemu', ['profiles' => $profiles, '--gpu' => true]))->toBe(Command::FAILURE); + + Process::assertNothingRan(); +})->with([ + 'non-root' => [['ubuntu-non-root']], + 'other distribution' => [['debian-root']], + 'two profiles' => [['ubuntu-root', 'debian-root']], +]); + +it('seeds a gpu vm through the container so it installs the driver', function () { + config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-seed-test-'.uniqid()]); + SeedDevelopmentQemuServer::mock()->shouldNotReceive('handle'); + fakeQemuHostWithGpu('shut off', ""); + + ManageDevelopmentQemuVm::run('ubuntu-root', true, false, true); + + Process::assertRan("docker exec 'coolify' php artisan dev:qemu:seed 'ubuntu-root' --as-localhost --gpu"); +});