mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
feat(dev): pass the host NVIDIA GPU through to the KVM dev VM
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
a3712f8902
commit
0eb78dcf9b
@@ -29,7 +29,7 @@ Docker Compose-based dev setup with services: coolify (app, which also runs Reve
|
||||
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
|
||||
```
|
||||
|
||||
The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.
|
||||
The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. To pass the host NVIDIA GPU through to the KVM localhost VM (`ubuntu-root` only), start with `COOLIFY_DEV_GPU=1 ./scripts/dev start` (or `php artisan dev:qemu ubuntu-root --as-localhost --gpu`). This attaches every PCI function of the GPU, gives the VM 4 GiB, stops `nvidia-persistenced` on the host, and installs the NVIDIA driver and container toolkit in the VM. The host cannot use the GPU while the VM runs. Remove it with `--fresh` without `--gpu`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.
|
||||
|
||||
## Testing the Self-Hosted Upgrade Process
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
<?php
|
||||
|
||||
namespace App\Actions\Development;
|
||||
|
||||
use App\Models\Server;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
|
||||
/**
|
||||
* Installs the NVIDIA driver and the NVIDIA Container Toolkit on an Ubuntu development VM with a passed-through GPU,
|
||||
* then refreshes the server details so Coolify shows the GPU. Steps that are already done are skipped.
|
||||
*
|
||||
* @see https://ubuntu.com/server/docs/how-to/graphics/install-nvidia-drivers/
|
||||
* @see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
|
||||
*/
|
||||
class InstallDevelopmentGpuDriver
|
||||
{
|
||||
use AsAction;
|
||||
|
||||
public function handle(Server $server): void
|
||||
{
|
||||
instant_remote_process([
|
||||
'set -e',
|
||||
'export DEBIAN_FRONTEND=noninteractive',
|
||||
'if ! nvidia-smi >/dev/null 2>&1; then',
|
||||
'apt-get update -qq',
|
||||
'apt-get install -y -qq ubuntu-drivers-common',
|
||||
'ubuntu-drivers install --gpgpu',
|
||||
// For example 580-server-open. The utilities package has no -open suffix.
|
||||
'flavor=$(dpkg-query -W -f=\'${Package}\n\' \'nvidia-headless-no-dkms-*\' | sed -n \'s/^nvidia-headless-no-dkms-//p\' | head -n1)',
|
||||
// ubuntu-drivers installs the modules for the newest kernel, which can differ from the running one.
|
||||
'apt-get install -y -qq "nvidia-utils-${flavor%-open}" "linux-modules-nvidia-${flavor}-$(uname -r)"',
|
||||
'modprobe nvidia',
|
||||
'nvidia-smi',
|
||||
'fi',
|
||||
'if ! command -v nvidia-ctk >/dev/null 2>&1; then',
|
||||
'curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --batch --yes --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg',
|
||||
'curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed \'s#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g\' > /etc/apt/sources.list.d/nvidia-container-toolkit.list',
|
||||
'apt-get update -qq',
|
||||
'apt-get install -y -qq nvidia-container-toolkit',
|
||||
'nvidia-ctk runtime configure --runtime=docker',
|
||||
'systemctl restart docker',
|
||||
'fi',
|
||||
], $server, timeout: 1800, disableMultiplexing: true, maxAttempts: 1);
|
||||
|
||||
$server->gatherServerMetadata();
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Actions\Development;
|
||||
|
||||
use Illuminate\Contracts\Process\ProcessResult;
|
||||
use Illuminate\Database\QueryException;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
@@ -12,28 +13,44 @@ class ManageDevelopmentQemuVm
|
||||
|
||||
/**
|
||||
* Start (or create) the selected VMs and seed their servers. Existing VMs are reused unless $fresh is set.
|
||||
* With $gpu, the host GPU is passed through and the container seeds the server, because it installs the
|
||||
* driver over Coolify's SSH connection.
|
||||
*
|
||||
* @param string|array<int, string> $profileNames
|
||||
*/
|
||||
public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false): void
|
||||
public function handle(string|array $profileNames, bool $asLocalhost = false, bool $fresh = false, bool $gpu = false): void
|
||||
{
|
||||
$profileNames = is_array($profileNames) ? array_values(array_unique($profileNames)) : [$profileNames];
|
||||
|
||||
foreach ($profileNames as $index => $profileName) {
|
||||
StartDevelopmentQemuVm::run($profileName, $fresh);
|
||||
StartDevelopmentQemuVm::run($profileName, $fresh, $gpu);
|
||||
|
||||
if ($gpu) {
|
||||
$this->seedInContainer($profileName, $index > 0, $asLocalhost, true)->throw();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
SeedDevelopmentQemuServer::run($profileName, $index === 0, $asLocalhost);
|
||||
} catch (QueryException $exception) {
|
||||
$keepOthers = $index === 0 ? '' : ' --keep-others';
|
||||
$localhostOption = $asLocalhost ? ' --as-localhost' : '';
|
||||
$container = escapeshellarg(config('development-qemu.coolify_container'));
|
||||
$result = Process::run("docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName).$keepOthers.$localhostOption);
|
||||
|
||||
if ($result->failed()) {
|
||||
if ($this->seedInContainer($profileName, $index > 0, $asLocalhost, false)->failed()) {
|
||||
throw $exception;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function seedInContainer(string $profileName, bool $keepOthers, bool $asLocalhost, bool $gpu): ProcessResult
|
||||
{
|
||||
$container = escapeshellarg(config('development-qemu.coolify_container'));
|
||||
$command = "docker exec {$container} php artisan dev:qemu:seed ".escapeshellarg($profileName)
|
||||
.($keepOthers ? ' --keep-others' : '')
|
||||
.($asLocalhost ? ' --as-localhost' : '')
|
||||
.($gpu ? ' --gpu' : '');
|
||||
|
||||
return $gpu
|
||||
? Process::forever()->run($command, fn (string $type, string $output) => fwrite(STDOUT, $output))
|
||||
: Process::run($command);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,8 +15,9 @@ class StartDevelopmentQemuVm
|
||||
/**
|
||||
* Start the VM of a profile, creating it only when its libvirt domain does not exist yet.
|
||||
* With $fresh, only this profile's VM is destroyed, undefined, and deleted before it is created again.
|
||||
* With $gpu, the NVIDIA GPU of the host is passed through to the VM. An existing VM is stopped to add it.
|
||||
*/
|
||||
public function handle(string $profileName, bool $fresh = false): void
|
||||
public function handle(string $profileName, bool $fresh = false, bool $gpu = false): void
|
||||
{
|
||||
$this->ensureDevelopmentEnvironment();
|
||||
$profile = config("development-qemu.profiles.{$profileName}");
|
||||
@@ -33,9 +34,12 @@ class StartDevelopmentQemuVm
|
||||
}
|
||||
|
||||
if (! $fresh && $this->domainExists($profile['domain'])) {
|
||||
if ($gpu) {
|
||||
$this->attachGpu($profile['domain']);
|
||||
}
|
||||
$this->startExistingVm($profile['domain']);
|
||||
} else {
|
||||
$this->createPreparedVm($profile);
|
||||
$this->createPreparedVm($profile, $gpu);
|
||||
}
|
||||
|
||||
ConfigureDevelopmentQemuHost::run();
|
||||
@@ -43,13 +47,13 @@ class StartDevelopmentQemuVm
|
||||
}
|
||||
|
||||
/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */
|
||||
private function createPreparedVm(array $profile): void
|
||||
private function createPreparedVm(array $profile, bool $gpu): void
|
||||
{
|
||||
$preparedImage = $this->preparedImage($profile['template']);
|
||||
|
||||
if (! File::exists($preparedImage)) {
|
||||
$this->createVm($profile, false);
|
||||
$this->waitForPreparation($profile['domain']);
|
||||
$this->waitForShutOff($profile['domain'], 900);
|
||||
$this->runOrFail('virsh undefine '.escapeshellarg($profile['domain']));
|
||||
$this->runOrFail('mv '.escapeshellarg($this->vmDisk($profile['domain'])).' '.escapeshellarg($preparedImage));
|
||||
if (File::exists($preparedImage)) {
|
||||
@@ -57,7 +61,7 @@ class StartDevelopmentQemuVm
|
||||
}
|
||||
}
|
||||
|
||||
$this->createVm($profile, true);
|
||||
$this->createVm($profile, true, $gpu);
|
||||
}
|
||||
|
||||
private function domainExists(string $domain): bool
|
||||
@@ -79,20 +83,84 @@ class StartDevelopmentQemuVm
|
||||
return;
|
||||
}
|
||||
|
||||
$this->applyMemory($domain);
|
||||
$hasGpu = $this->hasGpu($domain);
|
||||
$this->applyMemory($domain, $hasGpu);
|
||||
|
||||
if ($hasGpu) {
|
||||
$this->releaseHostGpu();
|
||||
}
|
||||
|
||||
$this->runOrFail('virsh start '.escapeshellarg($domain));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the configured memory on a stopped VM, so VMs created with a larger size shrink on their next start.
|
||||
*/
|
||||
private function applyMemory(string $domain): void
|
||||
private function applyMemory(string $domain, bool $hasGpu): void
|
||||
{
|
||||
$memory = (int) config('development-qemu.memory').'M';
|
||||
$memory = (int) config($hasGpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory').'M';
|
||||
$this->runOrFail(sprintf('virsh setmaxmem %s %s --config', escapeshellarg($domain), $memory));
|
||||
$this->runOrFail(sprintf('virsh setmem %s %s --config', escapeshellarg($domain), $memory));
|
||||
}
|
||||
|
||||
private function hasGpu(string $domain): bool
|
||||
{
|
||||
return str_contains(Process::run('virsh dumpxml --inactive '.escapeshellarg($domain))->output(), '<hostdev ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the GPU to the persistent definition of an existing VM. A running VM is shut down first.
|
||||
*/
|
||||
private function attachGpu(string $domain): void
|
||||
{
|
||||
if ($this->hasGpu($domain)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (trim(Process::run('virsh domstate '.escapeshellarg($domain))->output()) !== 'shut off') {
|
||||
$this->runOrFail('virsh shutdown '.escapeshellarg($domain));
|
||||
$this->waitForShutOff($domain, 120);
|
||||
}
|
||||
|
||||
foreach ($this->hostGpuDevices() as $device) {
|
||||
$this->runOrFail(sprintf('virt-xml %s --add-device --hostdev %s', escapeshellarg($domain), escapeshellarg($device)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the libvirt node device names of every PCI function of the first NVIDIA display controller,
|
||||
* for example pci_0000_01_00_0 (GPU) and pci_0000_01_00_1 (its audio device). All functions must move together.
|
||||
*
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function hostGpuDevices(): array
|
||||
{
|
||||
$devices = collect(explode("\n", trim(Process::run('lspci -Dnmm -d 10de:')->output())))
|
||||
->filter()
|
||||
->map(fn (string $line): array => str_getcsv($line, ' '));
|
||||
$gpu = $devices->first(fn (array $device): bool => in_array($device[1] ?? null, ['0300', '0302'], true));
|
||||
|
||||
if ($gpu === null) {
|
||||
throw new RuntimeException('No NVIDIA GPU found on this host.');
|
||||
}
|
||||
|
||||
$slot = substr($gpu[0], 0, strrpos($gpu[0], '.'));
|
||||
|
||||
return $devices
|
||||
->filter(fn (array $device): bool => str_starts_with($device[0], "{$slot}."))
|
||||
->map(fn (array $device): string => 'pci_'.str_replace([':', '.'], '_', $device[0]))
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* nvidia-persistenced keeps the GPU open, which blocks libvirt from moving it to vfio-pci.
|
||||
*/
|
||||
private function releaseHostGpu(): void
|
||||
{
|
||||
Process::run('systemctl stop nvidia-persistenced');
|
||||
}
|
||||
|
||||
private function destroyVm(string $domain): void
|
||||
{
|
||||
Process::run('virsh destroy '.escapeshellarg($domain));
|
||||
@@ -101,7 +169,7 @@ class StartDevelopmentQemuVm
|
||||
}
|
||||
|
||||
/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */
|
||||
private function createVm(array $profile, bool $prepared): void
|
||||
private function createVm(array $profile, bool $prepared, bool $gpu = false): void
|
||||
{
|
||||
$directory = config('development-qemu.storage_path');
|
||||
File::ensureDirectoryExists($directory);
|
||||
@@ -155,11 +223,17 @@ class StartDevelopmentQemuVm
|
||||
$seedDisk = $prepared ? '' : ' --disk path='.escapeshellarg($seedImage).',format=raw,bus=virtio,readonly=on';
|
||||
// Instance networks are isolated, so VMs of different instances may share the profile MAC.
|
||||
$macCheck = config('development-qemu.instance') === null ? '' : ' --check mac_in_use=off';
|
||||
$hostDevices = '';
|
||||
|
||||
if ($gpu) {
|
||||
$hostDevices = collect($this->hostGpuDevices())->map(fn (string $device): string => ' --hostdev '.escapeshellarg($device))->implode('');
|
||||
$this->releaseHostGpu();
|
||||
}
|
||||
|
||||
$this->runOrFail(sprintf(
|
||||
'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck,
|
||||
'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck.$hostDevices,
|
||||
escapeshellarg($profile['domain']),
|
||||
config('development-qemu.memory'),
|
||||
config($gpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory'),
|
||||
config('development-qemu.vcpus'),
|
||||
escapeshellarg($profile['os_variant']),
|
||||
escapeshellarg($disk),
|
||||
@@ -182,10 +256,10 @@ class StartDevelopmentQemuVm
|
||||
return config('development-qemu.storage_path')."/{$template}-prepared.qcow2";
|
||||
}
|
||||
|
||||
private function waitForPreparation(string $domain): void
|
||||
private function waitForShutOff(string $domain, int $timeoutSeconds): void
|
||||
{
|
||||
$check = 'while [ "$(virsh domstate '.escapeshellarg($domain).')" != "shut off" ]; do sleep 2; done';
|
||||
$this->runOrFail('timeout 900 bash -c '.escapeshellarg($check));
|
||||
$this->runOrFail("timeout {$timeoutSeconds} bash -c ".escapeshellarg($check));
|
||||
}
|
||||
|
||||
private function moveLegacyFiles(string $directory): void
|
||||
|
||||
@@ -9,7 +9,7 @@ use function Laravel\Prompts\multiselect;
|
||||
|
||||
class ManageDevelopmentQemuVmCommand extends Command
|
||||
{
|
||||
protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them}';
|
||||
protected $signature = 'dev:qemu {profiles?* : Profile keys from config/development-qemu.php} {--as-localhost : Use the VM for the localhost server (id 0)} {--fresh : Destroy and recreate the selected VMs instead of reusing them} {--gpu : Pass the NVIDIA GPU of the host through to the VM and install its driver}';
|
||||
|
||||
protected $description = 'Start (or create) selected development QEMU VMs and seed their Coolify servers';
|
||||
|
||||
@@ -34,7 +34,17 @@ class ManageDevelopmentQemuVmCommand extends Command
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'));
|
||||
if ($this->option('gpu')) {
|
||||
$profile = $profiles[$profileNames[0]] ?? null;
|
||||
|
||||
if (count($profileNames) !== 1 || $profile === null || $profile['user'] !== 'root' || ! str_starts_with($profile['os_variant'], 'ubuntu')) {
|
||||
$this->error('GPU passthrough needs exactly one Ubuntu root profile (ubuntu-root).');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
ManageDevelopmentQemuVm::run($profileNames, (bool) $this->option('as-localhost'), (bool) $this->option('fresh'), (bool) $this->option('gpu'));
|
||||
|
||||
foreach ($profileNames as $profileName) {
|
||||
$profile = $profiles[$profileName];
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Actions\Development\InstallDevelopmentGpuDriver;
|
||||
use App\Actions\Development\SeedDevelopmentQemuServer;
|
||||
use App\Actions\Development\StartDevelopmentInstanceDatabase;
|
||||
use Illuminate\Console\Command;
|
||||
@@ -9,7 +10,7 @@ use Throwable;
|
||||
|
||||
class SeedDevelopmentQemuServerCommand extends Command
|
||||
{
|
||||
protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port}';
|
||||
protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port} {--gpu : Install the NVIDIA driver and container toolkit in the VM}';
|
||||
|
||||
protected $description = 'Seed one development QEMU server in the Coolify database';
|
||||
|
||||
@@ -33,6 +34,12 @@ class SeedDevelopmentQemuServerCommand extends Command
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->option('gpu')) {
|
||||
$this->info('Installing the NVIDIA driver and container toolkit. This can take some minutes.');
|
||||
InstallDevelopmentGpuDriver::run($server);
|
||||
$this->info('The GPU is ready.');
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,6 +147,8 @@ return [
|
||||
'dns' => '1.1.1.1',
|
||||
// MiB per VM. Many dev instances share one host, so keep this small; existing VMs get it on their next start.
|
||||
'memory' => (int) env('DEVELOPMENT_QEMU_MEMORY', 1024),
|
||||
// MiB for a VM with a passed-through GPU (dev:qemu --gpu). The NVIDIA driver needs more than the default.
|
||||
'gpu_memory' => (int) env('DEVELOPMENT_QEMU_GPU_MEMORY', 4096),
|
||||
'vcpus' => 2,
|
||||
'disk_size' => '20G',
|
||||
'libvirt_network' => $developmentQemuInstance === null ? 'default' : "coolify-dev-{$developmentQemuSlot}",
|
||||
|
||||
+4
-1
@@ -23,6 +23,7 @@
|
||||
# redis +4, vite +5, mailpit +6/+7, minio +8/+9.
|
||||
#
|
||||
# Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE=<profile>
|
||||
# COOLIFY_DEV_GPU=1 passes the host NVIDIA GPU through to the KVM VM (ubuntu-root only)
|
||||
#
|
||||
# Side by side: COOLIFY_DEV_INSTANCE=<name> COOLIFY_DEV_SIDE_BY_SIDE=1 starts a second instance of
|
||||
# this checkout without stopping the other ones (for example a transfer target, see
|
||||
@@ -402,6 +403,8 @@ start_backend() {
|
||||
local host="${DEV_BIND_ADDRESS:-127.0.0.1}"
|
||||
[[ "$host" == 0.0.0.0 ]] && host=127.0.0.1
|
||||
echo "Using KVM profile: $profile"
|
||||
local gpu=()
|
||||
[[ "${COOLIFY_DEV_GPU:-}" == 1 ]] && gpu=(--gpu)
|
||||
# Use this instance database and encryption key, without a stale environment or config cache.
|
||||
# Send logs to stderr to avoid root-owned files in storage/logs.
|
||||
APP_ENV=local APP_CONFIG_CACHE="$ROOT/bootstrap/cache/dev-qemu-$$.php" \
|
||||
@@ -412,7 +415,7 @@ start_backend() {
|
||||
DEVELOPMENT_QEMU_SLOT="$(env_value "$instance" DEVELOPMENT_QEMU_SLOT)" \
|
||||
DEVELOPMENT_QEMU_DOCKER_NETWORK="$(project_name "$instance")" \
|
||||
DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$(project_name "$instance")" \
|
||||
"${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost
|
||||
"${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost "${gpu[@]}"
|
||||
elif [[ "$backend" == "lima" ]]; then
|
||||
echo "Using Lima profile: $profile"
|
||||
start_lima_vm
|
||||
|
||||
@@ -269,6 +269,26 @@ BASH);
|
||||
->and(devScriptLog())->toContain('php artisan dev:qemu ubuntu-root --as-localhost ENV=local CACHE=');
|
||||
});
|
||||
|
||||
it('passes the gpu option to the host qemu command when COOLIFY_DEV_GPU is 1', function () {
|
||||
if (! function_exists('posix_geteuid') || posix_geteuid() !== 0 || ! file_exists('/dev/kvm') || filetype('/dev/kvm') !== 'char' || ! is_readable('/dev/kvm') || ! is_writable('/dev/kvm')) {
|
||||
$this->markTestSkipped('KVM and root access are required for this launcher branch.');
|
||||
}
|
||||
|
||||
file_put_contents($this->devRoot.'/bin/php', <<<'BASH'
|
||||
#!/usr/bin/env bash
|
||||
printf 'php %s|\n' "$*" >> "$DEV_TEST_LOG"
|
||||
BASH);
|
||||
|
||||
$withGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '1']);
|
||||
$withoutGpu = runDevScript('wt', ['start', 'ubuntu-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto', 'COOLIFY_DEV_GPU' => '0']);
|
||||
|
||||
expect($withGpu->isSuccessful())->toBeTrue($withGpu->getErrorOutput())
|
||||
->and($withoutGpu->isSuccessful())->toBeTrue($withoutGpu->getErrorOutput())
|
||||
->and(devScriptLog())
|
||||
->toContain('php artisan dev:qemu ubuntu-root --as-localhost --gpu|')
|
||||
->toContain('php artisan dev:qemu ubuntu-root --as-localhost|');
|
||||
});
|
||||
|
||||
it('starts a lima vm as localhost on macos and seeds its forwarded ssh port', function () {
|
||||
($this->fakeLima)();
|
||||
runDevScript('main', ['start']);
|
||||
|
||||
@@ -690,3 +690,115 @@ function useDevelopmentQemuInstance(string $instance, int $slot, string $storage
|
||||
'storage_path' => $storagePath,
|
||||
]]);
|
||||
}
|
||||
|
||||
function fakeQemuHostWithGpu(string $domainState = 'running', string $domainXml = '<domain></domain>'): void
|
||||
{
|
||||
Process::fake([
|
||||
'* net-dumpxml *' => Process::result(output: '<network></network>'),
|
||||
'* network inspect *' => Process::result(output: "172.18.0.0/16\n"),
|
||||
'* dominfo *' => Process::result(output: 'exists'),
|
||||
'* domstate *' => Process::result(output: "{$domainState}\n"),
|
||||
'* dumpxml *' => Process::result(output: $domainXml),
|
||||
'lspci *' => Process::result(output: implode("\n", [
|
||||
'0000:01:00.0 "0300" "10de" "2584" -ra1 -p00 "1043" "8976"',
|
||||
'0000:01:00.1 "0403" "10de" "2291" -ra1 -p00 "1043" "8976"',
|
||||
])."\n"),
|
||||
'*' => Process::result(),
|
||||
]);
|
||||
}
|
||||
|
||||
it('creates a vm with every function of the host nvidia gpu and more memory', function () {
|
||||
$storagePath = sys_get_temp_dir().'/coolify-qemu-gpu-create-test-'.uniqid();
|
||||
config(['development-qemu.storage_path' => $storagePath, 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]);
|
||||
File::ensureDirectoryExists($storagePath);
|
||||
File::put("{$storagePath}/coolify-dev-ubuntu-root-prepared.qcow2", 'prepared image');
|
||||
fakeQemuHostWithGpu();
|
||||
|
||||
StartDevelopmentQemuVm::run('ubuntu-root', true, true);
|
||||
|
||||
Process::assertRan(fn ($process) => $process->command === 'systemctl stop nvidia-persistenced');
|
||||
Process::assertRan(fn ($process) => str_starts_with($process->command, 'virt-install ')
|
||||
&& str_contains($process->command, '--memory 4096 ')
|
||||
&& str_contains($process->command, "--hostdev 'pci_0000_01_00_0' --hostdev 'pci_0000_01_00_1'"));
|
||||
});
|
||||
|
||||
it('stops an existing vm to attach the host gpu and starts it with the gpu memory', function () {
|
||||
config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-attach-test-'.uniqid(), 'development-qemu.gpu_memory' => 4096]);
|
||||
$hostdevAttached = false;
|
||||
$ran = [];
|
||||
Process::fake(function ($process) use (&$hostdevAttached, &$ran) {
|
||||
$ran[] = $process->command;
|
||||
$hostdevAttached = $hostdevAttached || str_starts_with($process->command, 'virt-xml ');
|
||||
|
||||
return match (true) {
|
||||
str_contains($process->command, ' net-dumpxml ') => Process::result(output: '<network></network>'),
|
||||
str_contains($process->command, ' network inspect ') => Process::result(output: "172.18.0.0/16\n"),
|
||||
str_contains($process->command, ' dominfo ') => Process::result(output: 'exists'),
|
||||
str_contains($process->command, ' domstate ') => Process::result(output: $hostdevAttached ? "shut off\n" : "running\n"),
|
||||
str_contains($process->command, ' dumpxml ') => Process::result(output: $hostdevAttached ? "<domain><hostdev mode='subsystem'/></domain>" : '<domain></domain>'),
|
||||
str_starts_with($process->command, 'lspci ') => Process::result(output: "0000:01:00.0 \"0300\" \"10de\" \"2584\"\n0000:01:00.1 \"0403\" \"10de\" \"2291\"\n"),
|
||||
default => Process::result(),
|
||||
};
|
||||
});
|
||||
|
||||
StartDevelopmentQemuVm::run('ubuntu-root', false, true);
|
||||
|
||||
$commands = collect($ran);
|
||||
$shutdown = $commands->search("virsh shutdown 'coolify-dev-ubuntu-root'");
|
||||
$attach = $commands->search("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_0'");
|
||||
expect($shutdown)->not->toBeFalse()
|
||||
->and($attach)->toBeGreaterThan($shutdown)
|
||||
->and($commands)->toContain("virt-xml 'coolify-dev-ubuntu-root' --add-device --hostdev 'pci_0000_01_00_1'")
|
||||
->and($commands->search("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config"))->toBeGreaterThan($attach)
|
||||
->and($commands->search("virsh start 'coolify-dev-ubuntu-root'"))->toBeGreaterThan($commands->search('systemctl stop nvidia-persistenced'));
|
||||
Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-install '));
|
||||
});
|
||||
|
||||
it('keeps the gpu memory of a gpu vm that starts without the gpu option', function () {
|
||||
config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-restart-test-'.uniqid(), 'development-qemu.memory' => 1024, 'development-qemu.gpu_memory' => 4096]);
|
||||
fakeQemuHostWithGpu('shut off', "<domain><hostdev mode='subsystem'/></domain>");
|
||||
|
||||
StartDevelopmentQemuVm::run('ubuntu-root');
|
||||
|
||||
Process::assertRan("virsh setmem 'coolify-dev-ubuntu-root' 4096M --config");
|
||||
Process::assertRan('systemctl stop nvidia-persistenced');
|
||||
Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml '));
|
||||
Process::assertNotRan(fn ($process) => str_contains($process->command, '1024M'));
|
||||
});
|
||||
|
||||
it('fails before it changes the vm when the host has no nvidia gpu', function () {
|
||||
config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-no-gpu-test-'.uniqid()]);
|
||||
Process::fake([
|
||||
'* net-dumpxml *' => Process::result(output: '<network></network>'),
|
||||
'* network inspect *' => Process::result(output: "172.18.0.0/16\n"),
|
||||
'* dominfo *' => Process::result(output: 'exists'),
|
||||
'* domstate *' => Process::result(output: "shut off\n"),
|
||||
'*' => Process::result(),
|
||||
]);
|
||||
|
||||
expect(fn () => StartDevelopmentQemuVm::run('ubuntu-root', false, true))
|
||||
->toThrow(RuntimeException::class, 'No NVIDIA GPU found');
|
||||
Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-xml ') || str_contains($process->command, 'virsh start'));
|
||||
});
|
||||
|
||||
it('allows gpu passthrough only for one ubuntu root profile', function (array $profiles) {
|
||||
Process::fake();
|
||||
|
||||
expect(Artisan::call('dev:qemu', ['profiles' => $profiles, '--gpu' => true]))->toBe(Command::FAILURE);
|
||||
|
||||
Process::assertNothingRan();
|
||||
})->with([
|
||||
'non-root' => [['ubuntu-non-root']],
|
||||
'other distribution' => [['debian-root']],
|
||||
'two profiles' => [['ubuntu-root', 'debian-root']],
|
||||
]);
|
||||
|
||||
it('seeds a gpu vm through the container so it installs the driver', function () {
|
||||
config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-gpu-seed-test-'.uniqid()]);
|
||||
SeedDevelopmentQemuServer::mock()->shouldNotReceive('handle');
|
||||
fakeQemuHostWithGpu('shut off', "<domain><hostdev mode='subsystem'/></domain>");
|
||||
|
||||
ManageDevelopmentQemuVm::run('ubuntu-root', true, false, true);
|
||||
|
||||
Process::assertRan("docker exec 'coolify' php artisan dev:qemu:seed 'ubuntu-root' --as-localhost --gpu");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user