From ad6e929a933c55633ab5b2ae6e5c8d4d3f855ae9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:00:31 +0200 Subject: [PATCH] feat(gpu): fix GPU compose settings and show detected server GPUs - Build a valid compose device request (device_ids list, options mapping, blank driver) - Map --gpus N to count and drop --gpus when GPU settings are enabled - Validate only changed GPU values so stored legacy values keep working - Detect NVIDIA GPUs and the container toolkit with the server details - Show detected GPUs with a refresh button in the application GPU section Co-Authored-By: Claude Opus 5.5 --- app/Jobs/ApplicationDeploymentJob.php | 22 +-- app/Livewire/Project/Application/Advanced.php | 46 ++++- app/Models/ApplicationSetting.php | 79 ++++++++ app/Models/Server.php | 40 +++- bootstrap/helpers/api.php | 8 +- bootstrap/helpers/docker.php | 35 ++-- .../components/server/gpu-list.blade.php | 28 +++ .../project/application/advanced.blade.php | 29 ++- .../server/partials/server-details.blade.php | 4 + tests/Feature/ApplicationGpuSettingsTest.php | 175 ++++++++++++++++++ .../Helpers/DockerCustomCommandsTest.php | 19 ++ tests/Feature/ServerMetadataTest.php | 54 ++++++ 12 files changed, 497 insertions(+), 42 deletions(-) create mode 100644 resources/views/components/server/gpu-list.blade.php create mode 100644 tests/Feature/ApplicationGpuSettingsTest.php diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index acc4d3e7b6..25aec367bc 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -13,6 +13,7 @@ use App\Exceptions\DeploymentException; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Models\ApplicationPreview; +use App\Models\ApplicationSetting; use App\Models\EnvironmentVariable; use App\Models\GithubApp; use App\Models\GitlabApp; @@ -4265,21 +4266,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); } if ($this->application->settings->is_gpu_enabled) { $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [ - [ - 'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'), - 'capabilities' => ['gpu'], - 'options' => data_get($this->application, 'settings.gpu_options', []), - ], + $this->application->settings->gpuDeviceReservation(), ]; - if (data_get($this->application, 'settings.gpu_count')) { - $count = data_get($this->application, 'settings.gpu_count'); - if ($count === 'all') { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = $count; - } else { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count; - } - } elseif (data_get($this->application, 'settings.gpu_device_ids')) { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($this->application, 'settings.gpu_device_ids'); + if (ApplicationSetting::parseGpuOptions($this->application->settings->gpu_options) === null) { + $this->application_deployment_queue->addLogEntry('GPU options are not a valid key=value list or YAML mapping. They are ignored.'); } } if ($this->application->isHealthcheckDisabled()) { @@ -4309,6 +4299,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); if ($this->pull_request_id === 0) { $custom_compose = convertDockerRunToCompose($this->application->custom_docker_run_options); + if ($this->application->settings->is_gpu_enabled && data_get($custom_compose, 'deploy.resources.reservations.devices')) { + data_forget($custom_compose, 'deploy.resources.reservations.devices'); + $this->application_deployment_queue->addLogEntry('GPU access is enabled in the GPU settings, so --gpus in Custom Docker Options is ignored.'); + } if ((bool) $this->application->settings->is_consistent_container_name_enabled) { $docker_compose['services'][$this->application->uuid] = $docker_compose['services'][$this->container_name]; if ($this->container_name !== $this->application->uuid) { diff --git a/app/Livewire/Project/Application/Advanced.php b/app/Livewire/Project/Application/Advanced.php index 86b406a6b5..0075bdddd4 100644 --- a/app/Livewire/Project/Application/Advanced.php +++ b/app/Livewire/Project/Application/Advanced.php @@ -260,11 +260,21 @@ class Advanced extends Component { try { $this->authorize('update', $this->application); - if ($this->gpuCount && $this->gpuDeviceIds) { + // Check only changed GPU values, so stored legacy values do not block other settings. + $settings = $this->application->settings; + $gpuSelectionChanged = $this->gpuCount !== $settings->gpu_count || $this->gpuDeviceIds !== $settings->gpu_device_ids; + if ($gpuSelectionChanged && filled($this->gpuCount) && filled($this->gpuDeviceIds)) { $this->dispatch('error', 'You cannot set both GPU count and GPU device IDs.'); - $this->gpuCount = null; - $this->gpuDeviceIds = null; - $this->syncData(true); + + return; + } + if ($this->gpuCount !== $settings->gpu_count && filled($this->gpuCount) && ! preg_match(ApplicationSetting::GPU_COUNT_PATTERN, trim($this->gpuCount))) { + $this->dispatch('error', 'GPU count must be a number or "all".'); + + return; + } + if ($this->gpuOptions !== $settings->gpu_options && ApplicationSetting::parseGpuOptions($this->gpuOptions) === null) { + $this->dispatch('error', 'GPU options must be key=value pairs (one per line) or a YAML mapping.'); return; } @@ -437,6 +447,33 @@ class Advanced extends Component } } + public function refreshGpus(): void + { + try { + $servers = $this->application->maintenanceServers(); + foreach ($servers as $server) { + $this->authorize('update', $server); + } + $functionalServers = $servers->filter(fn ($server) => $server->isFunctional()); + if ($functionalServers->isEmpty()) { + $this->dispatch('error', 'Validate the server connection before you refresh GPUs.'); + + return; + } + + $failed = $functionalServers->reject(fn ($server) => $server->gatherServerMetadata())->pluck('name'); + $this->application->unsetRelation('destination')->unsetRelation('additional_servers'); + if ($failed->isNotEmpty()) { + $this->dispatch('error', 'Could not collect GPUs from '.$failed->implode(', ').'. Check the application logs for the remote command output.'); + + return; + } + $this->dispatch('success', 'GPUs refreshed.'); + } catch (\Throwable $e) { + handleError($e, $this); + } + } + public function render() { $server = data_get($this->application, 'destination.server'); @@ -444,6 +481,7 @@ class Advanced extends Component return view('livewire.project.application.advanced', [ 'isTraefikProxy' => $server?->proxyType() === ProxyTypes::TRAEFIK->value, 'serverCertificateResolver' => $server?->traefikCertificateResolver() ?? ServerSetting::TRAEFIK_CERTIFICATE_RESOLVER_NONE, + 'gpuServers' => $this->application->maintenanceServers(), ]); } diff --git a/app/Models/ApplicationSetting.php b/app/Models/ApplicationSetting.php index abe317530f..8ca5a0dbd3 100644 --- a/app/Models/ApplicationSetting.php +++ b/app/Models/ApplicationSetting.php @@ -5,6 +5,8 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; use OpenApi\Attributes as OA; +use Symfony\Component\Yaml\Exception\ParseException; +use Symfony\Component\Yaml\Yaml; #[OA\Schema( description: 'Application settings.', @@ -59,6 +61,8 @@ class ApplicationSetting extends Model */ public const MAX_CONTAINER_NAME_PREFIX_LENGTH = 30; + public const GPU_COUNT_PATTERN = '/^(all|\d+)$/'; + protected $casts = [ 'is_static' => 'boolean', 'is_spa' => 'boolean', @@ -170,6 +174,81 @@ class ApplicationSetting extends Model return $this->stopGracePeriodSeconds(); } + /** + * The compose device request (deploy.resources.reservations.devices item) for the GPU settings. + * A count wins over device IDs. Options that cannot be parsed are left out. + * + * @return array{capabilities: list, driver?: string, count?: int|string, device_ids?: list, options?: array} + */ + public function gpuDeviceReservation(): array + { + $device = ['capabilities' => ['gpu']]; + $driver = trim((string) $this->gpu_driver); + if ($driver !== '') { + $device['driver'] = $driver; + } + + $count = trim((string) $this->gpu_count); + $deviceIds = str((string) $this->gpu_device_ids)->explode(',')->map(fn (string $id) => trim($id))->filter(fn (string $id) => $id !== '')->values()->all(); + if ($count !== '') { + $device['count'] = $count === 'all' ? 'all' : (int) $count; + } elseif ($deviceIds !== []) { + $device['device_ids'] = $deviceIds; + } + + $options = self::parseGpuOptions($this->gpu_options); + if ($options) { + $device['options'] = $options; + } + + return $device; + } + + /** + * Parse GPU driver options written as a YAML/JSON mapping or as key=value pairs, one per line. + * + * @return array|null null when the value is not a flat mapping + */ + public static function parseGpuOptions(?string $options): ?array + { + $options = trim((string) $options); + if ($options === '') { + return []; + } + + try { + $parsed = Yaml::parse($options); + } catch (ParseException) { + $parsed = null; + } + if (is_array($parsed) && $parsed !== [] && ! array_is_list($parsed)) { + $mapping = []; + foreach ($parsed as $key => $value) { + if (! is_scalar($value)) { + return null; + } + $mapping[(string) $key] = is_bool($value) ? ($value ? 'true' : 'false') : (string) $value; + } + + return $mapping; + } + + $mapping = []; + foreach (preg_split('/\R/', $options) as $pair) { + $pair = trim($pair); + if ($pair === '') { + continue; + } + [$key, $value] = array_pad(explode('=', $pair, 2), 2, null); + if ($value === null || trim($key) === '') { + return null; + } + $mapping[trim($key)] = trim($value); + } + + return $mapping; + } + public function isStatic(): Attribute { return Attribute::make( diff --git a/app/Models/Server.php b/app/Models/Server.php index c1cebd7287..00b10235b3 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -2102,7 +2102,7 @@ $siteAddress { try { $output = instant_remote_process([ - 'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true)', + 'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true) && echo "---GPUS---" && (nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits 2>/dev/null || true) && echo "---NVIDIA_TOOLKIT---" && (command -v nvidia-container-runtime-hook nvidia-ctk 2>/dev/null || true)', ], $this, false); if (! $output) { @@ -2110,6 +2110,7 @@ $siteAddress { } $sections = []; + $sectionLines = []; $currentKey = null; foreach (explode("\n", trim($output)) as $line) { $line = trim($line); @@ -2117,6 +2118,9 @@ $siteAddress { $currentKey = $m[1]; } elseif ($currentKey) { $sections[$currentKey] = $line; + if ($line !== '') { + $sectionLines[$currentKey][] = $line; + } } } @@ -2127,6 +2131,8 @@ $siteAddress { 'cpus' => (int) ($sections['CPUS'] ?? 0), 'memory_bytes' => (int) ($sections['MEMORY'] ?? 0), 'uptime_since' => $sections['UPTIME_SINCE'] ?? null, + 'gpus' => self::parseNvidiaSmiGpus($sectionLines['GPUS'] ?? []), + 'nvidia_container_toolkit' => ($sectionLines['NVIDIA_TOOLKIT'] ?? []) !== [], 'collected_at' => now()->toIso8601String(), ]; @@ -2166,6 +2172,38 @@ $siteAddress { } } + /** + * Parse `nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits` lines. + * The name is the last column because it can contain commas. + * + * @param list $lines + * @return list + */ + public static function parseNvidiaSmiGpus(array $lines): array + { + $gpus = []; + foreach ($lines as $line) { + $columns = array_map('trim', explode(',', $line, 4)); + if (count($columns) !== 4 || ! ctype_digit($columns[0])) { + continue; + } + [$index, $uuid, $memory, $name] = $columns; + $gpus[] = ['index' => $index, 'uuid' => $uuid, 'memory_mb' => (int) $memory, 'name' => $name]; + } + + return $gpus; + } + + /** + * GPUs found when the server details were last collected, or null when they were never checked. + * + * @return list|null + */ + public function detectedGpus(): ?array + { + return data_get($this->server_metadata, 'gpus'); + } + public function isTerminalEnabled() { return $this->settings->is_terminal_enabled ?? false; diff --git a/bootstrap/helpers/api.php b/bootstrap/helpers/api.php index ca356fa16a..cebeef2d12 100644 --- a/bootstrap/helpers/api.php +++ b/bootstrap/helpers/api.php @@ -212,9 +212,13 @@ function sharedDataApplications(?Application $application = null): array 'is_log_drain_enabled' => 'boolean', 'is_gpu_enabled' => 'boolean', 'gpu_driver' => 'string|nullable', - 'gpu_count' => 'string|nullable', + 'gpu_count' => ['string', 'nullable', 'regex:'.ApplicationSetting::GPU_COUNT_PATTERN], 'gpu_device_ids' => 'string|nullable', - 'gpu_options' => 'string|nullable', + 'gpu_options' => ['string', 'nullable', function (string $attribute, mixed $value, Closure $fail): void { + if (ApplicationSetting::parseGpuOptions($value) === null) { + $fail('The gpu_options must be key=value pairs or a YAML mapping.'); + } + }], 'is_consistent_container_name_enabled' => 'boolean', 'custom_internal_name' => ['nullable', ...ValidationPatterns::containerNameRules()], 'custom_container_name_prefix' => 'string|nullable|max:'.ApplicationSetting::MAX_CONTAINER_NAME_PREFIX_LENGTH, diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 33dd277d03..e4548c0aed 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -1641,11 +1641,17 @@ function convertDockerRunToCompose(?string $custom_docker_run_options = null) foreach ($matches as $match) { $option = $match[1]; if ($option === '--gpus') { - $regexForParsingDeviceIds = '/device=([0-9A-Za-z-,]+)/'; - preg_match($regexForParsingDeviceIds, $custom_docker_run_options, $device_matches); - $value = $device_matches[1] ?? 'all'; - $options[$option][] = $value; - $options[$option] = array_unique($options[$option]); + // --gpus all | --gpus 2 | --gpus "device=0,1" + $gpuRequest = []; + $gpuValue = trim($match[2] ?? '', '\'"'); + if (preg_match('/device=([0-9A-Za-z-,]+)/', $custom_docker_run_options, $device_matches)) { + $gpuRequest['device_ids'] = explode(',', $device_matches[1]); + } elseif (ctype_digit($gpuValue)) { + $gpuRequest['count'] = (int) $gpuValue; + } + $options[$option] = $gpuRequest; + + continue; } if ($option === '--hostname') { // Match --hostname=value or --hostname value @@ -1736,23 +1742,14 @@ function convertDockerRunToCompose(?string $custom_docker_run_options = null) $compose_options->put($mapping[$option], $value[0]); } } elseif ($option === '--gpus') { - $payload = [ - 'driver' => 'nvidia', - 'capabilities' => ['gpu'], - ]; - if (! is_null($value) && is_array($value) && count($value) > 0 && ! empty(trim($value[0]))) { - if (str($value[0]) != 'all') { - if (str($value[0])->contains(',')) { - $payload['device_ids'] = str($value[0])->explode(',')->toArray(); - } else { - $payload['device_ids'] = [$value[0]]; - } - } - } $compose_options->put('deploy', [ 'resources' => [ 'reservations' => [ - 'devices' => [$payload], + 'devices' => [[ + 'driver' => 'nvidia', + 'capabilities' => ['gpu'], + ...$value, + ]], ], ], ]); diff --git a/resources/views/components/server/gpu-list.blade.php b/resources/views/components/server/gpu-list.blade.php new file mode 100644 index 0000000000..93c084dec5 --- /dev/null +++ b/resources/views/components/server/gpu-list.blade.php @@ -0,0 +1,28 @@ +@props(['server']) + +@php($gpus = $server->detectedGpus()) + +
merge(['class' => 'text-[12px] leading-5 text-neutral-600 dark:text-fg-dim']) }}> + @if ($gpus === null) +

Not checked yet. Refresh to find GPUs.

+ @elseif ($gpus === []) +

No NVIDIA GPU found (nvidia-smi is not installed or shows no GPU).

+ @else +
    + @foreach ($gpus as $gpu) +
  • + {{ $gpu['index'] }} + · {{ $gpu['name'] }} · {{ round($gpu['memory_mb'] / 1024, 1) }} GB · + {{ $gpu['uuid'] }} +
    • + @endforeach +
+ @unless (data_get($server->server_metadata, 'nvidia_container_toolkit')) +

+ NVIDIA Container Toolkit not found. Containers cannot use these GPUs until you + install it and restart Docker. +

+ @endunless + @endif +
diff --git a/resources/views/livewire/project/application/advanced.blade.php b/resources/views/livewire/project/application/advanced.blade.php index c5e5aa89c3..9ce4bc1ae9 100644 --- a/resources/views/livewire/project/application/advanced.blade.php +++ b/resources/views/livewire/project/application/advanced.blade.php @@ -203,6 +203,30 @@ @if ($application->build_pack !== 'dockercompose') + + @php($canRefreshGpus = $gpuServers->contains(fn ($gpuServer) => $gpuServer->isFunctional())) + + + Refresh GPUs + + + + @foreach ($gpuServers as $gpuServer) +
! $loop->first])> +

+ {{ $gpuServer->name }} + @if (data_get($gpuServer->server_metadata, 'collected_at')) + · checked + {{ \Illuminate\Support\Carbon::parse(data_get($gpuServer->server_metadata, 'collected_at'))->diffForHumans() }} + @endif +

+ +
+ @endforeach +
-
- @endif diff --git a/resources/views/livewire/server/partials/server-details.blade.php b/resources/views/livewire/server/partials/server-details.blade.php index be8d0236a0..262e54570d 100644 --- a/resources/views/livewire/server/partials/server-details.blade.php +++ b/resources/views/livewire/server/partials/server-details.blade.php @@ -21,3 +21,7 @@
@endforeach +
+

GPUs

+ +
diff --git a/tests/Feature/ApplicationGpuSettingsTest.php b/tests/Feature/ApplicationGpuSettingsTest.php new file mode 100644 index 0000000000..0f32b3f8b2 --- /dev/null +++ b/tests/Feature/ApplicationGpuSettingsTest.php @@ -0,0 +1,175 @@ +create(); + $team->members()->attach(auth()->id(), ['role' => 'owner']); + session(['currentTeam' => $team]); + $server = Server::factory()->create(['team_id' => $team->id]); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + + return Application::create([ + 'name' => 'gpu-settings-test-app', + 'git_repository' => 'https://github.com/coollabsio/coolify', + 'git_branch' => 'main', + 'build_pack' => 'nixpacks', + 'ports_exposes' => '3000', + 'environment_id' => $environment->id, + 'destination_id' => $server->standaloneDockers()->firstOrFail()->id, + 'destination_type' => $server->standaloneDockers()->firstOrFail()->getMorphClass(), + ]); +} + +it('builds a valid compose device request from the GPU settings', function (array $settings, array $expected) { + $setting = new ApplicationSetting(['gpu_driver' => 'nvidia', ...$settings]); + + expect($setting->gpuDeviceReservation())->toBe($expected); +})->with([ + 'all gpus' => [[], ['capabilities' => ['gpu'], 'driver' => 'nvidia']], + 'count' => [['gpu_count' => '2'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 2]], + 'count all' => [['gpu_count' => 'all'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 'all']], + 'device ids' => [['gpu_device_ids' => '0, 2,'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'device_ids' => ['0', '2']]], + 'count wins over device ids' => [['gpu_count' => '1', 'gpu_device_ids' => '0'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 1]], + 'blank driver' => [['gpu_driver' => ''], ['capabilities' => ['gpu']]], + 'key=value options' => [['gpu_options' => "virtualization=false\nfoo = bar"], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['virtualization' => 'false', 'foo' => 'bar']]], + 'option value with commas' => [['gpu_options' => 'capabilities=compute,utility'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['capabilities' => 'compute,utility']]], + 'yaml options' => [['gpu_options' => "virtualization: false\nlevel: 2"], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['virtualization' => 'false', 'level' => '2']]], + 'empty options' => [['gpu_options' => ''], ['capabilities' => ['gpu'], 'driver' => 'nvidia']], + 'invalid options are left out' => [['gpu_options' => 'not an option'], ['capabilities' => ['gpu'], 'driver' => 'nvidia']], +]); + +describe('advanced GPU form', function () { + beforeEach(function () { + $this->actingAs(User::factory()->create()); + }); + + it('saves valid GPU options', function () { + $application = createApplicationForGpuSettingsTest(); + + Livewire::test(Advanced::class, ['application' => $application]) + ->set('gpuCount', '2') + ->set('gpuOptions', 'virtualization=false') + ->call('submit') + ->assertDispatched('success'); + + $settings = $application->settings()->first(); + expect($settings->gpu_count)->toBe('2') + ->and($settings->gpu_options)->toBe('virtualization=false'); + }); + + it('rejects invalid GPU values', function (string $property, string $value) { + $application = createApplicationForGpuSettingsTest(); + + Livewire::test(Advanced::class, ['application' => $application]) + ->set($property, $value) + ->call('submit') + ->assertDispatched('error') + ->assertNotDispatched('success'); + })->with([ + 'count' => ['gpuCount', 'two'], + 'options' => ['gpuOptions', 'not an option'], + ]); + + it('does not block other settings when a stored GPU value is invalid', function () { + $application = createApplicationForGpuSettingsTest(); + $application->settings->update(['gpu_options' => 'legacy value', 'gpu_count' => 'some']); + + Livewire::test(Advanced::class, ['application' => $application->fresh()]) + ->set('includeSourceCommitInBuild', true) + ->call('submit') + ->assertDispatched('success'); + + expect($application->settings()->first()->include_source_commit_in_build)->toBeTrue(); + }); + + it('keeps a stored GPU count and device IDs pair when other settings are saved', function () { + $application = createApplicationForGpuSettingsTest(); + $application->settings->update(['gpu_count' => '1', 'gpu_device_ids' => '0']); + + Livewire::test(Advanced::class, ['application' => $application->fresh()]) + ->set('includeSourceCommitInBuild', true) + ->call('submit') + ->assertDispatched('success'); + + $settings = $application->settings()->first(); + expect($settings->gpu_count)->toBe('1') + ->and($settings->gpu_device_ids)->toBe('0') + ->and($settings->include_source_commit_in_build)->toBeTrue(); + }); + + it('refreshes the GPUs of the application server', function () { + $application = createApplicationForGpuSettingsTest(); + $server = $application->destination->server; + $server->update(['private_key_id' => PrivateKey::factory()->create(['team_id' => $server->team_id])->id]); + $server->settings->update(['is_reachable' => true, 'is_usable' => true]); + Process::fake(['*' => Process::result(output: "---GPUS---\n0, GPU-1a2b, 81559, NVIDIA H100\n---NVIDIA_TOOLKIT---\n/usr/bin/nvidia-ctk")]); + + Livewire::test(Advanced::class, ['application' => $application]) + ->call('refreshGpus') + ->assertDispatched('success') + ->assertSee('NVIDIA H100'); + + expect($server->fresh()->detectedGpus())->toBe([['index' => '0', 'uuid' => 'GPU-1a2b', 'memory_mb' => 81559, 'name' => 'NVIDIA H100']]); + }); + + it('does not refresh GPUs when no server is validated', function () { + $application = createApplicationForGpuSettingsTest(); + $application->destination->server->settings->update(['is_reachable' => false, 'is_usable' => false]); + Process::fake(); + + Livewire::test(Advanced::class, ['application' => $application]) + ->call('refreshGpus') + ->assertDispatched('error') + ->assertNotDispatched('success'); + + Process::assertNothingRan(); + }); + + it('does not let a team member refresh GPUs', function () { + $application = createApplicationForGpuSettingsTest(); + $server = $application->destination->server; + $server->settings->update(['is_reachable' => true, 'is_usable' => true]); + $member = User::factory()->create(); + $application->team()->members()->attach($member->id, ['role' => 'member']); + $this->actingAs($member); + Process::fake(); + + Livewire::test(Advanced::class, ['application' => $application]) + ->call('refreshGpus') + ->assertNotDispatched('success'); + + Process::assertNothingRan(); + expect($server->fresh()->detectedGpus())->toBeNull(); + }); + + it('rejects a change that sets both GPU count and device IDs without clearing stored values', function () { + $application = createApplicationForGpuSettingsTest(); + $application->settings->update(['gpu_device_ids' => '0']); + + Livewire::test(Advanced::class, ['application' => $application->fresh()]) + ->set('gpuCount', '2') + ->call('submit') + ->assertDispatched('error') + ->assertNotDispatched('success'); + + $settings = $application->settings()->first(); + expect($settings->gpu_count)->toBeNull() + ->and($settings->gpu_device_ids)->toBe('0'); + }); +}); diff --git a/tests/Feature/Helpers/DockerCustomCommandsTest.php b/tests/Feature/Helpers/DockerCustomCommandsTest.php index 299709ffd9..cf2408faf3 100644 --- a/tests/Feature/Helpers/DockerCustomCommandsTest.php +++ b/tests/Feature/Helpers/DockerCustomCommandsTest.php @@ -233,3 +233,22 @@ test('ConvertIpAndIp6Together', function () { 'ip6' => ['2001:db8::1'], ]); }); + +test('ConvertGpusCount', function () { + $output = convertDockerRunToCompose('--gpus 2'); + expect($output)->toBe([ + 'deploy' => [ + 'resources' => [ + 'reservations' => [ + 'devices' => [ + [ + 'driver' => 'nvidia', + 'capabilities' => ['gpu'], + 'count' => 2, + ], + ], + ], + ], + ], + ]); +}); diff --git a/tests/Feature/ServerMetadataTest.php b/tests/Feature/ServerMetadataTest.php index ec33d4f0e8..5d521a71e6 100644 --- a/tests/Feature/ServerMetadataTest.php +++ b/tests/Feature/ServerMetadataTest.php @@ -220,3 +220,57 @@ it('does not call gatherServerMetadata when docker version validation fails', fu ->instance() ->validateDockerVersion(); }); + +it('stores detected NVIDIA GPUs and the container toolkit when gathering server metadata', function (array $gpuLines, array $toolkitLines, array $expectedGpus, bool $expectedToolkit) { + $this->server->settings->update([ + 'is_reachable' => true, + 'is_usable' => true, + ]); + + Process::fake([ + '*' => Process::result(output: implode("\n", [ + '---PRETTY_NAME---', + 'Ubuntu 24.04 LTS', + '---CPUS---', + '8', + '---DOCKER---', + '29.4.3-ce', + '---COMPOSE---', + 'v2.32.4', + '---GPUS---', + ...$gpuLines, + '---NVIDIA_TOOLKIT---', + ...$toolkitLines, + ]), exitCode: 0), + ]); + + $this->server->gatherServerMetadata(); + $server = $this->server->fresh(); + + expect($server->detectedGpus())->toBe($expectedGpus) + ->and($server->server_metadata['nvidia_container_toolkit'])->toBe($expectedToolkit) + ->and($server->server_metadata['cpus'])->toBe(8); +})->with([ + 'two gpus' => [ + ['0, GPU-1a2b, 81559, NVIDIA H100 80GB HBM3', '1, GPU-3c4d, 24564, NVIDIA GeForce RTX 4090, Founders'], + ['/usr/bin/nvidia-container-runtime-hook', '/usr/bin/nvidia-ctk'], + [ + ['index' => '0', 'uuid' => 'GPU-1a2b', 'memory_mb' => 81559, 'name' => 'NVIDIA H100 80GB HBM3'], + ['index' => '1', 'uuid' => 'GPU-3c4d', 'memory_mb' => 24564, 'name' => 'NVIDIA GeForce RTX 4090, Founders'], + ], + true, + ], + 'driver installed without a working gpu' => [ + ['NVIDIA-SMI has failed because it could not communicate with the NVIDIA driver.'], + [], + [], + false, + ], + 'no nvidia-smi' => [[], [], [], false], +]); + +it('returns null detected GPUs when GPUs were never checked', function () { + $this->server->update(['server_metadata' => ['os' => 'Ubuntu 24.04 LTS']]); + + expect($this->server->fresh()->detectedGpus())->toBeNull(); +});