mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
feat(gpu): fix GPU compose settings and show detected server GPUs
- Build a valid compose device request (device_ids list, options mapping, blank driver) - Map --gpus N to count and drop --gpus when GPU settings are enabled - Validate only changed GPU values so stored legacy values keep working - Detect NVIDIA GPUs and the container toolkit with the server details - Show detected GPUs with a refresh button in the application GPU section Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3dce9bd960
commit
ad6e929a93
@@ -13,6 +13,7 @@ use App\Exceptions\DeploymentException;
|
||||
use App\Models\Application;
|
||||
use App\Models\ApplicationDeploymentQueue;
|
||||
use App\Models\ApplicationPreview;
|
||||
use App\Models\ApplicationSetting;
|
||||
use App\Models\EnvironmentVariable;
|
||||
use App\Models\GithubApp;
|
||||
use App\Models\GitlabApp;
|
||||
@@ -4265,21 +4266,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
||||
}
|
||||
if ($this->application->settings->is_gpu_enabled) {
|
||||
$docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [
|
||||
[
|
||||
'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'),
|
||||
'capabilities' => ['gpu'],
|
||||
'options' => data_get($this->application, 'settings.gpu_options', []),
|
||||
],
|
||||
$this->application->settings->gpuDeviceReservation(),
|
||||
];
|
||||
if (data_get($this->application, 'settings.gpu_count')) {
|
||||
$count = data_get($this->application, 'settings.gpu_count');
|
||||
if ($count === 'all') {
|
||||
$docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = $count;
|
||||
} else {
|
||||
$docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count;
|
||||
}
|
||||
} elseif (data_get($this->application, 'settings.gpu_device_ids')) {
|
||||
$docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($this->application, 'settings.gpu_device_ids');
|
||||
if (ApplicationSetting::parseGpuOptions($this->application->settings->gpu_options) === null) {
|
||||
$this->application_deployment_queue->addLogEntry('GPU options are not a valid key=value list or YAML mapping. They are ignored.');
|
||||
}
|
||||
}
|
||||
if ($this->application->isHealthcheckDisabled()) {
|
||||
@@ -4309,6 +4299,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
||||
|
||||
if ($this->pull_request_id === 0) {
|
||||
$custom_compose = convertDockerRunToCompose($this->application->custom_docker_run_options);
|
||||
if ($this->application->settings->is_gpu_enabled && data_get($custom_compose, 'deploy.resources.reservations.devices')) {
|
||||
data_forget($custom_compose, 'deploy.resources.reservations.devices');
|
||||
$this->application_deployment_queue->addLogEntry('GPU access is enabled in the GPU settings, so --gpus in Custom Docker Options is ignored.');
|
||||
}
|
||||
if ((bool) $this->application->settings->is_consistent_container_name_enabled) {
|
||||
$docker_compose['services'][$this->application->uuid] = $docker_compose['services'][$this->container_name];
|
||||
if ($this->container_name !== $this->application->uuid) {
|
||||
|
||||
@@ -260,11 +260,21 @@ class Advanced extends Component
|
||||
{
|
||||
try {
|
||||
$this->authorize('update', $this->application);
|
||||
if ($this->gpuCount && $this->gpuDeviceIds) {
|
||||
// Check only changed GPU values, so stored legacy values do not block other settings.
|
||||
$settings = $this->application->settings;
|
||||
$gpuSelectionChanged = $this->gpuCount !== $settings->gpu_count || $this->gpuDeviceIds !== $settings->gpu_device_ids;
|
||||
if ($gpuSelectionChanged && filled($this->gpuCount) && filled($this->gpuDeviceIds)) {
|
||||
$this->dispatch('error', 'You cannot set both GPU count and GPU device IDs.');
|
||||
$this->gpuCount = null;
|
||||
$this->gpuDeviceIds = null;
|
||||
$this->syncData(true);
|
||||
|
||||
return;
|
||||
}
|
||||
if ($this->gpuCount !== $settings->gpu_count && filled($this->gpuCount) && ! preg_match(ApplicationSetting::GPU_COUNT_PATTERN, trim($this->gpuCount))) {
|
||||
$this->dispatch('error', 'GPU count must be a number or "all".');
|
||||
|
||||
return;
|
||||
}
|
||||
if ($this->gpuOptions !== $settings->gpu_options && ApplicationSetting::parseGpuOptions($this->gpuOptions) === null) {
|
||||
$this->dispatch('error', 'GPU options must be key=value pairs (one per line) or a YAML mapping.');
|
||||
|
||||
return;
|
||||
}
|
||||
@@ -437,6 +447,33 @@ class Advanced extends Component
|
||||
}
|
||||
}
|
||||
|
||||
public function refreshGpus(): void
|
||||
{
|
||||
try {
|
||||
$servers = $this->application->maintenanceServers();
|
||||
foreach ($servers as $server) {
|
||||
$this->authorize('update', $server);
|
||||
}
|
||||
$functionalServers = $servers->filter(fn ($server) => $server->isFunctional());
|
||||
if ($functionalServers->isEmpty()) {
|
||||
$this->dispatch('error', 'Validate the server connection before you refresh GPUs.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$failed = $functionalServers->reject(fn ($server) => $server->gatherServerMetadata())->pluck('name');
|
||||
$this->application->unsetRelation('destination')->unsetRelation('additional_servers');
|
||||
if ($failed->isNotEmpty()) {
|
||||
$this->dispatch('error', 'Could not collect GPUs from '.$failed->implode(', ').'. Check the application logs for the remote command output.');
|
||||
|
||||
return;
|
||||
}
|
||||
$this->dispatch('success', 'GPUs refreshed.');
|
||||
} catch (\Throwable $e) {
|
||||
handleError($e, $this);
|
||||
}
|
||||
}
|
||||
|
||||
public function render()
|
||||
{
|
||||
$server = data_get($this->application, 'destination.server');
|
||||
@@ -444,6 +481,7 @@ class Advanced extends Component
|
||||
return view('livewire.project.application.advanced', [
|
||||
'isTraefikProxy' => $server?->proxyType() === ProxyTypes::TRAEFIK->value,
|
||||
'serverCertificateResolver' => $server?->traefikCertificateResolver() ?? ServerSetting::TRAEFIK_CERTIFICATE_RESOLVER_NONE,
|
||||
'gpuServers' => $this->application->maintenanceServers(),
|
||||
]);
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ namespace App\Models;
|
||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use OpenApi\Attributes as OA;
|
||||
use Symfony\Component\Yaml\Exception\ParseException;
|
||||
use Symfony\Component\Yaml\Yaml;
|
||||
|
||||
#[OA\Schema(
|
||||
description: 'Application settings.',
|
||||
@@ -59,6 +61,8 @@ class ApplicationSetting extends Model
|
||||
*/
|
||||
public const MAX_CONTAINER_NAME_PREFIX_LENGTH = 30;
|
||||
|
||||
public const GPU_COUNT_PATTERN = '/^(all|\d+)$/';
|
||||
|
||||
protected $casts = [
|
||||
'is_static' => 'boolean',
|
||||
'is_spa' => 'boolean',
|
||||
@@ -170,6 +174,81 @@ class ApplicationSetting extends Model
|
||||
return $this->stopGracePeriodSeconds();
|
||||
}
|
||||
|
||||
/**
|
||||
* The compose device request (deploy.resources.reservations.devices item) for the GPU settings.
|
||||
* A count wins over device IDs. Options that cannot be parsed are left out.
|
||||
*
|
||||
* @return array{capabilities: list<string>, driver?: string, count?: int|string, device_ids?: list<string>, options?: array<string, string>}
|
||||
*/
|
||||
public function gpuDeviceReservation(): array
|
||||
{
|
||||
$device = ['capabilities' => ['gpu']];
|
||||
$driver = trim((string) $this->gpu_driver);
|
||||
if ($driver !== '') {
|
||||
$device['driver'] = $driver;
|
||||
}
|
||||
|
||||
$count = trim((string) $this->gpu_count);
|
||||
$deviceIds = str((string) $this->gpu_device_ids)->explode(',')->map(fn (string $id) => trim($id))->filter(fn (string $id) => $id !== '')->values()->all();
|
||||
if ($count !== '') {
|
||||
$device['count'] = $count === 'all' ? 'all' : (int) $count;
|
||||
} elseif ($deviceIds !== []) {
|
||||
$device['device_ids'] = $deviceIds;
|
||||
}
|
||||
|
||||
$options = self::parseGpuOptions($this->gpu_options);
|
||||
if ($options) {
|
||||
$device['options'] = $options;
|
||||
}
|
||||
|
||||
return $device;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse GPU driver options written as a YAML/JSON mapping or as key=value pairs, one per line.
|
||||
*
|
||||
* @return array<string, string>|null null when the value is not a flat mapping
|
||||
*/
|
||||
public static function parseGpuOptions(?string $options): ?array
|
||||
{
|
||||
$options = trim((string) $options);
|
||||
if ($options === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
$parsed = Yaml::parse($options);
|
||||
} catch (ParseException) {
|
||||
$parsed = null;
|
||||
}
|
||||
if (is_array($parsed) && $parsed !== [] && ! array_is_list($parsed)) {
|
||||
$mapping = [];
|
||||
foreach ($parsed as $key => $value) {
|
||||
if (! is_scalar($value)) {
|
||||
return null;
|
||||
}
|
||||
$mapping[(string) $key] = is_bool($value) ? ($value ? 'true' : 'false') : (string) $value;
|
||||
}
|
||||
|
||||
return $mapping;
|
||||
}
|
||||
|
||||
$mapping = [];
|
||||
foreach (preg_split('/\R/', $options) as $pair) {
|
||||
$pair = trim($pair);
|
||||
if ($pair === '') {
|
||||
continue;
|
||||
}
|
||||
[$key, $value] = array_pad(explode('=', $pair, 2), 2, null);
|
||||
if ($value === null || trim($key) === '') {
|
||||
return null;
|
||||
}
|
||||
$mapping[trim($key)] = trim($value);
|
||||
}
|
||||
|
||||
return $mapping;
|
||||
}
|
||||
|
||||
public function isStatic(): Attribute
|
||||
{
|
||||
return Attribute::make(
|
||||
|
||||
+39
-1
@@ -2102,7 +2102,7 @@ $siteAddress {
|
||||
|
||||
try {
|
||||
$output = instant_remote_process([
|
||||
'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true)',
|
||||
'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true) && echo "---GPUS---" && (nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits 2>/dev/null || true) && echo "---NVIDIA_TOOLKIT---" && (command -v nvidia-container-runtime-hook nvidia-ctk 2>/dev/null || true)',
|
||||
], $this, false);
|
||||
|
||||
if (! $output) {
|
||||
@@ -2110,6 +2110,7 @@ $siteAddress {
|
||||
}
|
||||
|
||||
$sections = [];
|
||||
$sectionLines = [];
|
||||
$currentKey = null;
|
||||
foreach (explode("\n", trim($output)) as $line) {
|
||||
$line = trim($line);
|
||||
@@ -2117,6 +2118,9 @@ $siteAddress {
|
||||
$currentKey = $m[1];
|
||||
} elseif ($currentKey) {
|
||||
$sections[$currentKey] = $line;
|
||||
if ($line !== '') {
|
||||
$sectionLines[$currentKey][] = $line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2127,6 +2131,8 @@ $siteAddress {
|
||||
'cpus' => (int) ($sections['CPUS'] ?? 0),
|
||||
'memory_bytes' => (int) ($sections['MEMORY'] ?? 0),
|
||||
'uptime_since' => $sections['UPTIME_SINCE'] ?? null,
|
||||
'gpus' => self::parseNvidiaSmiGpus($sectionLines['GPUS'] ?? []),
|
||||
'nvidia_container_toolkit' => ($sectionLines['NVIDIA_TOOLKIT'] ?? []) !== [],
|
||||
'collected_at' => now()->toIso8601String(),
|
||||
];
|
||||
|
||||
@@ -2166,6 +2172,38 @@ $siteAddress {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits` lines.
|
||||
* The name is the last column because it can contain commas.
|
||||
*
|
||||
* @param list<string> $lines
|
||||
* @return list<array{index: string, uuid: string, memory_mb: int, name: string}>
|
||||
*/
|
||||
public static function parseNvidiaSmiGpus(array $lines): array
|
||||
{
|
||||
$gpus = [];
|
||||
foreach ($lines as $line) {
|
||||
$columns = array_map('trim', explode(',', $line, 4));
|
||||
if (count($columns) !== 4 || ! ctype_digit($columns[0])) {
|
||||
continue;
|
||||
}
|
||||
[$index, $uuid, $memory, $name] = $columns;
|
||||
$gpus[] = ['index' => $index, 'uuid' => $uuid, 'memory_mb' => (int) $memory, 'name' => $name];
|
||||
}
|
||||
|
||||
return $gpus;
|
||||
}
|
||||
|
||||
/**
|
||||
* GPUs found when the server details were last collected, or null when they were never checked.
|
||||
*
|
||||
* @return list<array{index: string, uuid: string, memory_mb: int, name: string}>|null
|
||||
*/
|
||||
public function detectedGpus(): ?array
|
||||
{
|
||||
return data_get($this->server_metadata, 'gpus');
|
||||
}
|
||||
|
||||
public function isTerminalEnabled()
|
||||
{
|
||||
return $this->settings->is_terminal_enabled ?? false;
|
||||
|
||||
@@ -212,9 +212,13 @@ function sharedDataApplications(?Application $application = null): array
|
||||
'is_log_drain_enabled' => 'boolean',
|
||||
'is_gpu_enabled' => 'boolean',
|
||||
'gpu_driver' => 'string|nullable',
|
||||
'gpu_count' => 'string|nullable',
|
||||
'gpu_count' => ['string', 'nullable', 'regex:'.ApplicationSetting::GPU_COUNT_PATTERN],
|
||||
'gpu_device_ids' => 'string|nullable',
|
||||
'gpu_options' => 'string|nullable',
|
||||
'gpu_options' => ['string', 'nullable', function (string $attribute, mixed $value, Closure $fail): void {
|
||||
if (ApplicationSetting::parseGpuOptions($value) === null) {
|
||||
$fail('The gpu_options must be key=value pairs or a YAML mapping.');
|
||||
}
|
||||
}],
|
||||
'is_consistent_container_name_enabled' => 'boolean',
|
||||
'custom_internal_name' => ['nullable', ...ValidationPatterns::containerNameRules()],
|
||||
'custom_container_name_prefix' => 'string|nullable|max:'.ApplicationSetting::MAX_CONTAINER_NAME_PREFIX_LENGTH,
|
||||
|
||||
@@ -1641,11 +1641,17 @@ function convertDockerRunToCompose(?string $custom_docker_run_options = null)
|
||||
foreach ($matches as $match) {
|
||||
$option = $match[1];
|
||||
if ($option === '--gpus') {
|
||||
$regexForParsingDeviceIds = '/device=([0-9A-Za-z-,]+)/';
|
||||
preg_match($regexForParsingDeviceIds, $custom_docker_run_options, $device_matches);
|
||||
$value = $device_matches[1] ?? 'all';
|
||||
$options[$option][] = $value;
|
||||
$options[$option] = array_unique($options[$option]);
|
||||
// --gpus all | --gpus 2 | --gpus "device=0,1"
|
||||
$gpuRequest = [];
|
||||
$gpuValue = trim($match[2] ?? '', '\'"');
|
||||
if (preg_match('/device=([0-9A-Za-z-,]+)/', $custom_docker_run_options, $device_matches)) {
|
||||
$gpuRequest['device_ids'] = explode(',', $device_matches[1]);
|
||||
} elseif (ctype_digit($gpuValue)) {
|
||||
$gpuRequest['count'] = (int) $gpuValue;
|
||||
}
|
||||
$options[$option] = $gpuRequest;
|
||||
|
||||
continue;
|
||||
}
|
||||
if ($option === '--hostname') {
|
||||
// Match --hostname=value or --hostname value
|
||||
@@ -1736,23 +1742,14 @@ function convertDockerRunToCompose(?string $custom_docker_run_options = null)
|
||||
$compose_options->put($mapping[$option], $value[0]);
|
||||
}
|
||||
} elseif ($option === '--gpus') {
|
||||
$payload = [
|
||||
'driver' => 'nvidia',
|
||||
'capabilities' => ['gpu'],
|
||||
];
|
||||
if (! is_null($value) && is_array($value) && count($value) > 0 && ! empty(trim($value[0]))) {
|
||||
if (str($value[0]) != 'all') {
|
||||
if (str($value[0])->contains(',')) {
|
||||
$payload['device_ids'] = str($value[0])->explode(',')->toArray();
|
||||
} else {
|
||||
$payload['device_ids'] = [$value[0]];
|
||||
}
|
||||
}
|
||||
}
|
||||
$compose_options->put('deploy', [
|
||||
'resources' => [
|
||||
'reservations' => [
|
||||
'devices' => [$payload],
|
||||
'devices' => [[
|
||||
'driver' => 'nvidia',
|
||||
'capabilities' => ['gpu'],
|
||||
...$value,
|
||||
]],
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
@props(['server'])
|
||||
|
||||
@php($gpus = $server->detectedGpus())
|
||||
|
||||
<div {{ $attributes->merge(['class' => 'text-[12px] leading-5 text-neutral-600 dark:text-fg-dim']) }}>
|
||||
@if ($gpus === null)
|
||||
<p>Not checked yet. Refresh to find GPUs.</p>
|
||||
@elseif ($gpus === [])
|
||||
<p>No NVIDIA GPU found (<span class="font-mono">nvidia-smi</span> is not installed or shows no GPU).</p>
|
||||
@else
|
||||
<ul>
|
||||
@foreach ($gpus as $gpu)
|
||||
<li wire:key="server-{{ $server->id }}-gpu-{{ $gpu['uuid'] }}">
|
||||
<span class="font-mono font-medium text-neutral-950 dark:text-fg">{{ $gpu['index'] }}</span>
|
||||
· {{ $gpu['name'] }} · {{ round($gpu['memory_mb'] / 1024, 1) }} GB ·
|
||||
<span class="font-mono">{{ $gpu['uuid'] }}</span>
|
||||
</li>
|
||||
@endforeach
|
||||
</ul>
|
||||
@unless (data_get($server->server_metadata, 'nvidia_container_toolkit'))
|
||||
<p class="mt-1 text-warning-900 dark:text-warning">
|
||||
NVIDIA Container Toolkit not found. Containers cannot use these GPUs until you
|
||||
<a href="https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html"
|
||||
class="underline" target="_blank">install it</a> and restart Docker.
|
||||
</p>
|
||||
@endunless
|
||||
@endif
|
||||
</div>
|
||||
@@ -203,6 +203,30 @@
|
||||
@if ($application->build_pack !== 'dockercompose')
|
||||
<x-application.settings-section id="advanced-gpu-section" title="GPU"
|
||||
helper="Give this application access to the host's GPUs. More info <a href='https://docs.docker.com/compose/gpu-support/' class='underline dark:text-white' target='_blank'>here</a>.">
|
||||
<x-slot:actions>
|
||||
@php($canRefreshGpus = $gpuServers->contains(fn ($gpuServer) => $gpuServer->isFunctional()))
|
||||
<x-forms.button type="button" wire:click="refreshGpus" canGate="update"
|
||||
:canResource="$gpuServers->first()" :disabled="! $canRefreshGpus"
|
||||
:tooltip="$canRefreshGpus ? null : 'Validate the server connection first.'">
|
||||
<x-reicon name="refresh" class="size-3.5" />
|
||||
Refresh GPUs
|
||||
</x-forms.button>
|
||||
</x-slot:actions>
|
||||
<x-callout type="info" title="Detected GPUs" class="mb-5">
|
||||
@foreach ($gpuServers as $gpuServer)
|
||||
<div wire:key="gpu-server-{{ $gpuServer->id }}" @class(['mt-2' => ! $loop->first])>
|
||||
<p class="font-medium text-neutral-700 dark:text-fg">
|
||||
<a href="{{ route('server.show', ['server_uuid' => $gpuServer->uuid]) }}"
|
||||
class="underline">{{ $gpuServer->name }}</a>
|
||||
@if (data_get($gpuServer->server_metadata, 'collected_at'))
|
||||
<span class="font-normal text-neutral-500 dark:text-fg-dim">· checked
|
||||
{{ \Illuminate\Support\Carbon::parse(data_get($gpuServer->server_metadata, 'collected_at'))->diffForHumans() }}</span>
|
||||
@endif
|
||||
</p>
|
||||
<x-server.gpu-list :server="$gpuServer" />
|
||||
</div>
|
||||
@endforeach
|
||||
</x-callout>
|
||||
<div class="grid w-full gap-4 sm:grid-cols-2">
|
||||
<x-forms.listbox id="isGpuEnabled" label="GPU access" onChange="instantSave"
|
||||
:options="[
|
||||
@@ -218,13 +242,14 @@
|
||||
targets="gpuDriver,gpuCount,gpuDeviceIds,gpuOptions" />
|
||||
<div class="grid gap-4 sm:grid-cols-2">
|
||||
<x-forms.input label="GPU driver" id="gpuDriver" canGate="update" :canResource="$application" />
|
||||
<x-forms.input label="GPU count" placeholder="Empty means use all GPUs" id="gpuCount"
|
||||
<x-forms.input label="GPU count" placeholder="Empty means use all GPUs" helper="A number, or all." id="gpuCount"
|
||||
canGate="update" :canResource="$application" />
|
||||
</div>
|
||||
<x-forms.input label="GPU device ids" placeholder="0,2"
|
||||
helper="Comma separated list of device ids. More info <a href='https://docs.docker.com/compose/gpu-support/#access-specific-devices' class='underline dark:text-white' target='_blank'>here</a>."
|
||||
id="gpuDeviceIds" canGate="update" :canResource="$application" />
|
||||
<x-forms.textarea rows="6" label="GPU options" id="gpuOptions" canGate="update"
|
||||
<x-forms.textarea rows="6" label="GPU options" id="gpuOptions" placeholder="key=value"
|
||||
helper="Driver options. Use one key=value pair per line, or a YAML mapping." canGate="update"
|
||||
:canResource="$application" />
|
||||
</form>
|
||||
@endif
|
||||
|
||||
@@ -21,3 +21,7 @@
|
||||
</div>
|
||||
@endforeach
|
||||
</dl>
|
||||
<div class="mt-5">
|
||||
<p class="text-xs font-medium text-neutral-500 dark:text-fg-dim">GPUs</p>
|
||||
<x-server.gpu-list :server="$server" class="mt-2" />
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
<?php
|
||||
|
||||
use App\Livewire\Project\Application\Advanced;
|
||||
use App\Models\Application;
|
||||
use App\Models\ApplicationSetting;
|
||||
use App\Models\Environment;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Livewire\Livewire;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
function createApplicationForGpuSettingsTest(): Application
|
||||
{
|
||||
$team = Team::factory()->create();
|
||||
$team->members()->attach(auth()->id(), ['role' => 'owner']);
|
||||
session(['currentTeam' => $team]);
|
||||
$server = Server::factory()->create(['team_id' => $team->id]);
|
||||
$project = Project::factory()->create(['team_id' => $team->id]);
|
||||
$environment = Environment::factory()->create(['project_id' => $project->id]);
|
||||
|
||||
return Application::create([
|
||||
'name' => 'gpu-settings-test-app',
|
||||
'git_repository' => 'https://github.com/coollabsio/coolify',
|
||||
'git_branch' => 'main',
|
||||
'build_pack' => 'nixpacks',
|
||||
'ports_exposes' => '3000',
|
||||
'environment_id' => $environment->id,
|
||||
'destination_id' => $server->standaloneDockers()->firstOrFail()->id,
|
||||
'destination_type' => $server->standaloneDockers()->firstOrFail()->getMorphClass(),
|
||||
]);
|
||||
}
|
||||
|
||||
it('builds a valid compose device request from the GPU settings', function (array $settings, array $expected) {
|
||||
$setting = new ApplicationSetting(['gpu_driver' => 'nvidia', ...$settings]);
|
||||
|
||||
expect($setting->gpuDeviceReservation())->toBe($expected);
|
||||
})->with([
|
||||
'all gpus' => [[], ['capabilities' => ['gpu'], 'driver' => 'nvidia']],
|
||||
'count' => [['gpu_count' => '2'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 2]],
|
||||
'count all' => [['gpu_count' => 'all'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 'all']],
|
||||
'device ids' => [['gpu_device_ids' => '0, 2,'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'device_ids' => ['0', '2']]],
|
||||
'count wins over device ids' => [['gpu_count' => '1', 'gpu_device_ids' => '0'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'count' => 1]],
|
||||
'blank driver' => [['gpu_driver' => ''], ['capabilities' => ['gpu']]],
|
||||
'key=value options' => [['gpu_options' => "virtualization=false\nfoo = bar"], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['virtualization' => 'false', 'foo' => 'bar']]],
|
||||
'option value with commas' => [['gpu_options' => 'capabilities=compute,utility'], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['capabilities' => 'compute,utility']]],
|
||||
'yaml options' => [['gpu_options' => "virtualization: false\nlevel: 2"], ['capabilities' => ['gpu'], 'driver' => 'nvidia', 'options' => ['virtualization' => 'false', 'level' => '2']]],
|
||||
'empty options' => [['gpu_options' => ''], ['capabilities' => ['gpu'], 'driver' => 'nvidia']],
|
||||
'invalid options are left out' => [['gpu_options' => 'not an option'], ['capabilities' => ['gpu'], 'driver' => 'nvidia']],
|
||||
]);
|
||||
|
||||
describe('advanced GPU form', function () {
|
||||
beforeEach(function () {
|
||||
$this->actingAs(User::factory()->create());
|
||||
});
|
||||
|
||||
it('saves valid GPU options', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application])
|
||||
->set('gpuCount', '2')
|
||||
->set('gpuOptions', 'virtualization=false')
|
||||
->call('submit')
|
||||
->assertDispatched('success');
|
||||
|
||||
$settings = $application->settings()->first();
|
||||
expect($settings->gpu_count)->toBe('2')
|
||||
->and($settings->gpu_options)->toBe('virtualization=false');
|
||||
});
|
||||
|
||||
it('rejects invalid GPU values', function (string $property, string $value) {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application])
|
||||
->set($property, $value)
|
||||
->call('submit')
|
||||
->assertDispatched('error')
|
||||
->assertNotDispatched('success');
|
||||
})->with([
|
||||
'count' => ['gpuCount', 'two'],
|
||||
'options' => ['gpuOptions', 'not an option'],
|
||||
]);
|
||||
|
||||
it('does not block other settings when a stored GPU value is invalid', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
$application->settings->update(['gpu_options' => 'legacy value', 'gpu_count' => 'some']);
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application->fresh()])
|
||||
->set('includeSourceCommitInBuild', true)
|
||||
->call('submit')
|
||||
->assertDispatched('success');
|
||||
|
||||
expect($application->settings()->first()->include_source_commit_in_build)->toBeTrue();
|
||||
});
|
||||
|
||||
it('keeps a stored GPU count and device IDs pair when other settings are saved', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
$application->settings->update(['gpu_count' => '1', 'gpu_device_ids' => '0']);
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application->fresh()])
|
||||
->set('includeSourceCommitInBuild', true)
|
||||
->call('submit')
|
||||
->assertDispatched('success');
|
||||
|
||||
$settings = $application->settings()->first();
|
||||
expect($settings->gpu_count)->toBe('1')
|
||||
->and($settings->gpu_device_ids)->toBe('0')
|
||||
->and($settings->include_source_commit_in_build)->toBeTrue();
|
||||
});
|
||||
|
||||
it('refreshes the GPUs of the application server', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
$server = $application->destination->server;
|
||||
$server->update(['private_key_id' => PrivateKey::factory()->create(['team_id' => $server->team_id])->id]);
|
||||
$server->settings->update(['is_reachable' => true, 'is_usable' => true]);
|
||||
Process::fake(['*' => Process::result(output: "---GPUS---\n0, GPU-1a2b, 81559, NVIDIA H100\n---NVIDIA_TOOLKIT---\n/usr/bin/nvidia-ctk")]);
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application])
|
||||
->call('refreshGpus')
|
||||
->assertDispatched('success')
|
||||
->assertSee('NVIDIA H100');
|
||||
|
||||
expect($server->fresh()->detectedGpus())->toBe([['index' => '0', 'uuid' => 'GPU-1a2b', 'memory_mb' => 81559, 'name' => 'NVIDIA H100']]);
|
||||
});
|
||||
|
||||
it('does not refresh GPUs when no server is validated', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
$application->destination->server->settings->update(['is_reachable' => false, 'is_usable' => false]);
|
||||
Process::fake();
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application])
|
||||
->call('refreshGpus')
|
||||
->assertDispatched('error')
|
||||
->assertNotDispatched('success');
|
||||
|
||||
Process::assertNothingRan();
|
||||
});
|
||||
|
||||
it('does not let a team member refresh GPUs', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
$server = $application->destination->server;
|
||||
$server->settings->update(['is_reachable' => true, 'is_usable' => true]);
|
||||
$member = User::factory()->create();
|
||||
$application->team()->members()->attach($member->id, ['role' => 'member']);
|
||||
$this->actingAs($member);
|
||||
Process::fake();
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application])
|
||||
->call('refreshGpus')
|
||||
->assertNotDispatched('success');
|
||||
|
||||
Process::assertNothingRan();
|
||||
expect($server->fresh()->detectedGpus())->toBeNull();
|
||||
});
|
||||
|
||||
it('rejects a change that sets both GPU count and device IDs without clearing stored values', function () {
|
||||
$application = createApplicationForGpuSettingsTest();
|
||||
$application->settings->update(['gpu_device_ids' => '0']);
|
||||
|
||||
Livewire::test(Advanced::class, ['application' => $application->fresh()])
|
||||
->set('gpuCount', '2')
|
||||
->call('submit')
|
||||
->assertDispatched('error')
|
||||
->assertNotDispatched('success');
|
||||
|
||||
$settings = $application->settings()->first();
|
||||
expect($settings->gpu_count)->toBeNull()
|
||||
->and($settings->gpu_device_ids)->toBe('0');
|
||||
});
|
||||
});
|
||||
@@ -233,3 +233,22 @@ test('ConvertIpAndIp6Together', function () {
|
||||
'ip6' => ['2001:db8::1'],
|
||||
]);
|
||||
});
|
||||
|
||||
test('ConvertGpusCount', function () {
|
||||
$output = convertDockerRunToCompose('--gpus 2');
|
||||
expect($output)->toBe([
|
||||
'deploy' => [
|
||||
'resources' => [
|
||||
'reservations' => [
|
||||
'devices' => [
|
||||
[
|
||||
'driver' => 'nvidia',
|
||||
'capabilities' => ['gpu'],
|
||||
'count' => 2,
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -220,3 +220,57 @@ it('does not call gatherServerMetadata when docker version validation fails', fu
|
||||
->instance()
|
||||
->validateDockerVersion();
|
||||
});
|
||||
|
||||
it('stores detected NVIDIA GPUs and the container toolkit when gathering server metadata', function (array $gpuLines, array $toolkitLines, array $expectedGpus, bool $expectedToolkit) {
|
||||
$this->server->settings->update([
|
||||
'is_reachable' => true,
|
||||
'is_usable' => true,
|
||||
]);
|
||||
|
||||
Process::fake([
|
||||
'*' => Process::result(output: implode("\n", [
|
||||
'---PRETTY_NAME---',
|
||||
'Ubuntu 24.04 LTS',
|
||||
'---CPUS---',
|
||||
'8',
|
||||
'---DOCKER---',
|
||||
'29.4.3-ce',
|
||||
'---COMPOSE---',
|
||||
'v2.32.4',
|
||||
'---GPUS---',
|
||||
...$gpuLines,
|
||||
'---NVIDIA_TOOLKIT---',
|
||||
...$toolkitLines,
|
||||
]), exitCode: 0),
|
||||
]);
|
||||
|
||||
$this->server->gatherServerMetadata();
|
||||
$server = $this->server->fresh();
|
||||
|
||||
expect($server->detectedGpus())->toBe($expectedGpus)
|
||||
->and($server->server_metadata['nvidia_container_toolkit'])->toBe($expectedToolkit)
|
||||
->and($server->server_metadata['cpus'])->toBe(8);
|
||||
})->with([
|
||||
'two gpus' => [
|
||||
['0, GPU-1a2b, 81559, NVIDIA H100 80GB HBM3', '1, GPU-3c4d, 24564, NVIDIA GeForce RTX 4090, Founders'],
|
||||
['/usr/bin/nvidia-container-runtime-hook', '/usr/bin/nvidia-ctk'],
|
||||
[
|
||||
['index' => '0', 'uuid' => 'GPU-1a2b', 'memory_mb' => 81559, 'name' => 'NVIDIA H100 80GB HBM3'],
|
||||
['index' => '1', 'uuid' => 'GPU-3c4d', 'memory_mb' => 24564, 'name' => 'NVIDIA GeForce RTX 4090, Founders'],
|
||||
],
|
||||
true,
|
||||
],
|
||||
'driver installed without a working gpu' => [
|
||||
['NVIDIA-SMI has failed because it could not communicate with the NVIDIA driver.'],
|
||||
[],
|
||||
[],
|
||||
false,
|
||||
],
|
||||
'no nvidia-smi' => [[], [], [], false],
|
||||
]);
|
||||
|
||||
it('returns null detected GPUs when GPUs were never checked', function () {
|
||||
$this->server->update(['server_metadata' => ['os' => 'Ubuntu 24.04 LTS']]);
|
||||
|
||||
expect($this->server->fresh()->detectedGpus())->toBeNull();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user