diff --git a/app/Actions/Service/DeployServiceApplication.php b/app/Actions/Service/DeployServiceApplication.php index 2a57b093de..78039a8252 100644 --- a/app/Actions/Service/DeployServiceApplication.php +++ b/app/Actions/Service/DeployServiceApplication.php @@ -34,6 +34,7 @@ class DeployServiceApplication 'echo '.escapeshellarg("Saved configuration files to {$workdir}."), 'touch '.escapeshellarg("{$workdir}/.env"), ...EnsureContentFilesOnServer::echoCommands($serviceApplication->fileStorages()->get(), $service->server), + ...StartService::composeVolumeWarningCommands($service), ]); if ($pullLatestImages) { diff --git a/app/Actions/Service/StartService.php b/app/Actions/Service/StartService.php index 09b63ceeb2..35a154b882 100644 --- a/app/Actions/Service/StartService.php +++ b/app/Actions/Service/StartService.php @@ -38,6 +38,7 @@ class StartService // but we guarantee it here in case of any edge cases or manual deployments $commands[] = "touch {$workdir}/.env"; $commands = array_merge($commands, EnsureContentFilesOnServer::echoCommands($this->contentFileStorages($service), $service->server)); + $commands = array_merge($commands, self::composeVolumeWarningCommands($service)); if ($pullLatestImages) { $commands[] = "echo 'Pulling images.'"; $commands[] = "docker compose --project-directory {$workdir} pull"; @@ -63,6 +64,20 @@ class StartService return remote_process($commands, $service->server, type_uuid: $service->uuid, callEventOnFinish: 'ServiceStatusChanged'); } + /** + * Shows the volume warnings of the last parse (for example an external volume that the + * service does not use yet) in the start log. + * + * @return list + */ + public static function composeVolumeWarningCommands(Service $service): array + { + return array_map( + fn (string $warning): string => 'echo '.escapeshellarg("Warning: {$warning}"), + $service->composeVolumeWarnings() + ); + } + /** * @return Collection */ diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 4678c7b214..b371452a74 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -743,7 +743,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->application_deployment_queue->addLogEntry('Build secrets are configured. Ensure your docker-compose file includes build.secrets configuration for services that need them.'); } } else { - $composeFile = $this->application->parse(pull_request_id: $this->pull_request_id, preview_id: data_get($this->preview, 'id'), commit: $this->commit); + $composeFile = $this->parseComposeFileForDeployment(); // Always add .env file to services $services = collect(data_get($composeFile, 'services', [])); $services = $services->map(function ($service, $name) { @@ -875,6 +875,22 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->application_deployment_queue->addLogEntry('New container started.'); } + /** + * Parses the Compose file for this deployment and writes the volume warnings of the parser + * (for example an external volume that the resource does not use yet) to the deployment log. + * + * @return Collection + */ + private function parseComposeFileForDeployment(): Collection + { + $composeFile = collect($this->application->parse(pull_request_id: $this->pull_request_id, preview_id: data_get($this->preview, 'id'), commit: $this->commit)); + foreach ($this->application->composeVolumeWarnings() as $warning) { + $this->application_deployment_queue->addLogEntry("Warning: {$warning}", 'stderr'); + } + + return $composeFile; + } + private function pull_docker_compose_images(): void { $this->application_deployment_queue->addLogEntry('Pulling image-based services before stopping the current deployment.'); diff --git a/app/Livewire/Project/Shared/Storages/All.php b/app/Livewire/Project/Shared/Storages/All.php index c71d411586..75d8a0dc2b 100644 --- a/app/Livewire/Project/Shared/Storages/All.php +++ b/app/Livewire/Project/Shared/Storages/All.php @@ -22,7 +22,7 @@ class All extends Component /** * Editable form state keyed by storage id. * - * @var array + * @var array */ public array $forms = []; @@ -207,6 +207,7 @@ class All extends Component 'canDeleteStale' => $this->canUpdate && ($storage->isServiceResource() || $storage->isDockerComposeResource()) && ! $storage->isDeclaredInCompose(), + 'replacedExternalVolume' => $this->isComposeOrService ? $storage->replacedExternalComposeVolume() : null, ]; } $this->forms = $forms; diff --git a/app/Models/Application.php b/app/Models/Application.php index 0bc88f592d..9db9d6d58d 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -13,6 +13,7 @@ use App\Support\DomainPortOverrides; use App\Support\DomainUrlParts; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; +use App\Traits\HasComposeVolumeWarnings; use App\Traits\HasConfiguration; use App\Traits\HasMetrics; use App\Traits\HasNoindexDomains; @@ -130,7 +131,7 @@ use Symfony\Component\Yaml\Yaml; class Application extends BaseModel { /** @use HasFactory */ - use Auditable, ClearsGlobalSearchCache, HasConfiguration, HasFactory, HasMetrics, HasNoindexDomains, HasSafeStringAttribute, HasSecretManager, ReleasesManagedDnsRecords, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasComposeVolumeWarnings, HasConfiguration, HasFactory, HasMetrics, HasNoindexDomains, HasSafeStringAttribute, HasSecretManager, ReleasesManagedDnsRecords, SoftDeletes; public const MAX_DOCKER_COMPOSE_SIZE_BYTES = 5 * 1024 * 1024; diff --git a/app/Models/LocalPersistentVolume.php b/app/Models/LocalPersistentVolume.php index ccd843d119..70a91be232 100644 --- a/app/Models/LocalPersistentVolume.php +++ b/app/Models/LocalPersistentVolume.php @@ -184,6 +184,7 @@ class LocalPersistentVolume extends BaseModel $compose = Yaml::parse($composeContent); $services = data_get($compose, 'services', []); + $topLevelVolumes = collect(data_get($compose, 'volumes') ?? []); if ($this->isServiceResource()) { $services = array_intersect_key($services, [$resource->name => true]); @@ -194,6 +195,11 @@ class LocalPersistentVolume extends BaseModel $parsedVolume = is_array($volume) ? $volume : parseDockerVolumeString($volume); $source = data_get($parsedVolume, 'source'); $target = data_get($parsedVolume, 'target'); + if ($source && isComposeExternalVolume($topLevelVolumes->get((string) $source))) { + // The parsers use an external volume as written. A storage entry with the generated + // name is an old entry that the user can delete (see replacedExternalComposeVolume()). + continue; + } $resourceUuid = $resource instanceof Application ? $resource->uuid : data_get($resource, 'service.uuid'); $generatedName = $source ? $resourceUuid.'_'.Str::slug($source, '-') : null; @@ -209,6 +215,50 @@ class LocalPersistentVolume extends BaseModel } } + /** + * The external Compose volume that this storage entry still replaces, or null. Before Coolify + * used external volumes as written, the parsers gave them a generated name, for example + * "{uuid}_{volume}" or "{uuid}_{volume}-pr-{id}". While this storage entry exists, the parsers + * keep that name so that the resource keeps its data (see useComposeExternalVolumeAsWritten()). + */ + public function replacedExternalComposeVolume(): ?string + { + try { + $resource = $this->resource; + if (! $resource) { + return null; + } + + $composeContent = $resource instanceof Application + ? $resource->docker_compose_raw + : data_get($resource, 'service.docker_compose_raw'); + if (blank($composeContent)) { + return null; + } + + foreach (data_get(Yaml::parse($composeContent), 'volumes') ?? [] as $key => $declaration) { + $key = (string) $key; + if (! isComposeExternalVolume($declaration)) { + continue; + } + + $legacyName = match (true) { + $resource instanceof Application && (int) $resource->compose_parsing_version < 3 => legacyApplicationComposeVolumeName($resource, $key, 0), + $resource instanceof Application => $resource->uuid.'_'.Str::slug($key, '-'), + default => data_get($resource, 'service.uuid').'_'.Str::slug($key, '-'), + }; + $isLegacyName = $legacyName !== $key && $this->name === $legacyName; + if ($isLegacyName || preg_match('/^'.preg_quote($legacyName, '/').'-pr-\d+$/', $this->name) === 1) { + return $key; + } + } + + return null; + } catch (\Throwable) { + return null; + } + } + // Check if this volume is read-only by parsing the docker-compose content public function isReadOnlyVolume(): bool { diff --git a/app/Models/Service.php b/app/Models/Service.php index 4cb8647f12..ecba7b383b 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -7,6 +7,7 @@ use App\Support\DomainPortOverrides; use App\Support\ResourceStartActivity; use App\Traits\Auditable; use App\Traits\ClearsGlobalSearchCache; +use App\Traits\HasComposeVolumeWarnings; use App\Traits\HasSafeStringAttribute; use App\Traits\HasSecretManager; use Illuminate\Database\Eloquent\Casts\Attribute; @@ -44,7 +45,7 @@ use Symfony\Component\Yaml\Yaml; )] class Service extends BaseModel { - use Auditable, ClearsGlobalSearchCache, HasFactory, HasSafeStringAttribute, HasSecretManager, SoftDeletes; + use Auditable, ClearsGlobalSearchCache, HasComposeVolumeWarnings, HasFactory, HasSafeStringAttribute, HasSecretManager, SoftDeletes; private static $parserVersion = '5'; diff --git a/app/Traits/HasComposeVolumeWarnings.php b/app/Traits/HasComposeVolumeWarnings.php new file mode 100644 index 0000000000..307119d0b8 --- /dev/null +++ b/app/Traits/HasComposeVolumeWarnings.php @@ -0,0 +1,33 @@ + */ + private array $composeVolumeWarnings = []; + + public function resetComposeVolumeWarnings(): void + { + $this->composeVolumeWarnings = []; + } + + public function addComposeVolumeWarning(string $warning): void + { + if (! in_array($warning, $this->composeVolumeWarnings, true)) { + $this->composeVolumeWarnings[] = $warning; + } + } + + /** + * @return list + */ + public function composeVolumeWarnings(): array + { + return $this->composeVolumeWarnings; + } +} diff --git a/bootstrap/helpers/parsers.php b/bootstrap/helpers/parsers.php index 3cb7cd6fba..bfd9af40df 100644 --- a/bootstrap/helpers/parsers.php +++ b/bootstrap/helpers/parsers.php @@ -349,6 +349,73 @@ function composeExternalVolumeDeclaration(iterable $topLevelVolumes, string $sou return $declaration; } +/** + * Tells if a parser uses the volume source as written because it is an external volume. + * + * Before Coolify used external volumes as written, the parsers renamed them like all other + * volumes (the old name, for example "{uuid}_{volume}" or "{uuid}_{volume}-pr-{id}"), so the + * resource wrote its data into the renamed volume. When the owner still has the storage entry + * with the old name, the parser must keep the old name, or the resource loses its data. Then + * this function records a warning on the resource and returns false. When the user deletes that + * storage entry, the next parse uses the external volume. + * + * Returns false also for a source that is not external. Then the parser continues as usual. + * + * @param iterable $topLevelVolumes + * @param string $legacyName the name that the parser gave the volume before it used external volumes as written + * + * @throws Exception If the external volume has a name that is not safe (see validateComposeExternalVolume()) + */ +function useComposeExternalVolumeAsWritten(Application|Service $resource, Application|ServiceApplication|ServiceDatabase $owner, iterable $topLevelVolumes, string $source, string $legacyName): bool +{ + $declaration = composeExternalVolumeDeclaration($topLevelVolumes, $source); + if ($declaration === null) { + return false; + } + if ($legacyName === $source) { + return true; + } + + $hasLegacyStorage = LocalPersistentVolume::query() + ->where('resource_type', get_class($owner)) + ->where('resource_id', $owner->getKey()) + ->where('name', $legacyName) + ->exists(); + if (! $hasLegacyStorage) { + return true; + } + + $resource->addComposeVolumeWarning(composeLegacyExternalVolumeWarning($source, $declaration, $legacyName)); + + return false; +} + +/** + * The name that the legacy application parsers (compose_parsing_version 1 and 2) gave a volume + * before they used external volumes as written: parser version 1 kept the name and added only the + * preview suffix, parser version 2 added the "{uuid}-" prefix too. + */ +function legacyApplicationComposeVolumeName(Application $resource, string $source, int $pull_request_id): string +{ + $name = addPreviewDeploymentSuffix($source, $pull_request_id); + + return $resource->compose_parsing_version === '2' ? "{$resource->uuid}-{$name}" : $name; +} + +/** + * The warning for an external volume that the resource does not use yet (see useComposeExternalVolumeAsWritten()). + * The volume names are validated and contain no secrets. + * + * @param array $declaration + */ +function composeLegacyExternalVolumeWarning(string $source, array $declaration, string $legacyName): string +{ + $dockerVolume = data_get($declaration, 'name') ?? data_get($declaration, 'external.name') ?? $source; + $external = $dockerVolume === $source ? 'external' : "external (Docker volume '{$dockerVolume}')"; + + return "Volume '{$source}' is declared as {$external}, but Coolify still uses '{$legacyName}' because this resource used it before. To use the external volume, copy your data into it and delete the storage entry '{$legacyName}', then redeploy."; +} + /** * The key and the name of an external volume must be literal Docker volume names. Docker Compose * can resolve variables in `name:`, but then only the deployment `.env` sets which existing volume @@ -740,6 +807,7 @@ function removeComposeVolumeFieldsPreservingComments(string $source, array $clea function applicationParser(Application $resource, int $pull_request_id = 0, ?int $preview_id = null, ?string $commit = null): Collection { + $resource->resetComposeVolumeWarnings(); $uuid = data_get($resource, 'uuid'); $compose = data_get($resource, 'docker_compose_raw'); // Store original compose for later use to update docker_compose_raw with content removed @@ -1201,7 +1269,11 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int } } } elseif ($type->value() === 'volume') { - if (composeExternalVolumeDeclaration($topLevel->get('volumes'), $source->value()) !== null) { + $legacyName = "{$uuid}_".Str::slug($source, '-'); + if ($isPullRequest) { + $legacyName = addPreviewDeploymentSuffix($legacyName, $pull_request_id); + } + if (useComposeExternalVolumeAsWritten($resource, $originalResource, $topLevel->get('volumes'), $source->value(), $legacyName)) { // Previews share the external volume. It gets no row, so Coolify never removes it. $volumesParsed->put($index, $volume); @@ -1913,6 +1985,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int function serviceParser(Service $resource): Collection { + $resource->resetComposeVolumeWarnings(); $uuid = data_get($resource, 'uuid'); $compose = data_get($resource, 'docker_compose_raw'); // Store original compose for later use to update docker_compose_raw with content removed @@ -2556,7 +2629,7 @@ function serviceParser(Service $resource): Collection } } } elseif ($type->value() === 'volume') { - if (composeExternalVolumeDeclaration($topLevel->get('volumes'), $source->value()) !== null) { + if (useComposeExternalVolumeAsWritten($resource, $originalResource, $topLevel->get('volumes'), $source->value(), "{$uuid}_".Str::slug($source, '-'))) { // The external volume gets no row, so Coolify never removes it. $volumesParsed->put($index, $volume); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index afc15e20e5..7f882bcaab 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2586,6 +2586,7 @@ function customApiValidator(Collection|array $item, array $rules, array $message } function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id = 0, ?int $preview_id = null) { + $resource->resetComposeVolumeWarnings(); if ($resource->getMorphClass() === Service::class) { if ($resource->docker_compose_raw) { // Extract inline comments from raw YAML before Symfony parser discards them @@ -2807,7 +2808,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal // Collect/create/update volumes if ($serviceVolumes->count() > 0) { - $serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) { + $serviceVolumes = $serviceVolumes->map(function ($volume) use ($resource, $savedService, $topLevelVolumes) { $type = null; $source = null; $target = null; @@ -2867,7 +2868,8 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal ] ); } elseif ($type->value() === 'volume') { - if (composeExternalVolumeDeclaration($topLevelVolumes, $source->value()) !== null) { + $legacyName = "{$savedService->service->uuid}_".Str::slug($source, '-'); + if (useComposeExternalVolumeAsWritten($resource, $savedService, $topLevelVolumes, $source->value(), $legacyName)) { // The external volume gets no row, so Coolify never removes it. return $volume; } @@ -3390,10 +3392,11 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal } $server = $resource->destination->server; $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); - // External volumes are used as written, also by previews (no prefix and no preview suffix). - $externalTopLevelVolumes = $topLevelVolumes->filter(fn (mixed $volume): bool => isComposeExternalVolume($volume)); + // Legacy Compose applications (parser versions 1 and 2) never stored their volumes, so Coolify + // cannot tell which external volume already holds data. They keep the old volume names. + $externalTopLevelVolumes = collect([]); if ($pull_request_id !== 0) { - $topLevelVolumes = collect($externalTopLevelVolumes->all()); + $topLevelVolumes = collect([]); } if ($topLevelVolumes->count() > 0) { @@ -3481,7 +3484,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $name = addPreviewDeploymentSuffix($name, $pull_request_id); } $volume = str("$name:$mount"); - } elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, $name->value()) !== null) { + } elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, $name->value(), legacyApplicationComposeVolumeName($resource, $name->value(), $pull_request_id))) { // An external volume is used as written. } else { if ($pull_request_id !== 0) { @@ -3550,7 +3553,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal } else { data_set($volume, 'source', $source.':'.$target); } - } elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, (string) $source) !== null) { + } elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, (string) $source, legacyApplicationComposeVolumeName($resource, (string) $source, $pull_request_id))) { // An external volume is used as written. data_set($volume, 'source', $source.':'.$target.($read_only ? ':ro' : '')); } else { @@ -3610,7 +3613,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $name = addPreviewDeploymentSuffix($name, $pull_request_id); } $volume = str("$name:$mount"); - } elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, $name->value()) !== null) { + } elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, $name->value(), legacyApplicationComposeVolumeName($resource, $name->value(), $pull_request_id))) { // An external volume is used as written. } else { if ($pull_request_id !== 0) { @@ -3681,7 +3684,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal } else { data_set($volume, 'source', $source.':'.$target); } - } elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, (string) $source) !== null) { + } elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, (string) $source, legacyApplicationComposeVolumeName($resource, (string) $source, $pull_request_id))) { // An external volume is used as written. data_set($volume, 'source', $source.':'.$target.($read_only ? ':ro' : '')); } else { diff --git a/resources/views/livewire/project/shared/storages/all.blade.php b/resources/views/livewire/project/shared/storages/all.blade.php index d38d7e605e..26c9a6cf1f 100644 --- a/resources/views/livewire/project/shared/storages/all.blade.php +++ b/resources/views/livewire/project/shared/storages/all.blade.php @@ -62,6 +62,11 @@ @if (blank($storage->host_path)) Volume mount @endif + @if ($form['replacedExternalVolume']) + + Replaces the external volume '{{ $form['replacedExternalVolume'] }}'. Copy the data into the external volume, then delete this entry to use it. + + @endif @if ($hasSourcePaths) @@ -149,9 +154,12 @@ 'label' => 'Also permanently delete the Docker volume and all its data.', 'default_warning' => 'The Docker volume and its data will not be deleted.', ]]" - :actions="[ + :actions="array_values(array_filter([ 'This removes only the stale volume entry from Coolify.', - ]" confirmationText="{{ $form['name'] }}" + $form['replacedExternalVolume'] + ? 'The next deployment uses the external volume \''.$form['replacedExternalVolume'].'\' instead of this volume.' + : null, + ]))" confirmationText="{{ $form['name'] }}" confirmationLabel="Please confirm by entering the Storage Name below" shortConfirmationLabel="Storage Name" /> @endif diff --git a/tests/Feature/ApplicationComposeInjectionValidationTest.php b/tests/Feature/ApplicationComposeInjectionValidationTest.php index a9f5ed20c8..5ed51f2160 100644 --- a/tests/Feature/ApplicationComposeInjectionValidationTest.php +++ b/tests/Feature/ApplicationComposeInjectionValidationTest.php @@ -343,7 +343,7 @@ test('the Compose deployment validates the repository file before any command us expect($loadPosition)->not->toBeFalse() ->and($body)->not->toContain('$this->application->loadComposeFile(') ->and($loadPosition)->toBeLessThan(strpos($body, '$this->application->oldRawParser()')) - ->and($loadPosition)->toBeLessThan(strpos($body, '$this->application->parse(')) + ->and($loadPosition)->toBeLessThan(strpos($body, '$this->parseComposeFileForDeployment()')) ->and($loadPosition)->toBeLessThan(strpos($body, 'base64 -d')) ->and($body)->toContain('"stat -c \'%F\' ".escapeshellarg($realPathInGit)'); }); diff --git a/tests/Feature/ComposeExternalVolumeTest.php b/tests/Feature/ComposeExternalVolumeTest.php index 7727d6f119..7cf697853f 100644 --- a/tests/Feature/ComposeExternalVolumeTest.php +++ b/tests/Feature/ComposeExternalVolumeTest.php @@ -1,7 +1,12 @@ pluck('name')->sort()->values()->all(); } +/** + * The storage entry that the parsers made for an external volume before they used it as written. + */ +function legacyExternalVolumeRow(Model $owner, string $name, string $mountPath = '/data'): LocalPersistentVolume +{ + return $owner->persistentStorages()->create(['name' => $name, 'mount_path' => $mountPath]); +} + +function legacyExternalVolumeServiceResource(Service $service, string $name, string $compose = EXTERNAL_VOLUME_SHORT_COMPOSE): ServiceApplication|ServiceDatabase +{ + $image = data_get(Yaml::parse($compose), "services.{$name}.image"); + + return isDatabaseImage($image) + ? ServiceDatabase::create(['name' => $name, 'service_id' => $service->id, 'image' => $image]) + : ServiceApplication::create(['name' => $name, 'service_id' => $service->id, 'image' => $image]); +} + +function legacyExternalVolumeWarning(string $oldName, ?string $dockerVolume = null): string +{ + $external = $dockerVolume === null ? 'external' : "external (Docker volume '{$dockerVolume}')"; + + return "Volume 'shared-data' is declared as {$external}, but Coolify still uses '{$oldName}' because this resource used it before. To use the external volume, copy your data into it and delete the storage entry '{$oldName}', then redeploy."; +} + +/** + * Runs the Compose parse step of a deployment job. + * + * @return array{0: Collection, 1: list} + */ +function parseExternalVolumeDeployment(Application $application, int $pullRequestId = 0, ?ApplicationPreview $preview = null): array +{ + $job = (new ReflectionClass(ApplicationDeploymentJob::class))->newInstanceWithoutConstructor(); + $logEntries = new ArrayObject; + $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial(); + $queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message, string $type = 'stdout') use ($logEntries): void { + $logEntries[] = [$message, $type]; + }); + + foreach ([ + 'application' => $application, + 'application_deployment_queue' => $queue, + 'pull_request_id' => $pullRequestId, + 'preview' => $preview, + 'commit' => 'HEAD', + ] as $property => $value) { + (new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($job, $value); + } + + $composeFile = (new ReflectionMethod(ApplicationDeploymentJob::class, 'parseComposeFileForDeployment'))->invoke($job); + + return [$composeFile, $logEntries->getArrayCopy()]; +} + +function loginExternalVolumeOwner(): void +{ + test()->withoutVite(); + $team = test()->environment->project->team; + $user = User::factory()->create(); + $team->members()->attach($user->id, ['role' => 'owner']); + test()->actingAs($user); + session(['currentTeam' => $team]); +} + /** * @param array $commands */ @@ -213,19 +298,81 @@ describe('applicationParser', function () { ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data-pr-42"]); }); - it('keeps an existing row of a volume that an earlier parse prefixed', function () { + it('does not show a warning when the resource has no old storage entry', function () { $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); - $uuid = $application->uuid; - $oldVolume = $application->persistentStorages()->create([ - 'name' => "{$uuid}_shared-data", - 'mount_path' => '/data', - ]); applicationParser($application); - expect(LocalPersistentVolume::find($oldVolume->id))->not->toBeNull() - ->and(LocalPersistentVolume::find($oldVolume->id)->name)->toBe("{$uuid}_shared-data") - ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data", "{$uuid}_shared-data"]); + expect($application->composeVolumeWarnings())->toBe([]); + }); + + it('keeps the old prefixed volume and shows a warning when the resource has its old storage entry', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $oldVolume = legacyExternalVolumeRow($application, "{$uuid}_shared-data"); + + $compose = applicationParser($application)->toArray(); + + expect($compose['services']['web']['volumes'])->toContain("{$uuid}_shared-data:/data:ro") + ->not->toContain('shared-data:/data:ro') + ->and($compose['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"]) + ->and(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data") + ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data", "{$uuid}_shared-data"]) + ->and($application->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data")]); + }); + + it('keeps the old prefixed volume in long syntax and names the external Docker volume in the warning', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_LONG_COMPOSE); + $uuid = $application->uuid; + legacyExternalVolumeRow($application, "{$uuid}_shared-data"); + + $compose = applicationParser($application)->toArray(); + $sources = collect($compose['services']['web']['volumes'])->pluck('source')->all(); + + expect($sources)->toBe(["{$uuid}_shared-data", "{$uuid}_app-data"]) + ->and($compose['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"]) + ->and($application->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data", 'existing-shared-volume')]); + }); + + it('keeps the old prefixed preview volume when the resource has its old storage entry', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $preview = externalVolumePreview($application); + legacyExternalVolumeRow($application, "{$uuid}_shared-data-pr-42"); + + $compose = applicationParser($application, 42, $preview->id)->toArray(); + + expect($compose['services']['web-pr-42']['volumes'])->toContain("{$uuid}_shared-data-pr-42:/data:ro") + ->and($compose['volumes']["{$uuid}_shared-data-pr-42"])->toBe(['name' => "{$uuid}_shared-data-pr-42"]) + ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data-pr-42", "{$uuid}_shared-data-pr-42"]) + ->and($application->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data-pr-42")]); + }); + + it('uses the external volume in a preview when only the production volume has an old storage entry', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $preview = externalVolumePreview($application); + legacyExternalVolumeRow($application, "{$uuid}_shared-data"); + + $compose = applicationParser($application, 42, $preview->id)->toArray(); + + expect($compose['services']['web-pr-42']['volumes'])->toContain('shared-data:/data:ro') + ->and($application->composeVolumeWarnings())->toBe([]); + }); + + it('uses the external volume after the old storage entry is deleted', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $oldVolume = legacyExternalVolumeRow($application, "{$uuid}_shared-data"); + applicationParser($application); + + $oldVolume->delete(); + $compose = applicationParser($application)->toArray(); + + expect($compose['services']['web']['volumes'])->toContain('shared-data:/data:ro') + ->and($compose['volumes'])->not->toHaveKey("{$uuid}_shared-data") + ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"]) + ->and($application->composeVolumeWarnings())->toBe([]); }); it('does not change nfs volumes', function () { @@ -265,19 +412,37 @@ describe('serviceParser', function () { ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"]); }); - it('keeps an existing row of a volume that an earlier parse prefixed', function () { + it('keeps the old prefixed volume and shows a warning when the service has its old storage entry', function (string $compose, string $serviceName, string $mountPath, string $mode) { + $service = externalVolumeService($compose); + $uuid = $service->uuid; + $owner = legacyExternalVolumeServiceResource($service, $serviceName, $compose); + $oldVolume = legacyExternalVolumeRow($owner, "{$uuid}_shared-data", $mountPath); + + $parsed = serviceParser($service)->toArray(); + + expect($parsed['services'][$serviceName]['volumes'])->toContain("{$uuid}_shared-data:{$mountPath}{$mode}") + ->and($parsed['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"]) + ->and(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data") + ->and(LocalPersistentVolume::query()->where('name', "{$uuid}_shared-data")->count())->toBe(1) + ->and($service->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data")]); + })->with([ + 'service application' => [EXTERNAL_VOLUME_SHORT_COMPOSE, 'web', '/data', ':ro'], + 'service database' => [EXTERNAL_VOLUME_DATABASE_COMPOSE, 'db', '/var/lib/postgresql/data', ''], + ]); + + it('uses the external volume after the old storage entry of the service is deleted', function () { $service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE); $uuid = $service->uuid; - $serviceApplication = ServiceApplication::create(['name' => 'web', 'service_id' => $service->id, 'image' => 'nginx:alpine']); - $oldVolume = $serviceApplication->persistentStorages()->create([ - 'name' => "{$uuid}_shared-data", - 'mount_path' => '/data', - ]); - + $oldVolume = legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data"); serviceParser($service); - expect(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data") - ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data", "{$uuid}_shared-data"]); + $oldVolume->delete(); + $parsed = serviceParser($service)->toArray(); + + expect($parsed['services']['web']['volumes'])->toContain('shared-data:/data:ro') + ->and($parsed['volumes'])->not->toHaveKey("{$uuid}_shared-data") + ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"]) + ->and($service->composeVolumeWarnings())->toBe([]); }); it('does not change nfs volumes', function () { @@ -310,43 +475,66 @@ describe('legacy parsers', function () { 'long syntax' => [EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data'], ]); - it('keeps an existing row of a volume that an earlier legacy service parse prefixed', function () { + it('keeps the old prefixed volume of a legacy service that has its old storage entry', function (string $compose, string $expectedVolume) { + $service = externalVolumeService($compose, '2'); + $uuid = $service->uuid; + $oldVolume = legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data"); + + $parsed = parseDockerComposeFile($service)->toArray(); + $volumes = collect($parsed['services']['web']['volumes']) + ->map(fn (mixed $volume): string => is_array($volume) ? $volume['source'] : $volume) + ->all(); + + expect($volumes)->toContain(str_replace('{uuid}', $uuid, $expectedVolume)) + ->and($parsed['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"]) + ->and(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data") + ->and(LocalPersistentVolume::query()->where('name', "{$uuid}_shared-data")->count())->toBe(1) + ->and($service->composeVolumeWarnings())->toHaveCount(1) + ->and($service->composeVolumeWarnings()[0])->toStartWith("Volume 'shared-data' is declared as external") + ->toContain("Coolify still uses '{$uuid}_shared-data'"); + })->with([ + 'short syntax' => [EXTERNAL_VOLUME_SHORT_COMPOSE, '{uuid}_shared-data:/data'], + 'long syntax' => [EXTERNAL_VOLUME_LONG_COMPOSE, '{uuid}_shared-data'], + ]); + + it('uses the external volume after the old storage entry of a legacy service is deleted', function () { $service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE, '2'); $uuid = $service->uuid; - $serviceApplication = ServiceApplication::create(['name' => 'web', 'service_id' => $service->id, 'image' => 'nginx:alpine']); - $oldVolume = $serviceApplication->persistentStorages()->create([ - 'name' => "{$uuid}_shared-data", - 'mount_path' => '/data', - ]); - + $oldVolume = legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data"); parseDockerComposeFile($service); - expect(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data"); + $oldVolume->delete(); + $parsed = parseDockerComposeFile($service)->toArray(); + + expect($parsed['services']['web']['volumes'])->toContain('shared-data:/data:ro') + ->and($parsed['volumes'])->not->toHaveKey("{$uuid}_shared-data") + ->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"]) + ->and($service->composeVolumeWarnings())->toBe([]); }); - it('keeps an external volume in a legacy application', function (string $parsingVersion, int $pullRequestId, string $compose, string $expectedExternal, string $expectedOther) { + it('keeps the old volume names of a legacy application, because it has no storage entries to check', function (string $parsingVersion, int $pullRequestId, string $compose, string $expectedVolume) { + // Legacy Compose applications (parser versions 1 and 2) never stored their volumes, so Coolify + // cannot tell which external volume already holds data: they always keep the old names. $application = externalVolumeApplication($compose, $parsingVersion); $uuid = $application->uuid; $previewId = $pullRequestId === 0 ? null : externalVolumePreview($application, $pullRequestId)->id; $parsed = parseDockerComposeFile($application, pull_request_id: $pullRequestId, preview_id: $previewId)->toArray(); $serviceName = $pullRequestId === 0 ? 'web' : "web-pr-{$pullRequestId}"; - $expectedOther = str_replace('{uuid}', $uuid, $expectedOther); + $expectedVolume = str_replace('{uuid}', $uuid, $expectedVolume); + $expectedName = explode(':', $expectedVolume)[0]; - expect($parsed['services'][$serviceName]['volumes'])->toContain($expectedExternal) - ->toContain($expectedOther) - ->and($parsed['volumes']['shared-data']['external'])->toBeTrue() - ->and(array_keys($parsed['volumes']))->not->toContain('shared-data-pr-42') - ->not->toContain("{$uuid}-shared-data") - ->not->toContain("{$uuid}-shared-data-pr-42"); + expect($parsed['services'][$serviceName]['volumes'])->toContain($expectedVolume) + ->and(array_keys($parsed['volumes'] ?? []))->toContain($expectedName) + ->and($application->composeVolumeWarnings())->toBe([]); })->with([ - 'v1 short' => ['1', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', 'app-data:/app'], - 'v1 short preview' => ['1', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', 'app-data-pr-42:/app'], - 'v1 long preview' => ['1', 42, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data:/data', 'app-data-pr-42:/app'], - 'v2 short' => ['2', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', '{uuid}-app-data:/app'], - 'v2 short preview' => ['2', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', '{uuid}-app-data-pr-42:/app'], - 'v2 long' => ['2', 0, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data:/data', '{uuid}-app-data:/app'], - 'v2 long preview' => ['2', 42, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data:/data', '{uuid}-app-data-pr-42:/app'], + 'v1 short' => ['1', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro'], + 'v1 short preview' => ['1', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data-pr-42:/data:ro'], + 'v1 long preview' => ['1', 42, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data-pr-42:/data'], + 'v2 short' => ['2', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, '{uuid}-shared-data:/data:ro'], + 'v2 short preview' => ['2', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, '{uuid}-shared-data-pr-42:/data:ro'], + 'v2 long' => ['2', 0, EXTERNAL_VOLUME_LONG_COMPOSE, '{uuid}-shared-data:/data'], + 'v2 long preview' => ['2', 42, EXTERNAL_VOLUME_LONG_COMPOSE, '{uuid}-shared-data-pr-42:/data'], ]); }); @@ -418,6 +606,69 @@ describe('delete', function () { ->and($application->docker_compose)->toContain("shared-data:\n external: true"); }); + it('removes only the old prefixed volume when a service with an old storage entry is deleted with its volumes', function () { + $service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $service->uuid; + legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data"); + serviceParser($service); + $service->delete(); + $commands = []; + fakeExternalVolumeServer($commands); + + (new DeleteResourceJob($service))->handle(); + + $all = implode("\n", $commands); + expect($all)->toContain("docker volume rm -f '{$uuid}_shared-data'") + ->toContain("docker volume rm -f '{$uuid}_app-data'") + ->not->toContain("'shared-data'"); + }); + + it('lets the storage list delete the old storage entry without removing the external volume', function (bool $deleteDockerVolume) { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $oldVolume = legacyExternalVolumeRow($application, "{$uuid}_shared-data"); + applicationParser($application); + loginExternalVolumeOwner(); + $commands = []; + fakeExternalVolumeServer($commands); + + Livewire::test(All::class, ['resource' => $application->fresh()]) + ->assertSee('Replaces the external volume') + ->call('delete', $oldVolume->id, 'password', $deleteDockerVolume ? ['deleteDockerVolume'] : []) + ->assertHasNoErrors() + ->assertReturned(true); + + $volumeCommands = collect($commands)->filter(fn (string $command): bool => str_contains($command, 'docker volume'))->values(); + expect(LocalPersistentVolume::find($oldVolume->id))->toBeNull() + ->and($volumeCommands)->toHaveCount($deleteDockerVolume ? 1 : 0) + ->and($volumeCommands->implode("\n"))->not->toContain("'shared-data'"); + if ($deleteDockerVolume) { + expect($volumeCommands->first())->toContain("docker volume rm -f '{$uuid}_shared-data'"); + } + + $compose = applicationParser($application)->toArray(); + expect($compose['services']['web']['volumes'])->toContain('shared-data:/data:ro') + ->and($application->composeVolumeWarnings())->toBe([]); + })->with([ + 'keep the Docker volume' => [false], + 'delete the old Docker volume' => [true], + ]); + + it('does not let the storage list delete a storage entry of a volume that is not external', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + applicationParser($application); + $volume = LocalPersistentVolume::query()->where('name', "{$application->uuid}_app-data")->firstOrFail(); + loginExternalVolumeOwner(); + + Livewire::test(All::class, ['resource' => $application->fresh()]) + ->assertDontSee('Replaces the external volume') + ->call('delete', $volume->id, 'password', []) + ->assertHasNoErrors() + ->assertReturned(false); + + expect(LocalPersistentVolume::find($volume->id))->not->toBeNull(); + }); + it('does not remove the external volume when a preview is deleted', function () { $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); $uuid = $application->uuid; @@ -431,4 +682,92 @@ describe('delete', function () { expect($volumeCommands)->toContain("docker volume rm -f '{$uuid}_app-data-pr-42'") ->not->toContain('shared-data'); }); + + it('removes the old prefixed preview volume, but not the external volume, when a preview is deleted', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $preview = externalVolumePreview($application); + legacyExternalVolumeRow($application, "{$uuid}_shared-data-pr-42"); + $commands = []; + fakeExternalVolumeServer($commands); + + $preview->forceDelete(); + + $volumeCommands = collect($commands)->filter(fn (string $command): bool => str_contains($command, 'docker volume'))->implode("\n"); + expect($volumeCommands)->toContain("docker volume rm -f '{$uuid}_shared-data-pr-42'") + ->not->toContain("'shared-data'"); + }); +}); + +describe('warnings', function () { + it('writes the warning to the deployment log after the parse', function (int $pullRequestId, string $oldName, string $expectedVolume) { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $application->uuid; + $preview = $pullRequestId === 0 ? null : externalVolumePreview($application, $pullRequestId); + legacyExternalVolumeRow($application, str_replace('{uuid}', $uuid, $oldName)); + + [$composeFile, $logEntries] = parseExternalVolumeDeployment($application, $pullRequestId, $preview); + + $serviceName = $pullRequestId === 0 ? 'web' : "web-pr-{$pullRequestId}"; + expect(data_get($composeFile, "services.{$serviceName}.volumes"))->toContain(str_replace('{uuid}', $uuid, $expectedVolume)) + ->and($logEntries)->toBe([['Warning: '.legacyExternalVolumeWarning(str_replace('{uuid}', $uuid, $oldName)), 'stderr']]); + })->with([ + 'production' => [0, '{uuid}_shared-data', '{uuid}_shared-data:/data:ro'], + 'preview' => [42, '{uuid}_shared-data-pr-42', '{uuid}_shared-data-pr-42:/data:ro'], + ]); + + it('writes no warning to the deployment log when the external volume is used as written', function () { + $application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE); + + [$composeFile, $logEntries] = parseExternalVolumeDeployment($application); + + expect(data_get($composeFile, 'services.web.volumes'))->toContain('shared-data:/data:ro') + ->and($logEntries)->toBe([]); + }); + + it('shows the warning before docker compose up when a service starts', function () { + $service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $service->uuid; + legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data"); + + $command = StartService::run($service->fresh())->getExtraProperty('command'); + + $echo = 'echo '.escapeshellarg('Warning: '.legacyExternalVolumeWarning("{$uuid}_shared-data")); + expect($command)->toContain($echo) + ->and(strpos($command, $echo))->toBeLessThan(strpos($command, ' up -d')); + }); + + it('shows the warning before docker compose up when a service application is deployed', function () { + $service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE); + $uuid = $service->uuid; + $serviceApplication = legacyExternalVolumeServiceResource($service, 'web'); + legacyExternalVolumeRow($serviceApplication, "{$uuid}_shared-data"); + + $command = DeployServiceApplication::run($serviceApplication->fresh())->getExtraProperty('command'); + + $echo = 'echo '.escapeshellarg('Warning: '.legacyExternalVolumeWarning("{$uuid}_shared-data")); + expect($command)->toContain($echo) + ->and(strpos($command, $echo))->toBeLessThan(strpos($command, ' up -d')); + }); + + it('keeps the warning line unchanged for a server with a non-root user', function () { + $service = externalVolumeService(EXTERNAL_VOLUME_LONG_COMPOSE); + $uuid = $service->uuid; + legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data"); + serviceParser($service); + $service->server->update(['user' => 'ubuntu']); + + $commands = StartService::composeVolumeWarningCommands($service); + + expect($commands)->toBe(['echo '.escapeshellarg('Warning: '.legacyExternalVolumeWarning("{$uuid}_shared-data", 'existing-shared-volume'))]) + ->and(parseCommandsByLineForSudo(collect($commands), $service->server->fresh()))->toBe($commands); + }); + + it('shows no warning when a service starts with the external volume as written', function () { + $service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE); + + $command = StartService::run($service->fresh())->getExtraProperty('command'); + + expect($command)->not->toContain('declared as external'); + }); });