starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-06 13:10:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(compose): keep the old volume of existing resources for external volumes

Browse Source 
Switching an external volume from the old <uuid>_<name> volume to the
declared external volume on redeploy could make an app look as if it
lost its data (for example a database that initializes a new, empty
data directory).

- If a resource already has the storage entry with the old generated
  name, the parsers keep using that volume and show a warning in the
  deployment log and the service start log, with the steps to switch.
- Without that entry (new resources, or external volumes added later)
  the external volume is used as declared.
- Deleting the old storage entry (storage page) switches to the
  external volume on the next deployment; the storage entry shows a
  note about it.
- Legacy Compose applications (parser versions 1 and 2) never stored
  their volumes, so they keep the old volume names.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2026-09-27 17:24:51 +02:00
co-authored by Claude Opus 5.5
parent ee62f64add
commit 49ae227f6e
13 changed files with 600 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Actions/Service/DeployServiceApplication.php
+1
View File
@@ -34,6 +34,7 @@ class DeployServiceApplication
'echo '.escapeshellarg("Saved configuration files to {$workdir}."),
'touch '.escapeshellarg("{$workdir}/.env"),
...EnsureContentFilesOnServer::echoCommands($serviceApplication->fileStorages()->get(), $service->server),
...StartService::composeVolumeWarningCommands($service),
]);
if ($pullLatestImages) {
app/Actions/Service/StartService.php
+15
View File
@@ -38,6 +38,7 @@ class StartService
// but we guarantee it here in case of any edge cases or manual deployments
$commands[] = "touch {$workdir}/.env";
$commands = array_merge($commands, EnsureContentFilesOnServer::echoCommands($this->contentFileStorages($service), $service->server));
$commands = array_merge($commands, self::composeVolumeWarningCommands($service));
if ($pullLatestImages) {
$commands[] = "echo 'Pulling images.'";
$commands[] = "docker compose --project-directory {$workdir} pull";
@@ -63,6 +64,20 @@ class StartService
return remote_process($commands, $service->server, type_uuid: $service->uuid, callEventOnFinish: 'ServiceStatusChanged');
}
/**
* Shows the volume warnings of the last parse (for example an external volume that the
* service does not use yet) in the start log.
*
* @return list<string>
*/
public static function composeVolumeWarningCommands(Service $service): array
{
return array_map(
fn (string $warning): string => 'echo '.escapeshellarg("Warning: {$warning}"),
$service->composeVolumeWarnings()
);
}
/**
* @return Collection<int, LocalFileVolume>
*/
app/Jobs/ApplicationDeploymentJob.php
+17 -1
View File
@@ -743,7 +743,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->application_deployment_queue->addLogEntry('Build secrets are configured. Ensure your docker-compose file includes build.secrets configuration for services that need them.');
}
} else {
$composeFile = $this->application->parse(pull_request_id: $this->pull_request_id, preview_id: data_get($this->preview, 'id'), commit: $this->commit);
$composeFile = $this->parseComposeFileForDeployment();
// Always add .env file to services
$services = collect(data_get($composeFile, 'services', []));
$services = $services->map(function ($service, $name) {
@@ -875,6 +875,22 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->application_deployment_queue->addLogEntry('New container started.');
}
/**
* Parses the Compose file for this deployment and writes the volume warnings of the parser
* (for example an external volume that the resource does not use yet) to the deployment log.
*
* @return Collection<array-key, mixed>
*/
private function parseComposeFileForDeployment(): Collection
{
$composeFile = collect($this->application->parse(pull_request_id: $this->pull_request_id, preview_id: data_get($this->preview, 'id'), commit: $this->commit));
foreach ($this->application->composeVolumeWarnings() as $warning) {
$this->application_deployment_queue->addLogEntry("Warning: {$warning}", 'stderr');
}
return $composeFile;
}
private function pull_docker_compose_images(): void
{
$this->application_deployment_queue->addLogEntry('Pulling image-based services before stopping the current deployment.');
app/Livewire/Project/Shared/Storages/All.php
+2 -1
View File
@@ -22,7 +22,7 @@ class All extends Component
/**
* Editable form state keyed by storage id.
*
* @var array<int|string, array{name: string, mountPath: string, isPreviewSuffixEnabled: bool, isReadOnly: bool, isShared: bool, canDeleteStale: bool}>
* @var array<int|string, array{name: string, mountPath: string, isPreviewSuffixEnabled: bool, isReadOnly: bool, isShared: bool, canDeleteStale: bool, replacedExternalVolume: ?string}>
*/
public array $forms = [];
@@ -207,6 +207,7 @@ class All extends Component
'canDeleteStale' => $this->canUpdate
&& ($storage->isServiceResource() || $storage->isDockerComposeResource())
&& ! $storage->isDeclaredInCompose(),
'replacedExternalVolume' => $this->isComposeOrService ? $storage->replacedExternalComposeVolume() : null,
];
}
$this->forms = $forms;
app/Models/Application.php
+2 -1
View File
@@ -13,6 +13,7 @@ use App\Support\DomainPortOverrides;
use App\Support\DomainUrlParts;
use App\Traits\Auditable;
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasComposeVolumeWarnings;
use App\Traits\HasConfiguration;
use App\Traits\HasMetrics;
use App\Traits\HasNoindexDomains;
@@ -130,7 +131,7 @@ use Symfony\Component\Yaml\Yaml;
class Application extends BaseModel
{
/** @use HasFactory<ApplicationFactory> */
use Auditable, ClearsGlobalSearchCache, HasConfiguration, HasFactory, HasMetrics, HasNoindexDomains, HasSafeStringAttribute, HasSecretManager, ReleasesManagedDnsRecords, SoftDeletes;
use Auditable, ClearsGlobalSearchCache, HasComposeVolumeWarnings, HasConfiguration, HasFactory, HasMetrics, HasNoindexDomains, HasSafeStringAttribute, HasSecretManager, ReleasesManagedDnsRecords, SoftDeletes;
public const MAX_DOCKER_COMPOSE_SIZE_BYTES = 5 * 1024 * 1024;
app/Models/LocalPersistentVolume.php
+50
View File
@@ -184,6 +184,7 @@ class LocalPersistentVolume extends BaseModel
$compose = Yaml::parse($composeContent);
$services = data_get($compose, 'services', []);
$topLevelVolumes = collect(data_get($compose, 'volumes') ?? []);
if ($this->isServiceResource()) {
$services = array_intersect_key($services, [$resource->name => true]);
@@ -194,6 +195,11 @@ class LocalPersistentVolume extends BaseModel
$parsedVolume = is_array($volume) ? $volume : parseDockerVolumeString($volume);
$source = data_get($parsedVolume, 'source');
$target = data_get($parsedVolume, 'target');
if ($source && isComposeExternalVolume($topLevelVolumes->get((string) $source))) {
// The parsers use an external volume as written. A storage entry with the generated
// name is an old entry that the user can delete (see replacedExternalComposeVolume()).
continue;
}
$resourceUuid = $resource instanceof Application ? $resource->uuid : data_get($resource, 'service.uuid');
$generatedName = $source ? $resourceUuid.'_'.Str::slug($source, '-') : null;
@@ -209,6 +215,50 @@ class LocalPersistentVolume extends BaseModel
}
}
/**
* The external Compose volume that this storage entry still replaces, or null. Before Coolify
* used external volumes as written, the parsers gave them a generated name, for example
* "{uuid}_{volume}" or "{uuid}_{volume}-pr-{id}". While this storage entry exists, the parsers
* keep that name so that the resource keeps its data (see useComposeExternalVolumeAsWritten()).
*/
public function replacedExternalComposeVolume(): ?string
{
try {
$resource = $this->resource;
if (! $resource) {
return null;
}
$composeContent = $resource instanceof Application
? $resource->docker_compose_raw
: data_get($resource, 'service.docker_compose_raw');
if (blank($composeContent)) {
return null;
}
foreach (data_get(Yaml::parse($composeContent), 'volumes') ?? [] as $key => $declaration) {
$key = (string) $key;
if (! isComposeExternalVolume($declaration)) {
continue;
}
$legacyName = match (true) {
$resource instanceof Application && (int) $resource->compose_parsing_version < 3 => legacyApplicationComposeVolumeName($resource, $key, 0),
$resource instanceof Application => $resource->uuid.'_'.Str::slug($key, '-'),
default => data_get($resource, 'service.uuid').'_'.Str::slug($key, '-'),
};
$isLegacyName = $legacyName !== $key && $this->name === $legacyName;
if ($isLegacyName || preg_match('/^'.preg_quote($legacyName, '/').'-pr-\d+$/', $this->name) === 1) {
return $key;
}
}
return null;
} catch (\Throwable) {
return null;
}
}
// Check if this volume is read-only by parsing the docker-compose content
public function isReadOnlyVolume(): bool
{
app/Models/Service.php
+2 -1
View File
@@ -7,6 +7,7 @@ use App\Support\DomainPortOverrides;
use App\Support\ResourceStartActivity;
use App\Traits\Auditable;
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasComposeVolumeWarnings;
use App\Traits\HasSafeStringAttribute;
use App\Traits\HasSecretManager;
use Illuminate\Database\Eloquent\Casts\Attribute;
@@ -44,7 +45,7 @@ use Symfony\Component\Yaml\Yaml;
)]
class Service extends BaseModel
{
use Auditable, ClearsGlobalSearchCache, HasFactory, HasSafeStringAttribute, HasSecretManager, SoftDeletes;
use Auditable, ClearsGlobalSearchCache, HasComposeVolumeWarnings, HasFactory, HasSafeStringAttribute, HasSecretManager, SoftDeletes;
private static $parserVersion = '5';
app/Traits/HasComposeVolumeWarnings.php
+33
View File
@@ -0,0 +1,33 @@
<?php
namespace App\Traits;
/**
* Keeps the Compose volume warnings of the last parse on the model instance. The warnings are not
* saved to the database. The deployment job and the service start actions show them to the user.
*/
trait HasComposeVolumeWarnings
{
/** @var list<string> */
private array $composeVolumeWarnings = [];
public function resetComposeVolumeWarnings(): void
{
$this->composeVolumeWarnings = [];
}
public function addComposeVolumeWarning(string $warning): void
{
if (! in_array($warning, $this->composeVolumeWarnings, true)) {
$this->composeVolumeWarnings[] = $warning;
}
}
/**
* @return list<string>
*/
public function composeVolumeWarnings(): array
{
return $this->composeVolumeWarnings;
}
}
bootstrap/helpers/parsers.php
+75 -2
View File
@@ -349,6 +349,73 @@ function composeExternalVolumeDeclaration(iterable $topLevelVolumes, string $sou
return $declaration;
}
/**
* Tells if a parser uses the volume source as written because it is an external volume.
*
* Before Coolify used external volumes as written, the parsers renamed them like all other
* volumes (the old name, for example "{uuid}_{volume}" or "{uuid}_{volume}-pr-{id}"), so the
* resource wrote its data into the renamed volume. When the owner still has the storage entry
* with the old name, the parser must keep the old name, or the resource loses its data. Then
* this function records a warning on the resource and returns false. When the user deletes that
* storage entry, the next parse uses the external volume.
*
* Returns false also for a source that is not external. Then the parser continues as usual.
*
* @param iterable<array-key, mixed> $topLevelVolumes
* @param string $legacyName the name that the parser gave the volume before it used external volumes as written
*
* @throws Exception If the external volume has a name that is not safe (see validateComposeExternalVolume())
*/
function useComposeExternalVolumeAsWritten(Application|Service $resource, Application|ServiceApplication|ServiceDatabase $owner, iterable $topLevelVolumes, string $source, string $legacyName): bool
{
$declaration = composeExternalVolumeDeclaration($topLevelVolumes, $source);
if ($declaration === null) {
return false;
}
if ($legacyName === $source) {
return true;
}
$hasLegacyStorage = LocalPersistentVolume::query()
->where('resource_type', get_class($owner))
->where('resource_id', $owner->getKey())
->where('name', $legacyName)
->exists();
if (! $hasLegacyStorage) {
return true;
}
$resource->addComposeVolumeWarning(composeLegacyExternalVolumeWarning($source, $declaration, $legacyName));
return false;
}
/**
* The name that the legacy application parsers (compose_parsing_version 1 and 2) gave a volume
* before they used external volumes as written: parser version 1 kept the name and added only the
* preview suffix, parser version 2 added the "{uuid}-" prefix too.
*/
function legacyApplicationComposeVolumeName(Application $resource, string $source, int $pull_request_id): string
{
$name = addPreviewDeploymentSuffix($source, $pull_request_id);
return $resource->compose_parsing_version === '2' ? "{$resource->uuid}-{$name}" : $name;
}
/**
* The warning for an external volume that the resource does not use yet (see useComposeExternalVolumeAsWritten()).
* The volume names are validated and contain no secrets.
*
* @param array<string, mixed> $declaration
*/
function composeLegacyExternalVolumeWarning(string $source, array $declaration, string $legacyName): string
{
$dockerVolume = data_get($declaration, 'name') ?? data_get($declaration, 'external.name') ?? $source;
$external = $dockerVolume === $source ? 'external' : "external (Docker volume '{$dockerVolume}')";
return "Volume '{$source}' is declared as {$external}, but Coolify still uses '{$legacyName}' because this resource used it before. To use the external volume, copy your data into it and delete the storage entry '{$legacyName}', then redeploy.";
}
/**
* The key and the name of an external volume must be literal Docker volume names. Docker Compose
* can resolve variables in `name:`, but then only the deployment `.env` sets which existing volume
@@ -740,6 +807,7 @@ function removeComposeVolumeFieldsPreservingComments(string $source, array $clea
function applicationParser(Application $resource, int $pull_request_id = 0, ?int $preview_id = null, ?string $commit = null): Collection
{
$resource->resetComposeVolumeWarnings();
$uuid = data_get($resource, 'uuid');
$compose = data_get($resource, 'docker_compose_raw');
// Store original compose for later use to update docker_compose_raw with content removed
@@ -1201,7 +1269,11 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
}
}
} elseif ($type->value() === 'volume') {
if (composeExternalVolumeDeclaration($topLevel->get('volumes'), $source->value()) !== null) {
$legacyName = "{$uuid}_".Str::slug($source, '-');
if ($isPullRequest) {
$legacyName = addPreviewDeploymentSuffix($legacyName, $pull_request_id);
}
if (useComposeExternalVolumeAsWritten($resource, $originalResource, $topLevel->get('volumes'), $source->value(), $legacyName)) {
// Previews share the external volume. It gets no row, so Coolify never removes it.
$volumesParsed->put($index, $volume);
@@ -1913,6 +1985,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int
function serviceParser(Service $resource): Collection
{
$resource->resetComposeVolumeWarnings();
$uuid = data_get($resource, 'uuid');
$compose = data_get($resource, 'docker_compose_raw');
// Store original compose for later use to update docker_compose_raw with content removed
@@ -2556,7 +2629,7 @@ function serviceParser(Service $resource): Collection
}
}
} elseif ($type->value() === 'volume') {
if (composeExternalVolumeDeclaration($topLevel->get('volumes'), $source->value()) !== null) {
if (useComposeExternalVolumeAsWritten($resource, $originalResource, $topLevel->get('volumes'), $source->value(), "{$uuid}_".Str::slug($source, '-'))) {
// The external volume gets no row, so Coolify never removes it.
$volumesParsed->put($index, $volume);
bootstrap/helpers/shared.php
+12 -9
View File
@@ -2586,6 +2586,7 @@ function customApiValidator(Collection|array $item, array $rules, array $message
}
function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id = 0, ?int $preview_id = null)
{
$resource->resetComposeVolumeWarnings();
if ($resource->getMorphClass() === Service::class) {
if ($resource->docker_compose_raw) {
// Extract inline comments from raw YAML before Symfony parser discards them
@@ -2807,7 +2808,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
// Collect/create/update volumes
if ($serviceVolumes->count() > 0) {
$serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) {
$serviceVolumes = $serviceVolumes->map(function ($volume) use ($resource, $savedService, $topLevelVolumes) {
$type = null;
$source = null;
$target = null;
@@ -2867,7 +2868,8 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
]
);
} elseif ($type->value() === 'volume') {
if (composeExternalVolumeDeclaration($topLevelVolumes, $source->value()) !== null) {
$legacyName = "{$savedService->service->uuid}_".Str::slug($source, '-');
if (useComposeExternalVolumeAsWritten($resource, $savedService, $topLevelVolumes, $source->value(), $legacyName)) {
// The external volume gets no row, so Coolify never removes it.
return $volume;
}
@@ -3390,10 +3392,11 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
}
$server = $resource->destination->server;
$topLevelVolumes = collect(data_get($yaml, 'volumes', []));
// External volumes are used as written, also by previews (no prefix and no preview suffix).
$externalTopLevelVolumes = $topLevelVolumes->filter(fn (mixed $volume): bool => isComposeExternalVolume($volume));
// Legacy Compose applications (parser versions 1 and 2) never stored their volumes, so Coolify
// cannot tell which external volume already holds data. They keep the old volume names.
$externalTopLevelVolumes = collect([]);
if ($pull_request_id !== 0) {
$topLevelVolumes = collect($externalTopLevelVolumes->all());
$topLevelVolumes = collect([]);
}
if ($topLevelVolumes->count() > 0) {
@@ -3481,7 +3484,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
$name = addPreviewDeploymentSuffix($name, $pull_request_id);
}
$volume = str("$name:$mount");
} elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, $name->value()) !== null) {
} elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, $name->value(), legacyApplicationComposeVolumeName($resource, $name->value(), $pull_request_id))) {
// An external volume is used as written.
} else {
if ($pull_request_id !== 0) {
@@ -3550,7 +3553,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
} else {
data_set($volume, 'source', $source.':'.$target);
}
} elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, (string) $source) !== null) {
} elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, (string) $source, legacyApplicationComposeVolumeName($resource, (string) $source, $pull_request_id))) {
// An external volume is used as written.
data_set($volume, 'source', $source.':'.$target.($read_only ? ':ro' : ''));
} else {
@@ -3610,7 +3613,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
$name = addPreviewDeploymentSuffix($name, $pull_request_id);
}
$volume = str("$name:$mount");
} elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, $name->value()) !== null) {
} elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, $name->value(), legacyApplicationComposeVolumeName($resource, $name->value(), $pull_request_id))) {
// An external volume is used as written.
} else {
if ($pull_request_id !== 0) {
@@ -3681,7 +3684,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
} else {
data_set($volume, 'source', $source.':'.$target);
}
} elseif (composeExternalVolumeDeclaration($externalTopLevelVolumes, (string) $source) !== null) {
} elseif (useComposeExternalVolumeAsWritten($resource, $resource, $externalTopLevelVolumes, (string) $source, legacyApplicationComposeVolumeName($resource, (string) $source, $pull_request_id))) {
// An external volume is used as written.
data_set($volume, 'source', $source.':'.$target.($read_only ? ':ro' : ''));
} else {
resources/views/livewire/project/shared/storages/all.blade.php
+10 -2
View File
@@ -62,6 +62,11 @@
@if (blank($storage->host_path))
<span class="block text-xs text-neutral-500 dark:text-fg-dim">Volume mount</span>
@endif
@if ($form['replacedExternalVolume'])
<span class="block text-xs text-amber-800 dark:text-amber-300/90">
Replaces the external volume '{{ $form['replacedExternalVolume'] }}'. Copy the data into the external volume, then delete this entry to use it.
</span>
@endif
</div>
@if ($hasSourcePaths)
@@ -149,9 +154,12 @@
'label' => 'Also permanently delete the Docker volume and all its data.',
'default_warning' => 'The Docker volume and its data will not be deleted.',
]]"
:actions="[
:actions="array_values(array_filter([
'This removes only the stale volume entry from Coolify.',
]" confirmationText="{{ $form['name'] }}"
$form['replacedExternalVolume']
? 'The next deployment uses the external volume \''.$form['replacedExternalVolume'].'\' instead of this volume.'
: null,
]))" confirmationText="{{ $form['name'] }}"
confirmationLabel="Please confirm by entering the Storage Name below"
shortConfirmationLabel="Storage Name" />
@endif
tests/Feature/ApplicationComposeInjectionValidationTest.php
+1 -1
View File
@@ -343,7 +343,7 @@ test('the Compose deployment validates the repository file before any command us
expect($loadPosition)->not->toBeFalse()
->and($body)->not->toContain('$this->application->loadComposeFile(')
->and($loadPosition)->toBeLessThan(strpos($body, '$this->application->oldRawParser()'))
->and($loadPosition)->toBeLessThan(strpos($body, '$this->application->parse('))
->and($loadPosition)->toBeLessThan(strpos($body, '$this->parseComposeFileForDeployment()'))
->and($loadPosition)->toBeLessThan(strpos($body, 'base64 -d'))
->and($body)->toContain('"stat -c \'%F\' ".escapeshellarg($realPathInGit)');
});
tests/Feature/ComposeExternalVolumeTest.php
+380 -41
View File
@@ -1,7 +1,12 @@
<?php
use App\Actions\Service\DeployServiceApplication;
use App\Actions\Service\StartService;
use App\Jobs\ApplicationDeploymentJob;
use App\Jobs\DeleteResourceJob;
use App\Livewire\Project\Shared\Storages\All;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationPreview;
use App\Models\Environment;
use App\Models\InstanceSettings;
@@ -11,12 +16,18 @@ use App\Models\Project;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Models\User;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
use Livewire\Livewire;
use Symfony\Component\Yaml\Yaml;
uses(RefreshDatabase::class);
@@ -63,6 +74,17 @@ volumes:
name: legacy-shared-volume
YAML;
const EXTERNAL_VOLUME_DATABASE_COMPOSE = <<<'YAML'
services:
db:
image: postgres:16-alpine
volumes:
- 'shared-data:/var/lib/postgresql/data'
volumes:
shared-data:
external: true
YAML;
const EXTERNAL_VOLUME_NFS_COMPOSE = <<<'YAML'
services:
web:
@@ -149,6 +171,69 @@ function persistentVolumeNames(): array
return LocalPersistentVolume::query()->pluck('name')->sort()->values()->all();
}
/**
* The storage entry that the parsers made for an external volume before they used it as written.
*/
function legacyExternalVolumeRow(Model $owner, string $name, string $mountPath = '/data'): LocalPersistentVolume
{
return $owner->persistentStorages()->create(['name' => $name, 'mount_path' => $mountPath]);
}
function legacyExternalVolumeServiceResource(Service $service, string $name, string $compose = EXTERNAL_VOLUME_SHORT_COMPOSE): ServiceApplication|ServiceDatabase
{
$image = data_get(Yaml::parse($compose), "services.{$name}.image");
return isDatabaseImage($image)
? ServiceDatabase::create(['name' => $name, 'service_id' => $service->id, 'image' => $image])
: ServiceApplication::create(['name' => $name, 'service_id' => $service->id, 'image' => $image]);
}
function legacyExternalVolumeWarning(string $oldName, ?string $dockerVolume = null): string
{
$external = $dockerVolume === null ? 'external' : "external (Docker volume '{$dockerVolume}')";
return "Volume 'shared-data' is declared as {$external}, but Coolify still uses '{$oldName}' because this resource used it before. To use the external volume, copy your data into it and delete the storage entry '{$oldName}', then redeploy.";
}
/**
* Runs the Compose parse step of a deployment job.
*
* @return array{0: Collection<array-key, mixed>, 1: list<array{0: string, 1: string}>}
*/
function parseExternalVolumeDeployment(Application $application, int $pullRequestId = 0, ?ApplicationPreview $preview = null): array
{
$job = (new ReflectionClass(ApplicationDeploymentJob::class))->newInstanceWithoutConstructor();
$logEntries = new ArrayObject;
$queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial();
$queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message, string $type = 'stdout') use ($logEntries): void {
$logEntries[] = [$message, $type];
});
foreach ([
'application' => $application,
'application_deployment_queue' => $queue,
'pull_request_id' => $pullRequestId,
'preview' => $preview,
'commit' => 'HEAD',
] as $property => $value) {
(new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($job, $value);
}
$composeFile = (new ReflectionMethod(ApplicationDeploymentJob::class, 'parseComposeFileForDeployment'))->invoke($job);
return [$composeFile, $logEntries->getArrayCopy()];
}
function loginExternalVolumeOwner(): void
{
test()->withoutVite();
$team = test()->environment->project->team;
$user = User::factory()->create();
$team->members()->attach($user->id, ['role' => 'owner']);
test()->actingAs($user);
session(['currentTeam' => $team]);
}
/**
* @param array<int, string> $commands
*/
@@ -213,19 +298,81 @@ describe('applicationParser', function () {
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data-pr-42"]);
});
it('keeps an existing row of a volume that an earlier parse prefixed', function () {
it('does not show a warning when the resource has no old storage entry', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$oldVolume = $application->persistentStorages()->create([
'name' => "{$uuid}_shared-data",
'mount_path' => '/data',
]);
applicationParser($application);
expect(LocalPersistentVolume::find($oldVolume->id))->not->toBeNull()
->and(LocalPersistentVolume::find($oldVolume->id)->name)->toBe("{$uuid}_shared-data")
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data", "{$uuid}_shared-data"]);
expect($application->composeVolumeWarnings())->toBe([]);
});
it('keeps the old prefixed volume and shows a warning when the resource has its old storage entry', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$oldVolume = legacyExternalVolumeRow($application, "{$uuid}_shared-data");
$compose = applicationParser($application)->toArray();
expect($compose['services']['web']['volumes'])->toContain("{$uuid}_shared-data:/data:ro")
->not->toContain('shared-data:/data:ro')
->and($compose['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"])
->and(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data")
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data", "{$uuid}_shared-data"])
->and($application->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data")]);
});
it('keeps the old prefixed volume in long syntax and names the external Docker volume in the warning', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_LONG_COMPOSE);
$uuid = $application->uuid;
legacyExternalVolumeRow($application, "{$uuid}_shared-data");
$compose = applicationParser($application)->toArray();
$sources = collect($compose['services']['web']['volumes'])->pluck('source')->all();
expect($sources)->toBe(["{$uuid}_shared-data", "{$uuid}_app-data"])
->and($compose['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"])
->and($application->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data", 'existing-shared-volume')]);
});
it('keeps the old prefixed preview volume when the resource has its old storage entry', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$preview = externalVolumePreview($application);
legacyExternalVolumeRow($application, "{$uuid}_shared-data-pr-42");
$compose = applicationParser($application, 42, $preview->id)->toArray();
expect($compose['services']['web-pr-42']['volumes'])->toContain("{$uuid}_shared-data-pr-42:/data:ro")
->and($compose['volumes']["{$uuid}_shared-data-pr-42"])->toBe(['name' => "{$uuid}_shared-data-pr-42"])
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data-pr-42", "{$uuid}_shared-data-pr-42"])
->and($application->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data-pr-42")]);
});
it('uses the external volume in a preview when only the production volume has an old storage entry', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$preview = externalVolumePreview($application);
legacyExternalVolumeRow($application, "{$uuid}_shared-data");
$compose = applicationParser($application, 42, $preview->id)->toArray();
expect($compose['services']['web-pr-42']['volumes'])->toContain('shared-data:/data:ro')
->and($application->composeVolumeWarnings())->toBe([]);
});
it('uses the external volume after the old storage entry is deleted', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$oldVolume = legacyExternalVolumeRow($application, "{$uuid}_shared-data");
applicationParser($application);
$oldVolume->delete();
$compose = applicationParser($application)->toArray();
expect($compose['services']['web']['volumes'])->toContain('shared-data:/data:ro')
->and($compose['volumes'])->not->toHaveKey("{$uuid}_shared-data")
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"])
->and($application->composeVolumeWarnings())->toBe([]);
});
it('does not change nfs volumes', function () {
@@ -265,19 +412,37 @@ describe('serviceParser', function () {
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"]);
});
it('keeps an existing row of a volume that an earlier parse prefixed', function () {
it('keeps the old prefixed volume and shows a warning when the service has its old storage entry', function (string $compose, string $serviceName, string $mountPath, string $mode) {
$service = externalVolumeService($compose);
$uuid = $service->uuid;
$owner = legacyExternalVolumeServiceResource($service, $serviceName, $compose);
$oldVolume = legacyExternalVolumeRow($owner, "{$uuid}_shared-data", $mountPath);
$parsed = serviceParser($service)->toArray();
expect($parsed['services'][$serviceName]['volumes'])->toContain("{$uuid}_shared-data:{$mountPath}{$mode}")
->and($parsed['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"])
->and(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data")
->and(LocalPersistentVolume::query()->where('name', "{$uuid}_shared-data")->count())->toBe(1)
->and($service->composeVolumeWarnings())->toBe([legacyExternalVolumeWarning("{$uuid}_shared-data")]);
})->with([
'service application' => [EXTERNAL_VOLUME_SHORT_COMPOSE, 'web', '/data', ':ro'],
'service database' => [EXTERNAL_VOLUME_DATABASE_COMPOSE, 'db', '/var/lib/postgresql/data', ''],
]);
it('uses the external volume after the old storage entry of the service is deleted', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $service->uuid;
$serviceApplication = ServiceApplication::create(['name' => 'web', 'service_id' => $service->id, 'image' => 'nginx:alpine']);
$oldVolume = $serviceApplication->persistentStorages()->create([
'name' => "{$uuid}_shared-data",
'mount_path' => '/data',
]);
$oldVolume = legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data");
serviceParser($service);
expect(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data")
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data", "{$uuid}_shared-data"]);
$oldVolume->delete();
$parsed = serviceParser($service)->toArray();
expect($parsed['services']['web']['volumes'])->toContain('shared-data:/data:ro')
->and($parsed['volumes'])->not->toHaveKey("{$uuid}_shared-data")
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"])
->and($service->composeVolumeWarnings())->toBe([]);
});
it('does not change nfs volumes', function () {
@@ -310,43 +475,66 @@ describe('legacy parsers', function () {
'long syntax' => [EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data'],
]);
it('keeps an existing row of a volume that an earlier legacy service parse prefixed', function () {
it('keeps the old prefixed volume of a legacy service that has its old storage entry', function (string $compose, string $expectedVolume) {
$service = externalVolumeService($compose, '2');
$uuid = $service->uuid;
$oldVolume = legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data");
$parsed = parseDockerComposeFile($service)->toArray();
$volumes = collect($parsed['services']['web']['volumes'])
->map(fn (mixed $volume): string => is_array($volume) ? $volume['source'] : $volume)
->all();
expect($volumes)->toContain(str_replace('{uuid}', $uuid, $expectedVolume))
->and($parsed['volumes']["{$uuid}_shared-data"])->toBe(['name' => "{$uuid}_shared-data"])
->and(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data")
->and(LocalPersistentVolume::query()->where('name', "{$uuid}_shared-data")->count())->toBe(1)
->and($service->composeVolumeWarnings())->toHaveCount(1)
->and($service->composeVolumeWarnings()[0])->toStartWith("Volume 'shared-data' is declared as external")
->toContain("Coolify still uses '{$uuid}_shared-data'");
})->with([
'short syntax' => [EXTERNAL_VOLUME_SHORT_COMPOSE, '{uuid}_shared-data:/data'],
'long syntax' => [EXTERNAL_VOLUME_LONG_COMPOSE, '{uuid}_shared-data'],
]);
it('uses the external volume after the old storage entry of a legacy service is deleted', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE, '2');
$uuid = $service->uuid;
$serviceApplication = ServiceApplication::create(['name' => 'web', 'service_id' => $service->id, 'image' => 'nginx:alpine']);
$oldVolume = $serviceApplication->persistentStorages()->create([
'name' => "{$uuid}_shared-data",
'mount_path' => '/data',
]);
$oldVolume = legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data");
parseDockerComposeFile($service);
expect(LocalPersistentVolume::find($oldVolume->id)?->name)->toBe("{$uuid}_shared-data");
$oldVolume->delete();
$parsed = parseDockerComposeFile($service)->toArray();
expect($parsed['services']['web']['volumes'])->toContain('shared-data:/data:ro')
->and($parsed['volumes'])->not->toHaveKey("{$uuid}_shared-data")
->and(persistentVolumeNames())->toBe(["{$uuid}_app-data"])
->and($service->composeVolumeWarnings())->toBe([]);
});
it('keeps an external volume in a legacy application', function (string $parsingVersion, int $pullRequestId, string $compose, string $expectedExternal, string $expectedOther) {
it('keeps the old volume names of a legacy application, because it has no storage entries to check', function (string $parsingVersion, int $pullRequestId, string $compose, string $expectedVolume) {
// Legacy Compose applications (parser versions 1 and 2) never stored their volumes, so Coolify
// cannot tell which external volume already holds data: they always keep the old names.
$application = externalVolumeApplication($compose, $parsingVersion);
$uuid = $application->uuid;
$previewId = $pullRequestId === 0 ? null : externalVolumePreview($application, $pullRequestId)->id;
$parsed = parseDockerComposeFile($application, pull_request_id: $pullRequestId, preview_id: $previewId)->toArray();
$serviceName = $pullRequestId === 0 ? 'web' : "web-pr-{$pullRequestId}";
$expectedOther = str_replace('{uuid}', $uuid, $expectedOther);
$expectedVolume = str_replace('{uuid}', $uuid, $expectedVolume);
$expectedName = explode(':', $expectedVolume)[0];
expect($parsed['services'][$serviceName]['volumes'])->toContain($expectedExternal)
->toContain($expectedOther)
->and($parsed['volumes']['shared-data']['external'])->toBeTrue()
->and(array_keys($parsed['volumes']))->not->toContain('shared-data-pr-42')
->not->toContain("{$uuid}-shared-data")
->not->toContain("{$uuid}-shared-data-pr-42");
expect($parsed['services'][$serviceName]['volumes'])->toContain($expectedVolume)
->and(array_keys($parsed['volumes'] ?? []))->toContain($expectedName)
->and($application->composeVolumeWarnings())->toBe([]);
})->with([
'v1 short' => ['1', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', 'app-data:/app'],
'v1 short preview' => ['1', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', 'app-data-pr-42:/app'],
'v1 long preview' => ['1', 42, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data:/data', 'app-data-pr-42:/app'],
'v2 short' => ['2', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', '{uuid}-app-data:/app'],
'v2 short preview' => ['2', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro', '{uuid}-app-data-pr-42:/app'],
'v2 long' => ['2', 0, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data:/data', '{uuid}-app-data:/app'],
'v2 long preview' => ['2', 42, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data:/data', '{uuid}-app-data-pr-42:/app'],
'v1 short' => ['1', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data:/data:ro'],
'v1 short preview' => ['1', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, 'shared-data-pr-42:/data:ro'],
'v1 long preview' => ['1', 42, EXTERNAL_VOLUME_LONG_COMPOSE, 'shared-data-pr-42:/data'],
'v2 short' => ['2', 0, EXTERNAL_VOLUME_SHORT_COMPOSE, '{uuid}-shared-data:/data:ro'],
'v2 short preview' => ['2', 42, EXTERNAL_VOLUME_SHORT_COMPOSE, '{uuid}-shared-data-pr-42:/data:ro'],
'v2 long' => ['2', 0, EXTERNAL_VOLUME_LONG_COMPOSE, '{uuid}-shared-data:/data'],
'v2 long preview' => ['2', 42, EXTERNAL_VOLUME_LONG_COMPOSE, '{uuid}-shared-data-pr-42:/data'],
]);
});
@@ -418,6 +606,69 @@ describe('delete', function () {
->and($application->docker_compose)->toContain("shared-data:\n external: true");
});
it('removes only the old prefixed volume when a service with an old storage entry is deleted with its volumes', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $service->uuid;
legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data");
serviceParser($service);
$service->delete();
$commands = [];
fakeExternalVolumeServer($commands);
(new DeleteResourceJob($service))->handle();
$all = implode("\n", $commands);
expect($all)->toContain("docker volume rm -f '{$uuid}_shared-data'")
->toContain("docker volume rm -f '{$uuid}_app-data'")
->not->toContain("'shared-data'");
});
it('lets the storage list delete the old storage entry without removing the external volume', function (bool $deleteDockerVolume) {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$oldVolume = legacyExternalVolumeRow($application, "{$uuid}_shared-data");
applicationParser($application);
loginExternalVolumeOwner();
$commands = [];
fakeExternalVolumeServer($commands);
Livewire::test(All::class, ['resource' => $application->fresh()])
->assertSee('Replaces the external volume')
->call('delete', $oldVolume->id, 'password', $deleteDockerVolume ? ['deleteDockerVolume'] : [])
->assertHasNoErrors()
->assertReturned(true);
$volumeCommands = collect($commands)->filter(fn (string $command): bool => str_contains($command, 'docker volume'))->values();
expect(LocalPersistentVolume::find($oldVolume->id))->toBeNull()
->and($volumeCommands)->toHaveCount($deleteDockerVolume ? 1 : 0)
->and($volumeCommands->implode("\n"))->not->toContain("'shared-data'");
if ($deleteDockerVolume) {
expect($volumeCommands->first())->toContain("docker volume rm -f '{$uuid}_shared-data'");
}
$compose = applicationParser($application)->toArray();
expect($compose['services']['web']['volumes'])->toContain('shared-data:/data:ro')
->and($application->composeVolumeWarnings())->toBe([]);
})->with([
'keep the Docker volume' => [false],
'delete the old Docker volume' => [true],
]);
it('does not let the storage list delete a storage entry of a volume that is not external', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
applicationParser($application);
$volume = LocalPersistentVolume::query()->where('name', "{$application->uuid}_app-data")->firstOrFail();
loginExternalVolumeOwner();
Livewire::test(All::class, ['resource' => $application->fresh()])
->assertDontSee('Replaces the external volume')
->call('delete', $volume->id, 'password', [])
->assertHasNoErrors()
->assertReturned(false);
expect(LocalPersistentVolume::find($volume->id))->not->toBeNull();
});
it('does not remove the external volume when a preview is deleted', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
@@ -431,4 +682,92 @@ describe('delete', function () {
expect($volumeCommands)->toContain("docker volume rm -f '{$uuid}_app-data-pr-42'")
->not->toContain('shared-data');
});
it('removes the old prefixed preview volume, but not the external volume, when a preview is deleted', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$preview = externalVolumePreview($application);
legacyExternalVolumeRow($application, "{$uuid}_shared-data-pr-42");
$commands = [];
fakeExternalVolumeServer($commands);
$preview->forceDelete();
$volumeCommands = collect($commands)->filter(fn (string $command): bool => str_contains($command, 'docker volume'))->implode("\n");
expect($volumeCommands)->toContain("docker volume rm -f '{$uuid}_shared-data-pr-42'")
->not->toContain("'shared-data'");
});
});
describe('warnings', function () {
it('writes the warning to the deployment log after the parse', function (int $pullRequestId, string $oldName, string $expectedVolume) {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $application->uuid;
$preview = $pullRequestId === 0 ? null : externalVolumePreview($application, $pullRequestId);
legacyExternalVolumeRow($application, str_replace('{uuid}', $uuid, $oldName));
[$composeFile, $logEntries] = parseExternalVolumeDeployment($application, $pullRequestId, $preview);
$serviceName = $pullRequestId === 0 ? 'web' : "web-pr-{$pullRequestId}";
expect(data_get($composeFile, "services.{$serviceName}.volumes"))->toContain(str_replace('{uuid}', $uuid, $expectedVolume))
->and($logEntries)->toBe([['Warning: '.legacyExternalVolumeWarning(str_replace('{uuid}', $uuid, $oldName)), 'stderr']]);
})->with([
'production' => [0, '{uuid}_shared-data', '{uuid}_shared-data:/data:ro'],
'preview' => [42, '{uuid}_shared-data-pr-42', '{uuid}_shared-data-pr-42:/data:ro'],
]);
it('writes no warning to the deployment log when the external volume is used as written', function () {
$application = externalVolumeApplication(EXTERNAL_VOLUME_SHORT_COMPOSE);
[$composeFile, $logEntries] = parseExternalVolumeDeployment($application);
expect(data_get($composeFile, 'services.web.volumes'))->toContain('shared-data:/data:ro')
->and($logEntries)->toBe([]);
});
it('shows the warning before docker compose up when a service starts', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $service->uuid;
legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data");
$command = StartService::run($service->fresh())->getExtraProperty('command');
$echo = 'echo '.escapeshellarg('Warning: '.legacyExternalVolumeWarning("{$uuid}_shared-data"));
expect($command)->toContain($echo)
->and(strpos($command, $echo))->toBeLessThan(strpos($command, ' up -d'));
});
it('shows the warning before docker compose up when a service application is deployed', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE);
$uuid = $service->uuid;
$serviceApplication = legacyExternalVolumeServiceResource($service, 'web');
legacyExternalVolumeRow($serviceApplication, "{$uuid}_shared-data");
$command = DeployServiceApplication::run($serviceApplication->fresh())->getExtraProperty('command');
$echo = 'echo '.escapeshellarg('Warning: '.legacyExternalVolumeWarning("{$uuid}_shared-data"));
expect($command)->toContain($echo)
->and(strpos($command, $echo))->toBeLessThan(strpos($command, ' up -d'));
});
it('keeps the warning line unchanged for a server with a non-root user', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_LONG_COMPOSE);
$uuid = $service->uuid;
legacyExternalVolumeRow(legacyExternalVolumeServiceResource($service, 'web'), "{$uuid}_shared-data");
serviceParser($service);
$service->server->update(['user' => 'ubuntu']);
$commands = StartService::composeVolumeWarningCommands($service);
expect($commands)->toBe(['echo '.escapeshellarg('Warning: '.legacyExternalVolumeWarning("{$uuid}_shared-data", 'existing-shared-volume'))])
->and(parseCommandsByLineForSudo(collect($commands), $service->server->fresh()))->toBe($commands);
});
it('shows no warning when a service starts with the external volume as written', function () {
$service = externalVolumeService(EXTERNAL_VOLUME_SHORT_COMPOSE);
$command = StartService::run($service->fresh())->getExtraProperty('command');
expect($command)->not->toContain('declared as external');
});
});