From 4accae95b5ca2167730fa162286639ca18de77aa Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:48:17 +0200 Subject: [PATCH] feat(database): add replace-existing option for PostgreSQL restores Allow single-database PostgreSQL imports to drop matching objects before restore. The API and import form accept replace_existing, which adds --clean --if-exists to pg_restore, and pg_restore now uses --exit-on-error. --- app/Actions/Database/StartDatabaseImport.php | 2 +- .../Concerns/HandlesDatabaseImportsApi.php | 6 ++-- app/Http/Controllers/Api/OpenApi.php | 6 ++-- app/Livewire/Project/Database/ImportForm.php | 27 ++++++++++++---- .../DatabaseImportCommandBuilder.php | 4 +-- .../DatabaseImport/DatabaseImportSource.php | 1 + openapi.json | 15 +++++++++ openapi.yaml | 12 +++++++ .../project/database/import-form.blade.php | 14 ++++++--- tests/Feature/Api/DatabaseImportApiTest.php | 31 +++++++++++++++++++ tests/Feature/DatabaseRestoreDialogTest.php | 9 ++++++ .../DatabaseImportCommandBuilderTest.php | 20 ++++++++++++ tests/Unit/DatabaseImportOpenApiTest.php | 5 +++ 13 files changed, 134 insertions(+), 18 deletions(-) diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index 0fb3168409..33e9e5e178 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -116,7 +116,7 @@ class StartDatabaseImport if ($safety = $this->commands->buildPostgresSafetyCommand($resource, $container, $containerPath)) { $commandList[] = $safety; } - $restore = base64_encode($this->commands->buildRestoreCommand($resource, $containerPath, $source->dumpAll)); + $restore = base64_encode($this->commands->buildRestoreCommand($resource, $containerPath, $source->dumpAll, $source->replaceExisting)); $commandList[] = 'echo '.escapeshellarg($restore).' | base64 -d > '.escapeshellarg($scriptPath); $commandList[] = 'chmod +x '.escapeshellarg($scriptPath); $commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}"); diff --git a/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php b/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php index 1280a1409b..c4dc363e6e 100644 --- a/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php +++ b/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php @@ -61,13 +61,14 @@ trait HandlesDatabaseImportsApi { $this->authorize('update', $resource); $payload = $request->json()->all() ?: $request->request->all(); - $allowed = ['source', 'upload_id', 's3_storage_uuid', 'path', 'dump_all']; + $allowed = ['source', 'upload_id', 's3_storage_uuid', 'path', 'dump_all', 'replace_existing']; $validator = Validator::make($payload, [ 'source' => ['required', Rule::in(['upload', 's3', 'server'])], 'upload_id' => ['required_if:source,upload', 'prohibited_unless:source,upload', 'uuid'], 's3_storage_uuid' => ['required_if:source,s3', 'prohibited_unless:source,s3', 'string'], 'path' => ['required_if:source,s3,server', 'prohibited_if:source,upload', 'string', 'max:4096'], 'dump_all' => ['sometimes', 'boolean'], + 'replace_existing' => ['sometimes', 'boolean'], ]); foreach (array_diff(array_keys($payload), $allowed) as $field) { $validator->errors()->add($field, 'This field is not allowed.'); @@ -77,7 +78,7 @@ trait HandlesDatabaseImportsApi } try { - $source = new DatabaseImportSource((string) $payload['source'], $payload['upload_id'] ?? null, $payload['path'] ?? null, $payload['s3_storage_uuid'] ?? null, (bool) ($payload['dump_all'] ?? false)); + $source = new DatabaseImportSource((string) $payload['source'], $payload['upload_id'] ?? null, $payload['path'] ?? null, $payload['s3_storage_uuid'] ?? null, (bool) ($payload['dump_all'] ?? false), (bool) ($payload['replace_existing'] ?? false)); $activity = app(StartDatabaseImport::class)->handle($resource, $source, $teamId); } catch (DatabaseImportException $exception) { return response()->json(['message' => $exception->getMessage()], $exception->status); @@ -87,6 +88,7 @@ trait HandlesDatabaseImportsApi 'database_uuid' => $resource->uuid, 'database_name' => $resource->name, 'source' => $source->type, + 'replace_existing' => $source->replaceExisting, 'activity_id' => $activity->id, ]); $url = route($statusRoute, [...$routeParameters, 'activity_id' => $activity->id], false); diff --git a/app/Http/Controllers/Api/OpenApi.php b/app/Http/Controllers/Api/OpenApi.php index 64b4121bcb..c63204a688 100644 --- a/app/Http/Controllers/Api/OpenApi.php +++ b/app/Http/Controllers/Api/OpenApi.php @@ -16,9 +16,9 @@ use OpenApi\Attributes as OA; new OA\Schema( schema: 'DatabaseImportRequest', oneOf: [ - new OA\Schema(required: ['source', 'upload_id'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['upload']), new OA\Property(property: 'upload_id', type: 'string', format: 'uuid'), new OA\Property(property: 'dump_all', type: 'boolean', default: false)]), - new OA\Schema(required: ['source', 's3_storage_uuid', 'path'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['s3']), new OA\Property(property: 's3_storage_uuid', type: 'string'), new OA\Property(property: 'path', type: 'string'), new OA\Property(property: 'dump_all', type: 'boolean', default: false)]), - new OA\Schema(required: ['source', 'path'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['server']), new OA\Property(property: 'path', type: 'string', example: '/var/backups/database.sql.gz'), new OA\Property(property: 'dump_all', type: 'boolean', default: false)]), + new OA\Schema(required: ['source', 'upload_id'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['upload']), new OA\Property(property: 'upload_id', type: 'string', format: 'uuid'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false)]), + new OA\Schema(required: ['source', 's3_storage_uuid', 'path'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['s3']), new OA\Property(property: 's3_storage_uuid', type: 'string'), new OA\Property(property: 'path', type: 'string'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false)]), + new OA\Schema(required: ['source', 'path'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['server']), new OA\Property(property: 'path', type: 'string', example: '/var/backups/database.sql.gz'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false)]), ], type: 'object', additionalProperties: false, diff --git a/app/Livewire/Project/Database/ImportForm.php b/app/Livewire/Project/Database/ImportForm.php index 66b2e72f44..8e0a665cbb 100644 --- a/app/Livewire/Project/Database/ImportForm.php +++ b/app/Livewire/Project/Database/ImportForm.php @@ -160,13 +160,15 @@ class ImportForm extends Component public bool $dumpAll = false; + public bool $replaceExisting = false; + public string $restoreCommandText = ''; public string $customLocation = ''; public ?int $activityId = null; - public string $postgresqlRestoreCommand = 'pg_restore -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; + public string $postgresqlRestoreCommand = 'pg_restore --exit-on-error -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; public string $mysqlRestoreCommand = 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE'; @@ -278,13 +280,24 @@ createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} EOD; $this->restoreCommandText = $this->postgresqlRestoreCommand.' && (gunzip -cf 2>/dev/null || cat ) | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; } else { - $this->postgresqlRestoreCommand = 'pg_restore -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; + $this->syncPostgresqlRestoreCommand(); } break; } } + public function updatedReplaceExisting(): void + { + $this->syncPostgresqlRestoreCommand(); + } + + private function syncPostgresqlRestoreCommand(): void + { + $replaceExisting = $this->replaceExisting ? ' --clean --if-exists' : ''; + $this->postgresqlRestoreCommand = 'pg_restore --exit-on-error'.$replaceExisting.' -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; + } + public function getContainers() { $this->containers = []; @@ -449,8 +462,8 @@ EOD; try { $this->importRunning = true; $source = Storage::exists("upload/{$this->resourceUuid}/restore") - ? new DatabaseImportSource('upload', dumpAll: $this->dumpAll) - : new DatabaseImportSource('server', path: $this->customLocation, dumpAll: $this->dumpAll); + ? new DatabaseImportSource('upload', dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting) + : new DatabaseImportSource('server', path: $this->customLocation, dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting); $activity = StartDatabaseImport::run($this->resource, $source, (int) currentTeam()->id); $this->activityId = $activity->id; $this->dispatch('activityMonitor', $activity->id); @@ -460,6 +473,7 @@ EOD; 'database_uuid' => $this->resource->uuid, 'database_name' => $this->resource->name, 'source' => 'file', + 'replace_existing' => $this->replaceExisting, ]); } catch (DatabaseImportException $e) { $this->dispatch('error', $e->getMessage()); @@ -606,7 +620,7 @@ EOD; try { $this->importRunning = true; - $source = new DatabaseImportSource('s3', path: $this->s3Path, s3StorageUuid: (string) $this->s3StorageId, dumpAll: $this->dumpAll); + $source = new DatabaseImportSource('s3', path: $this->s3Path, s3StorageUuid: (string) $this->s3StorageId, dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting); $activity = StartDatabaseImport::run($this->resource, $source, (int) currentTeam()->id); $this->activityId = $activity->id; $this->dispatch('activityMonitor', $activity->id); @@ -616,6 +630,7 @@ EOD; 'database_uuid' => $this->resource->uuid, 'database_name' => $this->resource->name, 'source' => 's3', + 'replace_existing' => $this->replaceExisting, 'storage_id' => $this->s3StorageId, ]); $this->dispatch('info', 'Restoring database from S3. Progress will be shown in the activity monitor...'); @@ -713,6 +728,6 @@ SH; public function buildRestoreCommand(string $tmpPath): string { - return app(DatabaseImportCommandBuilder::class)->buildRestoreCommand($this->resource, $tmpPath, $this->dumpAll); + return app(DatabaseImportCommandBuilder::class)->buildRestoreCommand($this->resource, $tmpPath, $this->dumpAll, $this->replaceExisting); } } diff --git a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php index ee35a8fef5..9217e71987 100644 --- a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php +++ b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php @@ -7,14 +7,14 @@ use InvalidArgumentException; class DatabaseImportCommandBuilder { - public function buildRestoreCommand(object $resource, string $path, bool $dumpAll): string + public function buildRestoreCommand(object $resource, string $path, bool $dumpAll, bool $replaceExisting = false): string { $path = escapeshellarg($path); return match ($this->databaseType($resource)) { 'postgresql' => $dumpAll ? 'psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} && createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} && (gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}' - : 'pg_restore -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$path, + : 'pg_restore --exit-on-error'.($replaceExisting ? ' --clean --if-exists' : '').' -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$path, 'mysql' => $dumpAll ? $this->mysqlDumpAll('mysql', 'MYSQL', $path) : 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE < '.$path, diff --git a/app/Support/DatabaseImport/DatabaseImportSource.php b/app/Support/DatabaseImport/DatabaseImportSource.php index 0ad7158bb2..12bbf343c2 100644 --- a/app/Support/DatabaseImport/DatabaseImportSource.php +++ b/app/Support/DatabaseImport/DatabaseImportSource.php @@ -12,6 +12,7 @@ readonly class DatabaseImportSource public ?string $path = null, public ?string $s3StorageUuid = null, public bool $dumpAll = false, + public bool $replaceExisting = false, ) { if (! in_array($type, ['upload', 's3', 'server'], true)) { throw new InvalidArgumentException('Invalid database import source.'); diff --git a/openapi.json b/openapi.json index 5b83605d54..415cca180c 100644 --- a/openapi.json +++ b/openapi.json @@ -21764,6 +21764,11 @@ "dump_all": { "type": "boolean", "default": false + }, + "replace_existing": { + "description": "Drop matching PostgreSQL objects before restoring a single-database archive.", + "type": "boolean", + "default": false } }, "type": "object" @@ -21790,6 +21795,11 @@ "dump_all": { "type": "boolean", "default": false + }, + "replace_existing": { + "description": "Drop matching PostgreSQL objects before restoring a single-database archive.", + "type": "boolean", + "default": false } }, "type": "object" @@ -21813,6 +21823,11 @@ "dump_all": { "type": "boolean", "default": false + }, + "replace_existing": { + "description": "Drop matching PostgreSQL objects before restoring a single-database archive.", + "type": "boolean", + "default": false } }, "type": "object" diff --git a/openapi.yaml b/openapi.yaml index e75c39118e..f4f906b37c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -13834,6 +13834,10 @@ components: dump_all: type: boolean default: false + replace_existing: + description: 'Drop matching PostgreSQL objects before restoring a single-database archive.' + type: boolean + default: false type: object - required: @@ -13852,6 +13856,10 @@ components: dump_all: type: boolean default: false + replace_existing: + description: 'Drop matching PostgreSQL objects before restoring a single-database archive.' + type: boolean + default: false type: object - required: @@ -13868,6 +13876,10 @@ components: dump_all: type: boolean default: false + replace_existing: + description: 'Drop matching PostgreSQL objects before restoring a single-database archive.' + type: boolean + default: false type: object additionalProperties: false DatabaseImportStatus: diff --git a/resources/views/livewire/project/database/import-form.blade.php b/resources/views/livewire/project/database/import-form.blade.php index 6bb5892dca..4dd771ac7e 100644 --- a/resources/views/livewire/project/database/import-form.blade.php +++ b/resources/views/livewire/project/database/import-form.blade.php @@ -48,8 +48,8 @@ @endscript
- - Restoring a backup is destructive. Review the source and import command before continuing. + + Review the source and import command before continuing. Existing objects can cause the import to fail unless replacement is enabled. @else - @endif @@ -95,6 +95,12 @@ ['value' => false, 'label' => 'Backup contains one database'], ]" />
+ @if (in_array($resourceDbType, ['standalone-postgresql', 'postgresql'], true) && ! $dumpAll) +
+ +
+ @endif diff --git a/tests/Feature/Api/DatabaseImportApiTest.php b/tests/Feature/Api/DatabaseImportApiTest.php index 52e2643c40..dc2c18d05c 100644 --- a/tests/Feature/Api/DatabaseImportApiTest.php +++ b/tests/Feature/Api/DatabaseImportApiTest.php @@ -71,6 +71,37 @@ test('audits a successfully queued standalone import', function () { ->and($event->metadata['activity_id'])->toBe($activity->id); }); +test('passes the replace existing option to a standalone database import', function () { + $database = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); + $activity = Activity::create(['log_name' => 'default', 'description' => 'queued', 'properties' => ['status' => 'queued']]); + $action = Mockery::mock(StartDatabaseImport::class); + $action->shouldReceive('handle')->once()->withArgs(fn ($resource, $source, $teamId) => $resource->is($database) + && $source->replaceExisting === true + && $teamId === $this->team->id)->andReturn($activity); + app()->instance(StartDatabaseImport::class, $action); + + $this->withHeaders($this->headers) + ->postJson("/api/v1/databases/{$database->uuid}/imports", [ + 'source' => 'server', + 'path' => '/tmp/backup.dump', + 'replace_existing' => true, + ]) + ->assertAccepted(); +}); + +test('validates replace existing as a boolean', function () { + $database = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); + + $this->withHeaders($this->headers) + ->postJson("/api/v1/databases/{$database->uuid}/imports", [ + 'source' => 'server', + 'path' => '/tmp/backup.dump', + 'replace_existing' => 'yes', + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors('replace_existing'); +}); + test('returns only a team and resource scoped import activity', function () { $database = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); $activity = Activity::create(['log_name' => 'default', 'description' => json_encode([['order' => 1, 'output' => 'restored', 'type' => 'stdout']]), 'properties' => ['team_id' => $this->team->id, 'type_uuid' => $database->uuid, 'operation' => 'database_import', 'status' => 'finished', 'exitCode' => 0]]); diff --git a/tests/Feature/DatabaseRestoreDialogTest.php b/tests/Feature/DatabaseRestoreDialogTest.php index 5f78706a8c..5989543cf1 100644 --- a/tests/Feature/DatabaseRestoreDialogTest.php +++ b/tests/Feature/DatabaseRestoreDialogTest.php @@ -18,3 +18,12 @@ test('postgresql dump all restore warns that administrator passwords are overwri ->toContain('The backup replaces PostgreSQL administrator role passwords, including the destination administrator password.') ->toContain("If the administrator password changes, update it in Coolify's database configuration after the restore."); }); + +test('postgresql single database restore offers explicit object replacement', function () { + $view = file_get_contents(resource_path('views/livewire/project/database/import-form.blade.php')); + + expect($view) + ->toContain('Replace objects that already exist') + ->toContain('wire:model="postgresqlRestoreCommand"') + ->toContain('label="Import command" readonly'); +}); diff --git a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php index e9ebffbcad..39a95513d4 100644 --- a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php +++ b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php @@ -48,6 +48,26 @@ test('builds dump-all commands and postgres safety scan', function () { ->toContain('docker exec postgres-safe'); }); +test('stops PostgreSQL restores on the first error without replacing existing objects by default', function () { + $builder = new DatabaseImportCommandBuilder; + $postgres = importResource(StandalonePostgresql::class); + + expect($builder->buildRestoreCommand($postgres, '/tmp/backup.dump', false, false)) + ->toContain('--exit-on-error') + ->not->toContain('--clean') + ->not->toContain('--if-exists'); +}); + +test('replaces existing PostgreSQL objects when requested', function () { + $builder = new DatabaseImportCommandBuilder; + $postgres = importResource(StandalonePostgresql::class); + + expect($builder->buildRestoreCommand($postgres, '/tmp/backup.dump', false, true)) + ->toContain('--clean') + ->toContain('--if-exists') + ->toContain('--exit-on-error'); +}); + test('rejects unsupported database types', function () { $builder = new DatabaseImportCommandBuilder; $redis = importResource(StandaloneRedis::class); diff --git a/tests/Unit/DatabaseImportOpenApiTest.php b/tests/Unit/DatabaseImportOpenApiTest.php index 9e54855a66..d8276f8ca0 100644 --- a/tests/Unit/DatabaseImportOpenApiTest.php +++ b/tests/Unit/DatabaseImportOpenApiTest.php @@ -10,4 +10,9 @@ test('documents standalone and service database import endpoints', function () { ->toHaveKey('/services/{uuid}/databases/{database_uuid}/imports/uploads') ->toHaveKey('/services/{uuid}/databases/{database_uuid}/imports') ->toHaveKey('/services/{uuid}/databases/{database_uuid}/imports/{activity_id}'); + + foreach ($document['components']['schemas']['DatabaseImportRequest']['oneOf'] as $source) { + expect($source['properties']['replace_existing']) + ->toMatchArray(['type' => 'boolean', 'default' => false]); + } });