From 523bf66908086828018155489d8aac5f2bafe455 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:05:57 +0200 Subject: [PATCH] feat(api): add database backup import endpoints --- app/Actions/Database/StartDatabaseImport.php | 155 ++++++ app/Events/DatabaseImportFinished.php | 34 ++ .../Concerns/HandlesDatabaseImportsApi.php | 113 ++++ .../Controllers/Api/DatabasesController.php | 28 + app/Http/Controllers/Api/OpenApi.php | 25 + .../Api/ServiceDatabasesController.php | 29 + app/Livewire/Project/Database/ImportForm.php | 270 +--------- .../DatabaseImportCommandBuilder.php | 70 +++ .../DatabaseImportException.php | 13 + .../DatabaseImport/DatabaseImportSource.php | 20 + openapi.json | 499 +++++++++++++++++- openapi.yaml | 331 +++++++++++- routes/api.php | 6 + tests/Feature/Api/DatabaseImportApiTest.php | 63 +++ .../Feature/Api/DatabaseImportRoutesTest.php | 14 + .../Api/ServiceDatabaseImportApiTest.php | 42 ++ .../DatabaseImportCommandBuilderTest.php | 59 +++ tests/Unit/DatabaseImportOpenApiTest.php | 13 + 18 files changed, 1533 insertions(+), 251 deletions(-) create mode 100644 app/Actions/Database/StartDatabaseImport.php create mode 100644 app/Events/DatabaseImportFinished.php create mode 100644 app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php create mode 100644 app/Support/DatabaseImport/DatabaseImportCommandBuilder.php create mode 100644 app/Support/DatabaseImport/DatabaseImportException.php create mode 100644 app/Support/DatabaseImport/DatabaseImportSource.php create mode 100644 tests/Feature/Api/DatabaseImportApiTest.php create mode 100644 tests/Feature/Api/DatabaseImportRoutesTest.php create mode 100644 tests/Feature/Api/ServiceDatabaseImportApiTest.php create mode 100644 tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php create mode 100644 tests/Unit/DatabaseImportOpenApiTest.php diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php new file mode 100644 index 0000000000..0fb3168409 --- /dev/null +++ b/app/Actions/Database/StartDatabaseImport.php @@ -0,0 +1,155 @@ +commands->supports($resource)) { + throw new DatabaseImportException('Database imports are not supported for this database type.'); + } + if (! str($resource->status)->startsWith('running')) { + throw new DatabaseImportException('The database must be running before an import can start.'); + } + + [$server, $container, $network] = $this->target($resource); + $destination = $resource instanceof ServiceDatabase ? $resource->service?->destination : $resource->destination; + if ($destination instanceof SwarmDocker) { + throw new DatabaseImportException('Database imports are not supported for Swarm servers yet.', 501); + } + if (! $server || ! ValidationPatterns::isValidContainerName($container)) { + throw new DatabaseImportException('The database server or container is invalid.', 400); + } + + $active = Activity::query()->where('properties->team_id', $teamId) + ->where('properties->type_uuid', $resource->uuid) + ->where('properties->operation', 'database_import') + ->whereIn('properties->status', [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value]) + ->exists(); + if ($active) { + throw new DatabaseImportException('A database import is already running.', 409); + } + + $operation = (string) Str::uuid(); + $containerPath = "/tmp/restore_{$operation}"; + $scriptPath = "/tmp/restore_{$operation}.sh"; + $commandList = []; + $cleanup = ['container' => $container, 'containerTmpPath' => $containerPath, 'scriptPath' => $scriptPath, 'serverId' => $server->id]; + + if ($source->type === 'upload') { + $staged = $source->uploadId + ? "upload/imports/{$teamId}/{$resource->uuid}/{$source->uploadId}/restore" + : "upload/{$resource->uuid}/restore"; + if (! Storage::exists($staged)) { + throw new DatabaseImportException('The completed upload was not found.'); + } + $local = Storage::path($staged); + if ($this->commands->databaseType($resource) === 'postgresql' && DatabaseBackupFileValidator::fileContainsPostgresqlProgramExecution($local)) { + Storage::delete($staged); + throw new DatabaseImportException('The uploaded backup contains disallowed PostgreSQL restore directives.'); + } + $serverPath = "/tmp/database-import-{$operation}"; + instant_scp($local, $serverPath, $server); + $source->uploadId ? Storage::deleteDirectory(dirname($staged)) : Storage::delete($staged); + $commandList[] = 'docker cp '.escapeshellarg($serverPath).' '.escapeshellarg("{$container}:{$containerPath}"); + $commandList[] = 'rm -f '.escapeshellarg($serverPath); + $cleanup['serverTmpPath'] = $serverPath; + } elseif ($source->type === 'server') { + $this->assertServerPath($source->path); + $size = (int) trim((string) instant_remote_process(['stat -c %s -- '.escapeshellarg($source->path)], $server)); + if ($size < 1 || $size > self::MAX_BYTES) { + throw new DatabaseImportException('The backup file is empty or exceeds the 10 GiB limit.'); + } + $commandList[] = 'docker cp '.escapeshellarg($source->path).' '.escapeshellarg("{$container}:{$containerPath}"); + } else { + $storage = S3Storage::ownedByCurrentTeamAPI($teamId) + ->where(fn ($query) => $query->whereUuid($source->s3StorageUuid)->orWhere('id', ctype_digit((string) $source->s3StorageUuid) ? (int) $source->s3StorageUuid : -1)) + ->where('is_usable', true)->first(); + if (! $storage || ! ValidationPatterns::isValidS3BucketName($storage->bucket)) { + throw new DatabaseImportException('S3 storage was not found or has an invalid bucket.'); + } + $key = ltrim((string) $source->path, '/'); + $this->assertS3Path($key); + $disk = Storage::build(['driver' => 's3', 'region' => $storage->region, 'key' => $storage->key, 'secret' => $storage->secret, 'bucket' => $storage->bucket, 'endpoint' => $storage->endpoint, 'use_path_style_endpoint' => true, 'http' => SafeWebhookUrl::httpClientOptions($storage->endpoint)]); + if (! $disk->exists($key) || $disk->size($key) > self::MAX_BYTES) { + throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.'); + } + $helper = "s3-restore-{$operation}"; + $serverPath = "/tmp/s3-restore-{$operation}"; + $sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}"); + $commandList = [ + 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true', + 'docker run -d --network '.escapeshellarg($network).' --name '.escapeshellarg($helper).' '.escapeshellarg(coolifyHelperImage().':'.getHelperVersion()).' sleep 3600', + 'docker exec '.escapeshellarg($helper).' mc alias set s3temp '.escapeshellarg($storage->endpoint).' '.escapeshellarg($storage->key).' '.escapeshellarg($storage->secret), + 'docker exec '.escapeshellarg($helper).' mc cp '.$sourceArg.' /tmp/restore', + 'docker cp '.escapeshellarg("{$helper}:/tmp/restore").' '.escapeshellarg($serverPath), + 'docker cp '.escapeshellarg($serverPath).' '.escapeshellarg("{$container}:{$containerPath}"), + 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true', + 'rm -f '.escapeshellarg($serverPath), + ]; + $cleanup += ['containerName' => $helper, 'serverTmpPath' => $serverPath]; + } + + if ($safety = $this->commands->buildPostgresSafetyCommand($resource, $container, $containerPath)) { + $commandList[] = $safety; + } + $restore = base64_encode($this->commands->buildRestoreCommand($resource, $containerPath, $source->dumpAll)); + $commandList[] = 'echo '.escapeshellarg($restore).' | base64 -d > '.escapeshellarg($scriptPath); + $commandList[] = 'chmod +x '.escapeshellarg($scriptPath); + $commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}"); + $commandList[] = 'rm -f '.escapeshellarg($scriptPath); + $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg($scriptPath); + + $activity = remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup); + $activity->properties = $activity->properties->merge(['operation' => 'database_import', 'resource_kind' => $resource instanceof ServiceDatabase ? 'service_database' : 'standalone_database', 'operation_uuid' => $operation]); + $activity->save(); + + return $activity; + } + + private function target(Model $resource): array + { + if ($resource instanceof ServiceDatabase) { + return [$resource->service?->server, $resource->name.'-'.$resource->service?->uuid, $resource->service?->destination?->network ?? 'coolify']; + } + + return [$resource->destination?->server, $resource->uuid, $resource->destination?->network ?? 'coolify']; + } + + private function assertServerPath(?string $path): void + { + if (! $path || ! str_starts_with($path, '/') || preg_match('/\.\.|[$()`|;&><\r\n\0\'"\\\\]/', $path) || ! DatabaseBackupFileValidator::hasAllowedExtension(basename($path))) { + throw new DatabaseImportException('The server path is invalid.'); + } + } + + private function assertS3Path(string $path): void + { + if ($path === '' || preg_match('/\.\.|[$()`|;&><\r\n\0\'"\\\\]/', $path) || ! DatabaseBackupFileValidator::hasAllowedExtension(basename($path))) { + throw new DatabaseImportException('The S3 path is invalid.'); + } + } +} diff --git a/app/Events/DatabaseImportFinished.php b/app/Events/DatabaseImportFinished.php new file mode 100644 index 0000000000..549cef3178 --- /dev/null +++ b/app/Events/DatabaseImportFinished.php @@ -0,0 +1,34 @@ +/dev/null || true'; + } + if (isSafeTmpPath($data['serverTmpPath'] ?? null)) { + $commands[] = 'rm -f '.escapeshellarg($data['serverTmpPath']).' 2>/dev/null || true'; + } + if (filled($data['container'] ?? null)) { + foreach (['containerTmpPath', 'scriptPath'] as $key) { + if (isSafeTmpPath($data[$key] ?? null)) { + $commands[] = 'docker exec '.escapeshellarg($data['container']).' rm -f '.escapeshellarg($data[$key]).' 2>/dev/null || true'; + } + } + } + $server = Server::find($data['serverId'] ?? null); + if ($server && $commands !== []) { + instant_remote_process($commands, $server, throwError: false); + } + } +} diff --git a/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php b/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php new file mode 100644 index 0000000000..21a339683b --- /dev/null +++ b/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php @@ -0,0 +1,113 @@ +authorize('uploadBackup', $resource); + $validator = Validator::make($request->all(), ['upload_id' => ['required', 'uuid'], 'file' => ['required', 'file']]); + if ($validator->fails()) { + return response()->json(['message' => 'Validation failed.', 'errors' => $validator->errors()], 422); + } + $originalName = $request->file('file')?->getClientOriginalName(); + if (! $originalName || ! DatabaseBackupFileValidator::hasAllowedExtension($originalName)) { + return response()->json(['message' => 'Validation failed.', 'errors' => ['file' => ['Unsupported backup file extension.']]], 422); + } + if ((int) $request->input('dzTotalFilesize', 0) > StartDatabaseImport::MAX_BYTES) { + return response()->json(['message' => 'Validation failed.', 'errors' => ['file' => ['The backup exceeds the 10 GiB limit.']]], 422); + } + + $request->merge(['dzuuid' => $request->input('dzuuid', $request->string('upload_id')->value())]); + $receiver = new FileReceiver('file', $request, HandlerFactory::classFromRequest($request)); + $save = $receiver->receive(); + if (! $save->isFinished()) { + return response()->json(['upload_id' => $request->string('upload_id')->value(), 'done' => $save->handler()->getPercentageDone(), 'status' => true]); + } + + $file = $save->getFile(); + if (! $file instanceof UploadedFile || ! DatabaseBackupFileValidator::isUploadAllowed($file, StartDatabaseImport::MAX_BYTES)) { + @unlink($file->getPathname()); + + return response()->json(['message' => 'Validation failed.', 'errors' => ['file' => ['Uploaded file failed validation.']]], 422); + } + $mimeType = $file->getMimeType(); + $size = $file->getSize(); + $directory = "upload/imports/{$teamId}/{$resource->uuid}/{$request->string('upload_id')->value()}"; + Storage::makeDirectory($directory); + $file->move(Storage::path($directory), 'restore'); + + return response()->json(['upload_id' => $request->string('upload_id')->value(), 'filename' => $originalName, 'mime_type' => $mimeType, 'size' => $size], 201); + } + + protected function startDatabaseImport(Request $request, Model $resource, int $teamId, string $statusRoute, array $routeParameters): JsonResponse + { + $this->authorize('update', $resource); + $payload = $request->json()->all() ?: $request->request->all(); + $allowed = ['source', 'upload_id', 's3_storage_uuid', 'path', 'dump_all']; + $validator = Validator::make($payload, [ + 'source' => ['required', Rule::in(['upload', 's3', 'server'])], + 'upload_id' => ['required_if:source,upload', 'prohibited_unless:source,upload', 'uuid'], + 's3_storage_uuid' => ['required_if:source,s3', 'prohibited_unless:source,s3', 'string'], + 'path' => ['required_if:source,s3,server', 'prohibited_if:source,upload', 'string', 'max:4096'], + 'dump_all' => ['sometimes', 'boolean'], + ]); + foreach (array_diff(array_keys($payload), $allowed) as $field) { + $validator->errors()->add($field, 'This field is not allowed.'); + } + if ($validator->fails() || $validator->errors()->isNotEmpty()) { + return response()->json(['message' => 'Validation failed.', 'errors' => $validator->errors()], 422); + } + + try { + $source = new DatabaseImportSource((string) $payload['source'], $payload['upload_id'] ?? null, $payload['path'] ?? null, $payload['s3_storage_uuid'] ?? null, (bool) ($payload['dump_all'] ?? false)); + $activity = app(StartDatabaseImport::class)->handle($resource, $source, $teamId); + } catch (DatabaseImportException $exception) { + return response()->json(['message' => $exception->getMessage()], $exception->status); + } + $url = route($statusRoute, [...$routeParameters, 'activity_id' => $activity->id], false); + + return response()->json(['id' => $activity->id, 'status' => data_get($activity, 'properties.status'), 'message' => 'Database import queued.', 'status_url' => $url], 202)->header('Location', $url); + } + + protected function showDatabaseImport(Model $resource, int $teamId, int $activityId): JsonResponse + { + $this->authorize('view', $resource); + $activity = Activity::query()->whereKey($activityId) + ->where('properties->team_id', $teamId) + ->where('properties->type_uuid', $resource->uuid) + ->where('properties->operation', 'database_import')->first(); + if (! $activity) { + return response()->json(['message' => 'Database import not found.'], 404); + } + $status = data_get($activity, 'properties.status'); + $terminal = in_array($status, ['finished', 'error', 'killed', 'cancelled', 'closed'], true); + + return response()->json([ + 'id' => $activity->id, + 'status' => $status, + 'exit_code' => data_get($activity, 'properties.exitCode'), + 'output' => remove_iip(RunRemoteProcess::decodeOutput($activity)), + 'created_at' => $activity->created_at, + 'updated_at' => $activity->updated_at, + 'finished_at' => $terminal ? $activity->updated_at : null, + ]); + } +} diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index 7b62b4980a..9b86cd98ef 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -32,8 +32,36 @@ use OpenApi\Attributes as OA; class DatabasesController extends Controller { + use Concerns\HandlesDatabaseImportsApi; use Concerns\HandlesTagsApi; + #[OA\Post(path: '/databases/{uuid}/imports/uploads', operationId: 'upload-database-import', summary: 'Upload database import', security: [['bearerAuth' => []]], tags: ['Databases'], responses: [new OA\Response(response: 201, description: 'Upload completed'), new OA\Response(response: 422, ref: '#/components/responses/422')])] + public function upload_import(Request $request, string $uuid): JsonResponse + { + $teamId = getTeamIdFromToken(); + $database = $teamId === null ? null : queryDatabaseByUuidWithinTeam($uuid, $teamId); + + return $database ? $this->uploadDatabaseImport($request, $database, $teamId) : response()->json(['message' => 'Database not found.'], 404); + } + + #[OA\Post(path: '/databases/{uuid}/imports', operationId: 'create-database-import', summary: 'Import database backup', requestBody: new OA\RequestBody(required: true, content: new OA\JsonContent(ref: '#/components/schemas/DatabaseImportRequest')), security: [['bearerAuth' => []]], tags: ['Databases'], responses: [new OA\Response(response: 202, description: 'Import queued'), new OA\Response(response: 409, description: 'Import already active'), new OA\Response(response: 422, ref: '#/components/responses/422')])] + public function create_import(Request $request, string $uuid): JsonResponse + { + $teamId = getTeamIdFromToken(); + $database = $teamId === null ? null : queryDatabaseByUuidWithinTeam($uuid, $teamId); + + return $database ? $this->startDatabaseImport($request, $database, $teamId, 'api.databases.imports.show', ['uuid' => $uuid]) : response()->json(['message' => 'Database not found.'], 404); + } + + #[OA\Get(path: '/databases/{uuid}/imports/{activity_id}', operationId: 'get-database-import', summary: 'Get database import status', security: [['bearerAuth' => []]], tags: ['Databases'], responses: [new OA\Response(response: 200, description: 'Import status'), new OA\Response(response: 404, ref: '#/components/responses/404')])] + public function show_import(Request $request, string $uuid, int $activity_id): JsonResponse + { + $teamId = getTeamIdFromToken(); + $database = $teamId === null ? null : queryDatabaseByUuidWithinTeam($uuid, $teamId); + + return $database ? $this->showDatabaseImport($database, $teamId, $activity_id) : response()->json(['message' => 'Database not found.'], 404); + } + protected function findTaggableResource(string $uuid, int|string $teamId): mixed { return queryDatabaseByUuidWithinTeam($uuid, $teamId); diff --git a/app/Http/Controllers/Api/OpenApi.php b/app/Http/Controllers/Api/OpenApi.php index 33d21ba5d0..64b4121bcb 100644 --- a/app/Http/Controllers/Api/OpenApi.php +++ b/app/Http/Controllers/Api/OpenApi.php @@ -12,6 +12,31 @@ use OpenApi\Attributes as OA; securityScheme: 'bearerAuth', description: 'Go to `Keys & Tokens` / `API tokens` and create a new token. Use the token as the bearer token.')] #[OA\Components( + schemas: [ + new OA\Schema( + schema: 'DatabaseImportRequest', + oneOf: [ + new OA\Schema(required: ['source', 'upload_id'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['upload']), new OA\Property(property: 'upload_id', type: 'string', format: 'uuid'), new OA\Property(property: 'dump_all', type: 'boolean', default: false)]), + new OA\Schema(required: ['source', 's3_storage_uuid', 'path'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['s3']), new OA\Property(property: 's3_storage_uuid', type: 'string'), new OA\Property(property: 'path', type: 'string'), new OA\Property(property: 'dump_all', type: 'boolean', default: false)]), + new OA\Schema(required: ['source', 'path'], properties: [new OA\Property(property: 'source', type: 'string', enum: ['server']), new OA\Property(property: 'path', type: 'string', example: '/var/backups/database.sql.gz'), new OA\Property(property: 'dump_all', type: 'boolean', default: false)]), + ], + type: 'object', + additionalProperties: false, + ), + new OA\Schema( + schema: 'DatabaseImportStatus', + type: 'object', + properties: [ + new OA\Property(property: 'id', type: 'integer'), + new OA\Property(property: 'status', type: 'string', enum: ['queued', 'in_progress', 'finished', 'error', 'killed', 'cancelled', 'closed']), + new OA\Property(property: 'exit_code', type: 'integer', nullable: true), + new OA\Property(property: 'output', type: 'string'), + new OA\Property(property: 'created_at', type: 'string', format: 'date-time'), + new OA\Property(property: 'updated_at', type: 'string', format: 'date-time'), + new OA\Property(property: 'finished_at', type: 'string', format: 'date-time', nullable: true), + ], + ), + ], responses: [ new OA\Response( response: 400, diff --git a/app/Http/Controllers/Api/ServiceDatabasesController.php b/app/Http/Controllers/Api/ServiceDatabasesController.php index 480ff4e557..331b11944a 100644 --- a/app/Http/Controllers/Api/ServiceDatabasesController.php +++ b/app/Http/Controllers/Api/ServiceDatabasesController.php @@ -18,6 +18,35 @@ use OpenApi\Attributes as OA; class ServiceDatabasesController extends Controller { + use Concerns\HandlesDatabaseImportsApi; + + #[OA\Post(path: '/services/{uuid}/databases/{database_uuid}/imports/uploads', operationId: 'upload-service-database-import', summary: 'Upload service database import', security: [['bearerAuth' => []]], tags: ['Service databases'], responses: [new OA\Response(response: 201, description: 'Upload completed'), new OA\Response(response: 422, ref: '#/components/responses/422')])] + public function upload_import(Request $request): JsonResponse + { + return $this->withImportDatabase($request, fn (ServiceDatabase $database, int $teamId) => $this->uploadDatabaseImport($request, $database, $teamId)); + } + + #[OA\Post(path: '/services/{uuid}/databases/{database_uuid}/imports', operationId: 'create-service-database-import', summary: 'Import service database backup', requestBody: new OA\RequestBody(required: true, content: new OA\JsonContent(ref: '#/components/schemas/DatabaseImportRequest')), security: [['bearerAuth' => []]], tags: ['Service databases'], responses: [new OA\Response(response: 202, description: 'Import queued'), new OA\Response(response: 409, description: 'Import already active'), new OA\Response(response: 422, ref: '#/components/responses/422')])] + public function create_import(Request $request): JsonResponse + { + return $this->withImportDatabase($request, fn (ServiceDatabase $database, int $teamId) => $this->startDatabaseImport($request, $database, $teamId, 'api.service-databases.imports.show', ['uuid' => $request->route('uuid'), 'database_uuid' => $database->uuid])); + } + + #[OA\Get(path: '/services/{uuid}/databases/{database_uuid}/imports/{activity_id}', operationId: 'get-service-database-import', summary: 'Get service database import status', security: [['bearerAuth' => []]], tags: ['Service databases'], responses: [new OA\Response(response: 200, description: 'Import status'), new OA\Response(response: 404, ref: '#/components/responses/404')])] + public function show_import(Request $request): JsonResponse + { + return $this->withImportDatabase($request, fn (ServiceDatabase $database, int $teamId) => $this->showDatabaseImport($database, $teamId, (int) $request->route('activity_id'))); + } + + private function withImportDatabase(Request $request, callable $callback): JsonResponse + { + $teamId = getTeamIdFromToken(); + $service = $teamId === null ? null : $this->resolveService($request, $teamId); + $database = $service ? $this->resolveServiceDatabase($request, $service) : null; + + return $database ? $callback($database, $teamId) : response()->json(['message' => 'Service database not found.'], 404); + } + private function removeSensitiveData(ServiceDatabase $serviceDatabase): array { $serviceDatabase->makeHidden([ diff --git a/app/Livewire/Project/Database/ImportForm.php b/app/Livewire/Project/Database/ImportForm.php index d6d713d801..8440de2efc 100644 --- a/app/Livewire/Project/Database/ImportForm.php +++ b/app/Livewire/Project/Database/ImportForm.php @@ -2,6 +2,7 @@ namespace App\Livewire\Project\Database; +use App\Actions\Database\StartDatabaseImport; use App\Models\S3Storage; use App\Models\Server; use App\Models\Service; @@ -10,12 +11,13 @@ use App\Models\StandaloneClickhouse; use App\Models\StandaloneDragonfly; use App\Models\StandaloneKeydb; use App\Models\StandaloneMariadb; -use App\Models\StandaloneMongodb; use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; use App\Rules\SafeWebhookUrl; -use App\Support\DatabaseBackupFileValidator; +use App\Support\DatabaseImport\DatabaseImportCommandBuilder; +use App\Support\DatabaseImport\DatabaseImportException; +use App\Support\DatabaseImport\DatabaseImportSource; use App\Support\ValidationPatterns; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Facades\Storage; @@ -446,77 +448,21 @@ EOD; try { $this->importRunning = true; - $this->importCommands = []; - $backupFileName = "upload/{$this->resourceUuid}/restore"; - - // Check if an uploaded file exists first (takes priority over custom location) - if (Storage::exists($backupFileName)) { - $path = Storage::path($backupFileName); - - // Reject malicious PostgreSQL payloads before transferring the file anywhere. - if ($this->isPostgresqlRestore() && DatabaseBackupFileValidator::fileContainsPostgresqlProgramExecution($path)) { - Storage::delete($backupFileName); - $this->dispatch('error', 'The uploaded backup contains disallowed PostgreSQL restore directives (COPY ... PROGRAM or psql shell commands) and was rejected.'); - - return true; - } - - $tmpPath = '/tmp/'.basename($backupFileName).'_'.$this->resourceUuid; - instant_scp($path, $tmpPath, $this->server); - Storage::delete($backupFileName); - $this->importCommands[] = "docker cp {$tmpPath} {$this->container}:{$tmpPath}"; - $this->addRestoreSafetyCheckCommand($this->importCommands, $tmpPath); - } elseif (filled($this->customLocation)) { - // Validate the custom location to prevent command injection - if (! $this->validateServerPath($this->customLocation)) { - $this->dispatch('error', 'Invalid file path. Path must be absolute and contain only safe characters.'); - - return true; - } - $tmpPath = '/tmp/restore_'.$this->resourceUuid; - $escapedCustomLocation = escapeshellarg($this->customLocation); - $this->importCommands[] = "docker cp {$escapedCustomLocation} {$this->container}:{$tmpPath}"; - $this->addRestoreSafetyCheckCommand($this->importCommands, $tmpPath); - } else { - $this->dispatch('error', 'The file does not exist or has been deleted.'); - - return true; - } - - // Copy the restore command to a script file - $scriptPath = "/tmp/restore_{$this->resourceUuid}.sh"; - - $restoreCommand = $this->buildRestoreCommand($tmpPath); - - $restoreCommandBase64 = base64_encode($restoreCommand); - $this->importCommands[] = "echo \"{$restoreCommandBase64}\" | base64 -d > {$scriptPath}"; - $this->importCommands[] = "chmod +x {$scriptPath}"; - $this->importCommands[] = "docker cp {$scriptPath} {$this->container}:{$scriptPath}"; - - $this->importCommands[] = "docker exec {$this->container} sh -c '{$scriptPath}'"; - $this->importCommands[] = "docker exec {$this->container} sh -c 'echo \"Import finished with exit code $?\"'"; - - if (! empty($this->importCommands)) { - $activity = remote_process($this->importCommands, $this->server, ignore_errors: true, callEventOnFinish: 'RestoreJobFinished', callEventData: [ - 'scriptPath' => $scriptPath, - 'tmpPath' => $tmpPath, - 'container' => $this->container, - 'serverId' => $this->server->id, - ]); - - // Track the activity ID - $this->activityId = $activity->id; - - // Dispatch activity to the monitor and open slide-over - $this->dispatch('activityMonitor', $activity->id); - $this->dispatch('databaserestore'); - auditLog('ui.database.import_started', [ - 'team_id' => $this->resource->team()?->id, - 'database_uuid' => $this->resource->uuid, - 'database_name' => $this->resource->name, - 'source' => 'file', - ]); - } + $source = Storage::exists("upload/{$this->resourceUuid}/restore") + ? new DatabaseImportSource('upload', dumpAll: $this->dumpAll) + : new DatabaseImportSource('server', path: $this->customLocation, dumpAll: $this->dumpAll); + $activity = StartDatabaseImport::run($this->resource, $source, (int) currentTeam()->id); + $this->activityId = $activity->id; + $this->dispatch('activityMonitor', $activity->id); + $this->dispatch('databaserestore'); + auditLog('ui.database.import_started', [ + 'team_id' => $this->resource->team()?->id, + 'database_uuid' => $this->resource->uuid, + 'database_name' => $this->resource->name, + 'source' => 'file', + ]); + } catch (DatabaseImportException $e) { + $this->dispatch('error', $e->getMessage()); } catch (\Throwable $e) { handleError($e, $this); @@ -660,118 +606,9 @@ EOD; try { $this->importRunning = true; - - $s3Storage = S3Storage::ownedByCurrentTeam()->findOrFail($this->s3StorageId); - - $key = $s3Storage->key; - $secret = $s3Storage->secret; - $bucket = $s3Storage->bucket; - $endpoint = $s3Storage->endpoint; - - // Validate bucket name to prevent command injection - if (! $this->validateBucketName($bucket)) { - $this->dispatch('error', 'Invalid S3 bucket name. Bucket name must contain only lowercase letters, numbers, dots, and dashes, and must follow S3 bucket naming rules.'); - - return true; - } - - // Clean the S3 path - $cleanPath = ltrim($this->s3Path, '/'); - - // Validate the S3 path to prevent command injection - if (! $this->validateS3Path($cleanPath)) { - $this->dispatch('error', 'Invalid S3 path. Path must contain only safe characters (alphanumerics, dots, dashes, underscores, slashes).'); - - return true; - } - - // Get helper image - $helperImage = coolifyHelperImage(); - $latestVersion = getHelperVersion(); - $fullImageName = "{$helperImage}:{$latestVersion}"; - - // Get the database destination network - if ($this->resource->getMorphClass() === ServiceDatabase::class) { - $destinationNetwork = $this->resource->service->destination->network ?? 'coolify'; - } else { - $destinationNetwork = $this->resource->destination->network ?? 'coolify'; - } - - // Generate unique names for this operation - $containerName = "s3-restore-{$this->resourceUuid}"; - $helperTmpPath = '/tmp/'.basename($cleanPath); - $serverTmpPath = "/tmp/s3-restore-{$this->resourceUuid}-".basename($cleanPath); - $containerTmpPath = "/tmp/restore_{$this->resourceUuid}-".basename($cleanPath); - $scriptPath = "/tmp/restore_{$this->resourceUuid}.sh"; - - $escapedServerTmpPath = escapeshellarg($serverTmpPath); - $escapedContainerTmpPath = escapeshellarg($containerTmpPath); - $escapedScriptPath = escapeshellarg($scriptPath); - $escapedHelperContainerPath = escapeshellarg("{$containerName}:{$helperTmpPath}"); - $escapedDatabaseContainerTmpPath = escapeshellarg("{$this->container}:{$containerTmpPath}"); - $escapedDatabaseContainerScriptPath = escapeshellarg("{$this->container}:{$scriptPath}"); - $restoreAndCleanupCommand = escapeshellarg("{$escapedScriptPath} && rm -f {$escapedContainerTmpPath} {$escapedScriptPath}"); - - // Prepare all commands in sequence - $commands = []; - - // 1. Clean up any existing helper container and temp files from previous runs - $commands[] = "docker rm -f {$containerName} 2>/dev/null || true"; - $commands[] = "rm -f {$escapedServerTmpPath} 2>/dev/null || true"; - $commands[] = "docker exec {$this->container} rm -f {$escapedContainerTmpPath} {$escapedScriptPath} 2>/dev/null || true"; - - // 2. Start helper container on the database network - $commands[] = "docker run -d --network {$destinationNetwork} --name {$containerName} {$fullImageName} sleep 3600"; - - // 3. Configure S3 access in helper container - $escapedEndpoint = escapeshellarg($endpoint); - $escapedKey = escapeshellarg($key); - $escapedSecret = escapeshellarg($secret); - $commands[] = "docker exec {$containerName} mc alias set s3temp {$escapedEndpoint} {$escapedKey} {$escapedSecret}"; - - // 4. Check file exists in S3 (bucket and path already validated above) - $escapedS3Source = escapeshellarg("s3temp/{$bucket}/{$cleanPath}"); - $commands[] = "docker exec {$containerName} mc stat {$escapedS3Source}"; - - // 5. Download from S3 to helper container (progress shown by default) - $escapedHelperTmpPath = escapeshellarg($helperTmpPath); - $commands[] = "docker exec {$containerName} mc cp {$escapedS3Source} {$escapedHelperTmpPath}"; - - // 6. Copy from helper to server, then immediately to database container - $commands[] = "docker cp {$escapedHelperContainerPath} {$escapedServerTmpPath}"; - $commands[] = "docker cp {$escapedServerTmpPath} {$escapedDatabaseContainerTmpPath}"; - $this->addRestoreSafetyCheckCommand($commands, $containerTmpPath); - - // 7. Cleanup helper container and server temp file immediately (no longer needed) - $commands[] = "docker rm -f {$containerName} 2>/dev/null || true"; - $commands[] = "rm -f {$escapedServerTmpPath} 2>/dev/null || true"; - - // 8. Build and execute restore command inside database container - $restoreCommand = $this->buildRestoreCommand($containerTmpPath); - - $restoreCommandBase64 = base64_encode($restoreCommand); - $commands[] = "echo \"{$restoreCommandBase64}\" | base64 -d > {$escapedScriptPath}"; - $commands[] = "chmod +x {$escapedScriptPath}"; - $commands[] = "docker cp {$escapedScriptPath} {$escapedDatabaseContainerScriptPath}"; - - // 9. Execute restore and cleanup temp files immediately after completion - $commands[] = "docker exec {$this->container} sh -c {$restoreAndCleanupCommand}"; - $commands[] = "docker exec {$this->container} sh -c 'echo \"Import finished with exit code $?\"'"; - - // Execute all commands with cleanup event (as safety net for edge cases) - $activity = remote_process($commands, $this->server, ignore_errors: true, callEventOnFinish: 'S3RestoreJobFinished', callEventData: [ - 'containerName' => $containerName, - 'serverTmpPath' => $serverTmpPath, - 'scriptPath' => $scriptPath, - 'containerTmpPath' => $containerTmpPath, - 'container' => $this->container, - 'serverId' => $this->server->id, - ]); - - // Track the activity ID + $source = new DatabaseImportSource('s3', path: $this->s3Path, s3StorageUuid: (string) $this->s3StorageId, dumpAll: $this->dumpAll); + $activity = StartDatabaseImport::run($this->resource, $source, (int) currentTeam()->id); $this->activityId = $activity->id; - - // Dispatch activity to the monitor and open slide-over $this->dispatch('activityMonitor', $activity->id); $this->dispatch('databaserestore'); auditLog('ui.database.restore_started', [ @@ -782,6 +619,8 @@ EOD; 'storage_id' => $this->s3StorageId, ]); $this->dispatch('info', 'Restoring database from S3. Progress will be shown in the activity monitor...'); + } catch (DatabaseImportException $e) { + $this->dispatch('error', $e->getMessage()); } catch (\Throwable $e) { $this->importRunning = false; handleError($e, $this); @@ -794,13 +633,7 @@ EOD; public function buildRestoreSafetyCheckCommand(string $tmpPath): ?string { - $script = $this->buildPostgresRestoreScanScript($tmpPath); - - if ($script === null) { - return null; - } - - return "docker exec {$this->container} sh -c ".escapeshellarg($script); + return app(DatabaseImportCommandBuilder::class)->buildPostgresSafetyCommand($this->resource, $this->container, $tmpPath); } /** @@ -856,59 +689,6 @@ EOD; public function buildRestoreCommand(string $tmpPath): string { - $escapedTmpPath = escapeshellarg($tmpPath); - $morphClass = $this->resource->getMorphClass(); - - // Handle ServiceDatabase by checking the database type - if ($morphClass === ServiceDatabase::class) { - $dbType = $this->resource->databaseType(); - if (str_contains($dbType, 'mysql')) { - $morphClass = 'mysql'; - } elseif (str_contains($dbType, 'mariadb')) { - $morphClass = 'mariadb'; - } elseif (str_contains($dbType, 'postgres')) { - $morphClass = 'postgresql'; - } elseif (str_contains($dbType, 'mongo')) { - $morphClass = 'mongodb'; - } - } - - switch ($morphClass) { - case StandaloneMariadb::class: - case 'mariadb': - $restoreCommand = $this->mariadbRestoreCommand; - if ($this->dumpAll) { - $restoreCommand .= " && (gunzip -cf {$escapedTmpPath} 2>/dev/null || cat {$escapedTmpPath}) | mariadb -u root -p\$MARIADB_ROOT_PASSWORD \${MARIADB_DATABASE:-default}"; - } else { - $restoreCommand .= " < {$escapedTmpPath}"; - } - break; - case StandaloneMysql::class: - case 'mysql': - $restoreCommand = $this->mysqlRestoreCommand; - if ($this->dumpAll) { - $restoreCommand .= " && (gunzip -cf {$escapedTmpPath} 2>/dev/null || cat {$escapedTmpPath}) | mysql -u root -p\$MYSQL_ROOT_PASSWORD \${MYSQL_DATABASE:-default}"; - } else { - $restoreCommand .= " < {$escapedTmpPath}"; - } - break; - case StandalonePostgresql::class: - case 'postgresql': - $restoreCommand = $this->postgresqlRestoreCommand; - if ($this->dumpAll) { - $restoreCommand .= " && (gunzip -cf {$escapedTmpPath} 2>/dev/null || cat {$escapedTmpPath}) | psql -U \${POSTGRES_USER} -d \${POSTGRES_DB:-\${POSTGRES_USER:-postgres}}"; - } else { - $restoreCommand .= " {$escapedTmpPath}"; - } - break; - case StandaloneMongodb::class: - case 'mongodb': - $restoreCommand = $this->mongodbRestoreCommand.$escapedTmpPath; - break; - default: - $restoreCommand = ''; - } - - return $restoreCommand; + return app(DatabaseImportCommandBuilder::class)->buildRestoreCommand($this->resource, $tmpPath, $this->dumpAll); } } diff --git a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php new file mode 100644 index 0000000000..1f69ce8c96 --- /dev/null +++ b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php @@ -0,0 +1,70 @@ +databaseType($resource)) { + 'postgresql' => $dumpAll + ? 'psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} && createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} && (gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}' + : 'pg_restore -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$path, + 'mysql' => $dumpAll + ? $this->mysqlDumpAll('mysql', 'MYSQL', $path) + : 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE < '.$path, + 'mariadb' => $dumpAll + ? $this->mysqlDumpAll('mariadb', 'MARIADB', $path) + : 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE < '.$path, + 'mongodb' => 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive='.$path, + default => throw new InvalidArgumentException('Database import is not supported for this database type.'), + }; + } + + public function buildPostgresSafetyCommand(object $resource, string $container, string $path): ?string + { + if ($this->databaseType($resource) !== 'postgresql') { + return null; + } + + $path = escapeshellarg($path); + $separator = '([[:space:]]|/\\*[^*]*\\*/)'; + $sqlPattern = escapeshellarg("(^|;){$separator}*copy{$separator}+[^;]*(from|to){$separator}+program"); + $psqlPattern = escapeshellarg("^{$separator}*\\\\(!|copy{$separator}+[^[:space:]]+.*{$separator}+program|(o|g){$separator}*\\|)"); + $contents = "{ gunzip -cf {$path} 2>/dev/null || cat {$path}; }"; + $script = "header=\$({$contents} | head -c 5); if [ \"\$header\" = 'PGDMP' ]; then exit 0; fi; if {$contents} | sed 's/--.*//' | grep -Eiq {$psqlPattern} || {$contents} | sed 's/--.*//' | tr '\n\r\t' ' ' | grep -Eiq {$sqlPattern}; then echo 'Blocked PostgreSQL restore: COPY ... PROGRAM and psql shell commands are not allowed.'; exit 1; fi"; + + return 'docker exec '.$container.' sh -c '.escapeshellarg($script); + } + + public function supports(object $resource): bool + { + return in_array($this->databaseType($resource), ['postgresql', 'mysql', 'mariadb', 'mongodb'], true); + } + + public function databaseType(object $resource): string + { + $class = $resource->getMorphClass(); + $type = ($resource instanceof ServiceDatabase || str_contains(strtolower($class), 'service')) + ? strtolower($resource->databaseType()) + : strtolower($class); + + return match (true) { + str_contains($type, 'postgres') => 'postgresql', + str_contains($type, 'mariadb') => 'mariadb', + str_contains($type, 'mysql') => 'mysql', + str_contains($type, 'mongo') => 'mongodb', + default => 'unsupported', + }; + } + + private function mysqlDumpAll(string $binary, string $prefix, string $path): string + { + return "for pid in \$({$binary} -u root -p\${{$prefix}_ROOT_PASSWORD} -N -e \"SELECT id FROM information_schema.processlist WHERE user != 'root';\"); do {$binary} -u root -p\${{$prefix}_ROOT_PASSWORD} -e \"KILL \$pid\" 2>/dev/null || true; done && {$binary} -u root -p\${{$prefix}_ROOT_PASSWORD} -N -e \"SELECT CONCAT('DROP DATABASE IF EXISTS \\`',schema_name,'\\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');\" | {$binary} -u root -p\${{$prefix}_ROOT_PASSWORD} && {$binary} -u root -p\${{$prefix}_ROOT_PASSWORD} -e \"CREATE DATABASE IF NOT EXISTS \\`\${{{$prefix}_DATABASE:-default}}\\`;\" && (gunzip -cf {$path} 2>/dev/null || cat {$path}) | {$binary} -u root -p\${{{$prefix}_ROOT_PASSWORD}} \${{{$prefix}_DATABASE:-default}}"; + } +} diff --git a/app/Support/DatabaseImport/DatabaseImportException.php b/app/Support/DatabaseImport/DatabaseImportException.php new file mode 100644 index 0000000000..aeef2067ba --- /dev/null +++ b/app/Support/DatabaseImport/DatabaseImportException.php @@ -0,0 +1,13 @@ +middleware(['api.ability:write']); Route::get('/databases/{uuid}', [DatabasesController::class, 'database_by_uuid'])->middleware(['api.ability:read']); + Route::post('/databases/{uuid}/imports/uploads', [DatabasesController::class, 'upload_import'])->middleware(['api.ability:deploy'])->name('api.databases.imports.upload'); + Route::post('/databases/{uuid}/imports', [DatabasesController::class, 'create_import'])->middleware(['api.ability:deploy'])->name('api.databases.imports.store'); + Route::get('/databases/{uuid}/imports/{activity_id}', [DatabasesController::class, 'show_import'])->middleware(['api.ability:read'])->name('api.databases.imports.show'); Route::get('/databases/{uuid}/backups', [DatabasesController::class, 'database_backup_details_uuid'])->middleware(['api.ability:read']); Route::get('/databases/{uuid}/backups/{scheduled_backup_uuid}/executions', [DatabasesController::class, 'list_backup_executions'])->middleware(['api.ability:read']); Route::patch('/databases/{uuid}', [DatabasesController::class, 'update_by_uuid'])->middleware(['api.ability:write']); @@ -407,6 +410,9 @@ Route::group([ Route::get('/services/{uuid}/databases', [ServiceDatabasesController::class, 'index'])->middleware(['api.ability:read']); Route::get('/services/{uuid}/databases/{database_uuid}', [ServiceDatabasesController::class, 'show'])->middleware(['api.ability:read']); + Route::post('/services/{uuid}/databases/{database_uuid}/imports/uploads', [ServiceDatabasesController::class, 'upload_import'])->middleware(['api.ability:deploy'])->name('api.service-databases.imports.upload'); + Route::post('/services/{uuid}/databases/{database_uuid}/imports', [ServiceDatabasesController::class, 'create_import'])->middleware(['api.ability:deploy'])->name('api.service-databases.imports.store'); + Route::get('/services/{uuid}/databases/{database_uuid}/imports/{activity_id}', [ServiceDatabasesController::class, 'show_import'])->middleware(['api.ability:read'])->name('api.service-databases.imports.show'); Route::patch('/services/{uuid}/databases/{database_uuid}', [ServiceDatabasesController::class, 'update'])->middleware(['api.ability:write']); Route::get('/services/{uuid}/databases/{database_uuid}/logs', [ServiceDatabasesController::class, 'logs'])->middleware(['api.ability:read']); Route::post('/services/{uuid}/databases/{database_uuid}/start', [ServiceDatabasesController::class, 'start'])->middleware(['api.ability:deploy']); diff --git a/tests/Feature/Api/DatabaseImportApiTest.php b/tests/Feature/Api/DatabaseImportApiTest.php new file mode 100644 index 0000000000..566ab0da8d --- /dev/null +++ b/tests/Feature/Api/DatabaseImportApiTest.php @@ -0,0 +1,63 @@ + 0, 'is_api_enabled' => true]); + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user, ['role' => 'owner']); + session(['currentTeam' => $this->team]); + $this->token = $this->user->tokens()->create(['name' => 'imports', 'token' => hash('sha256', 'secret'), 'abilities' => ['deploy', 'read'], 'team_id' => $this->team->id]); + $this->headers = ['Authorization' => 'Bearer '.$this->token->id.'|secret']; + $this->server = Server::factory()->create(['team_id' => $this->team->id]); + $this->destination = StandaloneDocker::firstOrCreate(['server_id' => $this->server->id, 'network' => 'coolify'], ['uuid' => (string) Str::uuid(), 'name' => 'docker']); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); +}); + +test('validates standalone import source and hides foreign databases', function () { + $database = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); + + $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$database->uuid}/imports", ['source' => 'upload', 'path' => '../bad']) + ->assertUnprocessable()->assertJsonValidationErrors(['upload_id', 'path']); + + $otherTeam = Team::factory()->create(); + $otherProject = Project::factory()->create(['team_id' => $otherTeam->id]); + $otherEnvironment = Environment::factory()->create(['project_id' => $otherProject->id]); + $foreign = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'foreign', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $otherEnvironment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); + + $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$foreign->uuid}/imports", ['source' => 'server', 'path' => '/tmp/a.sql'])->assertNotFound(); +}); + +test('requires deploy ability to start standalone import', function () { + $database = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); + $read = $this->user->createToken('read', ['read']); + + $this->withToken($read->plainTextToken)->postJson("/api/v1/databases/{$database->uuid}/imports", ['source' => 'server', 'path' => '/tmp/a.sql'])->assertForbidden(); +}); + +test('returns only a team and resource scoped import activity', function () { + $database = StandalonePostgresql::create(['uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass()]); + $activity = Activity::create(['log_name' => 'default', 'description' => json_encode([['order' => 1, 'output' => 'restored', 'type' => 'stdout']]), 'properties' => ['team_id' => $this->team->id, 'type_uuid' => $database->uuid, 'operation' => 'database_import', 'status' => 'finished', 'exitCode' => 0]]); + + $this->withHeaders($this->headers)->getJson("/api/v1/databases/{$database->uuid}/imports/{$activity->id}") + ->assertOk()->assertJson(['id' => $activity->id, 'status' => 'finished', 'exit_code' => 0, 'output' => 'restored']) + ->assertJsonMissingPath('command'); + + $activity->properties = $activity->properties->merge(['team_id' => $this->team->id + 1]); + $activity->save(); + $this->withHeaders($this->headers)->getJson("/api/v1/databases/{$database->uuid}/imports/{$activity->id}")->assertNotFound(); +}); diff --git a/tests/Feature/Api/DatabaseImportRoutesTest.php b/tests/Feature/Api/DatabaseImportRoutesTest.php new file mode 100644 index 0000000000..db62b9af19 --- /dev/null +++ b/tests/Feature/Api/DatabaseImportRoutesTest.php @@ -0,0 +1,14 @@ +getRoutes())->keyBy(fn ($route) => $route->getName()); + + foreach (['api.databases.imports.upload', 'api.databases.imports.store', 'api.databases.imports.show', 'api.service-databases.imports.upload', 'api.service-databases.imports.store', 'api.service-databases.imports.show'] as $name) { + expect($routes)->toHaveKey($name); + } + + expect($routes['api.databases.imports.store']->gatherMiddleware())->toContain('api.ability:deploy') + ->and($routes['api.databases.imports.show']->gatherMiddleware())->toContain('api.ability:read'); +}); diff --git a/tests/Feature/Api/ServiceDatabaseImportApiTest.php b/tests/Feature/Api/ServiceDatabaseImportApiTest.php new file mode 100644 index 0000000000..ae05455f5e --- /dev/null +++ b/tests/Feature/Api/ServiceDatabaseImportApiTest.php @@ -0,0 +1,42 @@ + 0, 'is_api_enabled' => true]); + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user, ['role' => 'owner']); + session(['currentTeam' => $this->team]); + $this->token = $this->user->tokens()->create(['name' => 'imports', 'token' => hash('sha256', 'secret'), 'abilities' => ['deploy', 'read'], 'team_id' => $this->team->id]); + $this->headers = ['Authorization' => 'Bearer '.$this->token->id.'|secret']; + $this->server = Server::factory()->create(['team_id' => $this->team->id]); + $this->destination = StandaloneDocker::firstOrCreate(['server_id' => $this->server->id, 'network' => 'coolify'], ['uuid' => (string) Str::uuid(), 'name' => 'docker']); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); +}); + +use App\Models\Service; +use App\Models\ServiceDatabase; + +test('validates service database imports and binds database to service', function () { + $service = Service::factory()->create(['environment_id' => $this->environment->id, 'server_id' => $this->server->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(), 'docker_compose_raw' => "services:\n postgres:\n image: postgres:17\n"]); + $database = ServiceDatabase::create(['uuid' => (string) Str::uuid(), 'name' => 'postgres', 'service_id' => $service->id, 'image' => 'postgres:17']); + + $url = "/api/v1/services/{$service->uuid}/databases/{$database->uuid}/imports"; + $this->withHeaders($this->headers)->postJson($url, ['source' => 's3', 'upload_id' => (string) Str::uuid()]) + ->assertUnprocessable()->assertJsonValidationErrors(['upload_id', 's3_storage_uuid', 'path']); + + $otherService = Service::factory()->create(['environment_id' => $this->environment->id, 'server_id' => $this->server->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(), 'docker_compose_raw' => "services: {}\n"]); + $this->withHeaders($this->headers)->postJson("/api/v1/services/{$otherService->uuid}/databases/{$database->uuid}/imports", ['source' => 'server', 'path' => '/tmp/a.sql'])->assertNotFound(); +}); diff --git a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php new file mode 100644 index 0000000000..43eeb1d333 --- /dev/null +++ b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php @@ -0,0 +1,59 @@ +shouldReceive('getMorphClass')->andReturn($class); + if ($class === ServiceDatabase::class) { + $resource->shouldReceive('databaseType')->andReturn($databaseType); + } + + return $resource; +} + +test('builds database-specific restore commands', function (string $class, ?string $type, string $needle) { + $builder = new DatabaseImportCommandBuilder; + + $command = $builder->buildRestoreCommand(importResource($class, $type), '/tmp/restore file', false); + + expect($command)->toContain($needle)->toContain("'/tmp/restore file'"); +})->with([ + 'postgresql' => [StandalonePostgresql::class, null, 'pg_restore'], + 'mysql' => [StandaloneMysql::class, null, 'mysql -u $MYSQL_USER'], + 'mariadb' => [StandaloneMariadb::class, null, 'mariadb -u $MARIADB_USER'], + 'mongodb' => [StandaloneMongodb::class, null, 'mongorestore'], + 'service postgres' => [ServiceDatabase::class, 'postgresql', 'pg_restore'], + 'service mysql' => [ServiceDatabase::class, 'mysql', 'mysql -u $MYSQL_USER'], + 'service mariadb' => [ServiceDatabase::class, 'mariadb', 'mariadb -u $MARIADB_USER'], + 'service mongo' => [ServiceDatabase::class, 'mongodb', 'mongorestore'], +]); + +test('builds dump-all commands and postgres safety scan', function () { + $builder = new DatabaseImportCommandBuilder; + $postgres = importResource(StandalonePostgresql::class); + + expect($builder->buildRestoreCommand($postgres, '/tmp/dump.sql.gz', true)) + ->toContain('pg_terminate_backend') + ->toContain("gunzip -cf '/tmp/dump.sql.gz'") + ->and($builder->buildPostgresSafetyCommand($postgres, 'postgres-safe', '/tmp/dump.sql.gz')) + ->toContain('COPY ... PROGRAM') + ->toContain('docker exec postgres-safe') + ->toContain('| tr') + ->toContain('/\\*[^*]*\\*/'); +}); + +test('rejects unsupported database types', function () { + $builder = new DatabaseImportCommandBuilder; + $redis = importResource(StandaloneRedis::class); + + expect(fn () => $builder->buildRestoreCommand($redis, '/tmp/backup', false)) + ->toThrow(InvalidArgumentException::class, 'not supported'); +}); diff --git a/tests/Unit/DatabaseImportOpenApiTest.php b/tests/Unit/DatabaseImportOpenApiTest.php new file mode 100644 index 0000000000..9e54855a66 --- /dev/null +++ b/tests/Unit/DatabaseImportOpenApiTest.php @@ -0,0 +1,13 @@ +toHaveKey('/databases/{uuid}/imports/uploads') + ->toHaveKey('/databases/{uuid}/imports') + ->toHaveKey('/databases/{uuid}/imports/{activity_id}') + ->toHaveKey('/services/{uuid}/databases/{database_uuid}/imports/uploads') + ->toHaveKey('/services/{uuid}/databases/{database_uuid}/imports') + ->toHaveKey('/services/{uuid}/databases/{database_uuid}/imports/{activity_id}'); +});