From 21d1ad4874eee2468935ae0e59333813f76371f7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:43:47 +0100 Subject: [PATCH] fix(backups): upload to S3 endpoints with a path, like Supabase Storage The MinIO client rejects endpoints such as `/storage/v1/s3`. Transfers to these endpoints now use curl with a presigned URL that Coolify signs, for database backups, volume backups, and S3 restores. Closes #8530 Co-Authored-By: Claude Opus 5.5 --- app/Actions/Database/StartDatabaseImport.php | 19 ++-- app/Jobs/DatabaseBackupJob.php | 16 ++-- app/Jobs/VolumeBackupJob.php | 20 +++-- app/Models/S3Storage.php | 84 ++++++++++++++++++ tests/Feature/BackupS3CredentialsArgvTest.php | 88 +++++++++++++++++++ .../DatabaseImportInterruptedCleanupTest.php | 4 + ...artDatabaseImportBackupPreparationTest.php | 4 + .../StartDatabaseImportS3CredentialsTest.php | 56 ++++++++++++ tests/Unit/S3StorageTest.php | 13 +++ 9 files changed, 281 insertions(+), 23 deletions(-) diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index e8fd053ec0..cb08f5a90c 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -194,10 +194,8 @@ class StartDatabaseImport throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.'); } $helper = "s3-restore-{$operation}"; - $resolveOptions = $this->startS3Helper($storage, $server, $helper, $network); - $sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}"); $commandList = [ - 'docker exec '.escapeshellarg($helper).' mc cp'.$resolveOptions.' '.$sourceArg.' /tmp/restore', + $this->startS3Helper($storage, $server, $helper, $network, $key), ...$this->streamFromHelper($helper, $container, $containerPath), 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true', ]; @@ -326,13 +324,13 @@ class StartDatabaseImport } /** - * Starts the S3 restore helper and sets its `s3temp` mc alias. The S3 keys go to mc through the - * stdin of a separate SSH process, so they are not in the arguments of any process on Coolify or - * on the server. The helper is removed when either step fails. + * Starts the S3 restore helper and sets its `s3temp` mc alias, or the curl config for endpoints with a path. + * The S3 keys go to mc through the stdin of a separate SSH process, so they are not in the arguments of any + * process on Coolify or on the server. The helper is removed when either step fails. * - * @return string The mc `--resolve` option that pins the endpoint host. + * @return string The command that downloads $key to /tmp/restore in the helper. */ - private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network): string + private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network, string $key): string { $image = coolifyHelperImage().':'.getHelperVersion(); @@ -344,10 +342,13 @@ class StartDatabaseImport .' '.$storage->minioClientEnvOptions() .' '.escapeshellarg($image).' sleep 86400', ], $server); + if ($storage->usesCurlTransfer()) { + return $storage->prepareCurlDownload($server, $helper, $key, '/tmp/restore'); + } $resolveOptions = $storage->minioClientResolveOptions($server, $image, $network); $this->setMinioClientAlias($storage, $server, $helper, $resolveOptions); - return $resolveOptions; + return 'docker exec '.escapeshellarg($helper).' mc cp'.$resolveOptions.' '.escapeshellarg("s3temp/{$storage->bucket}/{$key}").' /tmp/restore'; } catch (Throwable) { instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false); diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index d1c7e80ee6..776a17f58e 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -941,14 +941,18 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $mount = escapeshellarg($this->backupMountSource().':'.$this->backup_location.':ro'); $commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm {$s3->minioClientEnvOptions()} -v {$mount} {$fullImageName}"; - $escapedBackupLocation = escapeshellarg($this->backup_location); - $escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/"); - $resolveOptions = $s3->minioClientResolveOptions($this->server, $fullImageName, $network); - instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true); - $this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions); + if ($s3->usesCurlTransfer()) { + $uploadCommand = $s3->prepareCurlUpload($this->server, "backup-of-{$this->backup_log_uuid}", $this->backup_location, ltrim($this->backup_dir, '/').'/'.basename($this->backup_location)); + } else { + $escapedBackupLocation = escapeshellarg($this->backup_location); + $escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/"); + $resolveOptions = $s3->minioClientResolveOptions($this->server, $fullImageName, $network); + $this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions); + $uploadCommand = "docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}"; + } S3Storage::retryMinioClientUpload(fn () => instant_remote_process( - ["docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}"], + [$uploadCommand], $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true, )); diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index 916b8479a2..0216c8ee4a 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -90,8 +90,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz'; $backupLocation = $backupDirectory.'/'.$filename; $this->execution->update(['filename' => $backupLocation]); - // Streaming writes no local archive, so it can upload to one destination only. - $streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1; + // Streaming writes no local archive, so it can upload to one destination only. A curl upload needs the size + // of the file before it starts, so it cannot stream. + $streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1 && ! $destinations->first()->usesCurlTransfer(); try { $source = $this->backup->sourcePath(); @@ -468,7 +469,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $s3->testConnection(shouldSave: true); $containerName = 'volume-upload-'.$this->execution->uuid; $image = coolifyHelperImage().':'.getHelperVersion(); - $resolveOptions = $s3->minioClientResolveOptions($server, $image); try { instant_remote_process([ @@ -476,11 +476,15 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue 'docker run -d --name '.escapeshellarg($containerName).' --rm '.$s3->minioClientEnvOptions().' -v ' .escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image), ], $server, timeout: $this->timeout, disableMultiplexing: true); - $this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions); - S3Storage::retryMinioClientUpload(fn () => instant_remote_process([ - 'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' ' - .escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'), - ], $server, timeout: $this->timeout, disableMultiplexing: true)); + if ($s3->usesCurlTransfer()) { + $uploadCommand = $s3->prepareCurlUpload($server, $containerName, $backupLocation, ltrim($backupDirectory, '/').'/'.basename($backupLocation)); + } else { + $resolveOptions = $s3->minioClientResolveOptions($server, $image); + $this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions); + $uploadCommand = 'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' ' + .escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'); + } + S3Storage::retryMinioClientUpload(fn () => instant_remote_process([$uploadCommand], $server, timeout: $this->timeout, disableMultiplexing: true)); } finally { instant_remote_process( ['docker rm -f '.escapeshellarg($containerName)], diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index d063aa13b7..9c76104ac2 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -2,6 +2,8 @@ namespace App\Models; +use App\Helpers\SshMultiplexingHelper; +use App\Helpers\SshRetryHandler; use App\Rules\SafeWebhookUrl; use App\Rules\ValidS3BucketName; use App\Traits\Auditable; @@ -39,6 +41,14 @@ class S3Storage extends BaseModel */ private const MINIO_CLIENT_UPLOAD_RETRY_DELAYS_MS = [10_000, 30_000]; + /** + * How long the presigned URL of a curl transfer is valid. S3 checks it when the request starts, so it must + * cover the wait for the earlier transfers and the retries of the backup job. + */ + private const CURL_TRANSFER_URL_TTL_HOURS = 24; + + private const CURL_TRANSFER_CONFIG_PATH = '/tmp/s3-transfer.curl'; + /** * The exception code of runSshProcess() when a remote command times out. */ @@ -391,6 +401,80 @@ class S3Storage extends BaseModel return ' --api S3v4'.($this->usesPathStyleEndpoint() ? '' : ' --path off'); } + /** + * The MinIO client accepts only `scheme://host[:port]` endpoints. Providers like Supabase Storage need a path + * (`/storage/v1/s3`), so transfers to these endpoints use curl with a presigned URL instead. + */ + public function usesCurlTransfer(): bool + { + return trim((string) parse_url((string) $this->endpoint, PHP_URL_PATH), '/') !== ''; + } + + /** + * Prepares the upload of $localPath in $container to $key and returns the command that runs it. + */ + public function prepareCurlUpload(Server $server, string $container, string $localPath, string $key): string + { + $url = $this->filesystem()->temporaryUploadUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS))['url']; + + // An S3 error is an XML body. Write it to stderr, so it becomes the error message of the backup. + return $this->prepareCurlTransfer($server, $container, $url, ['upload-file = '.self::curlConfigValue($localPath), 'fail-with-body', 'output = "/dev/stderr"']); + } + + /** + * Prepares the download of $key to $localPath in $container and returns the command that runs it. + */ + public function prepareCurlDownload(Server $server, string $container, string $key, string $localPath): string + { + $url = $this->filesystem()->temporaryUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS)); + + return $this->prepareCurlTransfer($server, $container, $url, ['output = '.self::curlConfigValue($localPath), 'fail']); + } + + /** + * Writes the curl config of a transfer into $container through the stdin of the SSH process. The config holds + * the presigned URL, so the URL is not in the arguments of any process, and the S3 keys never reach the server. + * The `resolve` line pins the endpoint host to its validated IPs, as minioClientResolveOptions() does for mc. + * + * @param array $options + */ + private function prepareCurlTransfer(Server $server, string $container, string $url, array $options): string + { + $entries = SafeWebhookUrl::minioClientResolveOptions((string) $this->endpoint, $this->trustedInternalHosts()); + $config = collect(['url = '.self::curlConfigValue($url), 'silent', 'show-error', ...$options]) + ->when($entries !== [], fn ($lines) => $lines->push('resolve = '.self::curlConfigValue( + str($entries[0])->beforeLast('=')->toString().':'.collect($entries) + ->map(fn (string $entry): string => str($entry)->afterLast('=')->toString()) + ->map(fn (string $ip): string => str_contains($ip, ':') ? "[{$ip}]" : $ip) + ->implode(',') + ))) + ->when($this->skip_tls_verification, fn ($lines) => $lines->push('insecure')) + ->implode("\n")."\n"; + + $remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($container) + .' sh -c '.escapeshellarg('umask 077 && cat > '.self::CURL_TRANSFER_CONFIG_PATH); + $timeout = (int) config('constants.ssh.command_timeout'); + + SshRetryHandler::retry( + function () use ($server, $remoteCommand, $config, $timeout) { + $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout); + $process = runSshProcess($sshCommand, $timeout, $config); + + if ($process->exitCode() !== 0) { + excludeCertainErrors($process->errorOutput(), $process->exitCode()); + } + }, + ['server' => $server->ip, 'function' => 'prepareCurlTransfer'], + ); + + return 'docker exec '.escapeshellarg($container).' curl --config '.self::CURL_TRANSFER_CONFIG_PATH; + } + + private static function curlConfigValue(string $value): string + { + return '"'.addcslashes($value, '"\\').'"'; + } + /** * The mc `--resolve` option that pins the endpoint host to one of its validated IPs. mc keeps one IP per host, * so when the host has more than one IP, a helper container on the upload network connects to each IP, IPv4 diff --git a/tests/Feature/BackupS3CredentialsArgvTest.php b/tests/Feature/BackupS3CredentialsArgvTest.php index 2d8c8ebaaf..4343e53fb3 100644 --- a/tests/Feature/BackupS3CredentialsArgvTest.php +++ b/tests/Feature/BackupS3CredentialsArgvTest.php @@ -63,6 +63,8 @@ function s3ArgvTestSetup(string $sshUser): array $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]); $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + $disk->shouldReceive('temporaryUploadUrl')->zeroOrMoreTimes() + ->andReturnUsing(fn (string $key) => ['url' => "https://8.8.8.8/storage/v1/s3/bucket/{$key}?X-Amz-Signature=presigned", 'headers' => []]); $sshDisk = Storage::fake('ssh-keys'); Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk); @@ -267,3 +269,89 @@ it('passes the S3 keys of a volume backup to mc on stdin, not in a command line' ->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBe($streamToS3); s3ArgvTestAssertSecretOnlyOnStdin($processes, $sshUser !== 'root'); })->with(['root', 'deploy'])->with(['local archive upload' => false, 'S3-only stream' => true]); + +/** + * @param Collection $processes + */ +function s3ArgvTestAssertCurlUpload(Collection $processes, string $backupLocation, bool $nonRoot): void +{ + $configProcess = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl')); + $otherProcesses = $processes->reject(fn (array $process) => $process === $configProcess); + + expect($configProcess['input'])->toBe(implode("\n", [ + 'url = "https://8.8.8.8/storage/v1/s3/bucket/'.ltrim($backupLocation, '/').'?X-Amz-Signature=presigned"', + 'silent', + 'show-error', + 'upload-file = "'.$backupLocation.'"', + 'fail-with-body', + 'output = "/dev/stderr"', + ])."\n") + ->and(str_contains($configProcess['command'], 'sudo docker exec -i'))->toBe($nonRoot) + ->and($otherProcesses->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'curl --config /tmp/s3-transfer.curl')))->toBeTrue() + ->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc alias set')))->toBeFalse() + ->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBeFalse() + ->and($otherProcesses->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'X-Amz-Signature')))->toBeFalse() + ->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], S3_ARGV_TEST_SECRET)))->toBeFalse(); +} + +it('uploads a database backup with curl and a presigned URL when the endpoint has a path', function (string $sshUser) { + [$team, $server, $storage] = s3ArgvTestSetup($sshUser); + $storage->update(['endpoint' => 'https://8.8.8.8/storage/v1/s3']); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]); + $database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']); + $backup = new ScheduledDatabaseBackup; + $backup->forceFill([ + 'enabled' => true, + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $storage->id, + 'frequency' => 'daily', + 'databases_to_backup' => 'app', + 'database_id' => $database->id, + 'database_type' => $database->getMorphClass(), + 'team_id' => $team->id, + ])->save(); + $backup->syncS3Storages([$storage->id]); + $processes = s3ArgvTestFakeSsh(); + + (new DatabaseBackupJob($backup->fresh()))->handle(); + + $execution = $backup->executions()->sole(); + expect($execution->s3_uploaded)->toBeTrue(); + s3ArgvTestAssertCurlUpload($processes, $execution->filename, $sshUser !== 'root'); +})->with(['root', 'deploy']); + +it('writes a local archive and uploads it with curl when a volume backup endpoint has a path', function () { + [$team, $server, $storage] = s3ArgvTestSetup('root'); + $storage->update(['endpoint' => 'https://8.8.8.8/storage/v1/s3']); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $storage->id, + ]); + $processes = s3ArgvTestFakeSsh(); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + expect($execution->status)->toBe('success') + ->and($execution->s3_uploaded)->toBeTrue() + ->and($execution->local_storage_deleted)->toBeTrue(); + s3ArgvTestAssertCurlUpload($processes, $execution->filename, false); +}); diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.php index 8ee4d8d3ce..e956914b31 100644 --- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php +++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php @@ -16,6 +16,7 @@ use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\Team; +use App\Models\User; use App\Support\DatabaseImport\DatabaseImportCleanup; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; @@ -38,6 +39,9 @@ beforeEach(function () { InstanceSettings::forceCreate(['id' => 0]); $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + $this->actingAs($this->user); $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); $this->server = Server::factory()->create([ 'team_id' => $this->team->id, diff --git a/tests/Feature/StartDatabaseImportBackupPreparationTest.php b/tests/Feature/StartDatabaseImportBackupPreparationTest.php index 1741d157e7..34427f482d 100644 --- a/tests/Feature/StartDatabaseImportBackupPreparationTest.php +++ b/tests/Feature/StartDatabaseImportBackupPreparationTest.php @@ -11,6 +11,7 @@ use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\Team; +use App\Models\User; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; use App\Support\RemoteProcessCommand; @@ -32,6 +33,9 @@ beforeEach(function () { InstanceSettings::forceCreate(['id' => 0]); $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + $this->actingAs($this->user); $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); $this->server = Server::factory()->create([ 'team_id' => $this->team->id, diff --git a/tests/Feature/StartDatabaseImportS3CredentialsTest.php b/tests/Feature/StartDatabaseImportS3CredentialsTest.php index 045d5c1db9..04b79f5a3b 100644 --- a/tests/Feature/StartDatabaseImportS3CredentialsTest.php +++ b/tests/Feature/StartDatabaseImportS3CredentialsTest.php @@ -11,6 +11,7 @@ use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\Team; +use App\Models\User; use App\Support\DatabaseImport\DatabaseImportCleanup; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; @@ -34,6 +35,9 @@ beforeEach(function () { InstanceSettings::forceCreate(['id' => 0]); $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + $this->actingAs($this->user); $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); $this->server = Server::factory()->create([ 'team_id' => $this->team->id, @@ -358,3 +362,55 @@ it('removes the S3 restore helper when the mc alias cannot be set', function () expect($removedAfterAlias)->toBeTrue(); Queue::assertNothingPushed(); }); + +test('s3 import downloads with curl and a presigned URL when the endpoint has a path', function () { + $storage = S3Storage::create([ + 'name' => 'Supabase', + 'region' => 'eu-central-1', + 'key' => 'AKIA_PATH_ENDPOINT_KEY', + 'secret' => 'path/endpoint/SECRET', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://8.8.8.8/storage/v1/s3', + 'is_usable' => true, + 'team_id' => $this->team->id, + ]); + + $disk = Mockery::mock(FilesystemAdapter::class); + $disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true); + $disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024); + $disk->shouldReceive('temporaryUrl')->once()->with('backups/restore.sql', Mockery::any()) + ->andReturn('https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned'); + $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); + $filesystem->shouldReceive('build')->twice()->andReturn($disk); + Storage::swap($filesystem); + + $processes = collect(); + Process::fake(function ($process) use ($processes) { + $processes->push(['command' => $process->command, 'input' => $process->input]); + + return Process::result(); + }); + Queue::fake(); + + $activity = app(StartDatabaseImport::class)->handle( + $this->database, + new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), + $this->team->id, + ); + + $command = (string) RemoteProcessCommand::read($activity); + $config = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl'))['input']; + + expect($config)->toBe(implode("\n", [ + 'url = "https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned"', + 'silent', + 'show-error', + 'output = "/tmp/restore"', + 'fail', + ])."\n") + ->and($command)->toContain("docker exec 's3-restore-") + ->and($command)->toContain(' curl --config /tmp/s3-transfer.curl') + ->and($command)->not->toContain('mc cp') + ->and($command)->not->toContain('X-Amz-Signature') + ->and($command)->not->toContain('path/endpoint/SECRET'); +}); diff --git a/tests/Unit/S3StorageTest.php b/tests/Unit/S3StorageTest.php index 7ed53275f9..5718dadad4 100644 --- a/tests/Unit/S3StorageTest.php +++ b/tests/Unit/S3StorageTest.php @@ -298,3 +298,16 @@ test('S3Storage does not retry a MinIO client upload that timed out', function ( expect($attempts)->toBe(1); Sleep::assertNeverSlept(); }); + +test('S3Storage uses curl transfers only for endpoints with a path, which the MinIO client rejects', function (string $endpoint, bool $expected) { + $s3Storage = new S3Storage; + $s3Storage->setRawAttributes(['endpoint' => $endpoint]); + + expect($s3Storage->usesCurlTransfer())->toBe($expected); +})->with([ + 'Supabase Storage' => ['https://abcd.storage.supabase.co/storage/v1/s3', true], + 'path with trailing slash' => ['http://minio:9000/s3/', true], + 'host only' => ['https://s3.amazonaws.com', false], + 'host with trailing slash' => ['https://s3.amazonaws.com/', false], + 'host with port' => ['http://coolify-minio:9000', false], +]);