fix(backups): upload to S3 endpoints with a path, like Supabase Storage

The MinIO client rejects endpoints such as `/storage/v1/s3`. Transfers to
these endpoints now use curl with a presigned URL that Coolify signs, for
database backups, volume backups, and S3 restores.

Closes #8530

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2026-11-03 21:43:47 +01:00
co-authored by Claude Opus 5.5
parent 3dc89cfe05
commit 21d1ad4874
9 changed files with 281 additions and 23 deletions
+10 -9
View File
@@ -194,10 +194,8 @@ class StartDatabaseImport
throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.');
}
$helper = "s3-restore-{$operation}";
$resolveOptions = $this->startS3Helper($storage, $server, $helper, $network);
$sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}");
$commandList = [
'docker exec '.escapeshellarg($helper).' mc cp'.$resolveOptions.' '.$sourceArg.' /tmp/restore',
$this->startS3Helper($storage, $server, $helper, $network, $key),
...$this->streamFromHelper($helper, $container, $containerPath),
'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
];
@@ -326,13 +324,13 @@ class StartDatabaseImport
}
/**
* Starts the S3 restore helper and sets its `s3temp` mc alias. The S3 keys go to mc through the
* stdin of a separate SSH process, so they are not in the arguments of any process on Coolify or
* on the server. The helper is removed when either step fails.
* Starts the S3 restore helper and sets its `s3temp` mc alias, or the curl config for endpoints with a path.
* The S3 keys go to mc through the stdin of a separate SSH process, so they are not in the arguments of any
* process on Coolify or on the server. The helper is removed when either step fails.
*
* @return string The mc `--resolve` option that pins the endpoint host.
* @return string The command that downloads $key to /tmp/restore in the helper.
*/
private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network): string
private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network, string $key): string
{
$image = coolifyHelperImage().':'.getHelperVersion();
@@ -344,10 +342,13 @@ class StartDatabaseImport
.' '.$storage->minioClientEnvOptions()
.' '.escapeshellarg($image).' sleep 86400',
], $server);
if ($storage->usesCurlTransfer()) {
return $storage->prepareCurlDownload($server, $helper, $key, '/tmp/restore');
}
$resolveOptions = $storage->minioClientResolveOptions($server, $image, $network);
$this->setMinioClientAlias($storage, $server, $helper, $resolveOptions);
return $resolveOptions;
return 'docker exec '.escapeshellarg($helper).' mc cp'.$resolveOptions.' '.escapeshellarg("s3temp/{$storage->bucket}/{$key}").' /tmp/restore';
} catch (Throwable) {
instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false);
+10 -6
View File
@@ -941,14 +941,18 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$mount = escapeshellarg($this->backupMountSource().':'.$this->backup_location.':ro');
$commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm {$s3->minioClientEnvOptions()} -v {$mount} {$fullImageName}";
$escapedBackupLocation = escapeshellarg($this->backup_location);
$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
$resolveOptions = $s3->minioClientResolveOptions($this->server, $fullImageName, $network);
instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true);
$this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions);
if ($s3->usesCurlTransfer()) {
$uploadCommand = $s3->prepareCurlUpload($this->server, "backup-of-{$this->backup_log_uuid}", $this->backup_location, ltrim($this->backup_dir, '/').'/'.basename($this->backup_location));
} else {
$escapedBackupLocation = escapeshellarg($this->backup_location);
$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
$resolveOptions = $s3->minioClientResolveOptions($this->server, $fullImageName, $network);
$this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions);
$uploadCommand = "docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}";
}
S3Storage::retryMinioClientUpload(fn () => instant_remote_process(
["docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}"],
[$uploadCommand],
$this->server, true, false, $this->commandTimeout(), disableMultiplexing: true,
));
+12 -8
View File
@@ -90,8 +90,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
$backupLocation = $backupDirectory.'/'.$filename;
$this->execution->update(['filename' => $backupLocation]);
// Streaming writes no local archive, so it can upload to one destination only.
$streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1;
// Streaming writes no local archive, so it can upload to one destination only. A curl upload needs the size
// of the file before it starts, so it cannot stream.
$streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1 && ! $destinations->first()->usesCurlTransfer();
try {
$source = $this->backup->sourcePath();
@@ -468,7 +469,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$s3->testConnection(shouldSave: true);
$containerName = 'volume-upload-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
$resolveOptions = $s3->minioClientResolveOptions($server, $image);
try {
instant_remote_process([
@@ -476,11 +476,15 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
'docker run -d --name '.escapeshellarg($containerName).' --rm '.$s3->minioClientEnvOptions().' -v '
.escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image),
], $server, timeout: $this->timeout, disableMultiplexing: true);
$this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions);
S3Storage::retryMinioClientUpload(fn () => instant_remote_process([
'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' '
.escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'),
], $server, timeout: $this->timeout, disableMultiplexing: true));
if ($s3->usesCurlTransfer()) {
$uploadCommand = $s3->prepareCurlUpload($server, $containerName, $backupLocation, ltrim($backupDirectory, '/').'/'.basename($backupLocation));
} else {
$resolveOptions = $s3->minioClientResolveOptions($server, $image);
$this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions);
$uploadCommand = 'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' '
.escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/');
}
S3Storage::retryMinioClientUpload(fn () => instant_remote_process([$uploadCommand], $server, timeout: $this->timeout, disableMultiplexing: true));
} finally {
instant_remote_process(
['docker rm -f '.escapeshellarg($containerName)],
+84
View File
@@ -2,6 +2,8 @@
namespace App\Models;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SshRetryHandler;
use App\Rules\SafeWebhookUrl;
use App\Rules\ValidS3BucketName;
use App\Traits\Auditable;
@@ -39,6 +41,14 @@ class S3Storage extends BaseModel
*/
private const MINIO_CLIENT_UPLOAD_RETRY_DELAYS_MS = [10_000, 30_000];
/**
* How long the presigned URL of a curl transfer is valid. S3 checks it when the request starts, so it must
* cover the wait for the earlier transfers and the retries of the backup job.
*/
private const CURL_TRANSFER_URL_TTL_HOURS = 24;
private const CURL_TRANSFER_CONFIG_PATH = '/tmp/s3-transfer.curl';
/**
* The exception code of runSshProcess() when a remote command times out.
*/
@@ -391,6 +401,80 @@ class S3Storage extends BaseModel
return ' --api S3v4'.($this->usesPathStyleEndpoint() ? '' : ' --path off');
}
/**
* The MinIO client accepts only `scheme://host[:port]` endpoints. Providers like Supabase Storage need a path
* (`/storage/v1/s3`), so transfers to these endpoints use curl with a presigned URL instead.
*/
public function usesCurlTransfer(): bool
{
return trim((string) parse_url((string) $this->endpoint, PHP_URL_PATH), '/') !== '';
}
/**
* Prepares the upload of $localPath in $container to $key and returns the command that runs it.
*/
public function prepareCurlUpload(Server $server, string $container, string $localPath, string $key): string
{
$url = $this->filesystem()->temporaryUploadUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS))['url'];
// An S3 error is an XML body. Write it to stderr, so it becomes the error message of the backup.
return $this->prepareCurlTransfer($server, $container, $url, ['upload-file = '.self::curlConfigValue($localPath), 'fail-with-body', 'output = "/dev/stderr"']);
}
/**
* Prepares the download of $key to $localPath in $container and returns the command that runs it.
*/
public function prepareCurlDownload(Server $server, string $container, string $key, string $localPath): string
{
$url = $this->filesystem()->temporaryUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS));
return $this->prepareCurlTransfer($server, $container, $url, ['output = '.self::curlConfigValue($localPath), 'fail']);
}
/**
* Writes the curl config of a transfer into $container through the stdin of the SSH process. The config holds
* the presigned URL, so the URL is not in the arguments of any process, and the S3 keys never reach the server.
* The `resolve` line pins the endpoint host to its validated IPs, as minioClientResolveOptions() does for mc.
*
* @param array<int, string> $options
*/
private function prepareCurlTransfer(Server $server, string $container, string $url, array $options): string
{
$entries = SafeWebhookUrl::minioClientResolveOptions((string) $this->endpoint, $this->trustedInternalHosts());
$config = collect(['url = '.self::curlConfigValue($url), 'silent', 'show-error', ...$options])
->when($entries !== [], fn ($lines) => $lines->push('resolve = '.self::curlConfigValue(
str($entries[0])->beforeLast('=')->toString().':'.collect($entries)
->map(fn (string $entry): string => str($entry)->afterLast('=')->toString())
->map(fn (string $ip): string => str_contains($ip, ':') ? "[{$ip}]" : $ip)
->implode(',')
)))
->when($this->skip_tls_verification, fn ($lines) => $lines->push('insecure'))
->implode("\n")."\n";
$remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($container)
.' sh -c '.escapeshellarg('umask 077 && cat > '.self::CURL_TRANSFER_CONFIG_PATH);
$timeout = (int) config('constants.ssh.command_timeout');
SshRetryHandler::retry(
function () use ($server, $remoteCommand, $config, $timeout) {
$sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout);
$process = runSshProcess($sshCommand, $timeout, $config);
if ($process->exitCode() !== 0) {
excludeCertainErrors($process->errorOutput(), $process->exitCode());
}
},
['server' => $server->ip, 'function' => 'prepareCurlTransfer'],
);
return 'docker exec '.escapeshellarg($container).' curl --config '.self::CURL_TRANSFER_CONFIG_PATH;
}
private static function curlConfigValue(string $value): string
{
return '"'.addcslashes($value, '"\\').'"';
}
/**
* The mc `--resolve` option that pins the endpoint host to one of its validated IPs. mc keeps one IP per host,
* so when the host has more than one IP, a helper container on the upload network connects to each IP, IPv4
@@ -63,6 +63,8 @@ function s3ArgvTestSetup(string $sshUser): array
$disk = Mockery::mock(FilesystemAdapter::class);
$disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
$disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
$disk->shouldReceive('temporaryUploadUrl')->zeroOrMoreTimes()
->andReturnUsing(fn (string $key) => ['url' => "https://8.8.8.8/storage/v1/s3/bucket/{$key}?X-Amz-Signature=presigned", 'headers' => []]);
$sshDisk = Storage::fake('ssh-keys');
Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
@@ -267,3 +269,89 @@ it('passes the S3 keys of a volume backup to mc on stdin, not in a command line'
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBe($streamToS3);
s3ArgvTestAssertSecretOnlyOnStdin($processes, $sshUser !== 'root');
})->with(['root', 'deploy'])->with(['local archive upload' => false, 'S3-only stream' => true]);
/**
* @param Collection<int, array{command: string, input: ?string}> $processes
*/
function s3ArgvTestAssertCurlUpload(Collection $processes, string $backupLocation, bool $nonRoot): void
{
$configProcess = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl'));
$otherProcesses = $processes->reject(fn (array $process) => $process === $configProcess);
expect($configProcess['input'])->toBe(implode("\n", [
'url = "https://8.8.8.8/storage/v1/s3/bucket/'.ltrim($backupLocation, '/').'?X-Amz-Signature=presigned"',
'silent',
'show-error',
'upload-file = "'.$backupLocation.'"',
'fail-with-body',
'output = "/dev/stderr"',
])."\n")
->and(str_contains($configProcess['command'], 'sudo docker exec -i'))->toBe($nonRoot)
->and($otherProcesses->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'curl --config /tmp/s3-transfer.curl')))->toBeTrue()
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc alias set')))->toBeFalse()
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBeFalse()
->and($otherProcesses->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'X-Amz-Signature')))->toBeFalse()
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], S3_ARGV_TEST_SECRET)))->toBeFalse();
}
it('uploads a database backup with curl and a presigned URL when the endpoint has a path', function (string $sshUser) {
[$team, $server, $storage] = s3ArgvTestSetup($sshUser);
$storage->update(['endpoint' => 'https://8.8.8.8/storage/v1/s3']);
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
$environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
$database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']);
$backup = new ScheduledDatabaseBackup;
$backup->forceFill([
'enabled' => true,
'save_s3' => true,
'disable_local_backup' => true,
's3_storage_id' => $storage->id,
'frequency' => 'daily',
'databases_to_backup' => 'app',
'database_id' => $database->id,
'database_type' => $database->getMorphClass(),
'team_id' => $team->id,
])->save();
$backup->syncS3Storages([$storage->id]);
$processes = s3ArgvTestFakeSsh();
(new DatabaseBackupJob($backup->fresh()))->handle();
$execution = $backup->executions()->sole();
expect($execution->s3_uploaded)->toBeTrue();
s3ArgvTestAssertCurlUpload($processes, $execution->filename, $sshUser !== 'root');
})->with(['root', 'deploy']);
it('writes a local archive and uploads it with curl when a volume backup endpoint has a path', function () {
[$team, $server, $storage] = s3ArgvTestSetup('root');
$storage->update(['endpoint' => 'https://8.8.8.8/storage/v1/s3']);
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
$environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
$application = Application::factory()->create([
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
]);
$volume = LocalPersistentVolume::create([
'name' => 'app-data',
'mount_path' => '/data',
'resource_id' => $application->id,
'resource_type' => $application->getMorphClass(),
]);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
'frequency' => 'daily',
'save_s3' => true,
'disable_local_backup' => true,
's3_storage_id' => $storage->id,
]);
$processes = s3ArgvTestFakeSsh();
(new VolumeBackupJob($backup))->handle();
$execution = ScheduledVolumeBackupExecution::query()->sole();
expect($execution->status)->toBe('success')
->and($execution->s3_uploaded)->toBeTrue()
->and($execution->local_storage_deleted)->toBeTrue();
s3ArgvTestAssertCurlUpload($processes, $execution->filename, false);
});
@@ -16,6 +16,7 @@ use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Models\User;
use App\Support\DatabaseImport\DatabaseImportCleanup;
use App\Support\DatabaseImport\DatabaseImportException;
use App\Support\DatabaseImport\DatabaseImportSource;
@@ -38,6 +39,9 @@ beforeEach(function () {
InstanceSettings::forceCreate(['id' => 0]);
$this->team = Team::factory()->create();
$this->user = User::factory()->create();
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
$this->actingAs($this->user);
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
$this->server = Server::factory()->create([
'team_id' => $this->team->id,
@@ -11,6 +11,7 @@ use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Models\User;
use App\Support\DatabaseImport\DatabaseImportException;
use App\Support\DatabaseImport\DatabaseImportSource;
use App\Support\RemoteProcessCommand;
@@ -32,6 +33,9 @@ beforeEach(function () {
InstanceSettings::forceCreate(['id' => 0]);
$this->team = Team::factory()->create();
$this->user = User::factory()->create();
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
$this->actingAs($this->user);
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
$this->server = Server::factory()->create([
'team_id' => $this->team->id,
@@ -11,6 +11,7 @@ use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Models\User;
use App\Support\DatabaseImport\DatabaseImportCleanup;
use App\Support\DatabaseImport\DatabaseImportException;
use App\Support\DatabaseImport\DatabaseImportSource;
@@ -34,6 +35,9 @@ beforeEach(function () {
InstanceSettings::forceCreate(['id' => 0]);
$this->team = Team::factory()->create();
$this->user = User::factory()->create();
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
$this->actingAs($this->user);
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
$this->server = Server::factory()->create([
'team_id' => $this->team->id,
@@ -358,3 +362,55 @@ it('removes the S3 restore helper when the mc alias cannot be set', function ()
expect($removedAfterAlias)->toBeTrue();
Queue::assertNothingPushed();
});
test('s3 import downloads with curl and a presigned URL when the endpoint has a path', function () {
$storage = S3Storage::create([
'name' => 'Supabase',
'region' => 'eu-central-1',
'key' => 'AKIA_PATH_ENDPOINT_KEY',
'secret' => 'path/endpoint/SECRET',
'bucket' => 'test-bucket',
'endpoint' => 'https://8.8.8.8/storage/v1/s3',
'is_usable' => true,
'team_id' => $this->team->id,
]);
$disk = Mockery::mock(FilesystemAdapter::class);
$disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true);
$disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024);
$disk->shouldReceive('temporaryUrl')->once()->with('backups/restore.sql', Mockery::any())
->andReturn('https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned');
$filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial();
$filesystem->shouldReceive('build')->twice()->andReturn($disk);
Storage::swap($filesystem);
$processes = collect();
Process::fake(function ($process) use ($processes) {
$processes->push(['command' => $process->command, 'input' => $process->input]);
return Process::result();
});
Queue::fake();
$activity = app(StartDatabaseImport::class)->handle(
$this->database,
new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid),
$this->team->id,
);
$command = (string) RemoteProcessCommand::read($activity);
$config = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl'))['input'];
expect($config)->toBe(implode("\n", [
'url = "https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned"',
'silent',
'show-error',
'output = "/tmp/restore"',
'fail',
])."\n")
->and($command)->toContain("docker exec 's3-restore-")
->and($command)->toContain(' curl --config /tmp/s3-transfer.curl')
->and($command)->not->toContain('mc cp')
->and($command)->not->toContain('X-Amz-Signature')
->and($command)->not->toContain('path/endpoint/SECRET');
});
+13
View File
@@ -298,3 +298,16 @@ test('S3Storage does not retry a MinIO client upload that timed out', function (
expect($attempts)->toBe(1);
Sleep::assertNeverSlept();
});
test('S3Storage uses curl transfers only for endpoints with a path, which the MinIO client rejects', function (string $endpoint, bool $expected) {
$s3Storage = new S3Storage;
$s3Storage->setRawAttributes(['endpoint' => $endpoint]);
expect($s3Storage->usesCurlTransfer())->toBe($expected);
})->with([
'Supabase Storage' => ['https://abcd.storage.supabase.co/storage/v1/s3', true],
'path with trailing slash' => ['http://minio:9000/s3/', true],
'host only' => ['https://s3.amazonaws.com', false],
'host with trailing slash' => ['https://s3.amazonaws.com/', false],
'host with port' => ['http://coolify-minio:9000', false],
]);