mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
fix(backups): upload to S3 endpoints with a path, like Supabase Storage
The MinIO client rejects endpoints such as `/storage/v1/s3`. Transfers to these endpoints now use curl with a presigned URL that Coolify signs, for database backups, volume backups, and S3 restores. Closes #8530 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3dc89cfe05
commit
21d1ad4874
@@ -194,10 +194,8 @@ class StartDatabaseImport
|
||||
throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.');
|
||||
}
|
||||
$helper = "s3-restore-{$operation}";
|
||||
$resolveOptions = $this->startS3Helper($storage, $server, $helper, $network);
|
||||
$sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}");
|
||||
$commandList = [
|
||||
'docker exec '.escapeshellarg($helper).' mc cp'.$resolveOptions.' '.$sourceArg.' /tmp/restore',
|
||||
$this->startS3Helper($storage, $server, $helper, $network, $key),
|
||||
...$this->streamFromHelper($helper, $container, $containerPath),
|
||||
'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
|
||||
];
|
||||
@@ -326,13 +324,13 @@ class StartDatabaseImport
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the S3 restore helper and sets its `s3temp` mc alias. The S3 keys go to mc through the
|
||||
* stdin of a separate SSH process, so they are not in the arguments of any process on Coolify or
|
||||
* on the server. The helper is removed when either step fails.
|
||||
* Starts the S3 restore helper and sets its `s3temp` mc alias, or the curl config for endpoints with a path.
|
||||
* The S3 keys go to mc through the stdin of a separate SSH process, so they are not in the arguments of any
|
||||
* process on Coolify or on the server. The helper is removed when either step fails.
|
||||
*
|
||||
* @return string The mc `--resolve` option that pins the endpoint host.
|
||||
* @return string The command that downloads $key to /tmp/restore in the helper.
|
||||
*/
|
||||
private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network): string
|
||||
private function startS3Helper(S3Storage $storage, Server $server, string $helper, string $network, string $key): string
|
||||
{
|
||||
$image = coolifyHelperImage().':'.getHelperVersion();
|
||||
|
||||
@@ -344,10 +342,13 @@ class StartDatabaseImport
|
||||
.' '.$storage->minioClientEnvOptions()
|
||||
.' '.escapeshellarg($image).' sleep 86400',
|
||||
], $server);
|
||||
if ($storage->usesCurlTransfer()) {
|
||||
return $storage->prepareCurlDownload($server, $helper, $key, '/tmp/restore');
|
||||
}
|
||||
$resolveOptions = $storage->minioClientResolveOptions($server, $image, $network);
|
||||
$this->setMinioClientAlias($storage, $server, $helper, $resolveOptions);
|
||||
|
||||
return $resolveOptions;
|
||||
return 'docker exec '.escapeshellarg($helper).' mc cp'.$resolveOptions.' '.escapeshellarg("s3temp/{$storage->bucket}/{$key}").' /tmp/restore';
|
||||
} catch (Throwable) {
|
||||
instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false);
|
||||
|
||||
|
||||
@@ -941,14 +941,18 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$mount = escapeshellarg($this->backupMountSource().':'.$this->backup_location.':ro');
|
||||
$commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm {$s3->minioClientEnvOptions()} -v {$mount} {$fullImageName}";
|
||||
|
||||
$escapedBackupLocation = escapeshellarg($this->backup_location);
|
||||
$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
|
||||
$resolveOptions = $s3->minioClientResolveOptions($this->server, $fullImageName, $network);
|
||||
|
||||
instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true);
|
||||
$this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions);
|
||||
if ($s3->usesCurlTransfer()) {
|
||||
$uploadCommand = $s3->prepareCurlUpload($this->server, "backup-of-{$this->backup_log_uuid}", $this->backup_location, ltrim($this->backup_dir, '/').'/'.basename($this->backup_location));
|
||||
} else {
|
||||
$escapedBackupLocation = escapeshellarg($this->backup_location);
|
||||
$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
|
||||
$resolveOptions = $s3->minioClientResolveOptions($this->server, $fullImageName, $network);
|
||||
$this->setMinioClientAlias($this->server, "backup-of-{$this->backup_log_uuid}", $s3, $resolveOptions);
|
||||
$uploadCommand = "docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}";
|
||||
}
|
||||
S3Storage::retryMinioClientUpload(fn () => instant_remote_process(
|
||||
["docker exec backup-of-{$this->backup_log_uuid} mc cp{$resolveOptions} {$escapedBackupLocation} {$escapedS3Destination}"],
|
||||
[$uploadCommand],
|
||||
$this->server, true, false, $this->commandTimeout(), disableMultiplexing: true,
|
||||
));
|
||||
|
||||
|
||||
@@ -90,8 +90,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
|
||||
$backupLocation = $backupDirectory.'/'.$filename;
|
||||
$this->execution->update(['filename' => $backupLocation]);
|
||||
// Streaming writes no local archive, so it can upload to one destination only.
|
||||
$streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1;
|
||||
// Streaming writes no local archive, so it can upload to one destination only. A curl upload needs the size
|
||||
// of the file before it starts, so it cannot stream.
|
||||
$streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1 && ! $destinations->first()->usesCurlTransfer();
|
||||
|
||||
try {
|
||||
$source = $this->backup->sourcePath();
|
||||
@@ -468,7 +469,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$s3->testConnection(shouldSave: true);
|
||||
$containerName = 'volume-upload-'.$this->execution->uuid;
|
||||
$image = coolifyHelperImage().':'.getHelperVersion();
|
||||
$resolveOptions = $s3->minioClientResolveOptions($server, $image);
|
||||
|
||||
try {
|
||||
instant_remote_process([
|
||||
@@ -476,11 +476,15 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
'docker run -d --name '.escapeshellarg($containerName).' --rm '.$s3->minioClientEnvOptions().' -v '
|
||||
.escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image),
|
||||
], $server, timeout: $this->timeout, disableMultiplexing: true);
|
||||
$this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions);
|
||||
S3Storage::retryMinioClientUpload(fn () => instant_remote_process([
|
||||
'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' '
|
||||
.escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'),
|
||||
], $server, timeout: $this->timeout, disableMultiplexing: true));
|
||||
if ($s3->usesCurlTransfer()) {
|
||||
$uploadCommand = $s3->prepareCurlUpload($server, $containerName, $backupLocation, ltrim($backupDirectory, '/').'/'.basename($backupLocation));
|
||||
} else {
|
||||
$resolveOptions = $s3->minioClientResolveOptions($server, $image);
|
||||
$this->setMinioClientAlias($server, $containerName, $s3, $resolveOptions);
|
||||
$uploadCommand = 'docker exec '.escapeshellarg($containerName).' mc cp'.$resolveOptions.' '.escapeshellarg($backupLocation).' '
|
||||
.escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/');
|
||||
}
|
||||
S3Storage::retryMinioClientUpload(fn () => instant_remote_process([$uploadCommand], $server, timeout: $this->timeout, disableMultiplexing: true));
|
||||
} finally {
|
||||
instant_remote_process(
|
||||
['docker rm -f '.escapeshellarg($containerName)],
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use App\Helpers\SshMultiplexingHelper;
|
||||
use App\Helpers\SshRetryHandler;
|
||||
use App\Rules\SafeWebhookUrl;
|
||||
use App\Rules\ValidS3BucketName;
|
||||
use App\Traits\Auditable;
|
||||
@@ -39,6 +41,14 @@ class S3Storage extends BaseModel
|
||||
*/
|
||||
private const MINIO_CLIENT_UPLOAD_RETRY_DELAYS_MS = [10_000, 30_000];
|
||||
|
||||
/**
|
||||
* How long the presigned URL of a curl transfer is valid. S3 checks it when the request starts, so it must
|
||||
* cover the wait for the earlier transfers and the retries of the backup job.
|
||||
*/
|
||||
private const CURL_TRANSFER_URL_TTL_HOURS = 24;
|
||||
|
||||
private const CURL_TRANSFER_CONFIG_PATH = '/tmp/s3-transfer.curl';
|
||||
|
||||
/**
|
||||
* The exception code of runSshProcess() when a remote command times out.
|
||||
*/
|
||||
@@ -391,6 +401,80 @@ class S3Storage extends BaseModel
|
||||
return ' --api S3v4'.($this->usesPathStyleEndpoint() ? '' : ' --path off');
|
||||
}
|
||||
|
||||
/**
|
||||
* The MinIO client accepts only `scheme://host[:port]` endpoints. Providers like Supabase Storage need a path
|
||||
* (`/storage/v1/s3`), so transfers to these endpoints use curl with a presigned URL instead.
|
||||
*/
|
||||
public function usesCurlTransfer(): bool
|
||||
{
|
||||
return trim((string) parse_url((string) $this->endpoint, PHP_URL_PATH), '/') !== '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the upload of $localPath in $container to $key and returns the command that runs it.
|
||||
*/
|
||||
public function prepareCurlUpload(Server $server, string $container, string $localPath, string $key): string
|
||||
{
|
||||
$url = $this->filesystem()->temporaryUploadUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS))['url'];
|
||||
|
||||
// An S3 error is an XML body. Write it to stderr, so it becomes the error message of the backup.
|
||||
return $this->prepareCurlTransfer($server, $container, $url, ['upload-file = '.self::curlConfigValue($localPath), 'fail-with-body', 'output = "/dev/stderr"']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the download of $key to $localPath in $container and returns the command that runs it.
|
||||
*/
|
||||
public function prepareCurlDownload(Server $server, string $container, string $key, string $localPath): string
|
||||
{
|
||||
$url = $this->filesystem()->temporaryUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS));
|
||||
|
||||
return $this->prepareCurlTransfer($server, $container, $url, ['output = '.self::curlConfigValue($localPath), 'fail']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the curl config of a transfer into $container through the stdin of the SSH process. The config holds
|
||||
* the presigned URL, so the URL is not in the arguments of any process, and the S3 keys never reach the server.
|
||||
* The `resolve` line pins the endpoint host to its validated IPs, as minioClientResolveOptions() does for mc.
|
||||
*
|
||||
* @param array<int, string> $options
|
||||
*/
|
||||
private function prepareCurlTransfer(Server $server, string $container, string $url, array $options): string
|
||||
{
|
||||
$entries = SafeWebhookUrl::minioClientResolveOptions((string) $this->endpoint, $this->trustedInternalHosts());
|
||||
$config = collect(['url = '.self::curlConfigValue($url), 'silent', 'show-error', ...$options])
|
||||
->when($entries !== [], fn ($lines) => $lines->push('resolve = '.self::curlConfigValue(
|
||||
str($entries[0])->beforeLast('=')->toString().':'.collect($entries)
|
||||
->map(fn (string $entry): string => str($entry)->afterLast('=')->toString())
|
||||
->map(fn (string $ip): string => str_contains($ip, ':') ? "[{$ip}]" : $ip)
|
||||
->implode(',')
|
||||
)))
|
||||
->when($this->skip_tls_verification, fn ($lines) => $lines->push('insecure'))
|
||||
->implode("\n")."\n";
|
||||
|
||||
$remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($container)
|
||||
.' sh -c '.escapeshellarg('umask 077 && cat > '.self::CURL_TRANSFER_CONFIG_PATH);
|
||||
$timeout = (int) config('constants.ssh.command_timeout');
|
||||
|
||||
SshRetryHandler::retry(
|
||||
function () use ($server, $remoteCommand, $config, $timeout) {
|
||||
$sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout);
|
||||
$process = runSshProcess($sshCommand, $timeout, $config);
|
||||
|
||||
if ($process->exitCode() !== 0) {
|
||||
excludeCertainErrors($process->errorOutput(), $process->exitCode());
|
||||
}
|
||||
},
|
||||
['server' => $server->ip, 'function' => 'prepareCurlTransfer'],
|
||||
);
|
||||
|
||||
return 'docker exec '.escapeshellarg($container).' curl --config '.self::CURL_TRANSFER_CONFIG_PATH;
|
||||
}
|
||||
|
||||
private static function curlConfigValue(string $value): string
|
||||
{
|
||||
return '"'.addcslashes($value, '"\\').'"';
|
||||
}
|
||||
|
||||
/**
|
||||
* The mc `--resolve` option that pins the endpoint host to one of its validated IPs. mc keeps one IP per host,
|
||||
* so when the host has more than one IP, a helper container on the upload network connects to each IP, IPv4
|
||||
|
||||
@@ -63,6 +63,8 @@ function s3ArgvTestSetup(string $sshUser): array
|
||||
$disk = Mockery::mock(FilesystemAdapter::class);
|
||||
$disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
|
||||
$disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
|
||||
$disk->shouldReceive('temporaryUploadUrl')->zeroOrMoreTimes()
|
||||
->andReturnUsing(fn (string $key) => ['url' => "https://8.8.8.8/storage/v1/s3/bucket/{$key}?X-Amz-Signature=presigned", 'headers' => []]);
|
||||
$sshDisk = Storage::fake('ssh-keys');
|
||||
Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
|
||||
Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
|
||||
@@ -267,3 +269,89 @@ it('passes the S3 keys of a volume backup to mc on stdin, not in a command line'
|
||||
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBe($streamToS3);
|
||||
s3ArgvTestAssertSecretOnlyOnStdin($processes, $sshUser !== 'root');
|
||||
})->with(['root', 'deploy'])->with(['local archive upload' => false, 'S3-only stream' => true]);
|
||||
|
||||
/**
|
||||
* @param Collection<int, array{command: string, input: ?string}> $processes
|
||||
*/
|
||||
function s3ArgvTestAssertCurlUpload(Collection $processes, string $backupLocation, bool $nonRoot): void
|
||||
{
|
||||
$configProcess = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl'));
|
||||
$otherProcesses = $processes->reject(fn (array $process) => $process === $configProcess);
|
||||
|
||||
expect($configProcess['input'])->toBe(implode("\n", [
|
||||
'url = "https://8.8.8.8/storage/v1/s3/bucket/'.ltrim($backupLocation, '/').'?X-Amz-Signature=presigned"',
|
||||
'silent',
|
||||
'show-error',
|
||||
'upload-file = "'.$backupLocation.'"',
|
||||
'fail-with-body',
|
||||
'output = "/dev/stderr"',
|
||||
])."\n")
|
||||
->and(str_contains($configProcess['command'], 'sudo docker exec -i'))->toBe($nonRoot)
|
||||
->and($otherProcesses->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'curl --config /tmp/s3-transfer.curl')))->toBeTrue()
|
||||
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc alias set')))->toBeFalse()
|
||||
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'mc pipe')))->toBeFalse()
|
||||
->and($otherProcesses->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], 'X-Amz-Signature')))->toBeFalse()
|
||||
->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], S3_ARGV_TEST_SECRET)))->toBeFalse();
|
||||
}
|
||||
|
||||
it('uploads a database backup with curl and a presigned URL when the endpoint has a path', function (string $sshUser) {
|
||||
[$team, $server, $storage] = s3ArgvTestSetup($sshUser);
|
||||
$storage->update(['endpoint' => 'https://8.8.8.8/storage/v1/s3']);
|
||||
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
|
||||
$environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
|
||||
$database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']);
|
||||
$backup = new ScheduledDatabaseBackup;
|
||||
$backup->forceFill([
|
||||
'enabled' => true,
|
||||
'save_s3' => true,
|
||||
'disable_local_backup' => true,
|
||||
's3_storage_id' => $storage->id,
|
||||
'frequency' => 'daily',
|
||||
'databases_to_backup' => 'app',
|
||||
'database_id' => $database->id,
|
||||
'database_type' => $database->getMorphClass(),
|
||||
'team_id' => $team->id,
|
||||
])->save();
|
||||
$backup->syncS3Storages([$storage->id]);
|
||||
$processes = s3ArgvTestFakeSsh();
|
||||
|
||||
(new DatabaseBackupJob($backup->fresh()))->handle();
|
||||
|
||||
$execution = $backup->executions()->sole();
|
||||
expect($execution->s3_uploaded)->toBeTrue();
|
||||
s3ArgvTestAssertCurlUpload($processes, $execution->filename, $sshUser !== 'root');
|
||||
})->with(['root', 'deploy']);
|
||||
|
||||
it('writes a local archive and uploads it with curl when a volume backup endpoint has a path', function () {
|
||||
[$team, $server, $storage] = s3ArgvTestSetup('root');
|
||||
$storage->update(['endpoint' => 'https://8.8.8.8/storage/v1/s3']);
|
||||
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
|
||||
$environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
|
||||
$application = Application::factory()->create([
|
||||
'environment_id' => $environment->id,
|
||||
'destination_id' => $destination->id,
|
||||
'destination_type' => $destination->getMorphClass(),
|
||||
]);
|
||||
$volume = LocalPersistentVolume::create([
|
||||
'name' => 'app-data',
|
||||
'mount_path' => '/data',
|
||||
'resource_id' => $application->id,
|
||||
'resource_type' => $application->getMorphClass(),
|
||||
]);
|
||||
$backup = $volume->scheduledBackups()->create([
|
||||
'team_id' => $team->id,
|
||||
'frequency' => 'daily',
|
||||
'save_s3' => true,
|
||||
'disable_local_backup' => true,
|
||||
's3_storage_id' => $storage->id,
|
||||
]);
|
||||
$processes = s3ArgvTestFakeSsh();
|
||||
|
||||
(new VolumeBackupJob($backup))->handle();
|
||||
|
||||
$execution = ScheduledVolumeBackupExecution::query()->sole();
|
||||
expect($execution->status)->toBe('success')
|
||||
->and($execution->s3_uploaded)->toBeTrue()
|
||||
->and($execution->local_storage_deleted)->toBeTrue();
|
||||
s3ArgvTestAssertCurlUpload($processes, $execution->filename, false);
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@ use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Support\DatabaseImport\DatabaseImportCleanup;
|
||||
use App\Support\DatabaseImport\DatabaseImportException;
|
||||
use App\Support\DatabaseImport\DatabaseImportSource;
|
||||
@@ -38,6 +39,9 @@ beforeEach(function () {
|
||||
InstanceSettings::forceCreate(['id' => 0]);
|
||||
|
||||
$this->team = Team::factory()->create();
|
||||
$this->user = User::factory()->create();
|
||||
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
|
||||
$this->actingAs($this->user);
|
||||
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
|
||||
$this->server = Server::factory()->create([
|
||||
'team_id' => $this->team->id,
|
||||
|
||||
@@ -11,6 +11,7 @@ use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Support\DatabaseImport\DatabaseImportException;
|
||||
use App\Support\DatabaseImport\DatabaseImportSource;
|
||||
use App\Support\RemoteProcessCommand;
|
||||
@@ -32,6 +33,9 @@ beforeEach(function () {
|
||||
InstanceSettings::forceCreate(['id' => 0]);
|
||||
|
||||
$this->team = Team::factory()->create();
|
||||
$this->user = User::factory()->create();
|
||||
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
|
||||
$this->actingAs($this->user);
|
||||
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
|
||||
$this->server = Server::factory()->create([
|
||||
'team_id' => $this->team->id,
|
||||
|
||||
@@ -11,6 +11,7 @@ use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Support\DatabaseImport\DatabaseImportCleanup;
|
||||
use App\Support\DatabaseImport\DatabaseImportException;
|
||||
use App\Support\DatabaseImport\DatabaseImportSource;
|
||||
@@ -34,6 +35,9 @@ beforeEach(function () {
|
||||
InstanceSettings::forceCreate(['id' => 0]);
|
||||
|
||||
$this->team = Team::factory()->create();
|
||||
$this->user = User::factory()->create();
|
||||
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
|
||||
$this->actingAs($this->user);
|
||||
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
|
||||
$this->server = Server::factory()->create([
|
||||
'team_id' => $this->team->id,
|
||||
@@ -358,3 +362,55 @@ it('removes the S3 restore helper when the mc alias cannot be set', function ()
|
||||
expect($removedAfterAlias)->toBeTrue();
|
||||
Queue::assertNothingPushed();
|
||||
});
|
||||
|
||||
test('s3 import downloads with curl and a presigned URL when the endpoint has a path', function () {
|
||||
$storage = S3Storage::create([
|
||||
'name' => 'Supabase',
|
||||
'region' => 'eu-central-1',
|
||||
'key' => 'AKIA_PATH_ENDPOINT_KEY',
|
||||
'secret' => 'path/endpoint/SECRET',
|
||||
'bucket' => 'test-bucket',
|
||||
'endpoint' => 'https://8.8.8.8/storage/v1/s3',
|
||||
'is_usable' => true,
|
||||
'team_id' => $this->team->id,
|
||||
]);
|
||||
|
||||
$disk = Mockery::mock(FilesystemAdapter::class);
|
||||
$disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true);
|
||||
$disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024);
|
||||
$disk->shouldReceive('temporaryUrl')->once()->with('backups/restore.sql', Mockery::any())
|
||||
->andReturn('https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned');
|
||||
$filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial();
|
||||
$filesystem->shouldReceive('build')->twice()->andReturn($disk);
|
||||
Storage::swap($filesystem);
|
||||
|
||||
$processes = collect();
|
||||
Process::fake(function ($process) use ($processes) {
|
||||
$processes->push(['command' => $process->command, 'input' => $process->input]);
|
||||
|
||||
return Process::result();
|
||||
});
|
||||
Queue::fake();
|
||||
|
||||
$activity = app(StartDatabaseImport::class)->handle(
|
||||
$this->database,
|
||||
new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid),
|
||||
$this->team->id,
|
||||
);
|
||||
|
||||
$command = (string) RemoteProcessCommand::read($activity);
|
||||
$config = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl'))['input'];
|
||||
|
||||
expect($config)->toBe(implode("\n", [
|
||||
'url = "https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned"',
|
||||
'silent',
|
||||
'show-error',
|
||||
'output = "/tmp/restore"',
|
||||
'fail',
|
||||
])."\n")
|
||||
->and($command)->toContain("docker exec 's3-restore-")
|
||||
->and($command)->toContain(' curl --config /tmp/s3-transfer.curl')
|
||||
->and($command)->not->toContain('mc cp')
|
||||
->and($command)->not->toContain('X-Amz-Signature')
|
||||
->and($command)->not->toContain('path/endpoint/SECRET');
|
||||
});
|
||||
|
||||
@@ -298,3 +298,16 @@ test('S3Storage does not retry a MinIO client upload that timed out', function (
|
||||
expect($attempts)->toBe(1);
|
||||
Sleep::assertNeverSlept();
|
||||
});
|
||||
|
||||
test('S3Storage uses curl transfers only for endpoints with a path, which the MinIO client rejects', function (string $endpoint, bool $expected) {
|
||||
$s3Storage = new S3Storage;
|
||||
$s3Storage->setRawAttributes(['endpoint' => $endpoint]);
|
||||
|
||||
expect($s3Storage->usesCurlTransfer())->toBe($expected);
|
||||
})->with([
|
||||
'Supabase Storage' => ['https://abcd.storage.supabase.co/storage/v1/s3', true],
|
||||
'path with trailing slash' => ['http://minio:9000/s3/', true],
|
||||
'host only' => ['https://s3.amazonaws.com', false],
|
||||
'host with trailing slash' => ['https://s3.amazonaws.com/', false],
|
||||
'host with port' => ['http://coolify-minio:9000', false],
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user