set('cache.default', 'array'); config()->set('constants.ssh.mux_enabled', false); InstanceSettings::forceCreate(['id' => 0]); $this->team = Team::factory()->create(); $this->user = User::factory()->create(); $this->team->members()->attach($this->user->id, ['role' => 'owner']); $this->actingAs($this->user); $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); $this->server = Server::factory()->create([ 'team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id, 'user' => 'root', ]); $this->destination = StandaloneDocker::firstOrCreate( ['server_id' => $this->server->id, 'network' => 'coolify'], ['uuid' => (string) Str::uuid(), 'name' => 'docker'] ); $this->project = Project::factory()->create(['team_id' => $this->team->id]); $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); $this->database = StandalonePostgresql::create([ 'uuid' => (string) Str::uuid(), 'name' => 'db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'status' => 'running', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(), ]); }); test('s3 import activity command does not contain storage key or secret', function () { $accessKey = 'AKIA_TEST_ACCESS_KEY_LEAK'; $secret = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYTESTSECRET'; $storage = S3Storage::create([ 'name' => 'Import S3', 'region' => 'us-east-1', 'key' => $accessKey, 'secret' => $secret, 'bucket' => 'test-bucket', 'endpoint' => 'https://8.8.8.8', 'is_usable' => true, 'team_id' => $this->team->id, ]); $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true); $disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024); $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); $filesystem->shouldReceive('build')->once()->andReturn($disk); Storage::swap($filesystem); Process::fake(); Queue::fake(); $activity = app(StartDatabaseImport::class)->handle( $this->database, new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), $this->team->id, ); $command = (string) RemoteProcessCommand::read($activity); expect($command) ->not->toContain($accessKey) ->not->toContain($secret) ->not->toContain('.env') ->not->toContain('S3_ACCESS_KEY=') ->not->toContain('S3_SECRET_KEY=') ->not->toContain('mc alias set'); Queue::assertPushed(CoolifyTask::class, function (CoolifyTask $job) { $cleanup = $job->call_event_data; return is_array($cleanup) && ! array_key_exists('credentialTmpPath', $cleanup) && filled($cleanup['containerName'] ?? null); }); }); test('s3 import activity stores the cleanup names and paths but no credential values', function () { $accessKey = 'AKIA_STORED_CLEANUP_KEY'; $secret = 'stored/cleanup/SECRET/value'; $endpoint = 'https://8.8.4.4'; $storage = S3Storage::create([ 'name' => 'Import S3', 'region' => 'us-east-1', 'key' => $accessKey, 'secret' => $secret, 'bucket' => 'test-bucket', 'endpoint' => $endpoint, 'is_usable' => true, 'team_id' => $this->team->id, ]); $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true); $disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024); $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); $filesystem->shouldReceive('build')->once()->andReturn($disk); Storage::swap($filesystem); Process::fake(); Queue::fake(); $activity = app(StartDatabaseImport::class)->handle( $this->database, new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), $this->team->id, ); $stored = $activity->getExtraProperty(DatabaseImportCleanup::PROPERTY); $operation = $activity->getExtraProperty('operation_uuid'); expect($stored)->toBe([ 'container' => $this->database->uuid, 'containerTmpPath' => "/tmp/restore_{$operation}", 'scriptPath' => "/tmp/restore_{$operation}.sh", 'serverId' => $this->server->id, 'operationUuid' => $operation, 'containerName' => "s3-restore-{$operation}", ]); $properties = json_encode($activity->properties); expect($properties) ->not->toContain($accessKey) ->not->toContain($secret) ->not->toContain($endpoint) ->not->toContain('S3_ACCESS_KEY='); Queue::assertPushed(CoolifyTask::class, fn (CoolifyTask $job) => $job->call_event_data === $stored); }); test('s3 import helper signs requests with the storage region', function () { $storage = S3Storage::create([ 'name' => 'Import S3', 'region' => 'ru-central-1', 'key' => 'key', 'secret' => 'secret', 'bucket' => 'test-bucket', 'endpoint' => 'https://8.8.8.8', 'is_usable' => true, 'team_id' => $this->team->id, ]); $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('exists')->andReturn(true); $disk->shouldReceive('size')->andReturn(1024); $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); $filesystem->shouldReceive('build')->andReturn($disk); Storage::swap($filesystem); Process::fake(); Queue::fake(); app(StartDatabaseImport::class)->handle( $this->database, new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), $this->team->id, ); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "-e 'MC_REGION=ru-central-1'") && str_contains(remoteProcessScript($process), 'docker run -d')); }); const IMPORT_S3_ARGV_KEY = 'AK/+$x"y'; const IMPORT_S3_ARGV_SECRET = "s3c/r+e\$t 'q\"uo\\te`d` \\n"; /** * Starts an S3 import against a faked SSH layer and records every SSH process with its stdin. * * @return array{0: Collection, 1: ?Activity, 2: ?Throwable} */ function importS3ArgvTestRun(object $test, string $sshUser, bool $aliasFails = false): array { $test->server->update(['user' => $sshUser]); $storage = S3Storage::create([ 'name' => 'Import S3', 'region' => 'ru-central-1', 'key' => IMPORT_S3_ARGV_KEY, 'secret' => IMPORT_S3_ARGV_SECRET, 'bucket' => 'test-bucket', 'endpoint' => 'https://8.8.8.8', 'is_usable' => true, 'team_id' => $test->team->id, ]); $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('exists')->andReturn(true); $disk->shouldReceive('size')->andReturn(1024); $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); $filesystem->shouldReceive('build')->andReturn($disk); Storage::swap($filesystem); $processes = collect(); Process::fake(function ($process) use ($processes, $aliasFails) { $processes->push(['command' => $process->command, 'input' => $process->input]); return $aliasFails && str_contains($process->command, 'mc alias set') ? Process::result(errorOutput: 'mc: invalid credentials', exitCode: 1) : Process::result(); }); Queue::fake(); $activity = null; $error = null; try { $activity = app(StartDatabaseImport::class)->handle( $test->database->fresh(), new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), $test->team->id, ); } catch (Throwable $e) { $error = $e; } return [$processes, $activity, $error]; } /** * Runs the recorded `mc alias set` SSH process in a real shell with its stdin. Stubs for `timeout`, `ssh` * (runs the remote command in `sh -c`, like a login shell), `sudo`, `docker` and `mc` log their arguments. * * @param array{command: string, input: ?string} $aliasProcess * @return array{argv: string, mcStdin: string} */ function importS3ArgvTestRunAliasProcess(array $aliasProcess): array { $bin = sys_get_temp_dir().'/import-s3-argv-'.bin2hex(random_bytes(6)); mkdir($bin); $log = "{$bin}/argv.log"; $stubs = [ 'timeout' => "shift\nexec \"\$@\"", 'ssh' => "for last; do :; done\nexec sh -c \"\$last\"", 'sudo' => 'exec "$@"', 'docker' => "[ \"\$1\" = exec ] || exit 0\nshift\nwhile [ \"\${1#-}\" != \"\$1\" ]; do shift; done\nshift\nexec \"\$@\"", 'mc' => "cat > {$bin}/mc.stdin", ]; foreach ($stubs as $name => $body) { file_put_contents("{$bin}/{$name}", "#!/bin/sh\nprintf '%s\\n' \"{$name} \$*\" >> {$log}\n{$body}\n"); chmod("{$bin}/{$name}", 0755); } try { (new SymfonyProcess(['bash', '-c', $aliasProcess['command']], env: ['PATH' => "{$bin}:/usr/bin:/bin"], input: $aliasProcess['input']))->mustRun(); return ['argv' => (string) file_get_contents($log), 'mcStdin' => (string) file_get_contents("{$bin}/mc.stdin")]; } finally { array_map('unlink', glob("{$bin}/*")); rmdir($bin); } } it('pins the reachable IP of a dual-stack S3 host for the restore mc commands', function () { InstanceSettings::query()->whereKey(0)->update([ 'webhook_allowed_internal_hosts' => ['localhost'], 'webhook_allow_localhost' => true, ]); $storage = S3Storage::create([ 'name' => 'Import S3', 'region' => 'us-east-1', 'key' => 'key', 'secret' => 'secret', 'bucket' => 'test-bucket', 'endpoint' => 'http://localhost:9000', 'is_usable' => true, 'team_id' => $this->team->id, ]); $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('exists')->once()->andReturn(true); $disk->shouldReceive('size')->once()->andReturn(1024); $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); $filesystem->shouldReceive('build')->once()->andReturn($disk); Storage::swap($filesystem); fakeRemoteProcesses(['*/dev/tcp/*' => '::1', '*' => '']); Queue::fake(); $activity = app(StartDatabaseImport::class)->handle( $this->database, new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), $this->team->id, ); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker run --rm --network 'coolify'") && str_contains(remoteProcessScript($process), '/dev/tcp/$ip/9000')); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "mc alias set --resolve '\\''localhost:9000=::1'\\'' --api S3v4 s3temp")); expect((string) RemoteProcessCommand::read($activity)) ->toContain("mc cp --resolve 'localhost:9000=::1' ".escapeshellarg('s3temp/test-bucket/backups/restore.sql')); }); it('passes the S3 keys of an import to mc on stdin, not in a command line', function (string $sshUser) { [$processes, $activity, $error] = importS3ArgvTestRun($this, $sshUser); expect($error)->toBeNull(); $commands = $processes->pluck('command')->push((string) RemoteProcessCommand::read($activity)); foreach ($commands as $command) { expect($command) ->not->toContain(IMPORT_S3_ARGV_SECRET) ->not->toContain(escapeshellarg(IMPORT_S3_ARGV_SECRET)) ->not->toContain(escapeshellarg(IMPORT_S3_ARGV_KEY)) ->not->toContain(base64_encode(IMPORT_S3_ARGV_SECRET)) ->not->toContain(base64_encode(IMPORT_S3_ARGV_KEY)); } $aliasProcess = $processes->sole(fn (array $process) => str_contains($process['command'], 'mc alias set')); $run = importS3ArgvTestRunAliasProcess($aliasProcess); expect($run['mcStdin'])->toBe(IMPORT_S3_ARGV_KEY."\n".IMPORT_S3_ARGV_SECRET."\n") ->and($run['argv'])->toContain('mc alias set --api S3v4 s3temp https://8.8.8.8') ->and($run['argv'])->not->toContain(IMPORT_S3_ARGV_SECRET) ->and($run['argv'])->not->toContain(IMPORT_S3_ARGV_KEY) ->and($processes->contains(fn (array $process) => str_contains($process['command']."\n".$process['input'], "-e 'MC_REGION=ru-central-1'")))->toBeTrue() ->and((string) RemoteProcessCommand::read($activity))->toContain('mc cp '.escapeshellarg('s3temp/test-bucket/backups/restore.sql')); if ($sshUser !== 'root') { expect($run['argv'])->toContain('sudo docker exec -i'); } })->with(['root', 'deploy']); it('removes the S3 restore helper when the mc alias cannot be set', function () { [$processes, $activity, $error] = importS3ArgvTestRun($this, 'root', aliasFails: true); expect($activity)->toBeNull() ->and($error)->toBeInstanceOf(DatabaseImportException::class); $aliasIndex = $processes->search(fn (array $process) => str_contains($process['command'], 'mc alias set')); $removedAfterAlias = $processes->slice($aliasIndex + 1) ->contains(fn (array $process) => preg_match("/docker rm -f '?s3-restore-/", $process['command']."\n".$process['input']) === 1); expect($removedAfterAlias)->toBeTrue(); Queue::assertNothingPushed(); }); test('s3 import downloads with curl and a presigned URL when the endpoint has a path', function () { $storage = S3Storage::create([ 'name' => 'Supabase', 'region' => 'eu-central-1', 'key' => 'AKIA_PATH_ENDPOINT_KEY', 'secret' => 'path/endpoint/SECRET', 'bucket' => 'test-bucket', 'endpoint' => 'https://8.8.8.8/storage/v1/s3', 'is_usable' => true, 'team_id' => $this->team->id, ]); $disk = Mockery::mock(FilesystemAdapter::class); $disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true); $disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024); $disk->shouldReceive('temporaryUrl')->once()->with('backups/restore.sql', Mockery::any()) ->andReturn('https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned'); $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); $filesystem->shouldReceive('build')->twice()->andReturn($disk); Storage::swap($filesystem); $processes = collect(); Process::fake(function ($process) use ($processes) { $processes->push(['command' => $process->command, 'input' => $process->input]); return Process::result(); }); Queue::fake(); $activity = app(StartDatabaseImport::class)->handle( $this->database, new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid), $this->team->id, ); $command = (string) RemoteProcessCommand::read($activity); $config = $processes->sole(fn (array $process) => str_contains($process['command'], 'cat > /tmp/s3-transfer.curl'))['input']; expect($config)->toBe(implode("\n", [ 'url = "https://8.8.8.8/storage/v1/s3/test-bucket/backups/restore.sql?X-Amz-Signature=presigned"', 'silent', 'show-error', 'output = "/tmp/restore"', 'fail', ])."\n") ->and($command)->toContain("docker exec 's3-restore-") ->and($command)->toContain(' curl --config /tmp/s3-transfer.curl') ->and($command)->not->toContain('mc cp') ->and($command)->not->toContain('X-Amz-Signature') ->and($command)->not->toContain('path/endpoint/SECRET'); });