set('constants.ssh.mux_enabled', false); config()->set('constants.ssh.max_retries', 1); }); /** * @return array{0: SensitiveCommandFailureDeploymentJob, 1: ReflectionClass, 2: ApplicationDeploymentQueue, 3: Application} */ function makeSensitiveCommandFailureJob(array $environmentVariables = []): array { $user = User::factory()->create(); $team = $user->teams()->first(); $privateKeyContent = "-----BEGIN OPENSSH PRIVATE KEY-----\n" ."b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n" ."QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk\n" ."hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA\n" ."AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV\n" ."uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==\n" .'-----END OPENSSH PRIVATE KEY-----'; $privateKey = PrivateKey::create([ 'name' => 'sensitive-failure-key-'.uniqid(), 'private_key' => $privateKeyContent, 'team_id' => $team->id, ]); Storage::fake('ssh-keys'); Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent); $server = Server::factory()->create([ 'team_id' => $team->id, 'private_key_id' => $privateKey->id, 'user' => 'root', ]); $project = Project::create(['name' => 'Sensitive Failure Project', 'team_id' => $team->id]); $environment = Environment::where('project_id', $project->id)->firstOrFail(); $application = Application::factory()->create([ 'environment_id' => $environment->id, 'build_pack' => 'dockerfile', ]); $application->settings()->update([ 'include_source_commit_in_build' => false, 'is_env_sorting_enabled' => false, ]); foreach ($environmentVariables as $attributes) { EnvironmentVariable::create([ 'resourceable_type' => Application::class, 'resourceable_id' => $application->id, 'is_preview' => false, 'is_runtime' => true, 'is_buildtime' => true, 'is_multiline' => false, 'is_literal' => false, ...$attributes, ]); } $queue = ApplicationDeploymentQueue::create([ 'deployment_uuid' => 'sensitive-failure-deployment', 'application_id' => $application->id, 'server_id' => $server->id, 'status' => 'in_progress', ]); $job = new SensitiveCommandFailureDeploymentJob; $reflection = new ReflectionClass(ApplicationDeploymentJob::class); foreach ([ 'application' => $application->fresh(), 'application_deployment_queue' => $queue, 'server' => $server, 'mainServer' => $server, 'build_pack' => 'dockerfile', 'pull_request_id' => 0, 'commit' => 'HEAD', 'basedir' => '/artifacts/sensitive-failure', 'workdir' => '/artifacts/sensitive-failure', 'configuration_dir' => '/data/coolify/applications/sensitive-failure', 'deployment_uuid' => 'sensitive-failure-deployment', 'dockerfile_location' => '/Dockerfile', 'container_name' => 'sensitive-failure-app', 'coolify_variables' => null, 'dockerSecretsSupported' => false, 'saved_outputs' => new Collection, ] as $property => $value) { $reflectionProperty = $reflection->getProperty($property); $reflectionProperty->setAccessible(true); $reflectionProperty->setValue($job, $value); } return [$job, $reflection, $queue, $application]; } function invokeSensitiveFailureJobMethod(object $job, ReflectionClass $reflection, string $method, mixed ...$arguments): mixed { $reflectionMethod = $reflection->getMethod($method); $reflectionMethod->setAccessible(true); return $reflectionMethod->invoke($job, ...$arguments); } function captureDeploymentException(callable $callback): DeploymentException { try { $callback(); } catch (DeploymentException $exception) { return $exception; } throw new RuntimeException('Expected a DeploymentException.'); } /** * @return list */ function base64PayloadsInCommand(string $command): array { preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches); return array_values(array_filter( $matches[0], static fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), SENSITIVE_FAILURE_SECRET) )); } it('hides the .env write command and its base64 payload when the write fails', function () { [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([ ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET], ]); $payloads = []; Process::fake(function (PendingProcess $process) use (&$payloads) { $payloads = array_merge($payloads, base64PayloadsInCommand($process->command)); return Process::result(errorOutput: 'tee: /artifacts/sensitive-failure/.env: No space left on device', exitCode: 1); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables')); expect($payloads)->not->toBeEmpty(); expect($exception->getMessage()) ->toContain('Command execution failed (exit code 1): [command hidden because it contains sensitive data]') ->toContain('No space left on device') ->not->toContain(SENSITIVE_FAILURE_SECRET) ->not->toContain('base64 -d'); foreach ($payloads as $payload) { expect($exception->getMessage())->not->toContain($payload); } $storedLogs = (string) $queue->fresh()->logs; expect($storedLogs) ->toContain('No space left on device') ->not->toContain(SENSITIVE_FAILURE_SECRET); foreach ($payloads as $payload) { expect($storedLogs)->not->toContain($payload); } }); it('redacts the sensitive payload when the error output echoes the command', function () { [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([ ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET], ]); $payloads = []; Process::fake(function (PendingProcess $process) use (&$payloads) { $commandPayloads = base64PayloadsInCommand($process->command); $payloads = array_merge($payloads, $commandPayloads); return Process::result(errorOutput: "bash: line 1: echo '".($commandPayloads[0] ?? '')."': unexpected failure", exitCode: 2); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables')); expect($payloads)->not->toBeEmpty(); $storedLogs = (string) $queue->fresh()->logs; foreach ($payloads as $payload) { expect($exception->getMessage())->not->toContain($payload); expect($storedLogs)->not->toContain($payload); } expect($exception->getMessage())->toContain('unexpected failure'); }); it('hides the build-time env write command when the write fails', function () { [$job, $reflection] = makeSensitiveCommandFailureJob([ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false], ]); $payloads = []; Process::fake(function (PendingProcess $process) use (&$payloads) { $payloads = array_merge($payloads, base64PayloadsInCommand($process->command)); return Process::result(errorOutput: 'tee: /artifacts/build-time.env: No such file or directory', exitCode: 1); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables')); expect($payloads)->not->toBeEmpty(); expect($exception->getMessage()) ->toContain('[command hidden because it contains sensitive data]') ->toContain('No such file or directory') ->not->toContain(SENSITIVE_FAILURE_SECRET); foreach ($payloads as $payload) { expect($exception->getMessage())->not->toContain($payload); } }); it('keeps the command in the error message for failed commands that are not sensitive', function () { [$job] = makeSensitiveCommandFailureJob(); Process::fake(['*' => Process::result(errorOutput: 'ls: cannot access: No such file or directory', exitCode: 2)]); $exception = captureDeploymentException(fn () => $job->execute_remote_command([ 'command' => 'ls /artifacts/harmless-missing-directory', 'hidden' => true, ])); expect($exception->getMessage()) ->toContain('Command execution failed (exit code 2): ls /artifacts/harmless-missing-directory') ->toContain('No such file or directory') ->not->toContain('[command hidden because it contains sensitive data]'); }); it('redacts locked values from the error output of failed commands', function () { [$job] = makeSensitiveCommandFailureJob([ ['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true], ]); Process::fake(['*' => Process::result(errorOutput: 'failed with value '.SENSITIVE_FAILURE_SECRET, exitCode: 1)]); $exception = captureDeploymentException(fn () => $job->execute_remote_command([ 'command' => 'harmless-command --flag', ])); expect($exception->getMessage()) ->toContain('harmless-command --flag') ->toContain('failed with value '.REDACTED) ->not->toContain(SENSITIVE_FAILURE_SECRET); }); it('logs only the variable names of the .env files in development mode', function () { config()->set('app.env', 'local'); [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([ ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET], ]); $ranCommands = []; Process::fake(function (PendingProcess $process) use (&$ranCommands) { $ranCommands[] = $process->command; return Process::result(output: str_contains($process->command, 'cat ') ? 'APP_SECRET='.SENSITIVE_FAILURE_SECRET : ''); }); invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'); invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables'); $logEntries = collect(json_decode((string) $queue->fresh()->logs, true)); $keyListEntries = $logEntries->filter(fn (array $entry): bool => str_contains((string) $entry['output'], '[DEBUG] Variable names in')); expect($keyListEntries)->toHaveCount(2) ->each(fn ($entry) => $entry->hidden->toBeTrue()->output->toContain('APP_SECRET')); expect($logEntries->pluck('output')->merge($logEntries->pluck('command'))->filter()->implode("\n")) ->not->toContain('APP_SECRET='.SENSITIVE_FAILURE_SECRET) ->not->toContain('APP_SECRET="'.SENSITIVE_FAILURE_SECRET); expect(collect($ranCommands)->filter(fn (string $command): bool => str_contains($command, 'cat /artifacts/sensitive-failure/.env') || str_contains($command, 'cat '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH))) ->toBeEmpty(); }); function setSensitiveFailureJobProperties(object $job, ReflectionClass $reflection, array $properties): void { foreach ($properties as $property => $value) { $reflectionProperty = $reflection->getProperty($property); $reflectionProperty->setAccessible(true); $reflectionProperty->setValue($job, $value); } } it('hides the helper container command when build secrets are passed as environment flags', function () { [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false], ]); $application->settings()->update(['use_build_secrets' => true]); $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); setSensitiveFailureJobProperties($job, $reflection, [ 'application' => $application->fresh(), 'destination' => StandaloneDocker::forceCreate([ 'name' => 'sensitive-failure-network', 'network' => 'sensitive-failure-network', 'server_id' => $server->id, ]), ]); Process::fake(function (PendingProcess $process) { if (str_contains($process->command, 'docker run -d')) { return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125); } return Process::result(output: str_contains($process->command, 'echo $HOME') ? '/root' : 'NOK'); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image')); expect($exception->getMessage()) ->toContain('[command hidden because it contains sensitive data]') ->toContain('network not found') ->not->toContain(SENSITIVE_FAILURE_SECRET); expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET); }); it('hides the Railpack prepare command that passes build-time values', function () { [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false], ]); $application->update(['build_pack' => 'railpack']); setSensitiveFailureJobProperties($job, $reflection, [ 'application' => $application->fresh(), 'build_pack' => 'railpack', 'dockerBuildxAvailable' => true, ]); Process::fake(function (PendingProcess $process) { if (str_contains($process->command, 'railpack prepare')) { return Process::result(errorOutput: 'railpack: failed to detect a provider', exitCode: 1); } return Process::result(output: str_contains($process->command, 'buildx version') ? 'available' : 'missing'); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_railpack_image')); expect($exception->getMessage()) ->toContain('[command hidden because it contains sensitive data]') ->toContain('failed to detect a provider') ->not->toContain(SENSITIVE_FAILURE_SECRET); expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET); }); it('hides the Nixpacks plan write command that contains build-time values', function (bool $isStatic) { [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob(); $application->update(['build_pack' => 'nixpacks']); $application->settings()->update(['is_static' => $isStatic]); setSensitiveFailureJobProperties($job, $reflection, [ 'application' => $application->fresh(), 'build_pack' => 'nixpacks', 'build_args' => collect(), 'disableBuildCache' => false, 'force_rebuild' => false, 'nixpacks_plan' => json_encode(['variables' => ['BUILD_SECRET' => SENSITIVE_FAILURE_SECRET]]), ]); $payloads = []; Process::fake(function (PendingProcess $process) use (&$payloads) { $commandPayloads = base64PayloadsInCommand($process->command); if ($commandPayloads !== []) { $payloads = array_merge($payloads, $commandPayloads); return Process::result(errorOutput: 'tee: /artifacts/thegameplan.json: No space left on device', exitCode: 1); } return Process::result(); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_image')); expect($payloads)->not->toBeEmpty(); expect($exception->getMessage()) ->toContain('[command hidden because it contains sensitive data]') ->not->toContain(SENSITIVE_FAILURE_SECRET); $storedLogs = (string) $queue->fresh()->logs; foreach ($payloads as $payload) { expect($exception->getMessage())->not->toContain($payload); expect($storedLogs)->not->toContain($payload); } })->with([ 'static' => [true], 'not static' => [false], ]); it('hides the compose file write command because compose files can contain secrets', function () { [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob(); $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); setSensitiveFailureJobProperties($job, $reflection, [ 'destination' => $server->standaloneDockers()->firstOrFail(), 'production_image_name' => 'example/app:latest', ]); $payloads = []; Process::fake(function (PendingProcess $process) use (&$payloads) { preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $process->command, $matches); $composePayloads = array_filter($matches[0], fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), 'services:')); if ($composePayloads !== []) { $payloads = array_merge($payloads, $composePayloads); return Process::result(errorOutput: 'tee: docker-compose.yaml: No space left on device', exitCode: 1); } return Process::result(); }); $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'generate_compose_file')); expect($payloads)->not->toBeEmpty(); expect($exception->getMessage()) ->toContain('[command hidden because it contains sensitive data]') ->toContain('No space left on device'); $storedLogs = (string) $queue->fresh()->logs; foreach ($payloads as $payload) { expect($exception->getMessage())->not->toContain($payload); expect($storedLogs)->not->toContain($payload); } }); function readSensitiveFailureJobProperty(object $job, ReflectionClass $reflection, string $property): mixed { $reflectionProperty = $reflection->getProperty($property); $reflectionProperty->setAccessible(true); return $reflectionProperty->getValue($job); }