diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index db704f1dc4..994d9ad657 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -769,6 +769,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->execute_remote_command([ executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}{$this->docker_compose_location} > /dev/null"), 'hidden' => true, + 'skip_command_log' => true, ]); // Modify Dockerfiles for ARGs and build secrets @@ -1151,6 +1152,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue ], [ "echo '{$this->docker_compose_base64}' | base64 -d | tee $composeFileName > /dev/null", + 'skip_command_log' => true, ], [ "echo '{$readme}' > $mainDir/README.md", @@ -1774,12 +1776,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue ); if (isDev()) { - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "cat $this->workdir/.env"), - 'hidden' => true, - ] - ); + $this->logEnvironmentFileKeys("{$this->workdir}/.env", $environment_variables); } // Write .env file to configuration directory @@ -1802,6 +1799,20 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * Development aid: log which variables an env file contains, without their values. + * + * @param Collection $environmentVariables Lines in KEY=VALUE format. + */ + private function logEnvironmentFileKeys(string $path, Collection $environmentVariables): void + { + $keys = $environmentVariables + ->map(fn (string $line): string => explode('=', $line, 2)[0]) + ->implode(', '); + + $this->application_deployment_queue->addLogEntry("[DEBUG] Variable names in {$path}: {$keys}", hidden: true); + } + private function generate_buildtime_environment_variables() { if (isDev()) { @@ -2067,38 +2078,60 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } /** - * Build-time names go into shell and Docker build commands, so they must be valid. Runtime-only - * variables only go into the .env file: existing ones with names that new variables can no longer - * use (such as my-var) keep working, unless the name would break a .env line. + * Build-time names go into shell and Docker build commands, so they must be valid when the + * deployment builds an image. Runtime-only variables only go into the .env file: existing ones + * with names that new variables can no longer use (such as my-var) keep working, unless the + * name would break a .env line. Deployments without a build step (Docker image) treat + * build-time variables the same way, because no build receives them. */ private function validateDeploymentEnvironmentVariableKeys(): void { $environmentVariables = $this->pull_request_id === 0 - ? $this->application->environment_variables()->get(['key', 'is_buildtime']) - : $this->application->environment_variables_preview()->get(['key', 'is_buildtime']); + ? $this->application->environment_variables()->get(['key', 'is_buildtime', 'is_runtime']) + : $this->application->environment_variables_preview()->get(['key', 'is_buildtime', 'is_runtime']); + $passesBuildtimeVariables = $this->deploymentPassesBuildtimeVariables(); foreach ($environmentVariables as $environmentVariable) { $key = (string) $environmentVariable->key; $isEnvFileSafe = $key !== '' && strpbrk($key, "=\n\r\0") === false; - if ($environmentVariable->is_buildtime || ! $isEnvFileSafe) { + if (($environmentVariable->is_buildtime && $passesBuildtimeVariables) || ! $isEnvFileSafe) { $this->validatedBuildtimeEnvironmentVariableKey($key, 'the deployment environment'); continue; } if (! ValidationPatterns::isValidEnvironmentVariableKey($key)) { - $this->logLegacyRuntimeEnvironmentVariableKey($key); + $this->logLegacyEnvironmentVariableKey($key, (bool) $environmentVariable->is_buildtime, (bool) $environmentVariable->is_runtime); } } } - private function logLegacyRuntimeEnvironmentVariableKey(string $key): void + /** + * Only Docker image deployments never build an image. Other deployments (also restarts, + * rollbacks, and previews) can build and pass build-time variables to the build. + */ + private function deploymentPassesBuildtimeVariables(): bool + { + return $this->application->build_pack !== 'dockerimage'; + } + + private function logLegacyEnvironmentVariableKey(string $key, bool $isBuildtime, bool $isRuntime): void { $suggestedKey = (string) preg_replace('/[^A-Za-z0-9_]/', '_', $key); if (preg_match('/\A[0-9]/', $suggestedKey) === 1) { $suggestedKey = '_'.$suggestedKey; } - $this->application_deployment_queue->addLogEntry('⚠️ Runtime variable '.ValidationPatterns::displayShellEnvironmentVariableKey($key).' uses a name that new variables cannot use. It is still passed to the container, but shell scripts cannot read it.', 'stderr'); + $displayKey = ValidationPatterns::displayShellEnvironmentVariableKey($key); + + if ($isBuildtime) { + $this->application_deployment_queue->addLogEntry("⚠️ Build-time variable {$displayKey} uses a name that new variables cannot use. This deployment does not build an image, so it is not used as a build-time variable. Rename it before you use it in a build.", 'stderr'); + } + if ($isRuntime) { + $this->application_deployment_queue->addLogEntry("⚠️ Runtime variable {$displayKey} uses a name that new variables cannot use. It is still passed to the container, but shell scripts cannot read it.", 'stderr'); + } + if (! $isBuildtime && ! $isRuntime) { + $this->application_deployment_queue->addLogEntry("⚠️ Variable {$displayKey} uses a name that new variables cannot use. This deployment does not use it.", 'stderr'); + } $this->application_deployment_queue->addLogEntry(' Suggested name: '.ValidationPatterns::displayShellEnvironmentVariableKey($suggestedKey), type: 'info'); } @@ -2159,10 +2192,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue ]); if (isDev()) { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, 'cat '.self::BUILD_TIME_ENV_PATH), - 'hidden' => true, - ]); + $this->logEnvironmentFileKeys(self::BUILD_TIME_ENV_PATH, $environment_variables); } } elseif (in_array($this->build_pack, ['dockercompose', 'dockerfile', 'railpack'], true)) { $this->application_deployment_queue->addLogEntry('Creating empty build-time .env file in /artifacts (no build-time variables defined).', hidden: true); @@ -2527,6 +2557,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue [ $runCommand, 'hidden' => true, + 'skip_command_log' => filled($env_flags), ], [ 'command' => executeInDocker($this->deployment_uuid, "mkdir -p {$this->basedir}"), @@ -3325,7 +3356,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->application_deployment_queue->addLogEntry('Generating Railpack build plan.'); $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, $prepare_command), 'hidden' => true], + [executeInDocker($this->deployment_uuid, $prepare_command), 'hidden' => true, 'skip_command_log' => true], [ executeInDocker($this->deployment_uuid, 'cat /artifacts/railpack-plan.json'), 'hidden' => true, @@ -3813,7 +3844,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); $this->docker_compose = Yaml::dump($docker_compose, 10); $this->docker_compose_base64 = base64_encode($this->docker_compose); - $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true]); + $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true, 'skip_command_log' => true]); } private function generate_local_persistent_volumes() @@ -4044,7 +4075,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); } if ($this->application->build_pack === 'nixpacks') { $this->nixpacks_plan = base64_encode($this->nixpacks_plan); - $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true]); + $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true, 'skip_command_log' => true]); if ($this->force_rebuild) { $this->execute_remote_command([ executeInDocker($this->deployment_uuid, 'nixpacks build -c '.self::NIXPACKS_PLAN_PATH." --no-cache --no-error-without-start -n {$this->build_image_name} {$this->workdir} -o {$this->workdir}"), @@ -4230,7 +4261,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); } else { if ($this->application->build_pack === 'nixpacks') { $this->nixpacks_plan = base64_encode($this->nixpacks_plan); - $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true]); + $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true, 'skip_command_log' => true]); if ($this->force_rebuild) { $this->execute_remote_command([ executeInDocker($this->deployment_uuid, 'nixpacks build -c '.self::NIXPACKS_PLAN_PATH." --no-cache --no-error-without-start -n {$this->production_image_name} {$this->workdir} -o {$this->workdir}"), @@ -4579,8 +4610,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function generate_docker_env_flags_for_secrets() { - // Only generate env flags if build secrets are enabled - if (! $this->application->settings->use_build_secrets) { + // Only generate env flags if build secrets are enabled and the deployment builds an image + if (! $this->application->settings->use_build_secrets || ! $this->deploymentPassesBuildtimeVariables()) { return ''; } diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 0d8dfdc29f..5a4540ea3a 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -16,6 +16,8 @@ trait ExecuteRemoteCommand { use SshRetryable; + private const SENSITIVE_COMMAND_PLACEHOLDER = '[command hidden because it contains sensitive data]'; + public ?string $save = null; public static int $batch_counter = 0; @@ -184,7 +186,7 @@ trait ExecuteRemoteCommand $new_log_entry = [ 'command' => $skip_command_log || $command_hidden ? null : $this->redact_sensitive_info($command), - 'output' => $this->redact_sensitive_info($log_output), + 'output' => $this->redact_sensitive_info($skip_command_log ? $this->redactSensitiveCommandPayloads((string) $log_output, (string) $command) : $log_output), 'type' => $customType ?? ($type === 'err' ? 'stderr' : 'stdout'), 'timestamp' => Carbon::now('UTC'), 'hidden' => $hidden, @@ -241,12 +243,54 @@ trait ExecuteRemoteCommand if (empty($error)) { $error = $process_result->output() ?: 'Command failed with no error output'; } - $redactedCommand = $this->redact_sensitive_info($command); - throw new DeploymentException("Command execution failed (exit code {$process_result->exitCode()}): {$redactedCommand}\nError: {$error}"); + throw new DeploymentException($this->commandFailureMessage((string) $command, (int) $process_result->exitCode(), (string) $error, $skip_command_log)); } } } + /** + * Commands marked with skip_command_log embed secrets (for example base64 encoded .env files or + * private keys), so their text must never reach a log line or an exception message. + */ + private function commandFailureMessage(string $command, int $exitCode, string $error, bool $isSensitiveCommand): string + { + if ($isSensitiveCommand) { + $commandText = self::SENSITIVE_COMMAND_PLACEHOLDER; + $error = $this->redactSensitiveCommandPayloads($error, $command); + } else { + $commandText = $this->redact_sensitive_info($command); + } + + $error = $this->redact_sensitive_info($error); + + return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}"; + } + + /** + * Removes the encoded payloads of a sensitive command from output that could echo the command. + */ + private function redactSensitiveCommandPayloads(string $text, string $command): string + { + if ($text === '' || preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches) === false) { + return $text; + } + + $payloads = collect($matches[0]) + ->unique() + ->filter(function (string $candidate): bool { + $decoded = base64_decode($candidate, true); + + return $decoded !== false + && mb_check_encoding($decoded, 'UTF-8') + && preg_match('/[^\P{C}\t\n\r]/u', $decoded) !== 1; + }) + ->sortByDesc(fn (string $payload): int => strlen($payload)) + ->values() + ->all(); + + return $payloads === [] ? $text : str_replace($payloads, REDACTED, $text); + } + private function saveCommandOutput(string $output, bool $append): void { if (! $this->save) { diff --git a/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php b/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php index 1aa668e116..5f61f78c47 100644 --- a/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php +++ b/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php @@ -1030,6 +1030,94 @@ it('rejects existing variable names that would break the .env file or build comm 'build-time name with a hyphen' => ['my-var', true], ]); +it('warns instead of failing for invalid build-time names when the deployment does not build an image', function (bool $isRuntime) { + [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']); + $environmentVariable = createApplicationEnvironmentVariable($application, [ + 'key' => 'SAFE_KEY', + 'value' => 'secret', + 'is_buildtime' => true, + 'is_runtime' => $isRuntime, + ]); + DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']); + + [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server); + invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys'); + + expect(collect($job->recordedLogEntries)->implode("\n")) + ->toContain('my-var') + ->toContain('Suggested name: my_var') + ->not->toContain('Invalid environment variable name'); + expect($job->recordedCommands)->toBeEmpty(); +})->with([ + 'build-time only' => [false], + 'build-time and runtime' => [true], +]); + +it('warns instead of failing for invalid build-time preview names of Docker image deployments', function () { + [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']); + $environmentVariable = createApplicationEnvironmentVariable($application, [ + 'key' => 'SAFE_KEY', + 'value' => 'secret', + 'is_buildtime' => true, + 'is_preview' => true, + ]); + DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']); + + [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server, ['pull_request_id' => 7]); + invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys'); + + expect(collect($job->recordedLogEntries)->implode("\n")) + ->toContain('Suggested name: my_var') + ->not->toContain('Invalid environment variable name'); +}); + +it('still rejects Docker image variable names that would break the .env file', function () { + [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']); + $environmentVariable = createApplicationEnvironmentVariable($application, [ + 'key' => 'SAFE_KEY', + 'value' => 'secret', + 'is_buildtime' => true, + ]); + DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'A=B']); + + [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server); + + expect(fn () => invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys')) + ->toThrow(DeploymentException::class, 'Invalid environment variable name from the deployment environment'); +}); + +it('fails with a clear message for invalid build-time names when the deployment builds an image', function (string $buildPack) { + [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => $buildPack]); + $environmentVariable = createApplicationEnvironmentVariable($application, [ + 'key' => 'SAFE_KEY', + 'value' => 'secret', + 'is_buildtime' => true, + ]); + DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']); + + [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server, ['build_pack' => $buildPack]); + + expect(fn () => invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys')) + ->toThrow(DeploymentException::class, 'Invalid environment variable name from the deployment environment: my-var'); + expect(collect($job->recordedLogEntries)->implode("\n")) + ->toContain('Build-time variable names must start with a letter or underscore'); +})->with(['dockerfile', 'nixpacks', 'static', 'railpack', 'dockercompose']); + +it('does not pass build-time variables to the helper container of Docker image deployments', function () { + [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']); + $application->settings()->update(['use_build_secrets' => true]); + $environmentVariable = createApplicationEnvironmentVariable($application, [ + 'key' => 'SAFE_KEY', + 'value' => 'harmless-build-value', + 'is_buildtime' => true, + ]); + DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']); + + [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server); + + expect(invokeDeploymentJobMethod($job, $reflection, 'generate_docker_env_flags_for_secrets'))->toBe(''); +}); + it('injects raw escaped remote secrets into Dockerfile args and hashes the same values', function (int $pullRequestId, bool $isPreview) { [$application, $server] = makeDeploymentControlVarFixture(); diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.php new file mode 100644 index 0000000000..2fa3798d2e --- /dev/null +++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php @@ -0,0 +1,449 @@ +set('constants.ssh.mux_enabled', false); + config()->set('constants.ssh.max_retries', 1); +}); + +/** + * @return array{0: SensitiveCommandFailureDeploymentJob, 1: ReflectionClass, 2: ApplicationDeploymentQueue, 3: Application} + */ +function makeSensitiveCommandFailureJob(array $environmentVariables = []): array +{ + $user = User::factory()->create(); + $team = $user->teams()->first(); + + $privateKeyContent = "-----BEGIN OPENSSH PRIVATE KEY-----\n" + ."b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n" + ."QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk\n" + ."hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA\n" + ."AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV\n" + ."uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==\n" + .'-----END OPENSSH PRIVATE KEY-----'; + + $privateKey = PrivateKey::create([ + 'name' => 'sensitive-failure-key-'.uniqid(), + 'private_key' => $privateKeyContent, + 'team_id' => $team->id, + ]); + + Storage::fake('ssh-keys'); + Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent); + + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => $privateKey->id, + 'user' => 'root', + ]); + + $project = Project::create(['name' => 'Sensitive Failure Project', 'team_id' => $team->id]); + $environment = Environment::where('project_id', $project->id)->firstOrFail(); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'build_pack' => 'dockerfile', + ]); + $application->settings()->update([ + 'include_source_commit_in_build' => false, + 'is_env_sorting_enabled' => false, + ]); + + foreach ($environmentVariables as $attributes) { + EnvironmentVariable::create([ + 'resourceable_type' => Application::class, + 'resourceable_id' => $application->id, + 'is_preview' => false, + 'is_runtime' => true, + 'is_buildtime' => true, + 'is_multiline' => false, + 'is_literal' => false, + ...$attributes, + ]); + } + + $queue = ApplicationDeploymentQueue::create([ + 'deployment_uuid' => 'sensitive-failure-deployment', + 'application_id' => $application->id, + 'server_id' => $server->id, + 'status' => 'in_progress', + ]); + + $job = new SensitiveCommandFailureDeploymentJob; + $reflection = new ReflectionClass(ApplicationDeploymentJob::class); + + foreach ([ + 'application' => $application->fresh(), + 'application_deployment_queue' => $queue, + 'server' => $server, + 'mainServer' => $server, + 'build_pack' => 'dockerfile', + 'pull_request_id' => 0, + 'commit' => 'HEAD', + 'basedir' => '/artifacts/sensitive-failure', + 'workdir' => '/artifacts/sensitive-failure', + 'configuration_dir' => '/data/coolify/applications/sensitive-failure', + 'deployment_uuid' => 'sensitive-failure-deployment', + 'dockerfile_location' => '/Dockerfile', + 'container_name' => 'sensitive-failure-app', + 'coolify_variables' => null, + 'dockerSecretsSupported' => false, + 'saved_outputs' => new Collection, + ] as $property => $value) { + $reflectionProperty = $reflection->getProperty($property); + $reflectionProperty->setAccessible(true); + $reflectionProperty->setValue($job, $value); + } + + return [$job, $reflection, $queue, $application]; +} + +function invokeSensitiveFailureJobMethod(object $job, ReflectionClass $reflection, string $method, mixed ...$arguments): mixed +{ + $reflectionMethod = $reflection->getMethod($method); + $reflectionMethod->setAccessible(true); + + return $reflectionMethod->invoke($job, ...$arguments); +} + +function captureDeploymentException(callable $callback): DeploymentException +{ + try { + $callback(); + } catch (DeploymentException $exception) { + return $exception; + } + + throw new RuntimeException('Expected a DeploymentException.'); +} + +/** + * @return list + */ +function base64PayloadsInCommand(string $command): array +{ + preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches); + + return array_values(array_filter( + $matches[0], + static fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), SENSITIVE_FAILURE_SECRET) + )); +} + +it('hides the .env write command and its base64 payload when the write fails', function () { + [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([ + ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET], + ]); + + $payloads = []; + Process::fake(function (PendingProcess $process) use (&$payloads) { + $payloads = array_merge($payloads, base64PayloadsInCommand($process->command)); + + return Process::result(errorOutput: 'tee: /artifacts/sensitive-failure/.env: No space left on device', exitCode: 1); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables')); + + expect($payloads)->not->toBeEmpty(); + expect($exception->getMessage()) + ->toContain('Command execution failed (exit code 1): [command hidden because it contains sensitive data]') + ->toContain('No space left on device') + ->not->toContain(SENSITIVE_FAILURE_SECRET) + ->not->toContain('base64 -d'); + foreach ($payloads as $payload) { + expect($exception->getMessage())->not->toContain($payload); + } + + $storedLogs = (string) $queue->fresh()->logs; + expect($storedLogs) + ->toContain('No space left on device') + ->not->toContain(SENSITIVE_FAILURE_SECRET); + foreach ($payloads as $payload) { + expect($storedLogs)->not->toContain($payload); + } +}); + +it('redacts the sensitive payload when the error output echoes the command', function () { + [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([ + ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET], + ]); + + $payloads = []; + Process::fake(function (PendingProcess $process) use (&$payloads) { + $commandPayloads = base64PayloadsInCommand($process->command); + $payloads = array_merge($payloads, $commandPayloads); + + return Process::result(errorOutput: "bash: line 1: echo '".($commandPayloads[0] ?? '')."': unexpected failure", exitCode: 2); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables')); + + expect($payloads)->not->toBeEmpty(); + $storedLogs = (string) $queue->fresh()->logs; + foreach ($payloads as $payload) { + expect($exception->getMessage())->not->toContain($payload); + expect($storedLogs)->not->toContain($payload); + } + expect($exception->getMessage())->toContain('unexpected failure'); +}); + +it('hides the build-time env write command when the write fails', function () { + [$job, $reflection] = makeSensitiveCommandFailureJob([ + ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false], + ]); + + $payloads = []; + Process::fake(function (PendingProcess $process) use (&$payloads) { + $payloads = array_merge($payloads, base64PayloadsInCommand($process->command)); + + return Process::result(errorOutput: 'tee: /artifacts/build-time.env: No such file or directory', exitCode: 1); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables')); + + expect($payloads)->not->toBeEmpty(); + expect($exception->getMessage()) + ->toContain('[command hidden because it contains sensitive data]') + ->toContain('No such file or directory') + ->not->toContain(SENSITIVE_FAILURE_SECRET); + foreach ($payloads as $payload) { + expect($exception->getMessage())->not->toContain($payload); + } +}); + +it('keeps the command in the error message for failed commands that are not sensitive', function () { + [$job] = makeSensitiveCommandFailureJob(); + + Process::fake(['*' => Process::result(errorOutput: 'ls: cannot access: No such file or directory', exitCode: 2)]); + + $exception = captureDeploymentException(fn () => $job->execute_remote_command([ + 'command' => 'ls /artifacts/harmless-missing-directory', + 'hidden' => true, + ])); + + expect($exception->getMessage()) + ->toContain('Command execution failed (exit code 2): ls /artifacts/harmless-missing-directory') + ->toContain('No such file or directory') + ->not->toContain('[command hidden because it contains sensitive data]'); +}); + +it('redacts locked values from the error output of failed commands', function () { + [$job] = makeSensitiveCommandFailureJob([ + ['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true], + ]); + + Process::fake(['*' => Process::result(errorOutput: 'failed with value '.SENSITIVE_FAILURE_SECRET, exitCode: 1)]); + + $exception = captureDeploymentException(fn () => $job->execute_remote_command([ + 'command' => 'harmless-command --flag', + ])); + + expect($exception->getMessage()) + ->toContain('harmless-command --flag') + ->toContain('failed with value '.REDACTED) + ->not->toContain(SENSITIVE_FAILURE_SECRET); +}); + +it('logs only the variable names of the .env files in development mode', function () { + config()->set('app.env', 'local'); + [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([ + ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET], + ]); + + $ranCommands = []; + Process::fake(function (PendingProcess $process) use (&$ranCommands) { + $ranCommands[] = $process->command; + + return Process::result(output: str_contains($process->command, 'cat ') ? 'APP_SECRET='.SENSITIVE_FAILURE_SECRET : ''); + }); + + invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'); + invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables'); + + $logEntries = collect(json_decode((string) $queue->fresh()->logs, true)); + $keyListEntries = $logEntries->filter(fn (array $entry): bool => str_contains((string) $entry['output'], '[DEBUG] Variable names in')); + + expect($keyListEntries)->toHaveCount(2) + ->each(fn ($entry) => $entry->hidden->toBeTrue()->output->toContain('APP_SECRET')); + expect($logEntries->pluck('output')->merge($logEntries->pluck('command'))->filter()->implode("\n")) + ->not->toContain('APP_SECRET='.SENSITIVE_FAILURE_SECRET) + ->not->toContain('APP_SECRET="'.SENSITIVE_FAILURE_SECRET); + expect(collect($ranCommands)->filter(fn (string $command): bool => str_contains($command, 'cat /artifacts/sensitive-failure/.env') || str_contains($command, 'cat '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH))) + ->toBeEmpty(); +}); + +function setSensitiveFailureJobProperties(object $job, ReflectionClass $reflection, array $properties): void +{ + foreach ($properties as $property => $value) { + $reflectionProperty = $reflection->getProperty($property); + $reflectionProperty->setAccessible(true); + $reflectionProperty->setValue($job, $value); + } +} + +it('hides the helper container command when build secrets are passed as environment flags', function () { + [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([ + ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false], + ]); + $application->settings()->update(['use_build_secrets' => true]); + $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); + setSensitiveFailureJobProperties($job, $reflection, [ + 'application' => $application->fresh(), + 'destination' => StandaloneDocker::forceCreate([ + 'name' => 'sensitive-failure-network', + 'network' => 'sensitive-failure-network', + 'server_id' => $server->id, + ]), + ]); + + Process::fake(function (PendingProcess $process) { + if (str_contains($process->command, 'docker run -d')) { + return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125); + } + + return Process::result(output: str_contains($process->command, 'echo $HOME') ? '/root' : 'NOK'); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image')); + + expect($exception->getMessage()) + ->toContain('[command hidden because it contains sensitive data]') + ->toContain('network not found') + ->not->toContain(SENSITIVE_FAILURE_SECRET); + expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET); +}); + +it('hides the Railpack prepare command that passes build-time values', function () { + [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([ + ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false], + ]); + $application->update(['build_pack' => 'railpack']); + setSensitiveFailureJobProperties($job, $reflection, [ + 'application' => $application->fresh(), + 'build_pack' => 'railpack', + 'dockerBuildxAvailable' => true, + ]); + + Process::fake(function (PendingProcess $process) { + if (str_contains($process->command, 'railpack prepare')) { + return Process::result(errorOutput: 'railpack: failed to detect a provider', exitCode: 1); + } + + return Process::result(output: str_contains($process->command, 'buildx version') ? 'available' : 'missing'); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_railpack_image')); + + expect($exception->getMessage()) + ->toContain('[command hidden because it contains sensitive data]') + ->toContain('failed to detect a provider') + ->not->toContain(SENSITIVE_FAILURE_SECRET); + expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET); +}); + +it('hides the Nixpacks plan write command that contains build-time values', function (bool $isStatic) { + [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob(); + $application->update(['build_pack' => 'nixpacks']); + $application->settings()->update(['is_static' => $isStatic]); + setSensitiveFailureJobProperties($job, $reflection, [ + 'application' => $application->fresh(), + 'build_pack' => 'nixpacks', + 'build_args' => collect(), + 'disableBuildCache' => false, + 'force_rebuild' => false, + 'nixpacks_plan' => json_encode(['variables' => ['BUILD_SECRET' => SENSITIVE_FAILURE_SECRET]]), + ]); + + $payloads = []; + Process::fake(function (PendingProcess $process) use (&$payloads) { + $commandPayloads = base64PayloadsInCommand($process->command); + if ($commandPayloads !== []) { + $payloads = array_merge($payloads, $commandPayloads); + + return Process::result(errorOutput: 'tee: /artifacts/thegameplan.json: No space left on device', exitCode: 1); + } + + return Process::result(); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_image')); + + expect($payloads)->not->toBeEmpty(); + expect($exception->getMessage()) + ->toContain('[command hidden because it contains sensitive data]') + ->not->toContain(SENSITIVE_FAILURE_SECRET); + $storedLogs = (string) $queue->fresh()->logs; + foreach ($payloads as $payload) { + expect($exception->getMessage())->not->toContain($payload); + expect($storedLogs)->not->toContain($payload); + } +})->with([ + 'static' => [true], + 'not static' => [false], +]); + +it('hides the compose file write command because compose files can contain secrets', function () { + [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob(); + $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); + setSensitiveFailureJobProperties($job, $reflection, [ + 'destination' => $server->standaloneDockers()->firstOrFail(), + 'production_image_name' => 'example/app:latest', + ]); + + $payloads = []; + Process::fake(function (PendingProcess $process) use (&$payloads) { + preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $process->command, $matches); + $composePayloads = array_filter($matches[0], fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), 'services:')); + if ($composePayloads !== []) { + $payloads = array_merge($payloads, $composePayloads); + + return Process::result(errorOutput: 'tee: docker-compose.yaml: No space left on device', exitCode: 1); + } + + return Process::result(); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'generate_compose_file')); + + expect($payloads)->not->toBeEmpty(); + expect($exception->getMessage()) + ->toContain('[command hidden because it contains sensitive data]') + ->toContain('No space left on device'); + $storedLogs = (string) $queue->fresh()->logs; + foreach ($payloads as $payload) { + expect($exception->getMessage())->not->toContain($payload); + expect($storedLogs)->not->toContain($payload); + } +}); + +function readSensitiveFailureJobProperty(object $job, ReflectionClass $reflection, string $property): mixed +{ + $reflectionProperty = $reflection->getProperty($property); + $reflectionProperty->setAccessible(true); + + return $reflectionProperty->getValue($job); +}