fix(deployments): keep secrets out of failed command logs

- A failed command marked skip_command_log (for example the .env, build-time env, and SSH key writes) no longer puts its text into the exception, which was shown as the visible "Deployment failed" line with all secrets as base64. Its error output is also cleaned. - Mark more secret-carrying commands as sensitive: the helper container with build secrets, railpack prepare, the Nixpacks plan, and Compose file writes. - Dev debug lines list only variable names, not values. - Invalid build-time variable names such as my-var stop a deployment only when it builds an image. Docker image deployments log a warning with a suggested name instead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 14:36:44 -04:00 · 2026-09-26 10:40:53 +02:00
parent 510a2d838b
commit 1e207292b6
4 changed files with 640 additions and 28 deletions
@@ -769,6 +769,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
        $this->execute_remote_command([
            executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}{$this->docker_compose_location} > /dev/null"),
            'hidden' => true,
+            'skip_command_log' => true,
        ]);

        // Modify Dockerfiles for ARGs and build secrets
@@ -1151,6 +1152,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                ],
                [
                    "echo '{$this->docker_compose_base64}' | base64 -d | tee $composeFileName > /dev/null",
+                    'skip_command_log' => true,
                ],
                [
                    "echo '{$readme}' > $mainDir/README.md",
@@ -1774,12 +1776,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
        );

        if (isDev()) {
-            $this->execute_remote_command(
-                [
-                    executeInDocker($this->deployment_uuid, "cat $this->workdir/.env"),
-                    'hidden' => true,
-                ]
-            );
+            $this->logEnvironmentFileKeys("{$this->workdir}/.env", $environment_variables);
        }

        // Write .env file to configuration directory
@@ -1802,6 +1799,20 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
        }
    }

+    /**
+     * Development aid: log which variables an env file contains, without their values.
+     *
+     * @param  Collection<int, string>  $environmentVariables  Lines in KEY=VALUE format.
+     */
+    private function logEnvironmentFileKeys(string $path, Collection $environmentVariables): void
+    {
+        $keys = $environmentVariables
+            ->map(fn (string $line): string => explode('=', $line, 2)[0])
+            ->implode(', ');
+
+        $this->application_deployment_queue->addLogEntry("[DEBUG] Variable names in {$path}: {$keys}", hidden: true);
+    }
+
    private function generate_buildtime_environment_variables()
    {
        if (isDev()) {
@@ -2067,38 +2078,60 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
    }

    /**
-     * Build-time names go into shell and Docker build commands, so they must be valid. Runtime-only
-     * variables only go into the .env file: existing ones with names that new variables can no longer
-     * use (such as my-var) keep working, unless the name would break a .env line.
+     * Build-time names go into shell and Docker build commands, so they must be valid when the
+     * deployment builds an image. Runtime-only variables only go into the .env file: existing ones
+     * with names that new variables can no longer use (such as my-var) keep working, unless the
+     * name would break a .env line. Deployments without a build step (Docker image) treat
+     * build-time variables the same way, because no build receives them.
     */
    private function validateDeploymentEnvironmentVariableKeys(): void
    {
        $environmentVariables = $this->pull_request_id === 0
-            ? $this->application->environment_variables()->get(['key', 'is_buildtime'])
-            : $this->application->environment_variables_preview()->get(['key', 'is_buildtime']);
+            ? $this->application->environment_variables()->get(['key', 'is_buildtime', 'is_runtime'])
+            : $this->application->environment_variables_preview()->get(['key', 'is_buildtime', 'is_runtime']);
+        $passesBuildtimeVariables = $this->deploymentPassesBuildtimeVariables();

        foreach ($environmentVariables as $environmentVariable) {
            $key = (string) $environmentVariable->key;
            $isEnvFileSafe = $key !== '' && strpbrk($key, "=\n\r\0") === false;
-            if ($environmentVariable->is_buildtime || ! $isEnvFileSafe) {
+            if (($environmentVariable->is_buildtime && $passesBuildtimeVariables) || ! $isEnvFileSafe) {
                $this->validatedBuildtimeEnvironmentVariableKey($key, 'the deployment environment');

                continue;
            }
            if (! ValidationPatterns::isValidEnvironmentVariableKey($key)) {
-                $this->logLegacyRuntimeEnvironmentVariableKey($key);
+                $this->logLegacyEnvironmentVariableKey($key, (bool) $environmentVariable->is_buildtime, (bool) $environmentVariable->is_runtime);
            }
        }
    }

-    private function logLegacyRuntimeEnvironmentVariableKey(string $key): void
+    /**
+     * Only Docker image deployments never build an image. Other deployments (also restarts,
+     * rollbacks, and previews) can build and pass build-time variables to the build.
+     */
+    private function deploymentPassesBuildtimeVariables(): bool
+    {
+        return $this->application->build_pack !== 'dockerimage';
+    }
+
+    private function logLegacyEnvironmentVariableKey(string $key, bool $isBuildtime, bool $isRuntime): void
    {
        $suggestedKey = (string) preg_replace('/[^A-Za-z0-9_]/', '_', $key);
        if (preg_match('/\A[0-9]/', $suggestedKey) === 1) {
            $suggestedKey = '_'.$suggestedKey;
        }

-        $this->application_deployment_queue->addLogEntry('⚠️ Runtime variable '.ValidationPatterns::displayShellEnvironmentVariableKey($key).' uses a name that new variables cannot use. It is still passed to the container, but shell scripts cannot read it.', 'stderr');
+        $displayKey = ValidationPatterns::displayShellEnvironmentVariableKey($key);
+
+        if ($isBuildtime) {
+            $this->application_deployment_queue->addLogEntry("⚠️ Build-time variable {$displayKey} uses a name that new variables cannot use. This deployment does not build an image, so it is not used as a build-time variable. Rename it before you use it in a build.", 'stderr');
+        }
+        if ($isRuntime) {
+            $this->application_deployment_queue->addLogEntry("⚠️ Runtime variable {$displayKey} uses a name that new variables cannot use. It is still passed to the container, but shell scripts cannot read it.", 'stderr');
+        }
+        if (! $isBuildtime && ! $isRuntime) {
+            $this->application_deployment_queue->addLogEntry("⚠️ Variable {$displayKey} uses a name that new variables cannot use. This deployment does not use it.", 'stderr');
+        }
        $this->application_deployment_queue->addLogEntry('   Suggested name: '.ValidationPatterns::displayShellEnvironmentVariableKey($suggestedKey), type: 'info');
    }

@@ -2159,10 +2192,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                ]);

                if (isDev()) {
-                    $this->execute_remote_command([
-                        executeInDocker($this->deployment_uuid, 'cat '.self::BUILD_TIME_ENV_PATH),
-                        'hidden' => true,
-                    ]);
+                    $this->logEnvironmentFileKeys(self::BUILD_TIME_ENV_PATH, $environment_variables);
                }
            } elseif (in_array($this->build_pack, ['dockercompose', 'dockerfile', 'railpack'], true)) {
                $this->application_deployment_queue->addLogEntry('Creating empty build-time .env file in /artifacts (no build-time variables defined).', hidden: true);
@@ -2527,6 +2557,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
            [
                $runCommand,
                'hidden' => true,
+                'skip_command_log' => filled($env_flags),
            ],
            [
                'command' => executeInDocker($this->deployment_uuid, "mkdir -p {$this->basedir}"),
@@ -3325,7 +3356,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue

        $this->application_deployment_queue->addLogEntry('Generating Railpack build plan.');
        $this->execute_remote_command(
-            [executeInDocker($this->deployment_uuid, $prepare_command), 'hidden' => true],
+            [executeInDocker($this->deployment_uuid, $prepare_command), 'hidden' => true, 'skip_command_log' => true],
            [
                executeInDocker($this->deployment_uuid, 'cat /artifacts/railpack-plan.json'),
                'hidden' => true,
@@ -3813,7 +3844,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");

        $this->docker_compose = Yaml::dump($docker_compose, 10);
        $this->docker_compose_base64 = base64_encode($this->docker_compose);
-        $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true]);
+        $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true, 'skip_command_log' => true]);
    }

    private function generate_local_persistent_volumes()
@@ -4044,7 +4075,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
            }
            if ($this->application->build_pack === 'nixpacks') {
                $this->nixpacks_plan = base64_encode($this->nixpacks_plan);
-                $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true]);
+                $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true, 'skip_command_log' => true]);
                if ($this->force_rebuild) {
                    $this->execute_remote_command([
                        executeInDocker($this->deployment_uuid, 'nixpacks build -c '.self::NIXPACKS_PLAN_PATH." --no-cache --no-error-without-start -n {$this->build_image_name} {$this->workdir} -o {$this->workdir}"),
@@ -4230,7 +4261,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
            } else {
                if ($this->application->build_pack === 'nixpacks') {
                    $this->nixpacks_plan = base64_encode($this->nixpacks_plan);
-                    $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true]);
+                    $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true, 'skip_command_log' => true]);
                    if ($this->force_rebuild) {
                        $this->execute_remote_command([
                            executeInDocker($this->deployment_uuid, 'nixpacks build -c '.self::NIXPACKS_PLAN_PATH." --no-cache --no-error-without-start -n {$this->production_image_name} {$this->workdir} -o {$this->workdir}"),
@@ -4579,8 +4610,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");

    private function generate_docker_env_flags_for_secrets()
    {
-        // Only generate env flags if build secrets are enabled
-        if (! $this->application->settings->use_build_secrets) {
+        // Only generate env flags if build secrets are enabled and the deployment builds an image
+        if (! $this->application->settings->use_build_secrets || ! $this->deploymentPassesBuildtimeVariables()) {
            return '';
        }

@@ -16,6 +16,8 @@ trait ExecuteRemoteCommand
 {
    use SshRetryable;

+    private const SENSITIVE_COMMAND_PLACEHOLDER = '[command hidden because it contains sensitive data]';
+
    public ?string $save = null;

    public static int $batch_counter = 0;
@@ -184,7 +186,7 @@ trait ExecuteRemoteCommand

            $new_log_entry = [
                'command' => $skip_command_log || $command_hidden ? null : $this->redact_sensitive_info($command),
-                'output' => $this->redact_sensitive_info($log_output),
+                'output' => $this->redact_sensitive_info($skip_command_log ? $this->redactSensitiveCommandPayloads((string) $log_output, (string) $command) : $log_output),
                'type' => $customType ?? ($type === 'err' ? 'stderr' : 'stdout'),
                'timestamp' => Carbon::now('UTC'),
                'hidden' => $hidden,
@@ -241,12 +243,54 @@ trait ExecuteRemoteCommand
                if (empty($error)) {
                    $error = $process_result->output() ?: 'Command failed with no error output';
                }
-                $redactedCommand = $this->redact_sensitive_info($command);
-                throw new DeploymentException("Command execution failed (exit code {$process_result->exitCode()}): {$redactedCommand}\nError: {$error}");
+                throw new DeploymentException($this->commandFailureMessage((string) $command, (int) $process_result->exitCode(), (string) $error, $skip_command_log));
            }
        }
    }

+    /**
+     * Commands marked with skip_command_log embed secrets (for example base64 encoded .env files or
+     * private keys), so their text must never reach a log line or an exception message.
+     */
+    private function commandFailureMessage(string $command, int $exitCode, string $error, bool $isSensitiveCommand): string
+    {
+        if ($isSensitiveCommand) {
+            $commandText = self::SENSITIVE_COMMAND_PLACEHOLDER;
+            $error = $this->redactSensitiveCommandPayloads($error, $command);
+        } else {
+            $commandText = $this->redact_sensitive_info($command);
+        }
+
+        $error = $this->redact_sensitive_info($error);
+
+        return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}";
+    }
+
+    /**
+     * Removes the encoded payloads of a sensitive command from output that could echo the command.
+     */
+    private function redactSensitiveCommandPayloads(string $text, string $command): string
+    {
+        if ($text === '' || preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches) === false) {
+            return $text;
+        }
+
+        $payloads = collect($matches[0])
+            ->unique()
+            ->filter(function (string $candidate): bool {
+                $decoded = base64_decode($candidate, true);
+
+                return $decoded !== false
+                    && mb_check_encoding($decoded, 'UTF-8')
+                    && preg_match('/[^\P{C}\t\n\r]/u', $decoded) !== 1;
+            })
+            ->sortByDesc(fn (string $payload): int => strlen($payload))
+            ->values()
+            ->all();
+
+        return $payloads === [] ? $text : str_replace($payloads, REDACTED, $text);
+    }
+
    private function saveCommandOutput(string $output, bool $append): void
    {
        if (! $this->save) {
@@ -1030,6 +1030,94 @@ it('rejects existing variable names that would break the .env file or build comm
    'build-time name with a hyphen' => ['my-var', true],
 ]);

+it('warns instead of failing for invalid build-time names when the deployment does not build an image', function (bool $isRuntime) {
+    [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);
+    $environmentVariable = createApplicationEnvironmentVariable($application, [
+        'key' => 'SAFE_KEY',
+        'value' => 'secret',
+        'is_buildtime' => true,
+        'is_runtime' => $isRuntime,
+    ]);
+    DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);
+
+    [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server);
+    invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys');
+
+    expect(collect($job->recordedLogEntries)->implode("\n"))
+        ->toContain('my-var')
+        ->toContain('Suggested name: my_var')
+        ->not->toContain('Invalid environment variable name');
+    expect($job->recordedCommands)->toBeEmpty();
+})->with([
+    'build-time only' => [false],
+    'build-time and runtime' => [true],
+]);
+
+it('warns instead of failing for invalid build-time preview names of Docker image deployments', function () {
+    [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);
+    $environmentVariable = createApplicationEnvironmentVariable($application, [
+        'key' => 'SAFE_KEY',
+        'value' => 'secret',
+        'is_buildtime' => true,
+        'is_preview' => true,
+    ]);
+    DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);
+
+    [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server, ['pull_request_id' => 7]);
+    invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys');
+
+    expect(collect($job->recordedLogEntries)->implode("\n"))
+        ->toContain('Suggested name: my_var')
+        ->not->toContain('Invalid environment variable name');
+});
+
+it('still rejects Docker image variable names that would break the .env file', function () {
+    [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);
+    $environmentVariable = createApplicationEnvironmentVariable($application, [
+        'key' => 'SAFE_KEY',
+        'value' => 'secret',
+        'is_buildtime' => true,
+    ]);
+    DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'A=B']);
+
+    [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server);
+
+    expect(fn () => invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys'))
+        ->toThrow(DeploymentException::class, 'Invalid environment variable name from the deployment environment');
+});
+
+it('fails with a clear message for invalid build-time names when the deployment builds an image', function (string $buildPack) {
+    [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => $buildPack]);
+    $environmentVariable = createApplicationEnvironmentVariable($application, [
+        'key' => 'SAFE_KEY',
+        'value' => 'secret',
+        'is_buildtime' => true,
+    ]);
+    DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);
+
+    [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server, ['build_pack' => $buildPack]);
+
+    expect(fn () => invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys'))
+        ->toThrow(DeploymentException::class, 'Invalid environment variable name from the deployment environment: my-var');
+    expect(collect($job->recordedLogEntries)->implode("\n"))
+        ->toContain('Build-time variable names must start with a letter or underscore');
+})->with(['dockerfile', 'nixpacks', 'static', 'railpack', 'dockercompose']);
+
+it('does not pass build-time variables to the helper container of Docker image deployments', function () {
+    [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);
+    $application->settings()->update(['use_build_secrets' => true]);
+    $environmentVariable = createApplicationEnvironmentVariable($application, [
+        'key' => 'SAFE_KEY',
+        'value' => 'harmless-build-value',
+        'is_buildtime' => true,
+    ]);
+    DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);
+
+    [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server);
+
+    expect(invokeDeploymentJobMethod($job, $reflection, 'generate_docker_env_flags_for_secrets'))->toBe('');
+});
+
 it('injects raw escaped remote secrets into Dockerfile args and hashes the same values', function (int $pullRequestId, bool $isPreview) {
    [$application, $server] = makeDeploymentControlVarFixture();

@@ -0,0 +1,449 @@
+<?php
+
+use App\Exceptions\DeploymentException;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\EnvironmentVariable;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\PendingProcess;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+const SENSITIVE_FAILURE_SECRET = 'harmless-secret-marker-7431';
+
+class SensitiveCommandFailureDeploymentJob extends ApplicationDeploymentJob
+{
+    public function __construct() {}
+}
+
+beforeEach(function () {
+    config()->set('constants.ssh.mux_enabled', false);
+    config()->set('constants.ssh.max_retries', 1);
+});
+
+/**
+ * @return array{0: SensitiveCommandFailureDeploymentJob, 1: ReflectionClass, 2: ApplicationDeploymentQueue, 3: Application}
+ */
+function makeSensitiveCommandFailureJob(array $environmentVariables = []): array
+{
+    $user = User::factory()->create();
+    $team = $user->teams()->first();
+
+    $privateKeyContent = "-----BEGIN OPENSSH PRIVATE KEY-----\n"
+        ."b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n"
+        ."QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk\n"
+        ."hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA\n"
+        ."AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV\n"
+        ."uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==\n"
+        .'-----END OPENSSH PRIVATE KEY-----';
+
+    $privateKey = PrivateKey::create([
+        'name' => 'sensitive-failure-key-'.uniqid(),
+        'private_key' => $privateKeyContent,
+        'team_id' => $team->id,
+    ]);
+
+    Storage::fake('ssh-keys');
+    Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);
+
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'user' => 'root',
+    ]);
+
+    $project = Project::create(['name' => 'Sensitive Failure Project', 'team_id' => $team->id]);
+    $environment = Environment::where('project_id', $project->id)->firstOrFail();
+    $application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'build_pack' => 'dockerfile',
+    ]);
+    $application->settings()->update([
+        'include_source_commit_in_build' => false,
+        'is_env_sorting_enabled' => false,
+    ]);
+
+    foreach ($environmentVariables as $attributes) {
+        EnvironmentVariable::create([
+            'resourceable_type' => Application::class,
+            'resourceable_id' => $application->id,
+            'is_preview' => false,
+            'is_runtime' => true,
+            'is_buildtime' => true,
+            'is_multiline' => false,
+            'is_literal' => false,
+            ...$attributes,
+        ]);
+    }
+
+    $queue = ApplicationDeploymentQueue::create([
+        'deployment_uuid' => 'sensitive-failure-deployment',
+        'application_id' => $application->id,
+        'server_id' => $server->id,
+        'status' => 'in_progress',
+    ]);
+
+    $job = new SensitiveCommandFailureDeploymentJob;
+    $reflection = new ReflectionClass(ApplicationDeploymentJob::class);
+
+    foreach ([
+        'application' => $application->fresh(),
+        'application_deployment_queue' => $queue,
+        'server' => $server,
+        'mainServer' => $server,
+        'build_pack' => 'dockerfile',
+        'pull_request_id' => 0,
+        'commit' => 'HEAD',
+        'basedir' => '/artifacts/sensitive-failure',
+        'workdir' => '/artifacts/sensitive-failure',
+        'configuration_dir' => '/data/coolify/applications/sensitive-failure',
+        'deployment_uuid' => 'sensitive-failure-deployment',
+        'dockerfile_location' => '/Dockerfile',
+        'container_name' => 'sensitive-failure-app',
+        'coolify_variables' => null,
+        'dockerSecretsSupported' => false,
+        'saved_outputs' => new Collection,
+    ] as $property => $value) {
+        $reflectionProperty = $reflection->getProperty($property);
+        $reflectionProperty->setAccessible(true);
+        $reflectionProperty->setValue($job, $value);
+    }
+
+    return [$job, $reflection, $queue, $application];
+}
+
+function invokeSensitiveFailureJobMethod(object $job, ReflectionClass $reflection, string $method, mixed ...$arguments): mixed
+{
+    $reflectionMethod = $reflection->getMethod($method);
+    $reflectionMethod->setAccessible(true);
+
+    return $reflectionMethod->invoke($job, ...$arguments);
+}
+
+function captureDeploymentException(callable $callback): DeploymentException
+{
+    try {
+        $callback();
+    } catch (DeploymentException $exception) {
+        return $exception;
+    }
+
+    throw new RuntimeException('Expected a DeploymentException.');
+}
+
+/**
+ * @return list<string>
+ */
+function base64PayloadsInCommand(string $command): array
+{
+    preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches);
+
+    return array_values(array_filter(
+        $matches[0],
+        static fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), SENSITIVE_FAILURE_SECRET)
+    ));
+}
+
+it('hides the .env write command and its base64 payload when the write fails', function () {
+    [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
+        ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],
+    ]);
+
+    $payloads = [];
+    Process::fake(function (PendingProcess $process) use (&$payloads) {
+        $payloads = array_merge($payloads, base64PayloadsInCommand($process->command));
+
+        return Process::result(errorOutput: 'tee: /artifacts/sensitive-failure/.env: No space left on device', exitCode: 1);
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'));
+
+    expect($payloads)->not->toBeEmpty();
+    expect($exception->getMessage())
+        ->toContain('Command execution failed (exit code 1): [command hidden because it contains sensitive data]')
+        ->toContain('No space left on device')
+        ->not->toContain(SENSITIVE_FAILURE_SECRET)
+        ->not->toContain('base64 -d');
+    foreach ($payloads as $payload) {
+        expect($exception->getMessage())->not->toContain($payload);
+    }
+
+    $storedLogs = (string) $queue->fresh()->logs;
+    expect($storedLogs)
+        ->toContain('No space left on device')
+        ->not->toContain(SENSITIVE_FAILURE_SECRET);
+    foreach ($payloads as $payload) {
+        expect($storedLogs)->not->toContain($payload);
+    }
+});
+
+it('redacts the sensitive payload when the error output echoes the command', function () {
+    [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
+        ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],
+    ]);
+
+    $payloads = [];
+    Process::fake(function (PendingProcess $process) use (&$payloads) {
+        $commandPayloads = base64PayloadsInCommand($process->command);
+        $payloads = array_merge($payloads, $commandPayloads);
+
+        return Process::result(errorOutput: "bash: line 1: echo '".($commandPayloads[0] ?? '')."': unexpected failure", exitCode: 2);
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'));
+
+    expect($payloads)->not->toBeEmpty();
+    $storedLogs = (string) $queue->fresh()->logs;
+    foreach ($payloads as $payload) {
+        expect($exception->getMessage())->not->toContain($payload);
+        expect($storedLogs)->not->toContain($payload);
+    }
+    expect($exception->getMessage())->toContain('unexpected failure');
+});
+
+it('hides the build-time env write command when the write fails', function () {
+    [$job, $reflection] = makeSensitiveCommandFailureJob([
+        ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],
+    ]);
+
+    $payloads = [];
+    Process::fake(function (PendingProcess $process) use (&$payloads) {
+        $payloads = array_merge($payloads, base64PayloadsInCommand($process->command));
+
+        return Process::result(errorOutput: 'tee: /artifacts/build-time.env: No such file or directory', exitCode: 1);
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables'));
+
+    expect($payloads)->not->toBeEmpty();
+    expect($exception->getMessage())
+        ->toContain('[command hidden because it contains sensitive data]')
+        ->toContain('No such file or directory')
+        ->not->toContain(SENSITIVE_FAILURE_SECRET);
+    foreach ($payloads as $payload) {
+        expect($exception->getMessage())->not->toContain($payload);
+    }
+});
+
+it('keeps the command in the error message for failed commands that are not sensitive', function () {
+    [$job] = makeSensitiveCommandFailureJob();
+
+    Process::fake(['*' => Process::result(errorOutput: 'ls: cannot access: No such file or directory', exitCode: 2)]);
+
+    $exception = captureDeploymentException(fn () => $job->execute_remote_command([
+        'command' => 'ls /artifacts/harmless-missing-directory',
+        'hidden' => true,
+    ]));
+
+    expect($exception->getMessage())
+        ->toContain('Command execution failed (exit code 2): ls /artifacts/harmless-missing-directory')
+        ->toContain('No such file or directory')
+        ->not->toContain('[command hidden because it contains sensitive data]');
+});
+
+it('redacts locked values from the error output of failed commands', function () {
+    [$job] = makeSensitiveCommandFailureJob([
+        ['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true],
+    ]);
+
+    Process::fake(['*' => Process::result(errorOutput: 'failed with value '.SENSITIVE_FAILURE_SECRET, exitCode: 1)]);
+
+    $exception = captureDeploymentException(fn () => $job->execute_remote_command([
+        'command' => 'harmless-command --flag',
+    ]));
+
+    expect($exception->getMessage())
+        ->toContain('harmless-command --flag')
+        ->toContain('failed with value '.REDACTED)
+        ->not->toContain(SENSITIVE_FAILURE_SECRET);
+});
+
+it('logs only the variable names of the .env files in development mode', function () {
+    config()->set('app.env', 'local');
+    [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
+        ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],
+    ]);
+
+    $ranCommands = [];
+    Process::fake(function (PendingProcess $process) use (&$ranCommands) {
+        $ranCommands[] = $process->command;
+
+        return Process::result(output: str_contains($process->command, 'cat ') ? 'APP_SECRET='.SENSITIVE_FAILURE_SECRET : '');
+    });
+
+    invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables');
+    invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables');
+
+    $logEntries = collect(json_decode((string) $queue->fresh()->logs, true));
+    $keyListEntries = $logEntries->filter(fn (array $entry): bool => str_contains((string) $entry['output'], '[DEBUG] Variable names in'));
+
+    expect($keyListEntries)->toHaveCount(2)
+        ->each(fn ($entry) => $entry->hidden->toBeTrue()->output->toContain('APP_SECRET'));
+    expect($logEntries->pluck('output')->merge($logEntries->pluck('command'))->filter()->implode("\n"))
+        ->not->toContain('APP_SECRET='.SENSITIVE_FAILURE_SECRET)
+        ->not->toContain('APP_SECRET="'.SENSITIVE_FAILURE_SECRET);
+    expect(collect($ranCommands)->filter(fn (string $command): bool => str_contains($command, 'cat /artifacts/sensitive-failure/.env') || str_contains($command, 'cat '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH)))
+        ->toBeEmpty();
+});
+
+function setSensitiveFailureJobProperties(object $job, ReflectionClass $reflection, array $properties): void
+{
+    foreach ($properties as $property => $value) {
+        $reflectionProperty = $reflection->getProperty($property);
+        $reflectionProperty->setAccessible(true);
+        $reflectionProperty->setValue($job, $value);
+    }
+}
+
+it('hides the helper container command when build secrets are passed as environment flags', function () {
+    [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([
+        ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],
+    ]);
+    $application->settings()->update(['use_build_secrets' => true]);
+    $server = readSensitiveFailureJobProperty($job, $reflection, 'server');
+    setSensitiveFailureJobProperties($job, $reflection, [
+        'application' => $application->fresh(),
+        'destination' => StandaloneDocker::forceCreate([
+            'name' => 'sensitive-failure-network',
+            'network' => 'sensitive-failure-network',
+            'server_id' => $server->id,
+        ]),
+    ]);
+
+    Process::fake(function (PendingProcess $process) {
+        if (str_contains($process->command, 'docker run -d')) {
+            return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125);
+        }
+
+        return Process::result(output: str_contains($process->command, 'echo $HOME') ? '/root' : 'NOK');
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'));
+
+    expect($exception->getMessage())
+        ->toContain('[command hidden because it contains sensitive data]')
+        ->toContain('network not found')
+        ->not->toContain(SENSITIVE_FAILURE_SECRET);
+    expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET);
+});
+
+it('hides the Railpack prepare command that passes build-time values', function () {
+    [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([
+        ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],
+    ]);
+    $application->update(['build_pack' => 'railpack']);
+    setSensitiveFailureJobProperties($job, $reflection, [
+        'application' => $application->fresh(),
+        'build_pack' => 'railpack',
+        'dockerBuildxAvailable' => true,
+    ]);
+
+    Process::fake(function (PendingProcess $process) {
+        if (str_contains($process->command, 'railpack prepare')) {
+            return Process::result(errorOutput: 'railpack: failed to detect a provider', exitCode: 1);
+        }
+
+        return Process::result(output: str_contains($process->command, 'buildx version') ? 'available' : 'missing');
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_railpack_image'));
+
+    expect($exception->getMessage())
+        ->toContain('[command hidden because it contains sensitive data]')
+        ->toContain('failed to detect a provider')
+        ->not->toContain(SENSITIVE_FAILURE_SECRET);
+    expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET);
+});
+
+it('hides the Nixpacks plan write command that contains build-time values', function (bool $isStatic) {
+    [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();
+    $application->update(['build_pack' => 'nixpacks']);
+    $application->settings()->update(['is_static' => $isStatic]);
+    setSensitiveFailureJobProperties($job, $reflection, [
+        'application' => $application->fresh(),
+        'build_pack' => 'nixpacks',
+        'build_args' => collect(),
+        'disableBuildCache' => false,
+        'force_rebuild' => false,
+        'nixpacks_plan' => json_encode(['variables' => ['BUILD_SECRET' => SENSITIVE_FAILURE_SECRET]]),
+    ]);
+
+    $payloads = [];
+    Process::fake(function (PendingProcess $process) use (&$payloads) {
+        $commandPayloads = base64PayloadsInCommand($process->command);
+        if ($commandPayloads !== []) {
+            $payloads = array_merge($payloads, $commandPayloads);
+
+            return Process::result(errorOutput: 'tee: /artifacts/thegameplan.json: No space left on device', exitCode: 1);
+        }
+
+        return Process::result();
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_image'));
+
+    expect($payloads)->not->toBeEmpty();
+    expect($exception->getMessage())
+        ->toContain('[command hidden because it contains sensitive data]')
+        ->not->toContain(SENSITIVE_FAILURE_SECRET);
+    $storedLogs = (string) $queue->fresh()->logs;
+    foreach ($payloads as $payload) {
+        expect($exception->getMessage())->not->toContain($payload);
+        expect($storedLogs)->not->toContain($payload);
+    }
+})->with([
+    'static' => [true],
+    'not static' => [false],
+]);
+
+it('hides the compose file write command because compose files can contain secrets', function () {
+    [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();
+    $server = readSensitiveFailureJobProperty($job, $reflection, 'server');
+    setSensitiveFailureJobProperties($job, $reflection, [
+        'destination' => $server->standaloneDockers()->firstOrFail(),
+        'production_image_name' => 'example/app:latest',
+    ]);
+
+    $payloads = [];
+    Process::fake(function (PendingProcess $process) use (&$payloads) {
+        preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $process->command, $matches);
+        $composePayloads = array_filter($matches[0], fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), 'services:'));
+        if ($composePayloads !== []) {
+            $payloads = array_merge($payloads, $composePayloads);
+
+            return Process::result(errorOutput: 'tee: docker-compose.yaml: No space left on device', exitCode: 1);
+        }
+
+        return Process::result();
+    });
+
+    $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'generate_compose_file'));
+
+    expect($payloads)->not->toBeEmpty();
+    expect($exception->getMessage())
+        ->toContain('[command hidden because it contains sensitive data]')
+        ->toContain('No space left on device');
+    $storedLogs = (string) $queue->fresh()->logs;
+    foreach ($payloads as $payload) {
+        expect($exception->getMessage())->not->toContain($payload);
+        expect($storedLogs)->not->toContain($payload);
+    }
+});
+
+function readSensitiveFailureJobProperty(object $job, ReflectionClass $reflection, string $property): mixed
+{
+    $reflectionProperty = $reflection->getProperty($property);
+    $reflectionProperty->setAccessible(true);
+
+    return $reflectionProperty->getValue($job);
+}