fix(deploy): keep the healthy new container when a rolling update fails late

Laravel runs failed() on a fresh copy of the job, so the in-memory
newVersionIsHealthy flag was always false there and a failure after the
old container stopped removed the only running container. Keep the flag
in the cache, keyed by the deployment UUID.

Closes #12253

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/Feature/ApplicationDeploymentFailedKeepsHealthyContainerTest.php

@@ -0,0 +1,94 @@
+<?php
+
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\PendingProcess;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Sleep;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+    $team = Team::factory()->create();
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = $project->environments()->first()
+        ?? Environment::factory()->create(['project_id' => $project->id]);
+    $application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+        'build_pack' => 'dockerfile',
+    ]);
+
+    $this->deployment = ApplicationDeploymentQueue::create([
+        'application_id' => $application->id,
+        'deployment_uuid' => fake()->uuid(),
+        'status' => 'in_progress',
+        'server_id' => $server->id,
+        'destination_id' => $destination->id,
+        'commit' => 'HEAD',
+        'pull_request_id' => 0,
+    ]);
+
+    Process::fake([
+        '*State.Health.Status*' => Process::result(output: "\"healthy\"\n"),
+        '*' => Process::result(output: ''),
+    ]);
+    Queue::fake();
+    Notification::fake();
+    Sleep::fake();
+});
+
+/**
+ * Laravel calls failed() on a copy of the job that it unserializes from the payload taken at dispatch,
+ * so state that handle() keeps only in memory is lost there.
+ */
+function failQueuedCopy(string $queuedCopy): void
+{
+    unserialize($queuedCopy)->failed(new RuntimeException('Late failure'));
+}
+
+function removesContainer(string $containerName): Closure
+{
+    return fn (PendingProcess $process) => str_contains((string) $process->command, 'docker rm -f')
+        && str_contains((string) $process->command, $containerName);
+}
+
+it('keeps the new container when a deployment fails after the new container became healthy', function () {
+    $job = new ApplicationDeploymentJob($this->deployment->id);
+    $queuedCopy = serialize(clone $job);
+    $containerName = (fn () => $this->container_name)->call($job);
+    (fn () => $this->health_check())->call($job);
+    failQueuedCopy($queuedCopy);
+
+    Process::assertNotRan(removesContainer($containerName));
+    expect($this->deployment->fresh()->logs)->toContain('Keeping the new version of your application running.');
+});
+
+it('removes the new container when a deployment fails before the new container became healthy', function () {
+    $job = new ApplicationDeploymentJob($this->deployment->id);
+    $queuedCopy = serialize(clone $job);
+    $containerName = (fn () => $this->container_name)->call($job);
+
+    failQueuedCopy($queuedCopy);
+
+    Process::assertRan(removesContainer($containerName));
+    expect($this->deployment->fresh()->logs)->toContain('Removing the new version of your application.');
+});

tests/Unit/ApplicationDeploymentErrorLoggingTest.php

@@ -6,6 +6,9 @@ use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
 use Illuminate\Contracts\Queue\Job;
 use Illuminate\Queue\TimeoutExceededException;
+use Tests\TestCase;
+
+uses(TestCase::class);
 
 /**
  * Test to verify that deployment errors are properly logged with comprehensive details.
@@ -91,6 +94,7 @@ it('logs comprehensive error details when failed() is called', function () {
     $containerNameProperty = $reflection->getProperty('container_name');
     $containerNameProperty->setAccessible(true);
     $containerNameProperty->setValue($job, 'test-container');
+    $reflection->getProperty('deployment_uuid')->setValue($job, 'error-logging-test');
 
     // Mock the failDeployment method to prevent errors
     $job->shouldReceive('failDeployment')->andReturn();
@@ -197,6 +201,7 @@ it('handles exceptions with no message gracefully', function () {
     $containerNameProperty = $reflection->getProperty('container_name');
     $containerNameProperty->setAccessible(true);
     $containerNameProperty->setValue($job, 'test-container');
+    $reflection->getProperty('deployment_uuid')->setValue($job, 'error-logging-test');
 
     $job->shouldReceive('failDeployment')->andReturn();
     $job->shouldReceive('execute_remote_command')->andReturn();
@@ -288,6 +293,7 @@ it('logs error code 0 correctly', function () {
     $containerNameProperty = $reflection->getProperty('container_name');
     $containerNameProperty->setAccessible(true);
     $containerNameProperty->setValue($job, 'test-container');
+    $reflection->getProperty('deployment_uuid')->setValue($job, 'error-logging-test');
 
     $job->shouldReceive('failDeployment')->andReturn();
     $job->shouldReceive('execute_remote_command')->andReturn();
@@ -393,6 +399,7 @@ it('logs the configured timeout when the deployment times out', function () {
 
     $containerNameProperty = $reflection->getProperty('container_name');
     $containerNameProperty->setValue($job, 'test-container');
+    $reflection->getProperty('deployment_uuid')->setValue($job, 'error-logging-test');
 
     $job->shouldReceive('failDeployment')->andReturn();
     $job->shouldReceive('execute_remote_command')->andReturn();

tests/Unit/ApplicationDeploymentFailedContainerCleanupTest.php

@@ -4,11 +4,14 @@ use App\Jobs\ApplicationDeploymentJob;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
 use App\Models\ApplicationSetting;
+use Tests\TestCase;
+
+uses(TestCase::class);
 
 /**
  * @return array<int, array<int|string, mixed>>
  */
-function runFailedDeploymentCleanup(bool $consistentContainerName, ?string $customInternalName, int $pullRequestId, bool $newVersionIsHealthy = false): array
+function runFailedDeploymentCleanup(bool $consistentContainerName, ?string $customInternalName, int $pullRequestId): array
 {
     $settings = new ApplicationSetting;
     $settings->setRawAttributes([
@@ -36,8 +39,8 @@ function runFailedDeploymentCleanup(bool $consistentContainerName, ?string $cust
     $reflection->getProperty('application_deployment_queue')->setValue($job, $queue);
     $reflection->getProperty('application')->setValue($job, $application);
     $reflection->getProperty('pull_request_id')->setValue($job, $pullRequestId);
+    $reflection->getProperty('deployment_uuid')->setValue($job, 'cleanup-test-deployment');
     $reflection->getProperty('container_name')->setValue($job, 'new-container-20261001T120000');
-    $reflection->getProperty('newVersionIsHealthy')->setValue($job, $newVersionIsHealthy);
 
     $job->failed(new RuntimeException('Health check failed'));
 
@@ -62,14 +65,3 @@ it('keeps the running container when consistent container names are enabled', fu
 it('keeps the running container for pull request deployments', function () {
     expect(runFailedDeploymentCleanup(consistentContainerName: false, customInternalName: 'my-api', pullRequestId: 12))->toBeEmpty();
 });
-
-it('keeps the new container when the deployment fails after the new container became healthy', function () {
-    expect(runFailedDeploymentCleanup(consistentContainerName: false, customInternalName: null, pullRequestId: 0, newVersionIsHealthy: true))->toBeEmpty();
-});
-
-it('removes the new container when the deployment fails before the new container became healthy', function () {
-    $remoteCommands = runFailedDeploymentCleanup(consistentContainerName: false, customInternalName: null, pullRequestId: 0, newVersionIsHealthy: false);
-
-    expect($remoteCommands)->toHaveCount(1)
-        ->and($remoteCommands[0][0])->toContain('new-container-20261001T120000');
-});