fix(deploy): keep the queue moving and keep healthy containers on late failures

A queued deployment whose commit no longer validates (for example one
queued before an upgrade) made the job dispatch throw. The row stayed
in_progress, blocked every later deployment of the application, and
stopped the queue loop. Such a row is now marked failed with a log
entry and a failure notification, and the next queued deployment
starts.

An exception after the new container passed its health check (for
example in additional destinations, the next-deployment loop, or
notifications) made failed() remove the new container, which could be
the only one running. failed() now keeps a healthy new container, and
errors in the steps after a status change are logged without flipping
a finished deployment to failed.

app/Jobs/ApplicationDeploymentJob.php

@@ -5615,8 +5615,31 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
         }
 
         $this->updateDeploymentStatus($status);
-        $this->handleStatusTransition($status);
-        queue_next_deployment($this->application);
+
+        // The status is stored. Side effect failures are logged so they cannot flip a finished
+        // deployment to failed or stop the next queued deployment from starting.
+        try {
+            $this->handleStatusTransition($status);
+        } catch (Throwable $e) {
+            $this->logStatusTransitionSideEffectFailure("Post-{$status->value} actions failed", $e);
+        }
+
+        try {
+            queue_next_deployment($this->application);
+        } catch (Throwable $e) {
+            $this->logStatusTransitionSideEffectFailure('Starting the next queued deployment failed', $e);
+        }
+    }
+
+    private function logStatusTransitionSideEffectFailure(string $context, Throwable $e): void
+    {
+        \Log::warning("{$context} for deployment {$this->deployment_uuid}: {$e->getMessage()}");
+
+        try {
+            $this->application_deployment_queue->addLogEntry("Warning: {$context}: {$e->getMessage()}", 'stderr');
+        } catch (Throwable) {
+            // The warning is already in the application log.
+        }
     }
 
     /**
@@ -5845,6 +5868,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
                 // 69420 means failed to push the image to the registry, so we don't need to remove the new version as it is the currently running one
                 if ($this->application->settings->is_consistent_container_name_enabled || $this->pull_request_id !== 0) {
                     // do not remove already running container for PR deployments
+                } elseif ($this->newVersionIsHealthy) {
+                    // The new container passed its health check, so the old container may already be removed.
+                    // A later failure (additional destinations, queue advancement, ...) must not remove the only running version.
+                    $this->application_deployment_queue->addLogEntry('Deployment failed after the new version became healthy. Keeping the new version of your application running.', 'stderr');
                 } else {
                     $this->application_deployment_queue->addLogEntry('Deployment failed. Removing the new version of your application.', 'stderr');
                     $this->removeContainerWithTimeout($this->container_name);

bootstrap/helpers/applications.php

@@ -6,10 +6,13 @@ use App\Jobs\ApplicationDeploymentJob;
 use App\Jobs\VolumeCloneJob;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
+use App\Models\ApplicationPreview;
 use App\Models\EnvironmentVariable;
 use App\Models\Server;
 use App\Models\StandaloneDocker;
+use App\Notifications\Application\DeploymentFailed;
 use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
 use Spatie\Url\Url;
 
 function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null, ?string $parent_deployment_uuid = null)
@@ -173,13 +176,58 @@ function start_queued_deployment(ApplicationDeploymentQueue $deployment, bool $f
         return true;
     });
 
-    if ($started) {
+    if (! $started) {
+        return false;
+    }
+
+    try {
         ApplicationDeploymentJob::dispatch(
             application_deployment_queue_id: $deployment->id,
         );
+    } catch (Throwable $exception) {
+        fail_undispatchable_deployment($deployment, $exception);
+
+        return false;
     }
 
-    return $started;
+    return true;
+}
+
+/**
+ * Fail a deployment whose job could not be created or queued, for example a deployment queued
+ * before an upgrade with a commit that no longer validates. Leaving it in progress would block
+ * every later deployment of the application and stop the queue advancement loop.
+ */
+function fail_undispatchable_deployment(ApplicationDeploymentQueue $deployment, Throwable $exception): void
+{
+    $failed = ApplicationDeploymentQueue::query()
+        ->whereKey($deployment->id)
+        ->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)
+        ->update(['status' => ApplicationDeploymentStatus::FAILED->value]);
+
+    if ($failed === 0) {
+        return;
+    }
+
+    Log::warning("Deployment {$deployment->deployment_uuid} could not be started: {$exception->getMessage()}");
+
+    $deployment->refresh();
+    $deployment->addLogEntry('========================================', 'stderr');
+    $deployment->addLogEntry("Deployment could not be started: {$exception->getMessage()}", 'stderr');
+    $deployment->addLogEntry('========================================', 'stderr');
+
+    try {
+        $application = Application::query()->find($deployment->application_id);
+        if (! $application || filled($deployment->parent_deployment_uuid)) {
+            return;
+        }
+        $preview = $deployment->pull_request_id !== 0
+            ? ApplicationPreview::findPreviewByApplicationAndPullId($application->id, $deployment->pull_request_id)
+            : null;
+        $application->environment?->project?->team?->notify(new DeploymentFailed($application, $deployment->deployment_uuid, $preview));
+    } catch (Throwable $notificationException) {
+        Log::warning("Failed to send the failure notification for deployment {$deployment->deployment_uuid}: {$notificationException->getMessage()}");
+    }
 }
 
 function next_queuable(string $server_id, string $application_id, string $commit = 'HEAD', int $pull_request_id = 0): bool

tests/Feature/ApplicationDeploymentStatusSideEffectsTest.php

@@ -0,0 +1,71 @@
+<?php
+
+use App\Events\ApplicationConfigurationChanged;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+    $team = Team::factory()->create();
+    $this->server = Server::factory()->create(['team_id' => $team->id]);
+    $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = $project->environments()->first()
+        ?? Environment::factory()->create(['project_id' => $project->id]);
+    $this->application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $this->destination->id,
+        'destination_type' => $this->destination->getMorphClass(),
+    ]);
+
+    Process::fake(['*' => Process::result(output: '')]);
+    Queue::fake();
+    Notification::fake();
+});
+
+function createDeploymentForSideEffectsTest(string $status, string $createdAt): ApplicationDeploymentQueue
+{
+    $deployment = ApplicationDeploymentQueue::create([
+        'application_id' => test()->application->id,
+        'deployment_uuid' => fake()->uuid(),
+        'status' => $status,
+        'server_id' => test()->server->id,
+        'destination_id' => test()->destination->id,
+        'commit' => 'HEAD',
+        'pull_request_id' => 0,
+    ]);
+    $deployment->forceFill(['created_at' => $createdAt])->save();
+
+    return $deployment;
+}
+
+test('a failing post-success side effect keeps the deployment finished and starts the next queued deployment', function () {
+    $current = createDeploymentForSideEffectsTest('in_progress', '2026-10-01 10:00:00');
+    $next = createDeploymentForSideEffectsTest('queued', '2026-10-01 10:01:00');
+    Event::listen(ApplicationConfigurationChanged::class, fn () => throw new RuntimeException('broadcast failed'));
+
+    $job = new ApplicationDeploymentJob($current->id);
+    (fn () => $this->completeDeployment())->call($job);
+
+    $current->refresh();
+    expect($current->status)->toBe('finished')
+        ->and($current->logs)->toContain('broadcast failed')
+        ->and($next->fresh()->status)->toBe('in_progress');
+    Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $next->id);
+});

tests/Feature/QueuedDeploymentInvalidCommitTest.php

@@ -0,0 +1,89 @@
+<?php
+
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Notifications\Application\DeploymentFailed;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+    $this->team = Team::factory()->create();
+    $this->team->emailNotificationSettings->update([
+        'smtp_enabled' => true,
+        'deployment_failure_email_notifications' => true,
+    ]);
+    $this->server = Server::factory()->create(['team_id' => $this->team->id]);
+    $this->server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+        'force_disabled' => false,
+    ]);
+    $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $this->team->id]);
+    $environment = $project->environments()->first()
+        ?? Environment::factory()->create(['project_id' => $project->id]);
+    $this->application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $this->destination->id,
+        'destination_type' => $this->destination->getMorphClass(),
+    ]);
+
+    Process::fake(['*' => Process::result(output: '')]);
+    Queue::fake();
+    Notification::fake();
+});
+
+function createQueuedDeploymentWithCommit(string $commit, string $createdAt): ApplicationDeploymentQueue
+{
+    $deployment = ApplicationDeploymentQueue::create([
+        'application_id' => test()->application->id,
+        'deployment_uuid' => fake()->uuid(),
+        'status' => 'queued',
+        'server_id' => test()->server->id,
+        'destination_id' => test()->destination->id,
+        'commit' => $commit,
+        'pull_request_id' => 0,
+    ]);
+    $deployment->forceFill(['created_at' => $createdAt])->save();
+
+    return $deployment;
+}
+
+test('a queued deployment with an invalid commit fails and the next queued deployment starts', function () {
+    $invalidDeployment = createQueuedDeploymentWithCommit('abc;touch /tmp/pwned', '2026-10-01 10:00:00');
+    $validDeployment = createQueuedDeploymentWithCommit('HEAD', '2026-10-01 10:01:00');
+
+    queue_next_deployment($this->application);
+
+    $invalidDeployment->refresh();
+    expect($invalidDeployment->status)->toBe('failed')
+        ->and($invalidDeployment->logs)->toContain('Deployment could not be started')
+        ->and($invalidDeployment->logs)->toContain('Invalid deployment commit')
+        ->and($validDeployment->fresh()->status)->toBe('in_progress');
+
+    Queue::assertPushed(ApplicationDeploymentJob::class, 1);
+    Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $validDeployment->id);
+    Notification::assertSentTo($this->team, DeploymentFailed::class);
+});
+
+test('force starting a queued deployment with an invalid commit fails it instead of leaving it in progress', function () {
+    $invalidDeployment = createQueuedDeploymentWithCommit('$(id)', '2026-10-01 10:00:00');
+
+    expect(force_start_deployment($invalidDeployment))->toBeFalse()
+        ->and($invalidDeployment->fresh()->status)->toBe('failed');
+    Queue::assertNothingPushed();
+});

tests/Unit/ApplicationDeploymentFailedContainerCleanupTest.php

@@ -8,7 +8,7 @@ use App\Models\ApplicationSetting;
 /**
  * @return array<int, array<int|string, mixed>>
  */
-function runFailedDeploymentCleanup(bool $consistentContainerName, ?string $customInternalName, int $pullRequestId): array
+function runFailedDeploymentCleanup(bool $consistentContainerName, ?string $customInternalName, int $pullRequestId, bool $newVersionIsHealthy = false): array
 {
     $settings = new ApplicationSetting;
     $settings->setRawAttributes([
@@ -37,6 +37,7 @@ function runFailedDeploymentCleanup(bool $consistentContainerName, ?string $cust
     $reflection->getProperty('application')->setValue($job, $application);
     $reflection->getProperty('pull_request_id')->setValue($job, $pullRequestId);
     $reflection->getProperty('container_name')->setValue($job, 'new-container-20261001T120000');
+    $reflection->getProperty('newVersionIsHealthy')->setValue($job, $newVersionIsHealthy);
 
     $job->failed(new RuntimeException('Health check failed'));
 
@@ -61,3 +62,14 @@ it('keeps the running container when consistent container names are enabled', fu
 it('keeps the running container for pull request deployments', function () {
     expect(runFailedDeploymentCleanup(consistentContainerName: false, customInternalName: 'my-api', pullRequestId: 12))->toBeEmpty();
 });
+
+it('keeps the new container when the deployment fails after the new container became healthy', function () {
+    expect(runFailedDeploymentCleanup(consistentContainerName: false, customInternalName: null, pullRequestId: 0, newVersionIsHealthy: true))->toBeEmpty();
+});
+
+it('removes the new container when the deployment fails before the new container became healthy', function () {
+    $remoteCommands = runFailedDeploymentCleanup(consistentContainerName: false, customInternalName: null, pullRequestId: 0, newVersionIsHealthy: false);
+
+    expect($remoteCommands)->toHaveCount(1)
+        ->and($remoteCommands[0][0])->toContain('new-container-20261001T120000');
+});