fix(deployments): advance the queue of an additional server

When a deployment on an additional server finished, only the queue of
the primary server advanced. Deployments queued on the additional server
could wait with no limit.

- Also advance the queue of the server of the finished deployment

app/Jobs/ApplicationDeploymentJob.php

@@ -5667,7 +5667,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
         }
 
         try {
-            queue_next_deployment($this->application);
+            queue_next_deployment($this->application, $this->application_deployment_queue->server_id);
         } catch (Throwable $e) {
             $this->logStatusTransitionSideEffectFailure('Starting the next queued deployment failed', $e);
         }

bootstrap/helpers/applications.php

@@ -142,10 +142,17 @@ function force_start_deployment(ApplicationDeploymentQueue $deployment): bool
 {
     return start_queued_deployment($deployment, force: true);
 }
-function queue_next_deployment(Application $application)
+/**
+ * Start the queued deployments that can run now on the application's primary server and on the
+ * server of the deployment that just ended (an additional server).
+ */
+function queue_next_deployment(Application $application, ?int $finished_deployment_server_id = null)
 {
-    $server_id = $application->destination->server_id;
-    $queued_deployments = ApplicationDeploymentQueue::where('server_id', $server_id)
+    $server_ids = collect([$application->destination->server_id, $finished_deployment_server_id])
+        ->filter(fn ($server_id) => $server_id !== null)
+        ->unique()
+        ->values();
+    $queued_deployments = ApplicationDeploymentQueue::whereIn('server_id', $server_ids)
         ->where('status', ApplicationDeploymentStatus::QUEUED)
         ->get()
         ->sortBy('created_at');

tests/Feature/AdditionalServerDeploymentQueueAdvanceTest.php

@@ -0,0 +1,80 @@
+<?php
+
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+    $team = Team::factory()->create();
+    $primaryServer = Server::factory()->create(['team_id' => $team->id]);
+    $this->additionalServer = Server::factory()->create(['team_id' => $team->id]);
+    $this->additionalServer->settings->update(['concurrent_builds' => 1]);
+    $primaryDestination = StandaloneDocker::where('server_id', $primaryServer->id)->firstOrFail();
+    $this->additionalDestination = StandaloneDocker::where('server_id', $this->additionalServer->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = $project->environments()->first()
+        ?? Environment::factory()->create(['project_id' => $project->id]);
+
+    $makeApplication = function (Server $deploymentServer, StandaloneDocker $deploymentDestination) use ($environment): Application {
+        $application = Application::factory()->create([
+            'environment_id' => $environment->id,
+            'destination_id' => $deploymentDestination->id,
+            'destination_type' => $deploymentDestination->getMorphClass(),
+        ]);
+        $application->settings->update(['is_consistent_container_name_enabled' => true]);
+
+        return $application;
+    };
+    $this->multiServerApplication = $makeApplication($primaryServer, $primaryDestination);
+    $this->multiServerApplication->additional_servers()->attach($this->additionalServer->id, [
+        'standalone_docker_id' => $this->additionalDestination->id,
+    ]);
+    $this->otherApplication = $makeApplication($this->additionalServer, $this->additionalDestination);
+
+    Process::fake(['*' => Process::result(output: '')]);
+    Queue::fake();
+    Notification::fake();
+});
+
+test('a finished additional-server deployment starts the next deployment queued on that server', function () {
+    $childDeployment = ApplicationDeploymentQueue::create([
+        'application_id' => $this->multiServerApplication->id,
+        'deployment_uuid' => 'prr-child-deployment',
+        'status' => 'in_progress',
+        'server_id' => $this->additionalServer->id,
+        'destination_id' => $this->additionalDestination->id,
+        'commit' => 'HEAD',
+        'pull_request_id' => 0,
+        'parent_deployment_uuid' => 'prr-parent-deployment',
+    ]);
+    $waitingDeployment = ApplicationDeploymentQueue::create([
+        'application_id' => $this->otherApplication->id,
+        'deployment_uuid' => 'prr-waiting-deployment',
+        'status' => 'queued',
+        'server_id' => $this->additionalServer->id,
+        'destination_id' => $this->additionalDestination->id,
+        'commit' => 'HEAD',
+        'pull_request_id' => 0,
+    ]);
+
+    (new ApplicationDeploymentJob($childDeployment->id))->failed(new RuntimeException('build failed'));
+
+    expect($childDeployment->fresh()->status)->toBe('failed')
+        ->and($waitingDeployment->fresh()->status)->toBe('in_progress');
+    Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waitingDeployment->id);
+});