diff --git a/app/Actions/Application/CleanupPreviewDeployment.php b/app/Actions/Application/CleanupPreviewDeployment.php index 6c408bdf8b..661abc8d43 100644 --- a/app/Actions/Application/CleanupPreviewDeployment.php +++ b/app/Actions/Application/CleanupPreviewDeployment.php @@ -56,7 +56,7 @@ class CleanupPreviewDeployment if ($result['cancelled_deployments'] > 0) { try { - next_after_cancel($server); + next_after_cancel($server, $application); } catch (\Throwable $e) { \Log::warning("Failed to advance deployment queue after cleaning up preview for application {$application->id}: {$e->getMessage()}"); } diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php index fc7a9d5460..5c405f3e17 100644 --- a/app/Http/Controllers/Api/DeployController.php +++ b/app/Http/Controllers/Api/DeployController.php @@ -294,7 +294,7 @@ class DeployController extends Controller } finally { if ($cancelled) { try { - next_after_cancel($deploymentServer); + next_after_cancel($deploymentServer, $deployment->application); } catch (\Throwable $e) { \Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}"); } diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php index a5df1ebfa4..52fa592766 100644 --- a/app/Jobs/DeleteResourceJob.php +++ b/app/Jobs/DeleteResourceJob.php @@ -260,7 +260,7 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue if ($cancelledDeployments > 0) { try { - next_after_cancel($server); + next_after_cancel($server, $application); } catch (\Throwable $e) { \Log::warning("Failed to advance deployment queue after deleting preview {$this->resource->id}: {$e->getMessage()}"); } diff --git a/app/Livewire/Project/Application/DeploymentNavbar.php b/app/Livewire/Project/Application/DeploymentNavbar.php index 22f2b8043d..654b571db3 100644 --- a/app/Livewire/Project/Application/DeploymentNavbar.php +++ b/app/Livewire/Project/Application/DeploymentNavbar.php @@ -119,7 +119,7 @@ class DeploymentNavbar extends Component // Still mark as cancelled even if cleanup fails return handleError($e, $this); } finally { - next_after_cancel($server); + next_after_cancel($server, $this->application); } } } diff --git a/app/Mcp/Tools/CancelDeployment.php b/app/Mcp/Tools/CancelDeployment.php index 5dcee3ed38..c9913c4e5f 100644 --- a/app/Mcp/Tools/CancelDeployment.php +++ b/app/Mcp/Tools/CancelDeployment.php @@ -84,7 +84,7 @@ class CancelDeployment extends Tool try { $deploymentServer = Server::whereTeamId($teamId)->find($deployment->server_id); - next_after_cancel($deploymentServer); + next_after_cancel($deploymentServer, $application); } catch (\Throwable $e) { \Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}"); } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 8b90de0f40..c2a012692f 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -153,7 +153,7 @@ function queue_next_deployment(Application $application, ?int $finished_deployme ->where('status', ApplicationDeploymentStatus::QUEUED) ->distinct() ->pluck('server_id'); - $server_ids = collect([$application->destination->server_id, $finished_deployment_server_id]) + $server_ids = collect([$application->destination?->server_id, $finished_deployment_server_id]) ->merge($application_queued_server_ids) ->filter(fn ($server_id) => $server_id !== null) ->unique() @@ -164,7 +164,11 @@ function queue_next_deployment(Application $application, ?int $finished_deployme ->sortBy('created_at'); foreach ($queued_deployments as $next_deployment) { - start_queued_deployment($next_deployment); + try { + start_queued_deployment($next_deployment); + } catch (Throwable $e) { + Log::warning("Failed to start queued deployment {$next_deployment->deployment_uuid}: {$e->getMessage()}"); + } } } @@ -264,7 +268,11 @@ function next_queuable(string $server_id, string $application_id, string $commit } // Check server's concurrent build limit + // A deleted server keeps its queued deployments; they cannot run anymore. $server = Server::find($server_id); + if (! $server) { + return false; + } $concurrent_builds = $server->settings->concurrent_builds; $active_deployments = ApplicationDeploymentQueue::where('server_id', $server_id) ->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value) @@ -276,8 +284,17 @@ function next_queuable(string $server_id, string $application_id, string $commit return true; } -function next_after_cancel(?Server $server = null) +/** + * Start the queued deployments that can run after a cancellation. With the application of the + * cancelled deployment, its queued deployments on all servers start too, as when a deployment ends. + */ +function next_after_cancel(?Server $server = null, ?Application $application = null) { + if ($application) { + queue_next_deployment($application, $server?->id); + + return; + } if ($server) { $next_found = ApplicationDeploymentQueue::where('server_id', data_get($server, 'id')) ->where('status', ApplicationDeploymentStatus::QUEUED) diff --git a/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php b/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php new file mode 100644 index 0000000000..aac9f6c8d9 --- /dev/null +++ b/tests/Feature/DeploymentQueueAdvanceAcrossServersTest.php @@ -0,0 +1,136 @@ + Process::result(output: '')]); + config([ + 'constants.ssh.mux_enabled' => false, + 'cache.default' => 'array', + 'app.maintenance.store' => 'array', + ]); + + $settings = new InstanceSettings(['is_api_enabled' => true, 'is_mcp_server_enabled' => true]); + $settings->id = 0; + $settings->save(); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + session(['currentTeam' => $this->team]); + + $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); + $makeServer = function (string $ip) use ($privateKey): Server { + $server = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $privateKey->id, 'ip' => $ip]); + $server->settings->update(['concurrent_builds' => 1]); + + return $server; + }; + $this->primaryServer = $makeServer('203.0.113.31'); + $this->additionalServer = $makeServer('203.0.113.32'); + $this->deletedServer = $makeServer('203.0.113.33'); + + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $primaryDestination = StandaloneDocker::where('server_id', $this->primaryServer->id)->firstOrFail(); + $this->application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $primaryDestination->id, + 'destination_type' => $primaryDestination->getMorphClass(), + ]); + foreach ([$this->additionalServer, $this->deletedServer] as $server) { + $this->application->additional_servers()->attach($server->id, [ + 'standalone_docker_id' => StandaloneDocker::where('server_id', $server->id)->firstOrFail()->id, + ]); + } + + $this->makeQueueDeployment = fn (Server $server, string $status, string $uuid): ApplicationDeploymentQueue => ApplicationDeploymentQueue::create([ + 'application_id' => $this->application->id, + 'deployment_uuid' => $uuid, + 'status' => $status, + 'server_id' => $server->id, + 'destination_id' => StandaloneDocker::where('server_id', $server->id)->firstOrFail()->id, + 'commit' => 'HEAD', + 'pull_request_id' => 0, + 'only_this_server' => true, + ]); +}); + +afterEach(function () { + Server::flushIdentityMap(); +}); + +test('a finished deployment starts the queued deployment on another server when a deleted server also has one queued', function () { + $running = ($this->makeQueueDeployment)($this->primaryServer, 'in_progress', 'queue-across-running'); + $orphaned = ($this->makeQueueDeployment)($this->deletedServer, 'queued', 'queue-across-orphaned'); + $this->travel(1)->seconds(); + $waiting = ($this->makeQueueDeployment)($this->additionalServer, 'queued', 'queue-across-waiting'); + // A server deleted before deletion failed its queued deployments left them queued. + Server::withoutEvents(fn () => $this->deletedServer->delete()); + Server::flushIdentityMap(); + Log::spy(); + + (new ApplicationDeploymentJob($running->id))->failed(new RuntimeException('build failed')); + + Log::shouldNotHaveReceived('warning', [Mockery::pattern('/Starting the next queued deployment failed/')]); + expect($waiting->fresh()->status)->toBe('in_progress') + ->and($orphaned->fresh()->status)->toBe('queued'); + Queue::assertPushed(ApplicationDeploymentJob::class, 1); + Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waiting->id); +}); + +test('cancelling a deployment starts the queued deployment of the same application on another server', function (string $path) { + $running = ($this->makeQueueDeployment)($this->primaryServer, 'in_progress', 'queue-across-running'); + $waiting = ($this->makeQueueDeployment)($this->additionalServer, 'queued', 'queue-across-waiting'); + + if ($path === 'ui') { + $this->actingAs($this->user); + Livewire::test(DeploymentNavbar::class, ['application_deployment_queue' => $running])->call('cancel'); + } elseif ($path === 'api') { + $token = $this->user->createToken('queue-across', ['*'])->plainTextToken; + $this->withHeaders(['Authorization' => 'Bearer '.$token]) + ->postJson("/api/v1/deployments/{$running->deployment_uuid}/cancel") + ->assertSuccessful(); + } else { + $token = $this->user->createToken('queue-across', ['read', 'deploy'])->plainTextToken; + $this->withHeaders([ + 'Accept' => 'application/json, text/event-stream', + 'Authorization' => 'Bearer '.$token, + ])->postJson('/mcp', [ + 'jsonrpc' => '2.0', + 'id' => 1, + 'method' => 'tools/call', + 'params' => ['name' => 'cancel_deployment', 'arguments' => ['uuid' => $running->deployment_uuid]], + ])->assertOk(); + } + + expect($running->fresh()->status)->toBe(ApplicationDeploymentStatus::CANCELLED_BY_USER->value) + ->and($waiting->fresh()->status)->toBe(ApplicationDeploymentStatus::IN_PROGRESS->value); + Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $waiting->id); +})->with(['ui', 'api', 'mcp']);