From 2616b4bba12657d232d8d61fea09e7928cd01bbd Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:44:28 +0100 Subject: [PATCH 01/30] fix(deployments): index history ordering and skip payload columns in lists Co-Authored-By: Claude Opus 5.5 --- app/Livewire/Dashboard/ActiveDeployments.php | 69 ++++++----- .../Project/Application/Deployment/Index.php | 106 ++++++++--------- app/Models/Application.php | 3 +- ...dex_for_application_deployment_history.php | 41 +++++++ .../dashboard/active-deployments.blade.php | 8 +- tests/Feature/DashboardDeploymentsTest.php | 22 ++++ .../Feature/DeploymentHistoryPayloadTest.php | 111 ++++++++++++++++++ 7 files changed, 267 insertions(+), 93 deletions(-) create mode 100644 database/migrations/2026_10_28_214500_add_index_for_application_deployment_history.php create mode 100644 tests/Feature/DeploymentHistoryPayloadTest.php diff --git a/app/Livewire/Dashboard/ActiveDeployments.php b/app/Livewire/Dashboard/ActiveDeployments.php index 216c1f0bc4..00601df807 100644 --- a/app/Livewire/Dashboard/ActiveDeployments.php +++ b/app/Livewire/Dashboard/ActiveDeployments.php @@ -4,41 +4,39 @@ namespace App\Livewire\Dashboard; use App\Enums\ApplicationDeploymentStatus; use App\Models\ApplicationDeploymentQueue; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Support\Collection; +use Livewire\Attributes\Computed; use Livewire\Component; +/** + * The lists are computed, not public properties: Livewire would restore them with SELECT * on every poll. + */ class ActiveDeployments extends Component { - public Collection $activeDeployments; + private const COLUMNS = [ + 'id', + 'application_id', + 'application_name', + 'deployment_url', + 'deployment_uuid', + 'pull_request_id', + 'server_name', + 'server_id', + 'status', + 'created_at', + 'finished_at', + ]; - public Collection $recentDeployments; + /** + * Polling target. The lists are loaded again on every render. + */ + public function refreshDeployments(): void {} - public function mount(): void + #[Computed] + public function activeDeployments(): Collection { - $this->refreshDeployments(); - } - - public function refreshDeployments(): void - { - $columns = [ - 'id', - 'application_id', - 'application_name', - 'deployment_url', - 'deployment_uuid', - 'pull_request_id', - 'server_name', - 'server_id', - 'status', - 'created_at', - 'finished_at', - ]; - - $baseQuery = ApplicationDeploymentQueue::query() - ->with(['application.environment.project']) - ->ownedByTeam(currentTeam()->id); - - $this->activeDeployments = (clone $baseQuery) + return $this->deploymentsQuery() ->whereIn('status', [ ApplicationDeploymentStatus::IN_PROGRESS->value, ApplicationDeploymentStatus::QUEUED->value, @@ -46,16 +44,27 @@ class ActiveDeployments extends Component ->orderBy('status') ->orderBy('id') ->limit(5) - ->get($columns); + ->get(self::COLUMNS); + } - $this->recentDeployments = (clone $baseQuery) + #[Computed] + public function recentDeployments(): Collection + { + return $this->deploymentsQuery() ->whereNotIn('status', [ ApplicationDeploymentStatus::IN_PROGRESS->value, ApplicationDeploymentStatus::QUEUED->value, ]) ->orderByDesc('id') ->limit(5) - ->get($columns); + ->get(self::COLUMNS); + } + + private function deploymentsQuery(): Builder + { + return ApplicationDeploymentQueue::query() + ->with(['application.environment.project']) + ->ownedByTeam(currentTeam()->id); } public function render() diff --git a/app/Livewire/Project/Application/Deployment/Index.php b/app/Livewire/Project/Application/Deployment/Index.php index 08194f6bfc..40a9dc8e68 100644 --- a/app/Livewire/Project/Application/Deployment/Index.php +++ b/app/Livewire/Project/Application/Deployment/Index.php @@ -6,7 +6,6 @@ use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Traits\ListensToTeamChannel; -use Illuminate\Support\Collection; use Livewire\Attributes\On; use Livewire\Component; @@ -14,9 +13,28 @@ class Index extends Component { use ListensToTeamChannel; - public Application $application; + /** + * The history list never loads logs or configuration payloads. + */ + public const HISTORY_COLUMNS = [ + 'id', + 'application_id', + 'deployment_uuid', + 'pull_request_id', + 'status', + 'commit', + 'commit_message', + 'is_webhook', + 'is_api', + 'rollback', + 'server_id', + 'server_name', + 'created_at', + 'updated_at', + 'finished_at', + ]; - public ?Collection $deployments; + public Application $application; public int $deployments_count = 0; @@ -31,11 +49,8 @@ class Index extends Component $this->defaultTake = max(1, min(100, $this->defaultTake)); $this->skip = 0; - $this->loadDeployments(); } - public bool $showNext = false; - public bool $showPrev = false; public int $currentPage = 1; @@ -107,47 +122,24 @@ class Index extends Component } $this->loadPullRequestOptions(); $this->loadDeploymentFilterOptions(); - ['deployments' => $deployments, 'count' => $count] = $application->deployments( - search: $this->search, - filters: $this->deploymentFilters, - sort: $this->deploymentSort, - take: $this->defaultTake, - pullRequestId: $this->pull_request_id, - ); - $this->deployments = $deployments; - $this->deployments_count = $count; $this->current_url = route('project.application.deployment.index', [ 'project_uuid' => $project->uuid, 'environment_uuid' => $environment->uuid, 'application_uuid' => $application->uuid, ]); $this->updateCurrentPage(); - $this->showMore(); } - private function showMore() - { - if ($this->deployments->count() !== 0) { - $this->showNext = true; - if ($this->deployments->count() < $this->defaultTake) { - $this->showNext = false; - } - - return; - } - } - - public function reloadDeployments() - { - $this->loadDeployments(); - } + /** + * Polling target. The list is loaded again on every render. + */ + public function reloadDeployments(): void {} public function previousPage(): void { $this->skip = max(0, $this->skip - $this->defaultTake); $this->showPrev = $this->skip > 0; $this->updateCurrentPage(); - $this->loadDeployments(); } public function nextPage(): void @@ -162,27 +154,11 @@ class Index extends Component $this->skip = ($page - 1) * $this->defaultTake; $this->showPrev = $page > 1; $this->updateCurrentPage(); - $this->loadDeployments(); - } - - public function loadDeployments() - { - ['deployments' => $deployments, 'count' => $count] = $this->application->deployments( - skip: $this->skip, - take: $this->defaultTake, - pullRequestId: $this->pull_request_id, - search: $this->search, - filters: $this->deploymentFilters, - sort: $this->deploymentSort, - ); - $this->deployments = $deployments; - $this->deployments_count = $count; - $this->showMore(); } public function updatedSearch(): void { - $this->resetPaginationAndLoad(); + $this->resetPagination(); } public function toggleDeploymentFilter(string $filter): void @@ -204,7 +180,7 @@ class Index extends Component } $this->pull_request_id = null; - $this->resetPaginationAndLoad(); + $this->resetPagination(); } public function setPullRequestFilter(string $pullRequestId): void @@ -217,7 +193,7 @@ class Index extends Component $this->pull_request_id = $pullRequestId === '' ? null : $pullRequestId; $this->deploymentFilters = []; - $this->resetPaginationAndLoad(); + $this->resetPagination(); } public function setDeploymentSort(string $sort): void @@ -227,7 +203,7 @@ class Index extends Component } $this->deploymentSort = $sort; - $this->resetPaginationAndLoad(); + $this->resetPagination(); } public function updatedPullRequestId($value) @@ -248,14 +224,14 @@ class Index extends Component } $this->deploymentFilters = []; - $this->resetPaginationAndLoad(); + $this->resetPagination(); } public function clearFilter() { $this->pull_request_id = null; $this->deploymentFilters = []; - $this->resetPaginationAndLoad(); + $this->resetPagination(); } private function updateCurrentPage() @@ -367,16 +343,30 @@ class Index extends Component ->all(); } - private function resetPaginationAndLoad(): void + private function resetPagination(): void { $this->skip = 0; $this->showPrev = false; $this->updateCurrentPage(); - $this->loadDeployments(); } + /** + * The list is not a public property: Livewire would restore it with SELECT * on every request. + */ public function render() { - return view('livewire.project.application.deployment.index'); + ['deployments' => $deployments, 'count' => $this->deployments_count] = $this->application->deployments( + skip: $this->skip, + take: $this->defaultTake, + pullRequestId: $this->pull_request_id, + search: $this->search, + filters: $this->deploymentFilters, + sort: $this->deploymentSort, + columns: self::HISTORY_COLUMNS, + ); + + return view('livewire.project.application.deployment.index', [ + 'deployments' => $deployments, + ]); } } diff --git a/app/Models/Application.php b/app/Models/Application.php index 1fe908027b..93fd9d695f 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -1314,6 +1314,7 @@ class Application extends BaseModel ?string $search = null, array $filters = [], string $sort = 'newest', + array $columns = ['*'], ): array { $deployments = ApplicationDeploymentQueue::query() ->where('application_id', $this->id); @@ -1406,7 +1407,7 @@ class Application extends BaseModel ->orderBy('id', $sort === 'oldest' ? 'asc' : 'desc') ->skip($skip) ->take($take) - ->get(); + ->get($columns); return [ 'count' => $count, diff --git a/database/migrations/2026_10_28_214500_add_index_for_application_deployment_history.php b/database/migrations/2026_10_28_214500_add_index_for_application_deployment_history.php new file mode 100644 index 0000000000..134d74b85d --- /dev/null +++ b/database/migrations/2026_10_28_214500_add_index_for_application_deployment_history.php @@ -0,0 +1,41 @@ +index(['application_id', 'created_at', 'id'], 'idx_deployment_queues_app_created_id'); + }); + } + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS idx_deployment_queues_app_created_id'); + + return; + } + + if (Schema::hasIndex('application_deployment_queues', 'idx_deployment_queues_app_created_id')) { + Schema::table('application_deployment_queues', function (Blueprint $table) { + $table->dropIndex('idx_deployment_queues_app_created_id'); + }); + } + } +}; diff --git a/resources/views/livewire/dashboard/active-deployments.blade.php b/resources/views/livewire/dashboard/active-deployments.blade.php index b6cb5a8358..dc3ab8ed93 100644 --- a/resources/views/livewire/dashboard/active-deployments.blade.php +++ b/resources/views/livewire/dashboard/active-deployments.blade.php @@ -19,8 +19,8 @@ return [$label, $type]; }; - $hasActiveDeployments = $activeDeployments->isNotEmpty(); - $hasRecentDeployments = $recentDeployments->isNotEmpty(); + $hasActiveDeployments = $this->activeDeployments->isNotEmpty(); + $hasRecentDeployments = $this->recentDeployments->isNotEmpty(); $hasAnyDeployments = $hasActiveDeployments || $hasRecentDeployments; @endphp @@ -52,7 +52,7 @@ Started - @foreach ($activeDeployments as $deployment) + @foreach ($this->activeDeployments as $deployment) @php [$deploymentStatus, $deploymentStatusType] = $deploymentStatusMeta($deployment->status); $projectName = $deployment->application?->environment?->project?->name; @@ -112,7 +112,7 @@ Started - @foreach ($recentDeployments as $deployment) + @foreach ($this->recentDeployments as $deployment) @php [$deploymentStatus, $deploymentStatusType] = $deploymentStatusMeta($deployment->status); $projectName = $deployment->application?->environment?->project?->name; diff --git a/tests/Feature/DashboardDeploymentsTest.php b/tests/Feature/DashboardDeploymentsTest.php index d949a62e7e..e35d3a07c3 100644 --- a/tests/Feature/DashboardDeploymentsTest.php +++ b/tests/Feature/DashboardDeploymentsTest.php @@ -12,6 +12,7 @@ use App\Models\StandaloneDocker; use App\Models\Team; use App\Models\User; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Support\Facades\DB; use Livewire\Livewire; uses(RefreshDatabase::class); @@ -175,3 +176,24 @@ it('only includes deployments of applications owned by the current team', functi ->assertDontSee('Other Team Deployment') ->assertDontSee('Running or queued right now'); }); + +it('does not load deployment payload columns when the dashboard list polls', function () { + createDashboardDeployment(['logs' => '[{"output":"large deployment log"}]']); + + $deploymentQueries = []; + DB::listen(function ($query) use (&$deploymentQueries) { + if (str_contains($query->sql, 'application_deployment_queues')) { + $deploymentQueries[] = $query->sql; + } + }); + + Livewire::test(ActiveDeployments::class) + ->assertSee('Dashboard App') + ->call('refreshDeployments') + ->assertSee('Dashboard App'); + + expect($deploymentQueries)->not->toBeEmpty(); + foreach ($deploymentQueries as $sql) { + expect($sql)->not->toMatch('/select\s+("?\w+"?\.)?\*/i')->not->toContain('logs'); + } +}); diff --git a/tests/Feature/DeploymentHistoryPayloadTest.php b/tests/Feature/DeploymentHistoryPayloadTest.php new file mode 100644 index 0000000000..76a96006ee --- /dev/null +++ b/tests/Feature/DeploymentHistoryPayloadTest.php @@ -0,0 +1,111 @@ +user = User::factory()->create(); + $this->team = Team::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + + InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0])); + + $this->actingAs($this->user); + session(['currentTeam' => $this->team]); + + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail(); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); + $this->application = Application::factory()->create([ + 'environment_id' => $this->environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + $this->createDeployment = fn (string $uuid, string $createdAt) => ApplicationDeploymentQueue::query()->forceCreate([ + 'application_id' => $this->application->id, + 'deployment_uuid' => $uuid, + 'server_id' => $server->id, + 'server_name' => $server->name, + 'status' => ApplicationDeploymentStatus::FINISHED->value, + 'commit' => 'abcdef1234567890', + 'commit_message' => "Message of {$uuid}", + 'logs' => '[{"output":"large deployment log"}]', + 'configuration_snapshot' => ['fqdn' => 'https://example.com'], + 'configuration_diff' => ['fqdn' => ['old' => null, 'new' => 'https://example.com']], + 'created_at' => $createdAt, + 'updated_at' => $createdAt, + ]); +}); + +it('does not load deployment payload columns for the history list on any Livewire request', function () { + ($this->createDeployment)('deployment-history-row', '2026-10-01 10:00:00'); + + $historyQueries = []; + DB::listen(function ($query) use (&$historyQueries) { + if (str_contains($query->sql, 'application_deployment_queues')) { + $historyQueries[] = $query->sql; + } + }); + + Livewire::test(Index::class, [ + 'project_uuid' => $this->project->uuid, + 'environment_uuid' => $this->environment->uuid, + 'application_uuid' => $this->application->uuid, + 'embedded' => true, + ]) + ->assertSee('deployment-history-row') + ->assertSee('Message of deployment-history-row') + ->call('reloadDeployments') + ->assertSee('deployment-history-row') + ->call('setDeploymentSort', 'oldest') + ->assertSee('deployment-history-row'); + + expect($historyQueries)->not->toBeEmpty(); + foreach ($historyQueries as $sql) { + expect($sql) + ->not->toMatch('/select\s+("?\w+"?\.)?\*/i') + ->not->toContain('logs') + ->not->toContain('configuration_snapshot') + ->not->toContain('configuration_diff'); + } +}); + +it('orders and pages deployments with equal timestamps by id in both directions', function () { + $uuids = collect(['first', 'second', 'third', 'fourth', 'fifth']) + ->each(fn (string $uuid) => ($this->createDeployment)($uuid, '2026-10-01 10:00:00')); + + $page = fn (string $sort, int $skip) => $this->application + ->deployments(skip: $skip, take: 2, sort: $sort, columns: Index::HISTORY_COLUMNS)['deployments'] + ->pluck('deployment_uuid') + ->all(); + + expect([...$page('newest', 0), ...$page('newest', 2), ...$page('newest', 4)])->toBe($uuids->reverse()->values()->all()) + ->and([...$page('oldest', 0), ...$page('oldest', 2), ...$page('oldest', 4)])->toBe($uuids->all()); +}); + +it('keeps the full deployment payload for callers that do not restrict the columns', function () { + ($this->createDeployment)('full-deployment', '2026-10-01 10:00:00'); + + $listed = $this->application->deployments(columns: Index::HISTORY_COLUMNS)['deployments']->sole(); + $full = $this->application->deployments()['deployments']->sole(); + + expect($listed->getAttributes())->not->toHaveKeys(['logs', 'configuration_snapshot', 'configuration_diff']) + ->and($full->logs)->toBe('[{"output":"large deployment log"}]') + ->and($full->configuration_snapshot)->toBe(['fqdn' => 'https://example.com']); +}); From 68a08170e7a89b72652dbcfa8320a000264121d9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:53:10 +0100 Subject: [PATCH 02/30] fix(docker): index interrupted cleanup lookup and process it in batches Co-Authored-By: Claude Opus 5.5 --- app/Jobs/DockerCleanupJob.php | 42 +++++---- ..._interrupted_docker_cleanup_executions.php | 45 ++++++++++ tests/Feature/DockerCleanupJobTest.php | 90 +++++++++++++++++++ 3 files changed, 158 insertions(+), 19 deletions(-) create mode 100644 database/migrations/2026_10_28_214953_add_index_for_interrupted_docker_cleanup_executions.php diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index af1778eacf..3f70534301 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -13,6 +13,7 @@ use Carbon\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; @@ -43,6 +44,8 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue public const INTERRUPTED_MESSAGE = 'Docker cleanup was interrupted: its worker stopped before the cleanup finished.'; + public const INTERRUPTED_EXECUTIONS_CHUNK_SIZE = 100; + /** * Shares the per-server lock with queued CleanupDocker runs, so two cleanups never run on * one server at once. A scheduled run that finds the lock held marks its occurrence @@ -254,31 +257,32 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue */ public static function failInterruptedExecutions(): void { - $executions = DockerCleanupExecution::query() + DockerCleanupExecution::query() + ->select(['id', 'server_id']) ->with('server.team') ->where('status', 'running') ->whereNull('finished_at') ->where('created_at', '<', now()->subSeconds(CleanupDocker::overlapLockExpiresAfter())) - ->get(); + ->chunkById(self::INTERRUPTED_EXECUTIONS_CHUNK_SIZE, function (Collection $executions): void { + foreach ($executions as $execution) { + $updated = DockerCleanupExecution::query() + ->whereKey($execution->id) + ->where('status', 'running') + ->whereNull('finished_at') + ->update([ + 'status' => 'failed', + 'message' => self::INTERRUPTED_MESSAGE, + 'finished_at' => Carbon::now()->toImmutable(), + ]); - foreach ($executions as $execution) { - $updated = DockerCleanupExecution::query() - ->whereKey($execution->id) - ->where('status', 'running') - ->whereNull('finished_at') - ->update([ - 'status' => 'failed', - 'message' => self::INTERRUPTED_MESSAGE, - 'finished_at' => Carbon::now()->toImmutable(), - ]); + if ($updated === 0) { + continue; + } - if ($updated === 0) { - continue; - } - - event(new DockerCleanupDone($execution->refresh())); - $execution->server?->team?->notify(new DockerCleanupFailed($execution->server, 'Docker cleanup job failed with the following error: '.self::INTERRUPTED_MESSAGE)); - } + event(new DockerCleanupDone($execution->refresh())); + $execution->server?->team?->notify(new DockerCleanupFailed($execution->server, 'Docker cleanup job failed with the following error: '.self::INTERRUPTED_MESSAGE)); + } + }); } public function failed(?\Throwable $exception): void diff --git a/database/migrations/2026_10_28_214953_add_index_for_interrupted_docker_cleanup_executions.php b/database/migrations/2026_10_28_214953_add_index_for_interrupted_docker_cleanup_executions.php new file mode 100644 index 0000000000..8b3081d3d6 --- /dev/null +++ b/database/migrations/2026_10_28_214953_add_index_for_interrupted_docker_cleanup_executions.php @@ -0,0 +1,45 @@ +index(['status', 'created_at'], 'idx_docker_cleanup_executions_unfinished_status_created'); + }); + } + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS idx_docker_cleanup_executions_unfinished_status_created'); + + return; + } + + if (Schema::hasIndex('docker_cleanup_executions', 'idx_docker_cleanup_executions_unfinished_status_created')) { + Schema::table('docker_cleanup_executions', function (Blueprint $table) { + $table->dropIndex('idx_docker_cleanup_executions_unfinished_status_created'); + }); + } + } +}; diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php index 5d0b30b571..eb1d791818 100644 --- a/tests/Feature/DockerCleanupJobTest.php +++ b/tests/Feature/DockerCleanupJobTest.php @@ -883,3 +883,93 @@ it('fails an interrupted cleanup of a deleted server without breaking its broadc Queue::assertPushed(BroadcastEvent::class, fn (BroadcastEvent $job) => $job->event instanceof DockerCleanupDone && $job->event->broadcastOn() === []); }); + +/** + * Creates an execution at an exact age, so tests can place it on either side of the stale cutoff. + * + * @param array $attributes + */ +function dockerCleanupExecutionCreatedSecondsAgo(Server $server, int $secondsAgo, array $attributes = []): DockerCleanupExecution +{ + $execution = DockerCleanupExecution::create(['server_id' => $server->id, ...$attributes]); + $execution->forceFill(['created_at' => now()->subSeconds($secondsAgo)])->saveQuietly(); + + return $execution; +} + +it('fails only running, unfinished cleanups that are older than the server lock and sends one event and notification each', function () { + Carbon::setTestNow(Carbon::create(2026, 10, 28, 12, 0, 0, 'UTC')); + Event::fake([DockerCleanupDone::class]); + Notification::fake(); + $server = dockerCleanupReachableServer(); + $server->team->emailNotificationSettings->update(['smtp_enabled' => true, 'docker_cleanup_failure_email_notifications' => true]); + $cutoff = CleanupDocker::overlapLockExpiresAfter(); + + $stale = dockerCleanupExecutionCreatedSecondsAgo($server, $cutoff + 1, ['message' => 'still running', 'cleanup_log' => '["kept"]']); + $atCutoff = dockerCleanupExecutionCreatedSecondsAgo($server, $cutoff); + $recent = dockerCleanupExecutionCreatedSecondsAgo($server, 60); + $finished = dockerCleanupExecutionCreatedSecondsAgo($server, $cutoff + 3600, ['finished_at' => now()->subHour()]); + $succeeded = dockerCleanupExecutionCreatedSecondsAgo($server, $cutoff + 3600, ['status' => 'success', 'message' => 'Saved 3%']); + $failed = dockerCleanupExecutionCreatedSecondsAgo($server, $cutoff + 3600, ['status' => 'failed', 'message' => 'original error']); + + DockerCleanupJob::failInterruptedExecutions(); + + $stale->refresh(); + expect($stale->status)->toBe('failed') + ->and($stale->message)->toBe(DockerCleanupJob::INTERRUPTED_MESSAGE) + ->and($stale->finished_at)->not->toBeNull() + ->and($stale->cleanup_log)->toBe('["kept"]'); + foreach ([$atCutoff, $recent] as $running) { + expect($running->fresh()->only(['status', 'finished_at']))->toBe(['status' => 'running', 'finished_at' => null]); + } + expect($finished->fresh()->status)->toBe('running') + ->and($succeeded->fresh()->only(['status', 'message', 'finished_at']))->toBe(['status' => 'success', 'message' => 'Saved 3%', 'finished_at' => null]) + ->and($failed->fresh()->only(['status', 'message', 'finished_at']))->toBe(['status' => 'failed', 'message' => 'original error', 'finished_at' => null]); + + Event::assertDispatchedTimes(DockerCleanupDone::class, 1); + Event::assertDispatched(DockerCleanupDone::class, fn (DockerCleanupDone $event) => $event->execution->is($stale) + && $event->execution->status === 'failed' + && $event->teamId === $server->team_id); + Notification::assertSentToTimes($server->team, DockerCleanupFailed::class, 1); + Notification::assertSentTo($server->team, DockerCleanupFailed::class, fn (DockerCleanupFailed $notification) => $notification->server->is($server) + && str_contains($notification->message, DockerCleanupJob::INTERRUPTED_MESSAGE)); +}); + +it('fails interrupted cleanups across more than one batch', function () { + Carbon::setTestNow(Carbon::create(2026, 10, 28, 12, 0, 0, 'UTC')); + Event::fake([DockerCleanupDone::class]); + $server = dockerCleanupReachableServer(); + $count = DockerCleanupJob::INTERRUPTED_EXECUTIONS_CHUNK_SIZE * 2 + 1; + $staleAge = CleanupDocker::overlapLockExpiresAfter() + 1; + foreach (range(1, $count) as $ignored) { + dockerCleanupExecutionCreatedSecondsAgo($server, $staleAge); + } + + DockerCleanupJob::failInterruptedExecutions(); + + expect(DockerCleanupExecution::query()->where('status', 'failed')->whereNotNull('finished_at')->count())->toBe($count) + ->and(DockerCleanupExecution::query()->where('status', 'running')->count())->toBe(0); + Event::assertDispatchedTimes(DockerCleanupDone::class, $count); +}); + +it('does not fail a cleanup again when it finished after the stale lookup read it', function () { + Carbon::setTestNow(Carbon::create(2026, 10, 28, 12, 0, 0, 'UTC')); + Event::fake([DockerCleanupDone::class]); + $server = dockerCleanupReachableServer(); + $staleAge = CleanupDocker::overlapLockExpiresAfter() + 1; + $first = dockerCleanupExecutionCreatedSecondsAgo($server, $staleAge); + $finishesMeanwhile = dockerCleanupExecutionCreatedSecondsAgo($server, $staleAge); + DB::listen(function ($query) use ($first, $finishesMeanwhile) { + static $done = false; + if (! $done && str_starts_with(strtolower($query->sql), 'update') && in_array($first->id, $query->bindings, true)) { + $done = true; + DockerCleanupExecution::query()->whereKey($finishesMeanwhile->id)->update(['status' => 'success', 'message' => 'done', 'finished_at' => now()]); + } + }); + + DockerCleanupJob::failInterruptedExecutions(); + + expect($first->fresh()->status)->toBe('failed') + ->and($finishesMeanwhile->fresh()->only(['status', 'message']))->toBe(['status' => 'success', 'message' => 'done']); + Event::assertDispatchedTimes(DockerCleanupDone::class, 1); +}); From c23d3363f274a7d691dcf5f69fafedd167283532 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:53:30 +0100 Subject: [PATCH 03/30] fix(deployments): load only commit details for the application heading Co-Authored-By: Claude Opus 5.5 --- app/Livewire/Project/Application/Heading.php | 2 +- app/Models/Application.php | 4 +- tests/Feature/DashboardDeploymentsTest.php | 21 ++- .../Feature/DeploymentHistoryPayloadTest.php | 166 +++++++++++++++--- tests/Feature/DeploymentsIndicatorTest.php | 28 +++ 5 files changed, 191 insertions(+), 30 deletions(-) diff --git a/app/Livewire/Project/Application/Heading.php b/app/Livewire/Project/Application/Heading.php index 9b40240c19..182f96ae75 100644 --- a/app/Livewire/Project/Application/Heading.php +++ b/app/Livewire/Project/Application/Heading.php @@ -61,7 +61,7 @@ class Heading extends Component 'environment_uuid' => $this->application->environment->uuid, 'application_uuid' => $this->application->uuid, ]; - $lastDeployment = $this->application->get_last_successful_deployment(); + $lastDeployment = $this->application->get_last_successful_deployment(['commit', 'commit_message']); $this->lastDeploymentInfo = data_get_str($lastDeployment, 'commit')->limit(7).' '.data_get($lastDeployment, 'commit_message'); $this->lastDeploymentLink = $this->application->gitCommitLink(data_get($lastDeployment, 'commit')); $this->loadComposeOnOpen = $this->activeRouteName === 'project.application.configuration' diff --git a/app/Models/Application.php b/app/Models/Application.php index 93fd9d695f..084eb0fb25 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -1293,9 +1293,9 @@ class Application extends BaseModel return false; } - public function get_last_successful_deployment() + public function get_last_successful_deployment(array $columns = ['*']) { - return ApplicationDeploymentQueue::where('application_id', $this->id)->where('status', ApplicationDeploymentStatus::FINISHED->value)->where('pull_request_id', 0)->orderBy('created_at', 'desc')->first(); + return ApplicationDeploymentQueue::where('application_id', $this->id)->where('status', ApplicationDeploymentStatus::FINISHED->value)->where('pull_request_id', 0)->orderBy('created_at', 'desc')->first($columns); } public function get_last_days_deployments() diff --git a/tests/Feature/DashboardDeploymentsTest.php b/tests/Feature/DashboardDeploymentsTest.php index e35d3a07c3..bec45a4985 100644 --- a/tests/Feature/DashboardDeploymentsTest.php +++ b/tests/Feature/DashboardDeploymentsTest.php @@ -178,7 +178,16 @@ it('only includes deployments of applications owned by the current team', functi }); it('does not load deployment payload columns when the dashboard list polls', function () { - createDashboardDeployment(['logs' => '[{"output":"large deployment log"}]']); + createDashboardDeployment([ + 'logs' => json_encode([['output' => str_repeat('large deployment log ', 20000)]]), + 'configuration_snapshot' => ['docker_compose_raw' => str_repeat('x', 100000)], + 'configuration_diff' => ['fqdn' => ['old' => null, 'new' => 'https://example.com']], + ]); + createDashboardDeployment([ + 'application_name' => 'Dashboard App', + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + 'logs' => json_encode([['output' => str_repeat('large deployment log ', 20000)]]), + ]); $deploymentQueries = []; DB::listen(function ($query) use (&$deploymentQueries) { @@ -190,10 +199,16 @@ it('does not load deployment payload columns when the dashboard list polls', fun Livewire::test(ActiveDeployments::class) ->assertSee('Dashboard App') ->call('refreshDeployments') - ->assertSee('Dashboard App'); + ->assertSee('Dashboard App') + ->call('$refresh') + ->assertSee('Running or queued right now') + ->assertSee('Recent deployments'); expect($deploymentQueries)->not->toBeEmpty(); foreach ($deploymentQueries as $sql) { - expect($sql)->not->toMatch('/select\s+("?\w+"?\.)?\*/i')->not->toContain('logs'); + expect($sql)->not->toMatch('/select\s+("?\w+"?\.)?\*/i') + ->not->toContain('logs') + ->not->toContain('configuration_snapshot') + ->not->toContain('configuration_diff'); } }); diff --git a/tests/Feature/DeploymentHistoryPayloadTest.php b/tests/Feature/DeploymentHistoryPayloadTest.php index 76a96006ee..f5ac14a185 100644 --- a/tests/Feature/DeploymentHistoryPayloadTest.php +++ b/tests/Feature/DeploymentHistoryPayloadTest.php @@ -2,6 +2,8 @@ use App\Enums\ApplicationDeploymentStatus; use App\Livewire\Project\Application\Deployment\Index; +use App\Livewire\Project\Application\Deployment\Show; +use App\Livewire\Project\Application\Heading; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Models\Environment; @@ -37,7 +39,16 @@ beforeEach(function () { 'destination_type' => $destination->getMorphClass(), ]); - $this->createDeployment = fn (string $uuid, string $createdAt) => ApplicationDeploymentQueue::query()->forceCreate([ + $this->largeLog = json_encode([[ + 'command' => null, + 'output' => str_repeat('large deployment log ', 20000), + 'type' => 'stdout', + 'timestamp' => '2026-10-01T10:00:00.000000Z', + 'hidden' => false, + 'batch' => 1, + 'order' => 1, + ]]); + $this->createDeployment = fn (string $uuid, string $createdAt, array $overrides = []) => ApplicationDeploymentQueue::query()->forceCreate([ 'application_id' => $this->application->id, 'deployment_uuid' => $uuid, 'server_id' => $server->id, @@ -45,23 +56,67 @@ beforeEach(function () { 'status' => ApplicationDeploymentStatus::FINISHED->value, 'commit' => 'abcdef1234567890', 'commit_message' => "Message of {$uuid}", - 'logs' => '[{"output":"large deployment log"}]', - 'configuration_snapshot' => ['fqdn' => 'https://example.com'], + 'logs' => $this->largeLog, + 'configuration_snapshot' => ['fqdn' => 'https://example.com', 'docker_compose_raw' => str_repeat('x', 100000)], 'configuration_diff' => ['fqdn' => ['old' => null, 'new' => 'https://example.com']], 'created_at' => $createdAt, 'updated_at' => $createdAt, + ...$overrides, ]); -}); -it('does not load deployment payload columns for the history list on any Livewire request', function () { - ($this->createDeployment)('deployment-history-row', '2026-10-01 10:00:00'); - - $historyQueries = []; - DB::listen(function ($query) use (&$historyQueries) { + $this->deploymentQueries = collect(); + $this->recordDeploymentQueries = fn () => DB::listen(function ($query) { if (str_contains($query->sql, 'application_deployment_queues')) { - $historyQueries[] = $query->sql; + $this->deploymentQueries->push($query->sql); } }); + $this->assertNoPayloadQueries = function () { + expect($this->deploymentQueries)->not->toBeEmpty(); + $this->deploymentQueries->each(fn (string $sql) => expect($sql) + ->not->toMatch('/select\s+("?\w+"?\.)?\*/i') + ->not->toContain('logs') + ->not->toContain('configuration_snapshot') + ->not->toContain('configuration_diff')); + }; +}); + +function otherTeamDeployment(): array +{ + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); + $destination = StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $deployment = ApplicationDeploymentQueue::query()->forceCreate([ + 'application_id' => $application->id, + 'deployment_uuid' => 'other-team-deployment', + 'server_id' => $server->id, + 'status' => ApplicationDeploymentStatus::FINISHED->value, + 'logs' => '[{"output":"other team log"}]', + ]); + + return [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'application_uuid' => $application->uuid, + 'deployment_uuid' => $deployment->deployment_uuid, + ]; +} + +it('does not load deployment payload columns for the history list on any Livewire request', function () { + foreach (range(1, 4) as $number) { + ($this->createDeployment)("history-d{$number}", "2026-10-0{$number} 10:00:00"); + } + ($this->createDeployment)('history-d5', '2026-10-05 10:00:00', [ + 'status' => ApplicationDeploymentStatus::FAILED->value, + 'pull_request_id' => 7, + ]); + ($this->recordDeploymentQueries)(); Livewire::test(Index::class, [ 'project_uuid' => $this->project->uuid, @@ -69,21 +124,84 @@ it('does not load deployment payload columns for the history list on any Livewir 'application_uuid' => $this->application->uuid, 'embedded' => true, ]) - ->assertSee('deployment-history-row') - ->assertSee('Message of deployment-history-row') + ->assertSeeInOrder(['history-d5', 'history-d4', 'history-d3']) + ->assertSee('Message of history-d5') + ->assertDontSee('history-d2') ->call('reloadDeployments') - ->assertSee('deployment-history-row') + ->assertSee('history-d5') + ->call('nextPage') + ->assertSeeInOrder(['history-d2', 'history-d1']) + ->assertDontSee('history-d5') + ->call('previousPage') + ->assertSee('history-d5') + ->assertDontSee('history-d2') + ->call('goToPage', 2) + ->assertSee('history-d1') + ->call('toggleDeploymentFilter', 'status:failed') + ->assertSee('history-d5') + ->assertDontSee('history-d4') + ->call('toggleDeploymentFilter', 'status:failed') + ->call('setPullRequestFilter', '7') + ->assertSee('history-d5') + ->assertDontSee('history-d4') + ->call('clearFilter') + ->set('search', 'history-d3') + ->assertSee('history-d3') + ->assertDontSee('history-d4') + ->set('search', '') ->call('setDeploymentSort', 'oldest') - ->assertSee('deployment-history-row'); + ->assertSeeInOrder(['history-d1', 'history-d2', 'history-d3']) + ->assertDontSee('history-d5') + ->set('defaultTake', 10) + ->assertSee('history-d5') + ->assertSee('history-d1') + ->call('$refresh') + ->assertSee('history-d5') + ->dispatch('deploymentSelected', deploymentUuid: 'history-d2') + ->assertSet('selectedDeploymentUuid', 'history-d2') + ->assertSee('history-d2'); - expect($historyQueries)->not->toBeEmpty(); - foreach ($historyQueries as $sql) { - expect($sql) - ->not->toMatch('/select\s+("?\w+"?\.)?\*/i') - ->not->toContain('logs') - ->not->toContain('configuration_snapshot') - ->not->toContain('configuration_diff'); - } + ($this->assertNoPayloadQueries)(); +}); + +it('does not show the deployment history of another team', function () { + $otherTeam = otherTeamDeployment(); + unset($otherTeam['deployment_uuid']); + + Livewire::test(Index::class, $otherTeam) + ->assertRedirect(route('dashboard')) + ->assertDontSee('other-team-deployment'); +}); + +it('loads the full logs and configuration when an authorized user opens a deployment', function () { + ($this->createDeployment)('full-detail', '2026-10-01 10:00:00'); + + $component = Livewire::test(Show::class, [ + 'project_uuid' => $this->project->uuid, + 'environment_uuid' => $this->environment->uuid, + 'application_uuid' => $this->application->uuid, + 'deployment_uuid' => 'full-detail', + ])->call('polling'); + + $deployment = $component->get('application_deployment_queue'); + expect($deployment->logs)->toBe($this->largeLog) + ->and($deployment->configuration_snapshot['docker_compose_raw'])->toBe(str_repeat('x', 100000)); +}); + +it('does not open a deployment of another team', function () { + Livewire::test(Show::class, otherTeamDeployment()) + ->assertRedirect(route('dashboard')) + ->assertDontSee('other team log'); +}); + +it('loads only the commit of the last successful deployment for the application heading', function () { + ($this->createDeployment)('heading-deployment', '2026-10-01 10:00:00', ['commit_message' => 'Heading commit']); + ($this->recordDeploymentQueries)(); + + Livewire::test(Heading::class, ['application' => $this->application]) + ->assertSet('lastDeploymentInfo', 'abcdef1... Heading commit'); + + ($this->assertNoPayloadQueries)(); }); it('orders and pages deployments with equal timestamps by id in both directions', function () { @@ -106,6 +224,6 @@ it('keeps the full deployment payload for callers that do not restrict the colum $full = $this->application->deployments()['deployments']->sole(); expect($listed->getAttributes())->not->toHaveKeys(['logs', 'configuration_snapshot', 'configuration_diff']) - ->and($full->logs)->toBe('[{"output":"large deployment log"}]') - ->and($full->configuration_snapshot)->toBe(['fqdn' => 'https://example.com']); + ->and($full->logs)->toBe($this->largeLog) + ->and($full->configuration_snapshot['fqdn'])->toBe('https://example.com'); }); diff --git a/tests/Feature/DeploymentsIndicatorTest.php b/tests/Feature/DeploymentsIndicatorTest.php index c89ce6c12d..f0b54a9e26 100644 --- a/tests/Feature/DeploymentsIndicatorTest.php +++ b/tests/Feature/DeploymentsIndicatorTest.php @@ -12,6 +12,7 @@ use App\Models\StandaloneDocker; use App\Models\Team; use App\Models\User; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Support\Facades\DB; use Livewire\Livewire; uses(RefreshDatabase::class); @@ -113,3 +114,30 @@ it('keeps the indicator on an application page when another application is deplo ->assertSee('2 deployments', false) ->assertSee('aria-label="Active deployments"', false); }); + +it('does not load deployment payload columns when the indicator refreshes', function () { + ApplicationDeploymentQueue::query()->update([ + 'logs' => json_encode([['output' => str_repeat('large deployment log ', 20000)]]), + 'configuration_snapshot' => json_encode(['docker_compose_raw' => str_repeat('x', 100000)]), + ]); + + $deploymentQueries = []; + DB::listen(function ($query) use (&$deploymentQueries) { + if (str_contains($query->sql, 'application_deployment_queues')) { + $deploymentQueries[] = $query->sql; + } + }); + + Livewire::test(DeploymentsIndicator::class) + ->assertSee('Indicator App') + ->call('$refresh') + ->assertSee('Indicator App'); + + expect($deploymentQueries)->not->toBeEmpty(); + foreach ($deploymentQueries as $sql) { + expect($sql)->not->toMatch('/select\s+("?\w+"?\.)?\*/i') + ->not->toContain('logs') + ->not->toContain('configuration_snapshot') + ->not->toContain('configuration_diff'); + } +}); From 7324b5ca7b4ccf134cce3cf57d1c2d475541a096 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:54:34 +0100 Subject: [PATCH 04/30] fix(webhooks): filter manual webhook applications by repository in SQL Co-Authored-By: Claude Opus 5.5 --- .../MatchesManualWebhookApplications.php | 8 +- .../ManualWebhookApplicationLookupTest.php | 204 ++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 tests/Feature/Webhook/ManualWebhookApplicationLookupTest.php diff --git a/app/Http/Controllers/Webhook/Concerns/MatchesManualWebhookApplications.php b/app/Http/Controllers/Webhook/Concerns/MatchesManualWebhookApplications.php index decb6705fc..9fb910b49d 100644 --- a/app/Http/Controllers/Webhook/Concerns/MatchesManualWebhookApplications.php +++ b/app/Http/Controllers/Webhook/Concerns/MatchesManualWebhookApplications.php @@ -35,7 +35,13 @@ trait MatchesManualWebhookApplications */ protected function manualWebhookApplications(Builder $query, string $fullName): Collection { - return $query->get() + // Every repository form that manualWebhookRepositoryMatches() accepts + // contains the repository name, so the database discards unrelated + // applications that share the branch. The full match stays in PHP. + $pattern = '%'.str_replace('_', '\\_', mb_strtolower($fullName)).'%'; + + return $query->whereRaw("LOWER(git_repository) LIKE ? ESCAPE '\\'", [$pattern]) + ->get() ->filter(fn (Application $application): bool => $this->manualWebhookRepositoryMatches($application->git_repository, $fullName)) ->values(); } diff --git a/tests/Feature/Webhook/ManualWebhookApplicationLookupTest.php b/tests/Feature/Webhook/ManualWebhookApplicationLookupTest.php new file mode 100644 index 0000000000..cc9f9f634a --- /dev/null +++ b/tests/Feature/Webhook/ManualWebhookApplicationLookupTest.php @@ -0,0 +1,204 @@ +retrievedApplicationIds = collect(); + Event::listen('eloquent.retrieved: '.Application::class, fn (Application $application) => $this->retrievedApplicationIds->push($application->id)); +}); + +function manualLookupApplication(string $gitRepository, string $branch = 'main'): Application +{ + $team = Team::factory()->create(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $server = Server::factory()->create(['team_id' => $team->id]); + $server->settings->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); + $destination = $server->standaloneDockers()->firstOrFail(); + + return Application::create([ + 'name' => 'manual-lookup-app', + 'git_repository' => $gitRepository, + 'git_branch' => $branch, + 'build_pack' => 'nixpacks', + 'ports_exposes' => '3000', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ])->refresh(); +} + +function sendManualLookupPush(TestCase $test, string $provider, string $secret, string $repository = 'acme/app', string $branch = 'main'): TestResponse +{ + $server = ['REMOTE_ADDR' => '203.0.113.20', 'CONTENT_TYPE' => 'application/json']; + + if ($provider === 'gitlab') { + $payload = json_encode([ + 'object_kind' => 'push', + 'ref' => "refs/heads/{$branch}", + 'project' => ['path_with_namespace' => $repository], + 'after' => 'abc1234', + 'commits' => [], + ]); + + return $test->call('POST', '/webhooks/source/gitlab/events/manual', [], [], [], $server + ['HTTP_X-Gitlab-Token' => $secret], $payload); + } + + if ($provider === 'bitbucket') { + $payload = json_encode([ + 'push' => ['changes' => [['new' => ['name' => $branch, 'target' => ['hash' => 'abc1234']]]]], + 'repository' => ['full_name' => $repository], + ]); + + return $test->call('POST', '/webhooks/source/bitbucket/events/manual', [], [], [], $server + [ + 'HTTP_X-Event-Key' => 'repo:push', + 'HTTP_X-Hub-Signature' => 'sha256='.hash_hmac('sha256', $payload, $secret), + ], $payload); + } + + $payload = json_encode([ + 'ref' => "refs/heads/{$branch}", + 'repository' => ['full_name' => $repository], + 'after' => 'abc1234', + 'commits' => [], + ]); + + return $test->call('POST', "/webhooks/source/{$provider}/events/manual", [], [], [], $server + [ + $provider === 'github' ? 'HTTP_X-GitHub-Event' : 'HTTP_X-Gitea-Event' => 'push', + 'HTTP_X-Hub-Signature-256' => 'sha256='.hash_hmac('sha256', $payload, $secret), + ], $payload); +} + +function sendManualLookupGithubPullRequest(TestCase $test, Application $application, string $action, string $baseBranch = 'main'): TestResponse +{ + $payload = json_encode([ + 'action' => $action, + 'number' => 42, + 'repository' => ['id' => 987654321, 'full_name' => 'acme/app'], + 'pull_request' => [ + 'html_url' => 'https://github.com/acme/app/pull/42', + 'title' => 'Change', + 'head' => ['ref' => 'feature', 'sha' => 'e83c5163316f89bfbde7d9ab23ca2e25604af290', 'repo' => ['id' => 987654321]], + 'base' => ['ref' => $baseBranch, 'repo' => ['id' => 987654321]], + ], + ], JSON_THROW_ON_ERROR); + + return $test->call('POST', '/webhooks/source/github/events/manual', [], [], [], [ + 'HTTP_X-GitHub-Event' => 'pull_request', + 'HTTP_X-Hub-Signature-256' => 'sha256='.hash_hmac('sha256', $payload, $application->manual_webhook_secret_github), + 'CONTENT_TYPE' => 'application/json', + ], $payload); +} + +function manualLookupDeployed(Application $application): bool +{ + return ApplicationDeploymentQueue::query()->where('application_id', $application->id)->exists(); +} + +test('the candidate query filters by repository before applications are loaded', function (string $provider) { + $application = manualLookupApplication('https://github.com/acme/app'); + $unrelated = manualLookupApplication('https://github.com/other/service'); + $similarName = manualLookupApplication('https://github.com/acme/app-legacy'); + $this->retrievedApplicationIds = collect(); + + DB::enableQueryLog(); + sendManualLookupPush($this, $provider, $application->{"manual_webhook_secret_{$provider}"})->assertOk(); + $candidateQuery = collect(DB::getQueryLog())->first(fn (array $query): bool => str_contains($query['query'], 'from "applications" where "git_branch" = ?')); + + expect($candidateQuery['query'])->toContain('LOWER(git_repository) LIKE ?') + ->and($candidateQuery['bindings'])->toContain('%acme/app%') + ->and($this->retrievedApplicationIds)->not->toContain($unrelated->id) + ->and(manualLookupDeployed($application))->toBeTrue() + ->and(manualLookupDeployed($similarName))->toBeFalse(); +})->with(['github', 'gitlab', 'bitbucket', 'gitea']); + +test('every application of the pushed repository and branch is deployed', function (string $provider) { + $applications = collect([ + 'https://github.com/acme/app.git', + 'git@github.com:Acme/App.git', + 'ssh://git@git.example.com:2222/acme/app', + 'acme/app', + ])->map(fn (string $gitRepository): Application => manualLookupApplication($gitRepository)); + $secret = $applications->first()->{"manual_webhook_secret_{$provider}"}; + $applications->each(fn (Application $application) => $application->forceFill(["manual_webhook_secret_{$provider}" => $secret])->save()); + + sendManualLookupPush($this, $provider, $secret)->assertOk(); + + $applications->each(fn (Application $application) => expect(manualLookupDeployed($application))->toBeTrue()); +})->with(['github', 'gitlab', 'bitbucket', 'gitea']); + +test('identically named repositories on different git hosts deploy only the application whose secret signs the delivery', function () { + $githubApplication = manualLookupApplication('https://github.com/acme/app'); + $giteaApplication = manualLookupApplication('https://gitea.example.com/acme/app.git'); + + sendManualLookupPush($this, 'gitea', $giteaApplication->manual_webhook_secret_gitea)->assertOk(); + + expect(manualLookupDeployed($giteaApplication))->toBeTrue() + ->and(manualLookupDeployed($githubApplication))->toBeFalse(); +}); + +test('an underscore in the repository name matches only an underscore', function () { + $application = manualLookupApplication('https://github.com/acme/my_app'); + $wildcardMatch = manualLookupApplication('https://github.com/acme/myXapp'); + $this->retrievedApplicationIds = collect(); + + sendManualLookupPush($this, 'github', $application->manual_webhook_secret_github, 'acme/my_app')->assertOk(); + + expect(manualLookupDeployed($application))->toBeTrue() + ->and($this->retrievedApplicationIds)->not->toContain($wildcardMatch->id); +}); + +test('pushes that must not deploy an application', function (string $case, string $expectedMessage) { + $application = manualLookupApplication('https://github.com/acme/app', $case === 'other branch' ? 'develop' : 'main'); + $secret = $application->manual_webhook_secret_github; + match ($case) { + 'deleted application' => DB::table('applications')->where('id', $application->id)->update(['deleted_at' => now()]), + 'auto deploy disabled' => $application->settings->update(['is_auto_deploy_enabled' => false]), + 'invalid signature' => $secret = 'wrong-secret', + default => null, + }; + + $response = sendManualLookupPush($this, 'github', $secret)->assertOk(); + + expect($response->getContent())->toContain($expectedMessage) + ->and(manualLookupDeployed($application))->toBeFalse(); +})->with([ + 'other branch' => ['other branch', 'No matching application or invalid signature.'], + 'deleted application' => ['deleted application', 'No matching application or invalid signature.'], + 'auto deploy disabled' => ['auto deploy disabled', 'Deployments disabled.'], + 'invalid signature' => ['invalid signature', 'No matching application or invalid signature.'], +]); + +test('pull request events dispatch only applications of the repository', function (string $action) { + $application = manualLookupApplication('https://github.com/acme/app'); + $application->settings->update(['is_preview_deployments_enabled' => true]); + $unrelated = manualLookupApplication('https://github.com/other/service'); + $this->retrievedApplicationIds = collect(); + + sendManualLookupGithubPullRequest($this, $application, $action)->assertOk(); + + Queue::assertPushed(ProcessGithubPullRequestWebhook::class, 1); + Queue::assertPushed(ProcessGithubPullRequestWebhook::class, fn (ProcessGithubPullRequestWebhook $job): bool => $job->applicationId === $application->id && $job->action === $action); + expect($this->retrievedApplicationIds)->not->toContain($unrelated->id); +})->with(['opened', 'closed']); From 55605f65160d8aa70ac14d6ed68d043ed64c748f Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:43:16 +0100 Subject: [PATCH 05/30] fix(github-runners): select runner servers with a UNION of server ids An OR of two correlated EXISTS subqueries made Postgres estimate a per-server scan of the runner executions, so JIT compilation took most of the time of the per-minute reconcile query. Select the server ids from both runner tables with one UNION and index executions by status and server. Co-Authored-By: Claude Opus 5.5 --- app/Jobs/ReconcileGithubRunnersJob.php | 24 +++++-- ...ex_for_active_github_runner_executions.php | 41 ++++++++++++ ...concileGithubRunnersDisabledConfigTest.php | 67 +++++++++++++++++++ 3 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 database/migrations/2026_10_28_224219_add_index_for_active_github_runner_executions.php diff --git a/app/Jobs/ReconcileGithubRunnersJob.php b/app/Jobs/ReconcileGithubRunnersJob.php index f3adb513c4..e3447c87aa 100644 --- a/app/Jobs/ReconcileGithubRunnersJob.php +++ b/app/Jobs/ReconcileGithubRunnersJob.php @@ -43,11 +43,7 @@ class ReconcileGithubRunnersJob implements ShouldBeUnique, ShouldQueue public function handle(): void { - // Disabling keeps running jobs, so a disabled server is reconciled until its last runner is done. - Server::query() - ->where(fn (Builder $query) => $query - ->whereHas('githubRunnerConfig', fn (Builder $config) => $config->where('is_enabled', true)) - ->orWhereHas('githubRunnerExecutions', fn (Builder $execution) => $execution->whereIn('status', GithubRunnerStatus::occupying()))) + $this->serversToReconcile() ->with('settings') ->get() ->filter(fn (Server $server) => $server->isFunctional()) @@ -57,6 +53,24 @@ class ReconcileGithubRunnersJob implements ShouldBeUnique, ShouldQueue $this->reconcileQueued(); } + /** + * Servers with enabled runners or with executions that still occupy a runner. Disabling keeps + * running jobs, so a disabled server is reconciled until its last runner is done. The server ids + * come from one UNION: an OR of two EXISTS subqueries makes Postgres estimate a per-server scan + * of the executions and spend most of the query time on JIT compilation. + * + * @return Builder + */ + public function serversToReconcile(): Builder + { + return Server::query()->whereIn('id', GithubRunnerConfig::query() + ->select('server_id') + ->where('is_enabled', true) + ->union(GithubRunnerExecution::query() + ->select('server_id') + ->whereIn('status', GithubRunnerStatus::occupying()))); + } + /** * Removes executions that still occupy a runner of a deleted server (for example a server deleted * without model events), because they block deleting the GitHub App. diff --git a/database/migrations/2026_10_28_224219_add_index_for_active_github_runner_executions.php b/database/migrations/2026_10_28_224219_add_index_for_active_github_runner_executions.php new file mode 100644 index 0000000000..425849e2d7 --- /dev/null +++ b/database/migrations/2026_10_28_224219_add_index_for_active_github_runner_executions.php @@ -0,0 +1,41 @@ +index(['status', 'server_id'], 'idx_github_runner_executions_status_server_id'); + }); + } + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS idx_github_runner_executions_status_server_id'); + + return; + } + + if (Schema::hasIndex('github_runner_executions', 'idx_github_runner_executions_status_server_id')) { + Schema::table('github_runner_executions', function (Blueprint $table) { + $table->dropIndex('idx_github_runner_executions_status_server_id'); + }); + } + } +}; diff --git a/tests/Feature/ReconcileGithubRunnersDisabledConfigTest.php b/tests/Feature/ReconcileGithubRunnersDisabledConfigTest.php index 48cc9a95ce..8bdcda74ef 100644 --- a/tests/Feature/ReconcileGithubRunnersDisabledConfigTest.php +++ b/tests/Feature/ReconcileGithubRunnersDisabledConfigTest.php @@ -114,3 +114,70 @@ it('keeps reconciling a disabled server until its running jobs finish', function Process::assertRan(reconcileDisabledTestCommandFor($server)); expect($running->fresh()->status)->toBe(GithubRunnerStatus::Failed); }); + +function reconcileDisabledTestExecution(GithubApp $githubApp, Server $server, GithubRunnerStatus $status): GithubRunnerExecution +{ + static $jobId = 4000; + + return GithubRunnerExecution::create([ + 'github_app_id' => $githubApp->id, + 'server_id' => $server->id, + 'trigger_workflow_job_id' => ++$jobId, + 'labels' => ['self-hosted', 'coolify'], + 'status' => $status, + ]); +} + +function reconcileDisabledTestSelectedIds(): array +{ + return (new ReconcileGithubRunnersJob)->serversToReconcile()->pluck('id')->all(); +} + +it('selects servers with enabled runners or occupying executions', function () { + $enabled = reconcileDisabledTestServer($this->team, $this->githubApp, enabled: true); + $disabled = reconcileDisabledTestServer($this->team, $this->githubApp, enabled: false); + $disabledWithFinished = reconcileDisabledTestServer($this->team, $this->githubApp, enabled: false); + $executionOnly = Server::factory()->create(['team_id' => $this->team->id]); + $finishedOnly = Server::factory()->create(['team_id' => $this->team->id]); + + foreach ([GithubRunnerStatus::Queued, GithubRunnerStatus::Completed, GithubRunnerStatus::Failed, GithubRunnerStatus::Cancelled, GithubRunnerStatus::TimedOut] as $status) { + reconcileDisabledTestExecution($this->githubApp, $disabledWithFinished, $status); + reconcileDisabledTestExecution($this->githubApp, $finishedOnly, $status); + } + reconcileDisabledTestExecution($this->githubApp, $executionOnly, GithubRunnerStatus::Provisioning); + + expect(reconcileDisabledTestSelectedIds())->toEqualCanonicalizing([$enabled->id, $executionOnly->id]) + ->not->toContain($disabled->id); +}); + +it('selects each server once when several conditions match', function () { + $server = reconcileDisabledTestServer($this->team, $this->githubApp, enabled: true); + foreach (GithubRunnerStatus::occupying() as $status) { + reconcileDisabledTestExecution($this->githubApp, $server, $status); + } + + expect(reconcileDisabledTestSelectedIds())->toBe([$server->id]); +}); + +it('does not select soft-deleted servers', function () { + $server = reconcileDisabledTestServer($this->team, $this->githubApp, enabled: true); + reconcileDisabledTestExecution($this->githubApp, $server, GithubRunnerStatus::Running); + Server::query()->whereKey($server->id)->update(['deleted_at' => now()]); + + expect(reconcileDisabledTestSelectedIds())->toBe([]); +}); + +it('follows configuration and execution status changes', function () { + $server = reconcileDisabledTestServer($this->team, $this->githubApp, enabled: false); + expect(reconcileDisabledTestSelectedIds())->toBe([]); + + $server->githubRunnerConfig->update(['is_enabled' => true]); + expect(reconcileDisabledTestSelectedIds())->toBe([$server->id]); + + $server->githubRunnerConfig->update(['is_enabled' => false]); + $execution = reconcileDisabledTestExecution($this->githubApp, $server, GithubRunnerStatus::Idle); + expect(reconcileDisabledTestSelectedIds())->toBe([$server->id]); + + $execution->update(['status' => GithubRunnerStatus::Completed]); + expect(reconcileDisabledTestSelectedIds())->toBe([]); +}); From c34f5f8e3bbc52134e9aefbd605f4088a9e9931b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:47:55 +0100 Subject: [PATCH 06/30] fix(servers): index active server lookups by private key Co-Authored-By: Claude Opus 5.5 --- app/Livewire/Security/PrivateKey/Show.php | 1 - ...ndex_for_active_servers_by_private_key.php | 45 +++++ .../Security/PrivateKeyServerLookupTest.php | 184 ++++++++++++++++++ 3 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 database/migrations/2026_10_28_232000_add_index_for_active_servers_by_private_key.php create mode 100644 tests/Feature/Security/PrivateKeyServerLookupTest.php diff --git a/app/Livewire/Security/PrivateKey/Show.php b/app/Livewire/Security/PrivateKey/Show.php index 1b8f26ff28..1c9277d116 100644 --- a/app/Livewire/Security/PrivateKey/Show.php +++ b/app/Livewire/Security/PrivateKey/Show.php @@ -142,7 +142,6 @@ class Show extends Component $this->private_key->updatePrivateKey([ 'private_key' => formatPrivateKey($this->private_key->private_key), ]); - refresh_server_connection($this->private_key); $this->dispatch('success', 'Private key updated.'); if ($this->modalMode) { $this->dispatch('privateKeyUpdated'); diff --git a/database/migrations/2026_10_28_232000_add_index_for_active_servers_by_private_key.php b/database/migrations/2026_10_28_232000_add_index_for_active_servers_by_private_key.php new file mode 100644 index 0000000000..dd1187d9cc --- /dev/null +++ b/database/migrations/2026_10_28_232000_add_index_for_active_servers_by_private_key.php @@ -0,0 +1,45 @@ +index('private_key_id', 'idx_servers_private_key_id_active'); + }); + } + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS idx_servers_private_key_id_active'); + + return; + } + + if (Schema::hasIndex('servers', 'idx_servers_private_key_id_active')) { + Schema::table('servers', function (Blueprint $table) { + $table->dropIndex('idx_servers_private_key_id_active'); + }); + } + } +}; diff --git a/tests/Feature/Security/PrivateKeyServerLookupTest.php b/tests/Feature/Security/PrivateKeyServerLookupTest.php new file mode 100644 index 0000000000..3f8558b474 --- /dev/null +++ b/tests/Feature/Security/PrivateKeyServerLookupTest.php @@ -0,0 +1,184 @@ +whereKey(0)->exists()) { + $settings = new InstanceSettings; + $settings->id = 0; + $settings->save(); + } + Once::flush(); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + + session(['currentTeam' => $this->team]); + $this->actingAs($this->user); + + Config::set('cache.default', 'array'); + Config::set('constants.ssh.mux_enabled', true); + Storage::fake('ssh-keys'); + Process::fake(); + + $this->privateKey = teamKey($this->team); +}); + +function teamToken(User $user, Team $team): string +{ + $token = $user->createToken('test', ['write']); + $token->accessToken->forceFill(['team_id' => $team->id])->save(); + auth()->forgetGuards(); + + return $token->plainTextToken; +} + +function serverForKey(PrivateKey $privateKey, bool $deleted = false): Server +{ + $server = Server::factory()->create([ + 'team_id' => $privateKey->team_id, + 'private_key_id' => $privateKey->id, + ]); + if ($deleted) { + $server->delete(); + } + + return $server; +} + +function teamKey(Team $team): PrivateKey +{ + return PrivateKey::factory()->create([ + 'team_id' => $team->id, + 'private_key' => PrivateKey::generateNewKeyPair('ed25519')['private_key'], + ]); +} + +function assertMuxStops(Server $server, int $times): void +{ + Process::assertRanTimes(fn ($process) => str_contains($process->command, 'ssh -O stop') + && str_contains($process->command, 'mux_'.$server->uuid), $times); +} + +test('a key used by one or many active servers is in use', function () { + serverForKey($this->privateKey); + expect($this->privateKey->isInUse())->toBeTrue(); + + serverForKey($this->privateKey); + expect($this->privateKey->servers()->count())->toBe(2) + ->and($this->privateKey->isInUse())->toBeTrue(); +}); + +test('a key used only by soft-deleted servers is not in use and can be deleted', function () { + serverForKey($this->privateKey, deleted: true); + + expect($this->privateKey->servers()->exists())->toBeFalse() + ->and($this->privateKey->safeDelete())->toBeTrue() + ->and(PrivateKey::query()->whereKey($this->privateKey->id)->exists())->toBeFalse(); +}); + +test('an unreferenced key is not in use', function () { + serverForKey(teamKey($this->team)); + + expect($this->privateKey->isInUse())->toBeFalse(); +}); + +test('a key with id 0 is matched like any other key', function () { + $instanceKey = PrivateKey::factory()->make([ + 'team_id' => $this->team->id, + 'private_key' => PrivateKey::generateNewKeyPair('ed25519')['private_key'], + ]); + $instanceKey->id = 0; + $instanceKey->save(); + + expect($instanceKey->isInUse())->toBeFalse(); + + serverForKey($instanceKey); + + expect($instanceKey->isInUse())->toBeTrue() + ->and($instanceKey->safeDelete())->toBeFalse(); +}); + +test('a key without an id matches no servers', function () { + serverForKey($this->privateKey); + + expect((new PrivateKey)->isInUse())->toBeFalse(); + + refresh_server_connection(null); + refresh_server_connection(new PrivateKey); + + Process::assertNothingRan(); +}); + +test('refreshing connections stops the mux of every active server of the key only', function () { + $first = serverForKey($this->privateKey); + $second = serverForKey($this->privateKey); + $deleted = serverForKey($this->privateKey, deleted: true); + $other = serverForKey(teamKey($this->team)); + + refresh_server_connection($this->privateKey); + + assertMuxStops($first, 1); + assertMuxStops($second, 1); + assertMuxStops($deleted, 0); + assertMuxStops($other, 0); +}); + +test('changing the key content refreshes each server connection once', function () { + $server = serverForKey($this->privateKey); + $newKey = PrivateKey::generateNewKeyPair('ed25519')['private_key']; + + Livewire::test(Show::class, ['private_key_uuid' => $this->privateKey->uuid]) + ->set('privateKeyValue', $newKey) + ->call('changePrivateKey') + ->assertDispatched('success'); + + expect($this->privateKey->fresh()->private_key)->toBe(formatPrivateKey($newKey)); + assertMuxStops($server, 1); +}); + +test('a team member cannot change a key used by servers', function () { + $server = serverForKey($this->privateKey); + $member = User::factory()->create(); + $this->team->members()->attach($member->id, ['role' => 'member']); + $this->actingAs($member); + + Livewire::test(Show::class, ['private_key_uuid' => $this->privateKey->uuid]) + ->set('privateKeyValue', PrivateKey::generateNewKeyPair('ed25519')['private_key']) + ->call('changePrivateKey'); + + expect($this->privateKey->fresh()->private_key)->toBe($this->privateKey->private_key); + assertMuxStops($server, 0); +}); + +test('the api refuses to delete a key used by an active server of the team', function () { + serverForKey($this->privateKey); + $this->withToken(teamToken($this->user, $this->team)) + ->deleteJson("/api/v1/security/keys/{$this->privateKey->uuid}") + ->assertStatus(422); + + expect(PrivateKey::query()->whereKey($this->privateKey->id)->exists())->toBeTrue(); +}); + +test('the api does not reveal keys of another team', function () { + $otherKey = teamKey(Team::factory()->create()); + $this->withToken(teamToken($this->user, $this->team)) + ->deleteJson("/api/v1/security/keys/{$otherKey->uuid}") + ->assertNotFound(); + + expect(PrivateKey::query()->whereKey($otherKey->id)->exists())->toBeTrue(); +}); From 3d956c110ce38ebfdc81d11a4e9303eea7f96fb5 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:58:51 +0100 Subject: [PATCH 07/30] fix(servers): skip patch check results in bulk settings loads ServerManagerJob and the Docker cleanup step of ScheduledJobManager load the settings of every server with SELECT *. That reads the patch_check_result JSON, which holds the full update list of each server. With 12,709 rows, the scan took about 4 ms, but the full fetch took 390-536 ms and 294 MB of PHP memory. Without the column it took 74 ms and 111 MB. Add ServerSetting::withoutPatchCheckResult() and use it in both loops. Queued jobs reload the full settings through SerializesModels. Co-Authored-By: Claude Opus 5.5 --- .ai/lessons.md | 3 + app/Jobs/ScheduledJobManager.php | 3 +- app/Jobs/ServerManagerJob.php | 6 +- app/Models/ServerSetting.php | 16 +++ tests/Feature/ServerSettingsBulkLoadTest.php | 118 +++++++++++++++++++ 5 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 tests/Feature/ServerSettingsBulkLoadTest.php diff --git a/.ai/lessons.md b/.ai/lessons.md index 36a776d5a4..3452106a2e 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -154,3 +154,6 @@ ## Validate input safely - In PCRE, `$` also matches before a final `\n`. In validation regexes, use `\z` or reject control characters first, or a trailing newline passes. + +## Measure database reads end to end +- On PostgreSQL 15, `EXPLAIN (ANALYZE, BUFFERS)` does not detoast or send the selected columns, so a wide `jsonb`/`json`/`text` column does not show in it. Time the full fetch (`psql \timing` with `-o /dev/null`, or PDO `fetchAll`) as well before you blame or rule out an index. diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php index 7e137dadaa..48b4b0dae1 100644 --- a/app/Jobs/ScheduledJobManager.php +++ b/app/Jobs/ScheduledJobManager.php @@ -17,6 +17,7 @@ use Cron\CronExpression; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Relations\HasOne; use Illuminate\Database\Eloquent\Relations\MorphTo; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -741,7 +742,7 @@ class ScheduledJobManager implements ShouldQueue private function getServersForCleanupQuery(): Builder { - $query = Server::with('settings') + $query = Server::with(['settings' => fn (HasOne $query) => $query->withoutPatchCheckResult()]) ->whereNotNull('ip') ->where('ip', '!=', '') ->whereNotIn('ip', Server::PLACEHOLDER_IPS); diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php index 815bb09361..f21f66a3e3 100644 --- a/app/Jobs/ServerManagerJob.php +++ b/app/Jobs/ServerManagerJob.php @@ -8,6 +8,7 @@ use App\Models\Team; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Database\Eloquent\Relations\HasOne; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; @@ -78,11 +79,12 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue private function getServers(): Collection { - $allServers = Server::with(['settings', 'cloudProviderToken']); + $relations = ['settings' => fn (HasOne $query) => $query->withoutPatchCheckResult(), 'cloudProviderToken']; + $allServers = Server::with($relations); if (isCloud()) { $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get(); - $own = Team::find(0)->servers()->with(['settings', 'cloudProviderToken'])->get(); + $own = Team::find(0)->servers()->with($relations)->get(); return $servers->merge($own)->unique('id')->values(); } else { diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index f251b7fd5a..ae32835c10 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -5,10 +5,12 @@ namespace App\Models; use App\Enums\ServerRole; use App\Enums\TrafficIpMode; use Illuminate\Contracts\Encryption\DecryptException; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Facades\Schema; use OpenApi\Attributes as OA; #[OA\Schema( @@ -231,6 +233,9 @@ class ServerSetting extends Model 'geoip_maxmind_license_key', ]; + /** @var list|null */ + private static ?array $columnsWithoutPatchCheckResult = null; + protected static function booted() { static::creating(function ($setting) { @@ -465,6 +470,17 @@ class ServerSetting extends Model ); } + /** + * Selects every column except patch_check_result, which holds the full update list of the last patch + * check (often tens of kilobytes). Use it for instance-wide server loops that do not read the result. + */ + public function scopeWithoutPatchCheckResult(Builder $query): Builder + { + self::$columnsWithoutPatchCheckResult ??= array_values(array_diff(Schema::getColumnListing($this->getTable()), ['patch_check_result'])); + + return $query->select(self::$columnsWithoutPatchCheckResult); + } + /** * Summary of the saved patch check, or null when the server was never checked. * Results saved before security and reboot detection count as 0 and false. diff --git a/tests/Feature/ServerSettingsBulkLoadTest.php b/tests/Feature/ServerSettingsBulkLoadTest.php new file mode 100644 index 0000000000..befb493855 --- /dev/null +++ b/tests/Feature/ServerSettingsBulkLoadTest.php @@ -0,0 +1,118 @@ + true]); + InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']); + $this->team = Team::factory()->create(); + $this->patchResult = ['package_manager' => 'apt', 'total_updates' => 1, 'updates' => [['package' => 'openssl', 'new_version' => '3.0.13-2']]]; +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +function createBulkLoadServer(Team $team, array $settings): Server +{ + $server = Server::factory()->create(['team_id' => $team->id]); + $server->settings->forceFill([ + 'is_reachable' => true, + 'is_usable' => true, + 'server_timezone' => 'UTC', + ...$settings, + ])->saveQuietly(); + + return $server; +} + +/** + * @return list + */ +function serverSettingsQueriesDuring(Closure $callback): array +{ + $queries = []; + DB::listen(function ($query) use (&$queries) { + if (str_contains($query->sql, 'from "server_settings"')) { + $queries[] = $query->sql; + } + }); + $callback(); + + return $queries; +} + +it('loads every settings column except the patch result and matches each settings row to its server', function () { + $first = createBulkLoadServer($this->team, ['docker_cleanup_frequency' => '0 1 * * *', 'patch_check_result' => $this->patchResult]); + $second = createBulkLoadServer($this->team, ['docker_cleanup_frequency' => '0 2 * * *', 'patch_check_frequency' => 'daily']); + + $servers = Server::query() + ->whereKey([$first->id, $second->id]) + ->with(['settings' => fn (HasOne $query) => $query->withoutPatchCheckResult()]) + ->get() + ->keyBy('id'); + + foreach ([$first, $second] as $server) { + $full = ServerSetting::query()->where('server_id', $server->id)->firstOrFail()->getAttributes(); + unset($full['patch_check_result']); + + expect($servers[$server->id]->settings->getAttributes())->toBe($full); + } + expect($servers[$first->id]->settings->docker_cleanup_frequency)->toBe('0 1 * * *') + ->and($servers[$second->id]->settings->docker_cleanup_frequency)->toBe('0 2 * * *') + ->and($servers[$second->id]->settings->patch_check_frequency)->toBe('daily') + ->and($servers[$first->id]->settings->is_reachable)->toBeTrue(); +}); + +it('does not read the patch results of all servers in the server manager, and its jobs still get them', function () { + Queue::fake(); + Carbon::setTestNow(Carbon::create(2026, 10, 28, 12, 0, 0, 'UTC')); + $server = createBulkLoadServer($this->team, ['patch_check_frequency' => '* * * * *', 'patch_check_result' => $this->patchResult]); + + $queries = serverSettingsQueriesDuring(fn () => (new ServerManagerJob)->handle()); + + expect($queries)->not->toBeEmpty(); + foreach ($queries as $sql) { + expect($sql)->not->toContain('patch_check_result')->not->toContain('*'); + } + + Queue::assertPushed(ServerPatchCheckJob::class, function (ServerPatchCheckJob $job) use ($server) { + $restored = unserialize(serialize($job)); + + return $restored->server->id === $server->id + && $restored->server->settings->patch_check_result === $this->patchResult; + }); +}); + +it('does not read the patch results of all servers when it schedules Docker cleanups', function () { + Queue::fakeExcept(ScheduledJobManager::class); + Carbon::setTestNow(Carbon::create(2026, 10, 28, 12, 0, 0, 'UTC')); + $server = createBulkLoadServer($this->team, ['docker_cleanup_frequency' => '* * * * *', 'patch_check_result' => $this->patchResult]); + + $queries = serverSettingsQueriesDuring(fn () => (new ScheduledJobManager)->handle()); + + expect(collect($queries)->filter(fn (string $sql) => str_contains($sql, '"server_id" in')))->not->toBeEmpty(); + foreach ($queries as $sql) { + if (str_contains($sql, '"server_id" in')) { + expect($sql)->not->toContain('patch_check_result')->not->toContain('*'); + } + } + Queue::assertPushed(DockerCleanupJob::class, fn (DockerCleanupJob $job) => $job->server->id === $server->id); +}); From a12f62ed4fbc40b5276dfe96e44e73137d07bd5f Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:58:59 +0100 Subject: [PATCH 08/30] fix(dashboard): bound recent deployments query per application Co-Authored-By: Claude Opus 5.5 --- app/Livewire/Dashboard/ActiveDeployments.php | 63 ++++++++--- app/Models/ApplicationDeploymentQueue.php | 19 +++- ...index_for_dashboard_recent_deployments.php | 41 +++++++ tests/Feature/DashboardDeploymentsTest.php | 107 ++++++++++++++++++ 4 files changed, 213 insertions(+), 17 deletions(-) create mode 100644 database/migrations/2026_10_28_224423_add_index_for_dashboard_recent_deployments.php diff --git a/app/Livewire/Dashboard/ActiveDeployments.php b/app/Livewire/Dashboard/ActiveDeployments.php index 00601df807..b8d706bffe 100644 --- a/app/Livewire/Dashboard/ActiveDeployments.php +++ b/app/Livewire/Dashboard/ActiveDeployments.php @@ -6,6 +6,7 @@ use App\Enums\ApplicationDeploymentStatus; use App\Models\ApplicationDeploymentQueue; use Illuminate\Database\Eloquent\Builder; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\DB; use Livewire\Attributes\Computed; use Livewire\Component; @@ -28,6 +29,18 @@ class ActiveDeployments extends Component 'finished_at', ]; + private const ACTIVE_STATUSES = [ + ApplicationDeploymentStatus::IN_PROGRESS->value, + ApplicationDeploymentStatus::QUEUED->value, + ]; + + /** + * Not dehydrated, so it is loaded once per request and shared by both lists. + * + * @var Collection|null + */ + private ?Collection $teamApplicationIds = null; + /** * Polling target. The lists are loaded again on every render. */ @@ -37,10 +50,8 @@ class ActiveDeployments extends Component public function activeDeployments(): Collection { return $this->deploymentsQuery() - ->whereIn('status', [ - ApplicationDeploymentStatus::IN_PROGRESS->value, - ApplicationDeploymentStatus::QUEUED->value, - ]) + ->whereIn('application_id', $this->teamApplicationIds()) + ->whereIn('status', self::ACTIVE_STATUSES) ->orderBy('status') ->orderBy('id') ->limit(5) @@ -50,21 +61,47 @@ class ActiveDeployments extends Component #[Computed] public function recentDeployments(): Collection { - return $this->deploymentsQuery() - ->whereNotIn('status', [ - ApplicationDeploymentStatus::IN_PROGRESS->value, - ApplicationDeploymentStatus::QUEUED->value, - ]) + if (DB::connection()->getDriverName() !== 'pgsql') { + return $this->deploymentsQuery() + ->whereIn('application_id', $this->teamApplicationIds()) + ->whereNotIn('status', self::ACTIVE_STATUSES) + ->orderByDesc('id') + ->limit(5) + ->get(self::COLUMNS); + } + + // With "application_id IN (...) ORDER BY id DESC" PostgreSQL scans the primary key backwards and can + // read most of the table for a team without recent deployments. Take the five newest of each + // application from idx_deployment_queues_app_id instead; the five newest of the team are among them. + $newestPerApplication = ApplicationDeploymentQueue::query() + ->select(self::COLUMNS) + ->whereColumn('application_id', 'team_applications.id') + ->whereNotIn('status', self::ACTIVE_STATUSES) ->orderByDesc('id') + ->limit(5); + + return $this->deploymentsQuery() + ->fromRaw('unnest(?::varchar[]) as team_applications(id)', ['{'.$this->teamApplicationIds()->implode(',').'}']) + ->joinLateral($newestPerApplication, 'application_deployment_queues') + ->orderByDesc('application_deployment_queues.id') ->limit(5) - ->get(self::COLUMNS); + ->get(array_map(fn (string $column) => "application_deployment_queues.{$column}", self::COLUMNS)); } private function deploymentsQuery(): Builder { - return ApplicationDeploymentQueue::query() - ->with(['application.environment.project']) - ->ownedByTeam(currentTeam()->id); + return ApplicationDeploymentQueue::query()->with([ + 'application' => fn ($query) => $query->withoutGlobalScope('withRelations')->select(['id', 'environment_id']), + 'application.environment.project', + ]); + } + + /** + * @return Collection + */ + private function teamApplicationIds(): Collection + { + return $this->teamApplicationIds ??= ApplicationDeploymentQueue::applicationIdsOwnedByTeam(currentTeam()->id); } public function render() diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 3af70a5610..81019a288a 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -9,6 +9,7 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Carbon; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use OpenApi\Attributes as OA; @@ -166,13 +167,23 @@ class ApplicationDeploymentQueue extends Model */ public function scopeOwnedByTeam(Builder $query, int $teamId): Builder { - // application_id is varchar; whereHas joins to applications.id (bigint) and breaks on PostgreSQL. - $applicationIds = Application::withTrashed() + return $query->whereIn('application_id', self::applicationIdsOwnedByTeam($teamId)); + } + + /** + * IDs of the team's applications, deleted ones included, as strings to match application_id. + * + * application_id is varchar; whereHas joins to applications.id (bigint) and breaks on PostgreSQL. + * + * @return Collection + */ + public static function applicationIdsOwnedByTeam(int $teamId): Collection + { + return Application::withTrashed() + ->withoutGlobalScope('withRelations') ->whereRelation('environment.project.team', 'id', $teamId) ->pluck('id') ->map(fn ($id) => (string) $id); - - return $query->whereIn('application_id', $applicationIds); } public function server(): Attribute diff --git a/database/migrations/2026_10_28_224423_add_index_for_dashboard_recent_deployments.php b/database/migrations/2026_10_28_224423_add_index_for_dashboard_recent_deployments.php new file mode 100644 index 0000000000..982cb2f60b --- /dev/null +++ b/database/migrations/2026_10_28_224423_add_index_for_dashboard_recent_deployments.php @@ -0,0 +1,41 @@ +index(['application_id', 'id'], 'idx_deployment_queues_app_id'); + }); + } + } + + public function down(): void + { + if (DB::getDriverName() === 'pgsql') { + DB::statement('DROP INDEX CONCURRENTLY IF EXISTS idx_deployment_queues_app_id'); + + return; + } + + if (Schema::hasIndex('application_deployment_queues', 'idx_deployment_queues_app_id')) { + Schema::table('application_deployment_queues', function (Blueprint $table) { + $table->dropIndex('idx_deployment_queues_app_id'); + }); + } + } +}; diff --git a/tests/Feature/DashboardDeploymentsTest.php b/tests/Feature/DashboardDeploymentsTest.php index bec45a4985..41cd6f8da6 100644 --- a/tests/Feature/DashboardDeploymentsTest.php +++ b/tests/Feature/DashboardDeploymentsTest.php @@ -212,3 +212,110 @@ it('does not load deployment payload columns when the dashboard list polls', fun ->not->toContain('configuration_diff'); } }); + +function createDashboardApplication(array $overrides = []): Application +{ + return Application::factory()->create(array_merge([ + 'environment_id' => test()->environment->id, + 'destination_id' => test()->destination->id, + 'destination_type' => test()->destination->getMorphClass(), + ], $overrides)); +} + +it('lists the five newest eligible deployments of the team across its applications', function () { + $secondApplication = createDashboardApplication(); + $thirdApplication = createDashboardApplication(); + $otherTeam = Team::factory()->create(); + $otherProject = Project::factory()->create(['team_id' => $otherTeam->id]); + $otherApplication = createDashboardApplication([ + 'environment_id' => Environment::factory()->create(['project_id' => $otherProject->id])->id, + ]); + + $expected = []; + foreach (range(1, 4) as $round) { + foreach ([$this->application, $secondApplication, $thirdApplication] as $application) { + $expected[] = createDashboardDeployment([ + 'application_id' => $application->id, + 'status' => $round % 2 === 0 + ? ApplicationDeploymentStatus::FAILED->value + : ApplicationDeploymentStatus::FINISHED->value, + ])->id; + } + } + createDashboardDeployment(['application_id' => $otherApplication->id]); + createDashboardDeployment(['application_id' => $secondApplication->id, 'status' => ApplicationDeploymentStatus::QUEUED->value]); + createDashboardDeployment(['application_id' => $thirdApplication->id, 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value]); + + $component = Livewire::test(ActiveDeployments::class); + + expect($component->get('recentDeployments')->pluck('id')->all()) + ->toBe(array_slice(array_reverse($expected), 0, 5)); +}); + +it('lists fewer than five recent deployments when fewer are eligible', function () { + $finished = createDashboardDeployment()->id; + $failed = createDashboardDeployment(['status' => ApplicationDeploymentStatus::FAILED->value])->id; + createDashboardDeployment(['status' => ApplicationDeploymentStatus::QUEUED->value]); + + expect(Livewire::test(ActiveDeployments::class)->get('recentDeployments')->pluck('id')->all()) + ->toBe([$failed, $finished]); +}); + +it('keeps deployments of deleted applications and skips deployments without an application', function () { + $deletedApplication = createDashboardApplication(); + $deployment = createDashboardDeployment([ + 'application_id' => $deletedApplication->id, + 'application_name' => 'Deleted App Deployment', + ]); + $deletedApplication->delete(); + createDashboardDeployment([ + 'application_id' => '999999', + 'application_name' => 'Orphan Deployment', + ]); + + $component = Livewire::test(ActiveDeployments::class); + + expect($component->get('recentDeployments')->pluck('id')->all())->toBe([$deployment->id]) + ->and($component->get('recentDeployments')->first()->application)->toBeNull(); + $component->assertSee('Deleted App Deployment')->assertDontSee('Orphan Deployment'); +}); + +it('moves a deployment from the active to the recent list when its status changes', function () { + $deployment = createDashboardDeployment([ + 'application_name' => 'Status Change App', + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + ]); + + $component = Livewire::test(ActiveDeployments::class); + expect($component->get('activeDeployments')->pluck('id')->all())->toBe([$deployment->id]) + ->and($component->get('recentDeployments'))->toBeEmpty(); + + $deployment->setStatus(ApplicationDeploymentStatus::FINISHED->value); + $component->call('refreshDeployments'); + + expect($component->get('activeDeployments'))->toBeEmpty() + ->and($component->get('recentDeployments')->pluck('id')->all())->toBe([$deployment->id]); + $component->assertDontSee('Running or queued right now')->assertSee('Recent deployments'); +}); + +it('looks up the team applications once per request without loading full applications', function () { + createDashboardDeployment(); + createDashboardDeployment(['status' => ApplicationDeploymentStatus::IN_PROGRESS->value]); + + $applicationQueries = []; + DB::listen(function ($query) use (&$applicationQueries) { + if (str_contains($query->sql, 'from "applications"')) { + $applicationQueries[] = $query->sql; + } + }); + + Livewire::test(ActiveDeployments::class)->assertSee('Dashboard App'); + + $teamLookups = array_filter($applicationQueries, fn ($sql) => str_contains($sql, 'exists')); + expect($teamLookups)->toHaveCount(1); + foreach ($applicationQueries as $sql) { + expect($sql)->not->toMatch('/^select\s+("?\w+"?\.)?\*/i') + ->not->toContain('additional_servers_count') + ->not->toContain('additional_networks_count'); + } +}); From 3607abf7eb9c0b47e871f2fb87655e50af85a44c Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:59:34 +0100 Subject: [PATCH 09/30] fix(servers): load server manager servers in batches by id Co-Authored-By: Claude Opus 5.5 --- app/Jobs/ServerManagerJob.php | 120 +++++++++------- .../ServerManagerJobServerSelectionTest.php | 135 ++++++++++++++++++ 2 files changed, 201 insertions(+), 54 deletions(-) create mode 100644 tests/Feature/Server/ServerManagerJobServerSelectionTest.php diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php index f21f66a3e3..0d93b84d6b 100644 --- a/app/Jobs/ServerManagerJob.php +++ b/app/Jobs/ServerManagerJob.php @@ -4,7 +4,6 @@ namespace App\Jobs; use App\Models\InstanceSettings; use App\Models\Server; -use App\Models\Team; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -38,6 +37,11 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue */ private const CLOUD_PROVIDER_STATUS_CHECK_CRON = '*/5 * * * *'; + /** + * Number of servers loaded and processed at a time. + */ + public const SERVER_BATCH_SIZE = 500; + /** * Create a new job instance. */ @@ -60,44 +64,55 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue $this->instanceTimezone = config('app.timezone'); } - // Get all servers to process - $servers = $this->getServers(); + // Decide once per run: these cron keys are global, so a second call in the same run returns false. + $runCloudProviderStatusChecks = shouldRunCronNow(self::CLOUD_PROVIDER_STATUS_CHECK_CRON, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime); + $runConnectionChecks = shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime); - // Provider state checks run independently so slow APIs cannot block SSH checks. - $this->dispatchCloudProviderStatusChecks($servers); + $relations = ['settings' => fn (HasOne $query) => $query->withoutPatchCheckResult(), 'cloudProviderToken']; - $servers = $servers - ->reject(fn (Server $server) => $server->hasPlaceholderIp()) - ->values(); + foreach ($this->getServerIds()->chunk(self::SERVER_BATCH_SIZE) as $serverIds) { + $servers = Server::with($relations)->whereKey($serverIds)->orderBy('id')->get(); - // Dispatch ServerConnectionCheck for all servers efficiently - $this->dispatchConnectionChecks($servers); + // Provider state checks run independently so slow APIs cannot block SSH checks. + if ($runCloudProviderStatusChecks) { + $this->dispatchCloudProviderStatusChecks($servers); + } - // Process server-specific scheduled tasks - $this->processScheduledTasks($servers); + $servers = $servers + ->reject(fn (Server $server) => $server->hasPlaceholderIp()) + ->values(); + + if ($runConnectionChecks) { + $this->dispatchConnectionChecks($servers); + } + + // Process server-specific scheduled tasks + $this->processScheduledTasks($servers); + } } - private function getServers(): Collection + /** + * Selects only the IDs, so the paid-subscription filter runs once and the full server rows are loaded + * in batches by primary key. Loading every server in one query kept all of them in worker memory. + * toBase() skips model hydration, which an Eloquent pluck of the cast id column would do for every row. + * + * @return Collection + */ + private function getServerIds(): Collection { - $relations = ['settings' => fn (HasOne $query) => $query->withoutPatchCheckResult(), 'cloudProviderToken']; - $allServers = Server::with($relations); - - if (isCloud()) { - $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get(); - $own = Team::find(0)->servers()->with($relations)->get(); - - return $servers->merge($own)->unique('id')->values(); - } else { - return $allServers->get(); + if (! isCloud()) { + return Server::query()->orderBy('id')->toBase()->pluck('id'); } + + return Server::query()->whereRelation('team.subscription', 'stripe_invoice_paid', true)->toBase()->pluck('id') + ->merge(Server::query()->where('team_id', 0)->toBase()->pluck('id')) + ->unique() + ->sort() + ->values(); } private function dispatchCloudProviderStatusChecks(Collection $servers): void { - if (! shouldRunCronNow(self::CLOUD_PROVIDER_STATUS_CHECK_CRON, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime)) { - return; - } - $servers->each(function (Server $server) { $hasCloudResource = $server->hetzner_server_id || $server->vultr_instance_id @@ -112,34 +127,31 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue private function dispatchConnectionChecks(Collection $servers): void { - - if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) { - $servers->each(function (Server $server) { - try { - if ($server->hasPlaceholderIp()) { - return; - } - - // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity. - // Never skip while the server is marked unreachable or unusable: the heartbeat does not - // restore these flags, so deployments would fail with "Server is not functional" forever. - if ($server->isSentinelEnabled() && $server->isSentinelLive() - && $server->settings->is_reachable && $server->settings->is_usable) { - return; - } - if ($this->shouldSkipDueToBackoff($server)) { - return; - } - ServerConnectionCheckJob::dispatch($server); - } catch (\Exception $e) { - Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [ - 'server_id' => $server->id, - 'server_name' => $server->name, - 'error' => get_class($e).': '.$e->getMessage(), - ]); + $servers->each(function (Server $server) { + try { + if ($server->hasPlaceholderIp()) { + return; } - }); - } + + // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity. + // Never skip while the server is marked unreachable or unusable: the heartbeat does not + // restore these flags, so deployments would fail with "Server is not functional" forever. + if ($server->isSentinelEnabled() && $server->isSentinelLive() + && $server->settings->is_reachable && $server->settings->is_usable) { + return; + } + if ($this->shouldSkipDueToBackoff($server)) { + return; + } + ServerConnectionCheckJob::dispatch($server); + } catch (\Exception $e) { + Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [ + 'server_id' => $server->id, + 'server_name' => $server->name, + 'error' => get_class($e).': '.$e->getMessage(), + ]); + } + }); } private function processScheduledTasks(Collection $servers): void diff --git a/tests/Feature/Server/ServerManagerJobServerSelectionTest.php b/tests/Feature/Server/ServerManagerJobServerSelectionTest.php new file mode 100644 index 0000000000..e28573397f --- /dev/null +++ b/tests/Feature/Server/ServerManagerJobServerSelectionTest.php @@ -0,0 +1,135 @@ + 0, 'instance_timezone' => 'UTC']); + $this->rootTeam = Team::factory()->create(['id' => 0]); + config(['constants.coolify.self_hosted' => false]); + // Minute 0 is due for the Cloud connection check frequency (*/5). + Carbon::setTestNow('2026-07-11 12:00:00'); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +function createServerForSelectionTest(Team $team): Server +{ + $privateKey = PrivateKey::query()->firstWhere('team_id', $team->id) + ?? PrivateKey::factory()->create(['team_id' => $team->id]); + + return Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]); +} + +/** + * @return array server id => number of connection checks queued for it + */ +function connectionChecksPerServer(): array +{ + Queue::fake(); + (new ServerManagerJob)->handle(); + + return collect(Queue::pushed(ServerConnectionCheckJob::class)) + ->countBy(fn (ServerConnectionCheckJob $job) => $job->server->id) + ->sortKeys() + ->all(); +} + +it('processes each server of a paid team once, also with more than one paid subscription', function () { + $team = Team::factory()->create(); + Subscription::factory()->count(2)->create(['team_id' => $team->id, 'stripe_invoice_paid' => true]); + $server = createServerForSelectionTest($team); + + expect(connectionChecksPerServer())->toBe([$server->id => 1]); +}); + +it('skips servers of teams without a paid subscription', function () { + $unpaid = Team::factory()->create(); + Subscription::factory()->create(['team_id' => $unpaid->id, 'stripe_invoice_paid' => false]); + createServerForSelectionTest($unpaid); + + $pastDue = Team::factory()->create(); + Subscription::factory()->create(['team_id' => $pastDue->id, 'stripe_invoice_paid' => false, 'stripe_past_due' => true]); + createServerForSelectionTest($pastDue); + + createServerForSelectionTest(Team::factory()->create()); + + expect(connectionChecksPerServer())->toBe([]); +}); + +it('processes a team with one paid and one unpaid subscription', function () { + $team = Team::factory()->create(); + Subscription::factory()->create(['team_id' => $team->id, 'stripe_invoice_paid' => false]); + Subscription::factory()->create(['team_id' => $team->id, 'stripe_invoice_paid' => true]); + $server = createServerForSelectionTest($team); + + expect(connectionChecksPerServer())->toBe([$server->id => 1]); +}); + +it('skips soft-deleted servers of a paid team', function () { + $team = Team::factory()->create(); + Subscription::factory()->create(['team_id' => $team->id, 'stripe_invoice_paid' => true]); + $kept = createServerForSelectionTest($team); + createServerForSelectionTest($team)->delete(); + + expect(connectionChecksPerServer())->toBe([$kept->id => 1]); +}); + +it('processes root team servers once without a subscription', function () { + $rootServer = createServerForSelectionTest($this->rootTeam); + + expect(connectionChecksPerServer())->toBe([$rootServer->id => 1]); + + Subscription::factory()->create(['team_id' => 0, 'stripe_invoice_paid' => true]); + Carbon::setTestNow('2026-07-11 12:05:00'); + + expect(connectionChecksPerServer())->toBe([$rootServer->id => 1]); +}); + +it('follows subscription payment changes between runs', function () { + $team = Team::factory()->create(); + $subscription = Subscription::factory()->create(['team_id' => $team->id, 'stripe_invoice_paid' => false]); + $server = createServerForSelectionTest($team); + + expect(connectionChecksPerServer())->toBe([]); + + $subscription->update(['stripe_invoice_paid' => true]); + Carbon::setTestNow('2026-07-11 12:05:00'); + expect(connectionChecksPerServer())->toBe([$server->id => 1]); + + $subscription->update(['stripe_invoice_paid' => false]); + Carbon::setTestNow('2026-07-11 12:10:00'); + expect(connectionChecksPerServer())->toBe([]); +}); + +it('processes every server when they span more than one batch', function () { + $team = Team::factory()->create(); + Subscription::factory()->create(['team_id' => $team->id, 'stripe_invoice_paid' => true]); + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $servers = Server::factory() + ->count(ServerManagerJob::SERVER_BATCH_SIZE + 1) + ->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]); + + expect(connectionChecksPerServer()) + ->toBe($servers->pluck('id')->sort()->mapWithKeys(fn (int $id) => [$id => 1])->all()); +}); + +it('processes every server in self-hosted mode', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow('2026-07-11 12:01:00'); + $server = createServerForSelectionTest(Team::factory()->create()); + + expect(connectionChecksPerServer())->toBe([$server->id => 1]); +}); From f778942630954304c26fcb35ea70b365d807def7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:00:06 +0100 Subject: [PATCH 10/30] docs(ai): document hidden counts in application queries --- .ai/lessons.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ai/lessons.md b/.ai/lessons.md index 3452106a2e..7b3ba6f2be 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -155,5 +155,8 @@ ## Validate input safely - In PCRE, `$` also matches before a final `\n`. In validation regexes, use `\z` or reject control characters first, or a trailing newline passes. +## Application queries carry hidden counts +- `Application` has a global scope `withRelations` that adds two `withCount` subqueries and `applications.*`, also to `pluck('id')` and eager loads. For ID lookups and lean eager loads, call `withoutGlobalScope('withRelations')`. + ## Measure database reads end to end - On PostgreSQL 15, `EXPLAIN (ANALYZE, BUFFERS)` does not detoast or send the selected columns, so a wide `jsonb`/`json`/`text` column does not show in it. Time the full fetch (`psql \timing` with `-o /dev/null`, or PDO `fetchAll`) as well before you blame or rule out an index. From 0894251de94af7a51c5230495c397b2b7a9663e0 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:57:20 +0100 Subject: [PATCH 11/30] docs(api): remove unsupported preview and Highlight fields Remove is_preview from service environment variable request schemas and Highlight log drain fields from the generated OpenAPI specifications. --- .../Controllers/Api/ServicesController.php | 3 --- openapi.json | 18 ------------------ openapi.yaml | 12 +----------- 3 files changed, 1 insertion(+), 32 deletions(-) diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.php index 47f9d17db3..30d13ac286 100644 --- a/app/Http/Controllers/Api/ServicesController.php +++ b/app/Http/Controllers/Api/ServicesController.php @@ -1487,7 +1487,6 @@ class ServicesController extends Controller properties: [ 'key' => ['type' => 'string', 'description' => 'The key of the environment variable.'], 'value' => ['type' => 'string', 'description' => 'The value of the environment variable.'], - 'is_preview' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is used in preview deployments.'], 'is_literal' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is a literal, nothing espaced.'], 'is_multiline' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is multiline.'], 'is_shown_once' => ['type' => 'boolean', 'description' => 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'], @@ -1631,7 +1630,6 @@ class ServicesController extends Controller properties: [ 'key' => ['type' => 'string', 'description' => 'The key of the environment variable.'], 'value' => ['type' => 'string', 'description' => 'The value of the environment variable.'], - 'is_preview' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is used in preview deployments.'], 'is_literal' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is a literal, nothing espaced.'], 'is_multiline' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is multiline.'], 'is_shown_once' => ['type' => 'boolean', 'description' => 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'], @@ -1764,7 +1762,6 @@ class ServicesController extends Controller properties: [ 'key' => ['type' => 'string', 'description' => 'The key of the environment variable.'], 'value' => ['type' => 'string', 'description' => 'The value of the environment variable.'], - 'is_preview' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is used in preview deployments.'], 'is_literal' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is a literal, nothing espaced.'], 'is_multiline' => ['type' => 'boolean', 'description' => 'The flag to indicate if the environment variable is multiline.'], 'is_shown_once' => ['type' => 'boolean', 'description' => 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'], diff --git a/openapi.json b/openapi.json index e48c0d86e1..611d8f87af 100644 --- a/openapi.json +++ b/openapi.json @@ -20992,10 +20992,6 @@ "type": "string", "description": "The value of the environment variable." }, - "is_preview": { - "type": "boolean", - "description": "The flag to indicate if the environment variable is used in preview deployments." - }, "is_literal": { "type": "boolean", "description": "The flag to indicate if the environment variable is a literal, nothing espaced." @@ -21087,10 +21083,6 @@ "type": "string", "description": "The value of the environment variable." }, - "is_preview": { - "type": "boolean", - "description": "The flag to indicate if the environment variable is used in preview deployments." - }, "is_literal": { "type": "boolean", "description": "The flag to indicate if the environment variable is a literal, nothing espaced." @@ -21181,10 +21173,6 @@ "type": "string", "description": "The value of the environment variable." }, - "is_preview": { - "type": "boolean", - "description": "The flag to indicate if the environment variable is used in preview deployments." - }, "is_literal": { "type": "boolean", "description": "The flag to indicate if the environment variable is a literal, nothing espaced." @@ -25986,9 +25974,6 @@ "is_logdrain_custom_enabled": { "type": "boolean" }, - "is_logdrain_highlight_enabled": { - "type": "boolean" - }, "is_logdrain_newrelic_enabled": { "type": "boolean" }, @@ -26072,9 +26057,6 @@ "logdrain_custom_config_parser": { "type": "string" }, - "logdrain_highlight_project_id": { - "type": "string" - }, "logdrain_newrelic_base_uri": { "type": "string" }, diff --git a/openapi.yaml b/openapi.yaml index a0ed28c8e7..acaae528e9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -13221,9 +13221,6 @@ paths: value: type: string description: 'The value of the environment variable.' - is_preview: - type: boolean - description: 'The flag to indicate if the environment variable is used in preview deployments.' is_literal: type: boolean description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' @@ -13284,9 +13281,6 @@ paths: value: type: string description: 'The value of the environment variable.' - is_preview: - type: boolean - description: 'The flag to indicate if the environment variable is used in preview deployments.' is_literal: type: boolean description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' @@ -13341,7 +13335,7 @@ paths: properties: data: type: array - items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_preview: { type: boolean, description: 'The flag to indicate if the environment variable is used in preview deployments.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.' } }, type: object } + items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.' } }, type: object } type: object responses: '201': @@ -16630,8 +16624,6 @@ components: type: boolean is_logdrain_custom_enabled: type: boolean - is_logdrain_highlight_enabled: - type: boolean is_logdrain_newrelic_enabled: type: boolean is_metrics_enabled: @@ -16689,8 +16681,6 @@ components: type: string logdrain_custom_config_parser: type: string - logdrain_highlight_project_id: - type: string logdrain_newrelic_base_uri: type: string logdrain_newrelic_license_key: From cd992e72e10f96cf542909b535fd5ad6b5e7d8aa Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:53:44 +0100 Subject: [PATCH 12/30] feat(tables): filter tables by clicking a cell value Click a type, status, server, source, or role value in a table row to filter the table by that value. Click it again to remove the filter. Covers the environment resources, deployments, application and service backups, and team members tables. Co-Authored-By: Claude Opus 5.5 --- resources/css/app.css | 32 +++++++++++++++ .../application/backup/index.blade.php | 7 +++- .../application/deployment/index.blade.php | 39 ++++++++++++++++--- .../livewire/project/resource/index.blade.php | 23 ++++++++--- .../service/volume-backup/index.blade.php | 14 ++++++- .../views/livewire/team/member.blade.php | 8 ++-- 6 files changed, 105 insertions(+), 18 deletions(-) diff --git a/resources/css/app.css b/resources/css/app.css index 07aa70d5eb..453feea16d 100644 --- a/resources/css/app.css +++ b/resources/css/app.css @@ -3070,6 +3070,38 @@ input[type="search"]::-webkit-search-results-decoration { background: rgba(255, 255, 255, 0.02); } +/* A table cell value that filters the table by that value on click. */ +.table-filter-value { + position: relative; + z-index: 1; + max-width: 100%; + pointer-events: auto; + cursor: pointer; + text-align: left; + text-decoration-line: underline; + text-decoration-color: transparent; + text-underline-offset: 2px; + transition: color 0.15s, border-color 0.15s, text-decoration-color 0.15s; +} + +.table-filter-value:hover { + color: var(--color-black); + text-decoration-color: currentColor; +} + +.dark .table-filter-value:hover { + color: var(--color-white); +} + +.table-filter-value.table-filter-badge:hover { + text-decoration-color: transparent; + border-color: var(--color-neutral-400); +} + +.dark .table-filter-value.table-filter-badge:hover { + border-color: rgba(255, 255, 255, 0.3); +} + .env-table-grid { grid-template-columns: minmax(14rem, 2.5fr) 4.8rem 6rem 4rem 4.5rem 4.8rem 4.2rem 3rem; } diff --git a/resources/views/livewire/project/application/backup/index.blade.php b/resources/views/livewire/project/application/backup/index.blade.php index d60651df80..14745c8e7c 100644 --- a/resources/views/livewire/project/application/backup/index.blade.php +++ b/resources/views/livewire/project/application/backup/index.blade.php @@ -205,7 +205,12 @@ title="{{ $backup->targetName() }}"> {{ $backup->targetName() }} - {{ $backup->targetType() }} + + {{ $backup->targetType() }} + {{ $backup->frequency }} diff --git a/resources/views/livewire/project/application/deployment/index.blade.php b/resources/views/livewire/project/application/deployment/index.blade.php index f7875deba2..9e72863d45 100644 --- a/resources/views/livewire/project/application/deployment/index.blade.php +++ b/resources/views/livewire/project/application/deployment/index.blade.php @@ -236,6 +236,15 @@ (bool) data_get($deployment, 'is_api') => 'API', default => 'Manual', }; + $sourceKey = match (true) { + $pullRequestId > 0 => 'pull-request', + (bool) data_get($deployment, 'is_webhook') => 'webhook', + (bool) data_get($deployment, 'rollback') => 'rollback', + (bool) data_get($deployment, 'is_api') => 'api', + default => 'manual', + }; + $serverId = data_get($deployment, 'server_id'); + $serverName = data_get($deployment, 'server_name') ?: data_get($application, 'destination.server.name', '-'); $duration = match ($deploymentStatus) { 'queued' => 'Waiting', 'in_progress' => calculateDuration(data_get($deployment, 'created_at'), now()), @@ -264,9 +273,21 @@ x-bind:aria-busy="selecting === '{{ data_get($deployment, 'deployment_uuid') }}'" @endif> @unless ($embedded) - + + + @endunless - {{ $sourceLabel }} + @if ($embedded) + {{ $sourceLabel }} + @else + + + + @endif @if (data_get($deployment, 'commit')) @@ -285,7 +306,7 @@ @endif @if ($embedded) @endif @@ -295,9 +316,15 @@ @endunless {{ $duration }} - - {{ data_get($deployment, 'server_name') ?: data_get($application, 'destination.server.name', '-') }} - + @if (! $embedded && $serverId) + + + + @else + {{ $serverName }} + @endif @if ($embedded) -
+
+ +
-
- +
+ @@ -226,8 +232,13 @@ -
-
+
+ +