From 79cf62323a1342afaab6cc94e40d5d7f629cfb6b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:42:04 +0200 Subject: [PATCH] fix(metrics): load the Metrics page when a server is slow or unreachable Co-Authored-By: Claude Opus 5.5 --- app/Livewire/Fleet/Metrics.php | 15 +++++++-- app/Services/SentinelMetricsClient.php | 4 ++- .../Fleet/MetricsChartPayloadTest.php | 6 ++-- .../Livewire/Fleet/MetricsDataTest.php | 32 ++++++++++++++++--- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/app/Livewire/Fleet/Metrics.php b/app/Livewire/Fleet/Metrics.php index ccce6ae820..a8e11ee0fe 100644 --- a/app/Livewire/Fleet/Metrics.php +++ b/app/Livewire/Fleet/Metrics.php @@ -11,6 +11,7 @@ use Illuminate\Contracts\View\View; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Concurrency; +use Illuminate\Support\Facades\Log; use Livewire\Attributes\Lazy; use Livewire\Attributes\Url; use Livewire\Component; @@ -228,6 +229,10 @@ class Metrics extends Component $clients = []; $summaryTasks = []; foreach ($targets as $server) { + // Coolify does not connect to an unreachable server, and its SSH retries would hold a worker past the process timeout. + if (! $server->isFunctional()) { + continue; + } $client = $clients[$server->uuid] = $this->metricsClient($server); $summaryTasks[$server->uuid] = static fn () => $client->summary(); } @@ -247,7 +252,7 @@ class Metrics extends Component $results = $this->runMetricTasks($tasks); foreach ($targets as $server) { - $summary = $summaries[$server->uuid]; + $summary = $summaries[$server->uuid] ?? null; foreach (['cpu', 'memory', 'disk', 'load'] as $metric) { $series[$metric][] = $results[$server->uuid.':'.$metric] ?? []; } @@ -303,7 +308,13 @@ class Metrics extends Component } $results = []; foreach (array_chunk($tasks, 8, true) as $batch) { - $results += Concurrency::run($batch); + try { + $results += Concurrency::run($batch); + } catch (\Throwable $e) { + // A worker that runs past the process timeout fails its batch. Show those metrics as missing, not a server error. + Log::warning('Fleet metrics batch failed', ['error' => $e->getMessage()]); + $results += array_fill_keys(array_keys($batch), null); + } } return $results; diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php index 615adc5357..6980861e6e 100644 --- a/app/Services/SentinelMetricsClient.php +++ b/app/Services/SentinelMetricsClient.php @@ -229,7 +229,9 @@ class SentinelMetricsClient ["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"], $this->server, false, - timeout: 15 + timeout: 15, + // Metrics are read again on the next refresh. SSH retries would keep a worker past its timeout. + maxAttempts: 1 ); } diff --git a/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php b/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php index 130b4c5ebe..f46da0a532 100644 --- a/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php +++ b/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php @@ -79,7 +79,7 @@ beforeEach(function () { it('dispatches a chart payload with every trend series', function () { $s = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id]); - $s->settings->update(['is_metrics_enabled' => true]); + $s->settings->forceFill(['is_metrics_enabled' => true, 'is_reachable' => true, 'is_usable' => true])->saveQuietly(); Livewire::test(ChartTestableMetrics::class) ->call('loadData') @@ -96,7 +96,7 @@ it('dispatches a chart payload with every trend series', function () { it('includes KPI-tile spark series aligned to one shared time axis', function () { $s = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id]); - $s->settings->update(['is_metrics_enabled' => true]); + $s->settings->forceFill(['is_metrics_enabled' => true, 'is_reachable' => true, 'is_usable' => true])->saveQuietly(); Livewire::test(ChartTestableMetrics::class) ->call('loadData') @@ -115,7 +115,7 @@ it('includes KPI-tile spark series aligned to one shared time axis', function () it('caps KPI-tile sparks to a readable point count on a shared axis', function () { $s = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id]); - $s->settings->update(['is_metrics_enabled' => true]); + $s->settings->forceFill(['is_metrics_enabled' => true, 'is_reachable' => true, 'is_usable' => true])->saveQuietly(); Livewire::test(DenseTestableMetrics::class) ->call('loadData') diff --git a/tests/Feature/Livewire/Fleet/MetricsDataTest.php b/tests/Feature/Livewire/Fleet/MetricsDataTest.php index c304fa2610..79154709d8 100644 --- a/tests/Feature/Livewire/Fleet/MetricsDataTest.php +++ b/tests/Feature/Livewire/Fleet/MetricsDataTest.php @@ -93,10 +93,11 @@ beforeEach(function () { $this->team = $team; }); -function metricsServer($team, $privateKey): Server +function metricsServer($team, $privateKey, bool $reachable = true): Server { $server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]); $server->settings->update(['is_metrics_enabled' => true]); + $server->settings->forceFill(['is_reachable' => $reachable, 'is_usable' => $reachable])->saveQuietly(); return $server; } @@ -112,7 +113,7 @@ function onlineSummary(array $overrides = []): array it('keeps an unreachable server in the table but excludes it from KPIs', function () { $ok = metricsServer($this->team, $this->privateKey); - $dead = metricsServer($this->team, $this->privateKey); + $dead = metricsServer($this->team, $this->privateKey, reachable: false); StubMetricsClient::$summaries = [ $ok->uuid => onlineSummary(['cpu' => 30.0]), @@ -134,7 +135,6 @@ it('keeps an unreachable server in the table but excludes it from KPIs', functio it('counts a reachable server as online when Sentinel returns no summary', function () { $server = metricsServer($this->team, $this->privateKey); $server->update(['name' => 'web-1']); - $server->settings->forceFill(['is_reachable' => true, 'is_usable' => true])->saveQuietly(); StubMetricsClient::$summaries = [$server->uuid => null]; $component = Livewire::test(TestableMetrics::class) @@ -165,7 +165,7 @@ it('ranks containers across servers by the selected metric', function () { it('shows the KPI tiles, a stale badge for offline servers, and the container toggle', function () { $ok = metricsServer($this->team, $this->privateKey); $ok->update(['name' => 'web-1']); - $dead = metricsServer($this->team, $this->privateKey); + $dead = metricsServer($this->team, $this->privateKey, reachable: false); $dead->update(['name' => 'web-2']); StubMetricsClient::$summaries = [ @@ -299,6 +299,30 @@ it('permits team members to read metrics', function () { expect(Livewire::test(TestableMetrics::class)->get('kpis')['serversOnline'])->toBe(1); }); +it('does not query Sentinel on a server that Coolify cannot reach', function () { + $server = metricsServer($this->team, $this->privateKey, reachable: false); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]]]]; + + $component = Livewire::test(TestableMetrics::class); + + $row = collect($component->get('serverRows'))->firstWhere('uuid', $server->uuid); + expect($row['online'])->toBeFalse(); + expect($row['reporting'])->toBeFalse(); + expect($component->get('chartData')['cpu'])->toBe([]); +}); + +it('shows the page when a metrics worker runs past the process timeout', function () { + $server = metricsServer($this->team, $this->privateKey); + Concurrency::shouldReceive('run')->andThrow(new RuntimeException('The process exceeded the timeout of 60 seconds.')); + + $component = Livewire::test(TestableMetrics::class)->assertOk(); + + $row = collect($component->get('serverRows'))->firstWhere('uuid', $server->uuid); + expect($row['reporting'])->toBeFalse(); + expect($component->get('kpis')['serversReporting'])->toBe(0); +}); + it('keeps host trends when an older Sentinel has no summary endpoint', function () { $server = metricsServer($this->team, $this->privateKey); StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]], 'memory' => [[1000, 100.0]]]];