From d77580cb35f4e9bf731369ad5dec8f02a2362098 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:56:39 +0200 Subject: [PATCH] fix(metrics): fetch fleet data concurrently and refresh server access Run metric requests in batches of eight workers, isolate endpoint failures, and preserve CPU and memory history when Sentinel lacks a summary endpoint. Repair Sentinel tokens before creating workers and bound SSH fetch timeouts. Reauthorize team servers on refresh and hydration, clear stale metrics after transfers or deletions, and resolve container metadata for team ID zero. Use only root filesystem samples when mount fields are present, while supporting legacy disk data. Add regression coverage for concurrent workers, access changes, endpoint failures, and disk mapping. --- .ai/lessons.md | 1 + app/Livewire/Fleet/Metrics.php | 113 ++++++++++--- app/Services/SentinelMetricsClient.php | 21 ++- app/Traits/HasMetrics.php | 6 +- config/app.php | 2 + .../views/livewire/fleet/metrics.blade.php | 3 +- .../Fleet/MetricsChartPayloadTest.php | 2 + .../Livewire/Fleet/MetricsDataTest.php | 155 +++++++++++++++++- tests/Support/ConcurrentMetricsClient.php | 36 ++++ tests/Unit/Metrics/HasMetricsMappingTest.php | 14 ++ .../Services/SentinelMetricsClientTest.php | 27 +++ 11 files changed, 343 insertions(+), 37 deletions(-) create mode 100644 tests/Support/ConcurrentMetricsClient.php diff --git a/.ai/lessons.md b/.ai/lessons.md index 43de6bf878..3298a28d68 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -4,6 +4,7 @@ - Reproduce the reported failure on the unchanged baseline before adding a fix. - When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround. - Test old reports against the current branch because later changes can make the report obsolete. +- Test serialized worker tasks with the real process driver. An in-process test does not prove that captured models or nested closures survive serialization. - Use the same regression test before and after the production change so the result shows the behavior difference. ## Verify the complete user flow diff --git a/app/Livewire/Fleet/Metrics.php b/app/Livewire/Fleet/Metrics.php index 99c596d716..bd9580a8cc 100644 --- a/app/Livewire/Fleet/Metrics.php +++ b/app/Livewire/Fleet/Metrics.php @@ -8,7 +8,9 @@ use App\Models\Server; use App\Services\FleetMetricsAggregator; use App\Services\SentinelMetricsClient; use Illuminate\Contracts\View\View; +use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Concurrency; use Livewire\Attributes\Lazy; use Livewire\Attributes\Url; use Livewire\Component; @@ -16,6 +18,7 @@ use Livewire\Component; #[Lazy] class Metrics extends Component { + use AuthorizesRequests; use BuildsMetricsChartPayload; public string $chartId = 'fleet-metrics'; @@ -59,12 +62,39 @@ class Metrics extends Component public function mount(): void { - $allServers = Server::ownedByCurrentTeamCached(); + $this->refreshServers(); - $this->servers = $allServers + $this->normalizeContainerMetric(); + + if ($this->servers->isNotEmpty()) { + $this->loadData(); + } + } + + public function hydrate(): void + { + $previous = array_column($this->serverRows, 'uuid'); + $this->refreshServers(); + $current = $this->targetServers()->pluck('uuid')->all(); + sort($previous); + sort($current); + + if ($previous !== $current) { + $this->loadData(); + } + } + + private function refreshServers(): void + { + abort_unless(currentTeam() !== null, 403); + $this->servers = Server::ownedByCurrentTeam()->get() ->filter(fn (Server $server) => $server->isMetricsEnabled()) ->values(); + foreach ($this->servers as $server) { + $this->authorize('view', $server); + } + $this->serverOptions = $this->servers ->mapWithKeys(fn (Server $server) => [$server->uuid => $server->name]) ->all(); @@ -72,12 +102,6 @@ class Metrics extends Component if ($this->serverUuid !== '' && ! array_key_exists($this->serverUuid, $this->serverOptions)) { $this->serverUuid = ''; } - - $this->normalizeContainerMetric(); - - if ($this->servers->isNotEmpty()) { - $this->loadData(); - } } public function updatedServerUuid(): void @@ -125,21 +149,39 @@ class Metrics extends Component public function loadData(): void { - if ($this->servers->isEmpty()) { - return; - } + $this->refreshServers(); + $this->normalizeContainerMetric(); $rows = []; $containers = []; $series = ['cpu' => [], 'memory' => [], 'disk' => [], 'load' => [], 'networkRx' => [], 'networkTx' => []]; - foreach ($this->targetServers() as $server) { - $client = $this->metricsClient($server); + $targets = $this->targetServers(); + $clients = []; + $summaryTasks = []; + foreach ($targets as $server) { + $client = $clients[$server->uuid] = $this->metricsClient($server); + $summaryTasks[$server->uuid] = static fn () => $client->summary(); + } + $summaries = $this->runMetricTasks($summaryTasks); + $tasks = []; + $range = $this->range; + foreach ($clients as $uuid => $client) { + $metrics = $summaries[$uuid] === null ? ['cpu', 'memory'] : ['cpu', 'memory', 'disk', 'load']; + foreach ($metrics as $metric) { + $tasks[$uuid.':'.$metric] = static fn () => $client->history($metric, $range); + } + if ($summaries[$uuid] !== null) { + $tasks[$uuid.':containers'] = static fn () => $client->containersCurrent(); + $tasks[$uuid.':network'] = static fn () => $client->networkHistory($range); + } + } + $results = $this->runMetricTasks($tasks); - try { - $summary = $client->summary(); - } catch (\Throwable) { - $summary = null; + foreach ($targets as $server) { + $summary = $summaries[$server->uuid]; + foreach (['cpu', 'memory', 'disk', 'load'] as $metric) { + $series[$metric][] = $results[$server->uuid.':'.$metric] ?? []; } if ($summary === null) { @@ -148,7 +190,7 @@ class Metrics extends Component continue; } - $current = $client->containersCurrent(); + $current = $results[$server->uuid.':containers'] ?? []; $rows[] = array_merge( ['uuid' => $server->uuid, 'name' => $server->name, 'online' => true, 'containers' => count($current)], $summary @@ -160,11 +202,7 @@ class Metrics extends Component $containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]); } - $series['cpu'][] = $client->history('cpu', $this->range); - $series['memory'][] = $client->history('memory', $this->range); - $series['disk'][] = $client->history('disk', $this->range); - $series['load'][] = $client->history('load', $this->range); - $net = $client->networkHistory($this->range); + $net = $results[$server->uuid.':network'] ?? ['rx' => [], 'tx' => []]; $series['networkRx'][] = $net['rx']; $series['networkTx'][] = $net['tx']; } @@ -178,6 +216,31 @@ class Metrics extends Component $this->dispatch("refreshChartData-{$this->chartId}", $this->chartData); } + /** + * Bound the number of PHP workers and SSH connections for large fleets. + * + * @param array $tasks + * @return array + */ + protected function runMetricTasks(array $tasks): array + { + foreach ($tasks as $key => $task) { + $tasks[$key] = static function () use ($task) { + try { + return $task(); + } catch (\Throwable) { + return null; + } + }; + } + $results = []; + foreach (array_chunk($tasks, 8, true) as $batch) { + $results += Concurrency::run($batch); + } + + return $results; + } + /** * @return array */ @@ -193,6 +256,8 @@ class Metrics extends Component protected function metricsClient(Server $server): SentinelMetricsClient { + $server->settings->ensureValidSentinelToken(); + return app(SentinelMetricsClient::class, ['server' => $server]); } @@ -211,7 +276,7 @@ class Metrics extends Component } $teamId = currentTeam()?->id; - $resource = $teamId ? getResourceByUuid($id, $teamId) : null; + $resource = $teamId !== null ? getResourceByUuid($id, $teamId) : null; $image = null; $link = null; diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php index ab942e6e06..7d11edd852 100644 --- a/app/Services/SentinelMetricsClient.php +++ b/app/Services/SentinelMetricsClient.php @@ -108,7 +108,7 @@ class SentinelMetricsClient default => ['/cpu/history', 'percent'], }; - return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range); + return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range, $metric === 'disk'); } /** @@ -127,7 +127,7 @@ class SentinelMetricsClient /** * @return array */ - private function seriesRows(string $url, string $field, string $range): array + private function seriesRows(string $url, string $field, string $range, bool $rootOnly = false): array { try { $rows = json_decode($this->raw($url), true); @@ -140,9 +140,8 @@ class SentinelMetricsClient } // Disk history interleaves every mount; keep the root filesystem only. - $root = array_filter($rows, fn ($r) => ($r['mount'] ?? null) === '/'); - if ($root !== []) { - $rows = $root; + if ($rootOnly && array_any($rows, fn ($row) => array_key_exists('mount', $row))) { + $rows = array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/'); } // Average raw samples into epoch-aligned buckets so every server shares the same @@ -166,11 +165,14 @@ class SentinelMetricsClient */ private function rootMount(array $mounts): array { - if ($mounts === []) { - return ['used' => null, 'total' => null, 'percent' => null]; + $root = collect($mounts)->firstWhere('mount', '/'); + if ($root === null && ! array_any($mounts, fn ($mount) => array_key_exists('mount', $mount))) { + $root = $mounts[0] ?? null; } - $root = collect($mounts)->firstWhere('mount', '/') ?? $mounts[0]; + if ($root === null) { + return ['used' => null, 'total' => null, 'percent' => null]; + } return [ 'used' => (int) ($root['used'] ?? 0), @@ -206,7 +208,8 @@ class SentinelMetricsClient return instant_remote_process( ["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"], $this->server, - false + false, + timeout: 15 ); } diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php index 0cba5af8ab..cdf5a9c2e9 100644 --- a/app/Traits/HasMetrics.php +++ b/app/Traits/HasMetrics.php @@ -38,9 +38,11 @@ trait HasMetrics return null; } - $root = collect($rows)->filter(fn ($r) => ($r['mount'] ?? null) === '/')->values()->all(); + if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) { + $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/')); + } - return $this->mapRows($root ?: $rows, 'usedPercent', $mins); + return $this->mapRows($rows, 'usedPercent', $mins); } /** diff --git a/config/app.php b/config/app.php index 59aa6f4c28..43ce289433 100644 --- a/config/app.php +++ b/config/app.php @@ -11,6 +11,7 @@ use Illuminate\Auth\Passwords\PasswordResetServiceProvider; use Illuminate\Broadcasting\BroadcastServiceProvider; use Illuminate\Bus\BusServiceProvider; use Illuminate\Cache\CacheServiceProvider; +use Illuminate\Concurrency\ConcurrencyServiceProvider; use Illuminate\Cookie\CookieServiceProvider; use Illuminate\Database\DatabaseServiceProvider; use Illuminate\Encryption\EncryptionServiceProvider; @@ -217,6 +218,7 @@ return [ BroadcastServiceProvider::class, BusServiceProvider::class, CacheServiceProvider::class, + ConcurrencyServiceProvider::class, ConsoleSupportServiceProvider::class, CookieServiceProvider::class, DatabaseServiceProvider::class, diff --git a/resources/views/livewire/fleet/metrics.blade.php b/resources/views/livewire/fleet/metrics.blade.php index 6a1e7355f5..4ff5b14ddb 100644 --- a/resources/views/livewire/fleet/metrics.blade.php +++ b/resources/views/livewire/fleet/metrics.blade.php @@ -29,7 +29,8 @@ $serverListboxOptions = array_merge(
- +