diff --git a/.ai/lessons.md b/.ai/lessons.md index 43de6bf878..3298a28d68 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -4,6 +4,7 @@ - Reproduce the reported failure on the unchanged baseline before adding a fix. - When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround. - Test old reports against the current branch because later changes can make the report obsolete. +- Test serialized worker tasks with the real process driver. An in-process test does not prove that captured models or nested closures survive serialization. - Use the same regression test before and after the production change so the result shows the behavior difference. ## Verify the complete user flow diff --git a/app/Livewire/Fleet/Metrics.php b/app/Livewire/Fleet/Metrics.php index 99c596d716..bd9580a8cc 100644 --- a/app/Livewire/Fleet/Metrics.php +++ b/app/Livewire/Fleet/Metrics.php @@ -8,7 +8,9 @@ use App\Models\Server; use App\Services\FleetMetricsAggregator; use App\Services\SentinelMetricsClient; use Illuminate\Contracts\View\View; +use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Concurrency; use Livewire\Attributes\Lazy; use Livewire\Attributes\Url; use Livewire\Component; @@ -16,6 +18,7 @@ use Livewire\Component; #[Lazy] class Metrics extends Component { + use AuthorizesRequests; use BuildsMetricsChartPayload; public string $chartId = 'fleet-metrics'; @@ -59,12 +62,39 @@ class Metrics extends Component public function mount(): void { - $allServers = Server::ownedByCurrentTeamCached(); + $this->refreshServers(); - $this->servers = $allServers + $this->normalizeContainerMetric(); + + if ($this->servers->isNotEmpty()) { + $this->loadData(); + } + } + + public function hydrate(): void + { + $previous = array_column($this->serverRows, 'uuid'); + $this->refreshServers(); + $current = $this->targetServers()->pluck('uuid')->all(); + sort($previous); + sort($current); + + if ($previous !== $current) { + $this->loadData(); + } + } + + private function refreshServers(): void + { + abort_unless(currentTeam() !== null, 403); + $this->servers = Server::ownedByCurrentTeam()->get() ->filter(fn (Server $server) => $server->isMetricsEnabled()) ->values(); + foreach ($this->servers as $server) { + $this->authorize('view', $server); + } + $this->serverOptions = $this->servers ->mapWithKeys(fn (Server $server) => [$server->uuid => $server->name]) ->all(); @@ -72,12 +102,6 @@ class Metrics extends Component if ($this->serverUuid !== '' && ! array_key_exists($this->serverUuid, $this->serverOptions)) { $this->serverUuid = ''; } - - $this->normalizeContainerMetric(); - - if ($this->servers->isNotEmpty()) { - $this->loadData(); - } } public function updatedServerUuid(): void @@ -125,21 +149,39 @@ class Metrics extends Component public function loadData(): void { - if ($this->servers->isEmpty()) { - return; - } + $this->refreshServers(); + $this->normalizeContainerMetric(); $rows = []; $containers = []; $series = ['cpu' => [], 'memory' => [], 'disk' => [], 'load' => [], 'networkRx' => [], 'networkTx' => []]; - foreach ($this->targetServers() as $server) { - $client = $this->metricsClient($server); + $targets = $this->targetServers(); + $clients = []; + $summaryTasks = []; + foreach ($targets as $server) { + $client = $clients[$server->uuid] = $this->metricsClient($server); + $summaryTasks[$server->uuid] = static fn () => $client->summary(); + } + $summaries = $this->runMetricTasks($summaryTasks); + $tasks = []; + $range = $this->range; + foreach ($clients as $uuid => $client) { + $metrics = $summaries[$uuid] === null ? ['cpu', 'memory'] : ['cpu', 'memory', 'disk', 'load']; + foreach ($metrics as $metric) { + $tasks[$uuid.':'.$metric] = static fn () => $client->history($metric, $range); + } + if ($summaries[$uuid] !== null) { + $tasks[$uuid.':containers'] = static fn () => $client->containersCurrent(); + $tasks[$uuid.':network'] = static fn () => $client->networkHistory($range); + } + } + $results = $this->runMetricTasks($tasks); - try { - $summary = $client->summary(); - } catch (\Throwable) { - $summary = null; + foreach ($targets as $server) { + $summary = $summaries[$server->uuid]; + foreach (['cpu', 'memory', 'disk', 'load'] as $metric) { + $series[$metric][] = $results[$server->uuid.':'.$metric] ?? []; } if ($summary === null) { @@ -148,7 +190,7 @@ class Metrics extends Component continue; } - $current = $client->containersCurrent(); + $current = $results[$server->uuid.':containers'] ?? []; $rows[] = array_merge( ['uuid' => $server->uuid, 'name' => $server->name, 'online' => true, 'containers' => count($current)], $summary @@ -160,11 +202,7 @@ class Metrics extends Component $containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]); } - $series['cpu'][] = $client->history('cpu', $this->range); - $series['memory'][] = $client->history('memory', $this->range); - $series['disk'][] = $client->history('disk', $this->range); - $series['load'][] = $client->history('load', $this->range); - $net = $client->networkHistory($this->range); + $net = $results[$server->uuid.':network'] ?? ['rx' => [], 'tx' => []]; $series['networkRx'][] = $net['rx']; $series['networkTx'][] = $net['tx']; } @@ -178,6 +216,31 @@ class Metrics extends Component $this->dispatch("refreshChartData-{$this->chartId}", $this->chartData); } + /** + * Bound the number of PHP workers and SSH connections for large fleets. + * + * @param array $tasks + * @return array + */ + protected function runMetricTasks(array $tasks): array + { + foreach ($tasks as $key => $task) { + $tasks[$key] = static function () use ($task) { + try { + return $task(); + } catch (\Throwable) { + return null; + } + }; + } + $results = []; + foreach (array_chunk($tasks, 8, true) as $batch) { + $results += Concurrency::run($batch); + } + + return $results; + } + /** * @return array */ @@ -193,6 +256,8 @@ class Metrics extends Component protected function metricsClient(Server $server): SentinelMetricsClient { + $server->settings->ensureValidSentinelToken(); + return app(SentinelMetricsClient::class, ['server' => $server]); } @@ -211,7 +276,7 @@ class Metrics extends Component } $teamId = currentTeam()?->id; - $resource = $teamId ? getResourceByUuid($id, $teamId) : null; + $resource = $teamId !== null ? getResourceByUuid($id, $teamId) : null; $image = null; $link = null; diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php index ab942e6e06..7d11edd852 100644 --- a/app/Services/SentinelMetricsClient.php +++ b/app/Services/SentinelMetricsClient.php @@ -108,7 +108,7 @@ class SentinelMetricsClient default => ['/cpu/history', 'percent'], }; - return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range); + return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range, $metric === 'disk'); } /** @@ -127,7 +127,7 @@ class SentinelMetricsClient /** * @return array */ - private function seriesRows(string $url, string $field, string $range): array + private function seriesRows(string $url, string $field, string $range, bool $rootOnly = false): array { try { $rows = json_decode($this->raw($url), true); @@ -140,9 +140,8 @@ class SentinelMetricsClient } // Disk history interleaves every mount; keep the root filesystem only. - $root = array_filter($rows, fn ($r) => ($r['mount'] ?? null) === '/'); - if ($root !== []) { - $rows = $root; + if ($rootOnly && array_any($rows, fn ($row) => array_key_exists('mount', $row))) { + $rows = array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/'); } // Average raw samples into epoch-aligned buckets so every server shares the same @@ -166,11 +165,14 @@ class SentinelMetricsClient */ private function rootMount(array $mounts): array { - if ($mounts === []) { - return ['used' => null, 'total' => null, 'percent' => null]; + $root = collect($mounts)->firstWhere('mount', '/'); + if ($root === null && ! array_any($mounts, fn ($mount) => array_key_exists('mount', $mount))) { + $root = $mounts[0] ?? null; } - $root = collect($mounts)->firstWhere('mount', '/') ?? $mounts[0]; + if ($root === null) { + return ['used' => null, 'total' => null, 'percent' => null]; + } return [ 'used' => (int) ($root['used'] ?? 0), @@ -206,7 +208,8 @@ class SentinelMetricsClient return instant_remote_process( ["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"], $this->server, - false + false, + timeout: 15 ); } diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php index 0cba5af8ab..cdf5a9c2e9 100644 --- a/app/Traits/HasMetrics.php +++ b/app/Traits/HasMetrics.php @@ -38,9 +38,11 @@ trait HasMetrics return null; } - $root = collect($rows)->filter(fn ($r) => ($r['mount'] ?? null) === '/')->values()->all(); + if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) { + $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/')); + } - return $this->mapRows($root ?: $rows, 'usedPercent', $mins); + return $this->mapRows($rows, 'usedPercent', $mins); } /** diff --git a/config/app.php b/config/app.php index 59aa6f4c28..43ce289433 100644 --- a/config/app.php +++ b/config/app.php @@ -11,6 +11,7 @@ use Illuminate\Auth\Passwords\PasswordResetServiceProvider; use Illuminate\Broadcasting\BroadcastServiceProvider; use Illuminate\Bus\BusServiceProvider; use Illuminate\Cache\CacheServiceProvider; +use Illuminate\Concurrency\ConcurrencyServiceProvider; use Illuminate\Cookie\CookieServiceProvider; use Illuminate\Database\DatabaseServiceProvider; use Illuminate\Encryption\EncryptionServiceProvider; @@ -217,6 +218,7 @@ return [ BroadcastServiceProvider::class, BusServiceProvider::class, CacheServiceProvider::class, + ConcurrencyServiceProvider::class, ConsoleSupportServiceProvider::class, CookieServiceProvider::class, DatabaseServiceProvider::class, diff --git a/resources/views/livewire/fleet/metrics.blade.php b/resources/views/livewire/fleet/metrics.blade.php index 6a1e7355f5..4ff5b14ddb 100644 --- a/resources/views/livewire/fleet/metrics.blade.php +++ b/resources/views/livewire/fleet/metrics.blade.php @@ -29,7 +29,8 @@ $serverListboxOptions = array_merge(
- +