diff --git a/.ai/lessons.md b/.ai/lessons.md index 69790b0ee1..d1e140e5eb 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -4,6 +4,7 @@ - Reproduce the reported failure on the unchanged baseline before adding a fix. - When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround. - Test old reports against the current branch because later changes can make the report obsolete. +- Test serialized worker tasks with the real process driver. An in-process test does not prove that captured models or nested closures survive serialization. - Use the same regression test before and after the production change so the result shows the behavior difference. - Several dev instances can run from one checkout (see `./scripts/dev urls`). After you add a migration, run it on each running instance of that checkout, or those instances fail on the new code. - The checkout is shared with other sessions. When the full suite fails, rerun each failing file alone and compare with a clean `git archive HEAD` copy before you connect a failure to your change. diff --git a/DESIGN.md b/DESIGN.md index 1226dd0587..a482fa3e50 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -736,10 +736,14 @@ Charts follow the application metrics implementation: - downsample long time ranges before rendering. Only add a metric if Sentinel exposes historical data for it. Current Sentinel -history endpoints store CPU and memory. Root filesystem usage is included in -the periodic push payload for threshold notifications, but it is not stored as -a historical Sentinel metric and has no history endpoint, so it cannot power a -disk-usage graph yet. +history endpoints store CPU, memory, disk, network throughput, and load average +(Sentinel PR #47 added disk, network, and load). Bulk snapshot endpoints +(`/api/summary`, `/api/containers/current`) return the latest host and +per-container values in one call for fleet-wide views. + +The fleet Metrics page (`/metrics`, top-level sidebar) aggregates these across +every team server: KPI tiles, per-metric trend cards, a threshold-colored +servers table, and a fleet-wide hottest-containers list. --- @@ -759,6 +763,7 @@ Use these as implementation references: | Dense environment table and footer | `resources/views/livewire/project/shared/environment-variable/all.blade.php` | | Standard table toolbar controls | `resources/views/components/table/*` | | Application metrics charts | `resources/views/livewire/project/shared/metrics.blade.php` | +| Fleet metrics page | `resources/views/livewire/fleet/metrics.blade.php` | | Browser terminal workspace | `resources/views/livewire/terminal/index.blade.php` | | Layer card | `resources/views/components/application/settings-section.blade.php` | | Custom dropdown | `resources/views/components/forms/listbox.blade.php` | diff --git a/app/Livewire/Concerns/BuildsMetricsChartPayload.php b/app/Livewire/Concerns/BuildsMetricsChartPayload.php new file mode 100644 index 0000000000..12796c81e6 --- /dev/null +++ b/app/Livewire/Concerns/BuildsMetricsChartPayload.php @@ -0,0 +1,135 @@ + + */ + protected function buildChartPayload(array $seriesByServer): array + { + $cpu = FleetMetricsAggregator::sumSeriesByBucket($seriesByServer['cpu'], 'avg'); + $memory = FleetMetricsAggregator::sumSeriesByBucket($seriesByServer['memory'], 'sum'); + $disk = FleetMetricsAggregator::sumSeriesByBucket($seriesByServer['disk'], 'avg'); + $load = FleetMetricsAggregator::sumSeriesByBucket($seriesByServer['load'], 'avg'); + $networkRx = FleetMetricsAggregator::sumSeriesByBucket($seriesByServer['networkRx'], 'sum'); + $networkTx = FleetMetricsAggregator::sumSeriesByBucket($seriesByServer['networkTx'], 'sum'); + + // Throughput spark = rx + tx per bucket. + $network = $this->combineSeries($networkRx, $networkTx); + $categories = $this->sparkCategories($cpu, $memory, $disk, $network); + + [$sparkCategories, $sparks] = $this->capSparks($categories, [ + 'cpuSpark' => $this->alignSpark($cpu, $categories), + 'memSpark' => $this->alignSpark($memory, $categories), + 'diskSpark' => $this->alignSpark($disk, $categories), + 'netSpark' => $this->alignSpark($network, $categories), + ], self::SPARK_MAX_POINTS); + + return [ + 'range' => $this->range, + 'cpu' => $cpu, + 'memory' => $memory, + 'disk' => $disk, + 'load' => $load, + 'networkRx' => $networkRx, + 'networkTx' => $networkTx, + // KPI-tile sparklines: values-only series on one shared, capped time axis. + 'sparkCategories' => $sparkCategories, + ] + $sparks; + } + + /** + * Thin every spark and the shared axis to at most $max points using the same + * evenly-spaced indices (first and last always kept), so the series stay aligned. + * + * @param array $categories + * @param array> $sparks + * @return array{0: array, 1: array>} + */ + private function capSparks(array $categories, array $sparks, int $max): array + { + $count = count($categories); + if ($count <= $max || $max < 2) { + return [$categories, $sparks]; + } + + $indices = []; + for ($i = 0; $i < $max; $i++) { + $indices[] = (int) round($i * ($count - 1) / ($max - 1)); + } + $indices = array_values(array_unique($indices)); + + $pick = fn (array $values): array => array_values(array_map(fn (int $i) => $values[$i], $indices)); + + return [$pick($categories), array_map($pick, $sparks)]; + } + + /** + * Sum two [ts, value] series bucket-by-bucket into one sorted [ts, value] series. + * + * @param array $a + * @param array $b + * @return array + */ + private function combineSeries(array $a, array $b): array + { + $map = []; + foreach ([$a, $b] as $series) { + foreach ($series as [$ts, $value]) { + $map[$ts] = ($map[$ts] ?? 0.0) + (float) $value; + } + } + ksort($map); + + return array_map(fn ($ts, $value) => [(int) $ts, $value], array_keys($map), array_values($map)); + } + + /** + * Sorted union of bucket timestamps across every series (the shared spark x-axis). + * + * @param array ...$seriesList + * @return array + */ + private function sparkCategories(array ...$seriesList): array + { + $set = []; + foreach ($seriesList as $series) { + foreach ($series as [$ts]) { + $set[(int) $ts] = true; + } + } + $categories = array_keys($set); + sort($categories); + + return $categories; + } + + /** + * Project a [ts, value] series onto the shared category axis, leaving gaps as null. + * + * @param array $series + * @param array $categories + * @return array + */ + private function alignSpark(array $series, array $categories): array + { + $map = []; + foreach ($series as [$ts, $value]) { + $map[(int) $ts] = $value; + } + + return array_map(fn ($ts) => $map[$ts] ?? null, $categories); + } +} diff --git a/app/Livewire/Fleet/Metrics.php b/app/Livewire/Fleet/Metrics.php new file mode 100644 index 0000000000..bd9580a8cc --- /dev/null +++ b/app/Livewire/Fleet/Metrics.php @@ -0,0 +1,311 @@ + uuid => name */ + public array $serverOptions = []; + + #[Url(as: 'range')] + public string $range = '24h'; + + #[Url(as: 'server')] + public string $serverUuid = ''; + + #[Url(as: 'cmetric')] + public string $containerMetric = 'cpu'; + + private const CONTAINER_METRICS = ['cpu', 'memory', 'disk', 'network']; + + public bool $live = false; + + /** @var array */ + public array $kpis = []; + + /** @var array> */ + public array $serverRows = []; + + /** @var array> All fetched containers, kept so the metric toggle re-ranks without re-fetching. */ + public array $containersRaw = []; + + /** @var array> */ + public array $topContainers = []; + + /** @var array Trend series for the charts, also embedded for first paint. */ + public array $chartData = []; + + /** @var array */ + protected array $containerMetaCache = []; + + public function mount(): void + { + $this->refreshServers(); + + $this->normalizeContainerMetric(); + + if ($this->servers->isNotEmpty()) { + $this->loadData(); + } + } + + public function hydrate(): void + { + $previous = array_column($this->serverRows, 'uuid'); + $this->refreshServers(); + $current = $this->targetServers()->pluck('uuid')->all(); + sort($previous); + sort($current); + + if ($previous !== $current) { + $this->loadData(); + } + } + + private function refreshServers(): void + { + abort_unless(currentTeam() !== null, 403); + $this->servers = Server::ownedByCurrentTeam()->get() + ->filter(fn (Server $server) => $server->isMetricsEnabled()) + ->values(); + + foreach ($this->servers as $server) { + $this->authorize('view', $server); + } + + $this->serverOptions = $this->servers + ->mapWithKeys(fn (Server $server) => [$server->uuid => $server->name]) + ->all(); + + if ($this->serverUuid !== '' && ! array_key_exists($this->serverUuid, $this->serverOptions)) { + $this->serverUuid = ''; + } + } + + public function updatedServerUuid(): void + { + $this->loadData(); + } + + public function updatedContainerMetric(): void + { + $this->normalizeContainerMetric(); + + // Re-rank the already-fetched containers; no need to re-hit Sentinel. + $this->topContainers = FleetMetricsAggregator::rankContainers($this->containersRaw, $this->containerMetric); + } + + private function normalizeContainerMetric(): void + { + if (! in_array($this->containerMetric, self::CONTAINER_METRICS, true)) { + $this->containerMetric = 'cpu'; + } + } + + public function setRange(string $range): void + { + $this->range = in_array($range, ['24h', '7d', '30d'], true) ? $range : '24h'; + $this->loadData(); + } + + public function isLivePollable(): bool + { + return $this->live && $this->range === '24h'; + } + + /** + * @return Collection + */ + protected function targetServers(): Collection + { + if ($this->serverUuid !== '') { + return $this->servers->filter(fn (Server $s) => $s->uuid === $this->serverUuid)->values(); + } + + return $this->servers; + } + + public function loadData(): void + { + $this->refreshServers(); + $this->normalizeContainerMetric(); + + $rows = []; + $containers = []; + $series = ['cpu' => [], 'memory' => [], 'disk' => [], 'load' => [], 'networkRx' => [], 'networkTx' => []]; + + $targets = $this->targetServers(); + $clients = []; + $summaryTasks = []; + foreach ($targets as $server) { + $client = $clients[$server->uuid] = $this->metricsClient($server); + $summaryTasks[$server->uuid] = static fn () => $client->summary(); + } + $summaries = $this->runMetricTasks($summaryTasks); + $tasks = []; + $range = $this->range; + foreach ($clients as $uuid => $client) { + $metrics = $summaries[$uuid] === null ? ['cpu', 'memory'] : ['cpu', 'memory', 'disk', 'load']; + foreach ($metrics as $metric) { + $tasks[$uuid.':'.$metric] = static fn () => $client->history($metric, $range); + } + if ($summaries[$uuid] !== null) { + $tasks[$uuid.':containers'] = static fn () => $client->containersCurrent(); + $tasks[$uuid.':network'] = static fn () => $client->networkHistory($range); + } + } + $results = $this->runMetricTasks($tasks); + + foreach ($targets as $server) { + $summary = $summaries[$server->uuid]; + foreach (['cpu', 'memory', 'disk', 'load'] as $metric) { + $series[$metric][] = $results[$server->uuid.':'.$metric] ?? []; + } + + if ($summary === null) { + $rows[] = $this->offlineRow($server); + + continue; + } + + $current = $results[$server->uuid.':containers'] ?? []; + $rows[] = array_merge( + ['uuid' => $server->uuid, 'name' => $server->name, 'online' => true, 'containers' => count($current)], + $summary + ); + + foreach ($current as $c) { + // Names/images are resolved lazily for the displayed rows only (containerMeta), + // so a server with many containers doesn't trigger a resolve storm here. + $containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]); + } + + $net = $results[$server->uuid.':network'] ?? ['rx' => [], 'tx' => []]; + $series['networkRx'][] = $net['rx']; + $series['networkTx'][] = $net['tx']; + } + + $this->serverRows = $rows; + $this->kpis = FleetMetricsAggregator::fleetKpis($rows); + $this->containersRaw = $containers; + $this->topContainers = FleetMetricsAggregator::rankContainers($containers, $this->containerMetric); + $this->chartData = $this->buildChartPayload($series); + + $this->dispatch("refreshChartData-{$this->chartId}", $this->chartData); + } + + /** + * Bound the number of PHP workers and SSH connections for large fleets. + * + * @param array $tasks + * @return array + */ + protected function runMetricTasks(array $tasks): array + { + foreach ($tasks as $key => $task) { + $tasks[$key] = static function () use ($task) { + try { + return $task(); + } catch (\Throwable) { + return null; + } + }; + } + $results = []; + foreach (array_chunk($tasks, 8, true) as $batch) { + $results += Concurrency::run($batch); + } + + return $results; + } + + /** + * @return array + */ + protected function offlineRow(Server $server): array + { + return [ + 'uuid' => $server->uuid, 'name' => $server->name, 'online' => false, + 'cpu' => null, 'memUsed' => null, 'memTotal' => null, + 'diskUsed' => null, 'diskTotal' => null, 'diskPercent' => null, + 'load1' => null, 'netRx' => null, 'netTx' => null, 'containers' => null, + ]; + } + + protected function metricsClient(Server $server): SentinelMetricsClient + { + $server->settings->ensureValidSentinelToken(); + + return app(SentinelMetricsClient::class, ['server' => $server]); + } + + /** + * Resolve a Sentinel container id (the resource uuid Sentinel keys by) to a + * team-owned resource name, docker image, and metrics link, memoized. Shows the raw + * id and no image when no team-owned resource matches, so a Sentinel id never + * discloses another team's resource. + * + * @return array{name: string, image: ?string, link: ?string} + */ + public function containerMeta(string $id): array + { + if (isset($this->containerMetaCache[$id])) { + return $this->containerMetaCache[$id]; + } + + $teamId = currentTeam()?->id; + $resource = $teamId !== null ? getResourceByUuid($id, $teamId) : null; + + $image = null; + $link = null; + if ($resource) { + $image = data_get($resource, 'docker_registry_image_name') ?: data_get($resource, 'image'); + + if ($resource instanceof Application && data_get($resource, 'environment.project.uuid')) { + $link = route('project.application.metrics', [ + 'project_uuid' => $resource->environment->project->uuid, + 'environment_uuid' => $resource->environment->uuid, + 'application_uuid' => $resource->uuid, + ]); + } + } + + return $this->containerMetaCache[$id] = [ + 'name' => $resource?->name ?? $id, + 'image' => $image ?: null, + 'link' => $link, + ]; + } + + public function placeholder(): View + { + return view('livewire.fleet.metrics-placeholder'); + } + + public function render(): View + { + return view('livewire.fleet.metrics'); + } +} diff --git a/app/Livewire/Server/Charts.php b/app/Livewire/Server/Charts.php index c77662315a..6ff1f682fa 100644 --- a/app/Livewire/Server/Charts.php +++ b/app/Livewire/Server/Charts.php @@ -104,15 +104,37 @@ class Charts extends Component try { $cpuMetrics = $this->server->getCpuMetrics($this->interval); $memoryMetrics = $this->server->getMemoryMetrics($this->interval); + // Disk/network/load arrived in a later Sentinel; fetch each optionally so an + // older Sentinel (which 404s them) still shows CPU and memory. + $diskMetrics = $this->optionalMetric(fn () => $this->server->getDiskMetrics($this->interval)); + $loadMetrics = $this->optionalMetric(fn () => $this->server->getLoadMetrics($this->interval)); + $networkMetrics = $this->optionalMetric(fn () => $this->server->getNetworkMetrics($this->interval)) ?? []; $this->dispatch("refreshChartData-{$this->chartId}-metrics", [ 'cpuSeries' => $cpuMetrics, 'memorySeries' => $memoryMetrics, + 'diskSeries' => $diskMetrics, + 'loadSeries' => $loadMetrics, + 'networkRxSeries' => $networkMetrics['rx'] ?? [], + 'networkTxSeries' => $networkMetrics['tx'] ?? [], ]); } catch (\Throwable $e) { return handleError($e, $this); } } + /** + * Fetch a metric that may not exist on an older Sentinel; a failure yields null + * rather than breaking the whole chart refresh. + */ + private function optionalMetric(callable $fetch): mixed + { + try { + return $fetch(); + } catch (\Throwable) { + return null; + } + } + /** * Metrics settings belong to Sentinel, so they use the same event as the Sentinel page and API. * diff --git a/app/Services/FleetMetricsAggregator.php b/app/Services/FleetMetricsAggregator.php new file mode 100644 index 0000000000..29e0459d3b --- /dev/null +++ b/app/Services/FleetMetricsAggregator.php @@ -0,0 +1,136 @@ +> $rows + * @return array + */ + public static function fleetKpis(array $rows): array + { + $online = array_values(array_filter($rows, fn ($r) => $r['online'] === true)); + + $cpuValues = self::present($online, 'cpu'); + $cpuAvg = $cpuValues === [] ? 0.0 : round(array_sum($cpuValues) / count($cpuValues), 1); + + $busiest = null; + foreach ($online as $r) { + if ($r['cpu'] !== null && ($busiest === null || $r['cpu'] > $busiest['percent'])) { + $busiest = ['name' => $r['name'], 'percent' => (float) $r['cpu']]; + } + } + + $memUsed = (int) self::sum($online, 'memUsed'); + $memTotal = (int) self::sum($online, 'memTotal'); + $diskUsed = (int) self::sum($online, 'diskUsed'); + $diskTotal = (int) self::sum($online, 'diskTotal'); + $load1 = self::present($online, 'load1'); + + $needsAttention = 0; + foreach ($online as $r) { + $hot = PressureLevel::for($r['cpu'], 'cpu') !== 'ok' + || PressureLevel::for(self::percent($r['memUsed'], $r['memTotal']), 'memory') !== 'ok' + || PressureLevel::for($r['diskPercent'], 'disk') !== 'ok'; + if ($hot) { + $needsAttention++; + } + } + + return [ + 'serversOnline' => count($online), + 'serversTotal' => count($rows), + 'needsAttention' => $needsAttention, + 'cpuAvg' => $cpuAvg, + 'cpuApproximate' => count($cpuValues) > 1, + 'cpuBusiest' => $busiest, + 'memUsed' => $memUsed, + 'memTotal' => $memTotal, + 'memPercent' => self::percent($memUsed, $memTotal), + 'diskUsed' => $diskUsed, + 'diskTotal' => $diskTotal, + 'diskPercent' => self::percent($diskUsed, $diskTotal), + 'netRx' => (float) self::sum($online, 'netRx'), + 'netTx' => (float) self::sum($online, 'netTx'), + 'loadBusiest' => $load1 === [] ? 0.0 : max($load1), + 'loadAvg' => $load1 === [] ? 0.0 : round(array_sum($load1) / count($load1), 2), + 'containers' => (int) self::sum($online, 'containers'), + ]; + } + + /** + * @param array> $seriesPerServer + * @return array + */ + public static function sumSeriesByBucket(array $seriesPerServer, string $mode): array + { + $totals = []; + $counts = []; + + foreach ($seriesPerServer as $series) { + foreach ($series as [$ts, $value]) { + $totals[$ts] = ($totals[$ts] ?? 0.0) + (float) $value; + $counts[$ts] = ($counts[$ts] ?? 0) + 1; + } + } + + ksort($totals); + + $out = []; + foreach ($totals as $ts => $sum) { + $out[] = [(int) $ts, $mode === 'avg' ? round($sum / $counts[$ts], 2) : round($sum, 2)]; + } + + return $out; + } + + /** + * @param array> $containers + * @return array> + */ + public static function rankContainers(array $containers, string $metric, int $limit = 20): array + { + $key = match ($metric) { + 'memory' => 'memUsed', + 'disk' => 'diskBytes', + 'network' => 'net', + default => 'cpu', + }; + + usort($containers, fn ($a, $b) => ($b[$key] ?? 0) <=> ($a[$key] ?? 0)); + + return array_slice($containers, 0, $limit); + } + + /** + * @param array> $rows + * @return array + */ + private static function present(array $rows, string $key): array + { + return array_values(array_map( + fn ($r) => (float) $r[$key], + array_filter($rows, fn ($r) => $r[$key] !== null) + )); + } + + /** + * @param array> $rows + */ + private static function sum(array $rows, string $key): float + { + return array_sum(array_map(fn ($r) => $r[$key] ?? 0, $rows)); + } + + private static function percent(int|float|null $used, int|float|null $total): float + { + if (! $total) { + return 0.0; + } + + return round(($used / $total) * 100, 1); + } +} diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php new file mode 100644 index 0000000000..7d11edd852 --- /dev/null +++ b/app/Services/SentinelMetricsClient.php @@ -0,0 +1,230 @@ +startOfMinute(); + + $from = match ($range) { + '7d' => $now->subDays(7), + '30d' => $now->subDays(30), + default => $now->subDay(), + }; + + return $from->toIso8601ZuluString(); + } + + /** Bucket width per range, kept at roughly 300 points per series. */ + public static function bucketMs(string $range): int + { + return match ($range) { + '7d' => 30 * 60_000, + '30d' => 2 * 60 * 60_000, + default => 5 * 60_000, + }; + } + + /** + * @return array|null + */ + public function summary(): ?array + { + try { + $json = json_decode($this->raw($this->url('/summary')), true); + } catch (\Throwable) { + return null; + } + + if (! is_array($json)) { + return null; + } + + $disk = $this->rootMount($json['disk'] ?? []); + + return [ + 'online' => true, + 'cpu' => isset($json['cpu']['percent']) ? (float) $json['cpu']['percent'] : null, + 'memUsed' => isset($json['memory']['used']) ? (int) $json['memory']['used'] : null, + 'memTotal' => isset($json['memory']['total']) ? (int) $json['memory']['total'] : null, + 'diskUsed' => $disk['used'], + 'diskTotal' => $disk['total'], + 'diskPercent' => $disk['percent'], + 'load1' => isset($json['load']['load1']) ? (float) $json['load']['load1'] : null, + 'netRx' => isset($json['network']['rxBytesPerSec']) ? (float) $json['network']['rxBytesPerSec'] : null, + 'netTx' => isset($json['network']['txBytesPerSec']) ? (float) $json['network']['txBytesPerSec'] : null, + ]; + } + + /** + * @return array> + */ + public function containersCurrent(): array + { + try { + $rows = json_decode($this->raw($this->url('/containers/current')), true); + } catch (\Throwable) { + return []; + } + + if (! is_array($rows)) { + return []; + } + + return array_values(array_map(function ($c) { + $disk = ($c['disk']['writableLayer'] ?? 0) + ($c['disk']['volumesTotal'] ?? 0); + $net = ($c['network']['rxBytesPerSec'] ?? 0) + ($c['network']['txBytesPerSec'] ?? 0); + + return [ + 'id' => (string) ($c['id'] ?? ''), + 'cpu' => isset($c['cpu']['percent']) ? (float) $c['cpu']['percent'] : null, + 'memUsed' => isset($c['memory']['used']) ? (int) $c['memory']['used'] : null, + 'memPercent' => isset($c['memory']['usedPercent']) ? (float) $c['memory']['usedPercent'] : null, + 'diskBytes' => isset($c['disk']) ? (int) $disk : null, + 'net' => isset($c['network']) ? (float) $net : null, + ]; + }, $rows)); + } + + /** + * @return array + */ + public function history(string $metric, string $range): array + { + [$path, $field] = match ($metric) { + 'memory' => ['/memory/history', 'used'], + 'disk' => ['/disk/history', 'usedPercent'], + 'load' => ['/load/history', 'load1'], + default => ['/cpu/history', 'percent'], + }; + + return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range, $metric === 'disk'); + } + + /** + * @return array{rx: array, tx: array} + */ + public function networkHistory(string $range): array + { + $url = $this->url('/network/history', ['from' => self::rangeFrom($range)]); + + return [ + 'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range), + 'tx' => $this->seriesRows($url, 'txBytesPerSec', $range), + ]; + } + + /** + * @return array + */ + private function seriesRows(string $url, string $field, string $range, bool $rootOnly = false): array + { + try { + $rows = json_decode($this->raw($url), true); + } catch (\Throwable) { + return []; + } + + if (! is_array($rows)) { + return []; + } + + // Disk history interleaves every mount; keep the root filesystem only. + if ($rootOnly && array_any($rows, fn ($row) => array_key_exists('mount', $row))) { + $rows = array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/'); + } + + // Average raw samples into epoch-aligned buckets so every server shares the same + // timestamps and the fleet aggregator can sum them point by point. + $bucketMs = self::bucketMs($range); + $sums = []; + $counts = []; + foreach ($rows as $r) { + $bucket = intdiv((int) ($r['time'] ?? 0), $bucketMs) * $bucketMs; + $sums[$bucket] = ($sums[$bucket] ?? 0.0) + (float) ($r[$field] ?? 0); + $counts[$bucket] = ($counts[$bucket] ?? 0) + 1; + } + ksort($sums); + + return array_map(fn ($bucket) => [$bucket, round($sums[$bucket] / $counts[$bucket], 2)], array_keys($sums)); + } + + /** + * @param array> $mounts + * @return array{used: ?int, total: ?int, percent: ?float} + */ + private function rootMount(array $mounts): array + { + $root = collect($mounts)->firstWhere('mount', '/'); + if ($root === null && ! array_any($mounts, fn ($mount) => array_key_exists('mount', $mount))) { + $root = $mounts[0] ?? null; + } + + if ($root === null) { + return ['used' => null, 'total' => null, 'percent' => null]; + } + + return [ + 'used' => (int) ($root['used'] ?? 0), + 'total' => (int) ($root['total'] ?? 0), + 'percent' => (float) ($root['usedPercent'] ?? 0), + ]; + } + + /** + * @param array $query + */ + private function url(string $path, array $query = []): string + { + $qs = empty($query) ? '' : '?'.str_replace('%3A', ':', http_build_query($query)); + + return $this->base.$path.$qs; + } + + private function cacheKey(string $url): string + { + return 'metrics:'.$this->server->uuid.':'.md5($url); + } + + protected function raw(string $url): string + { + return Cache::remember($this->cacheKey($url), 60, fn () => $this->guard($this->remoteFetch($url))); + } + + protected function remoteFetch(string $url): string + { + $token = $this->server->settings->ensureValidSentinelToken(); + + return instant_remote_process( + ["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"], + $this->server, + false, + timeout: 15 + ); + } + + private function guard(string $response): string + { + $payload = json_decode($response, true); + + if ($payload === null) { + throw new \RuntimeException('Metrics returned an invalid response.'); + } + + if (is_array($payload) && array_key_exists('error', $payload)) { + throw new \RuntimeException('Metrics request failed.'); + } + + return $response; + } +} diff --git a/app/Support/Metrics/PressureLevel.php b/app/Support/Metrics/PressureLevel.php new file mode 100644 index 0000000000..2904c56504 --- /dev/null +++ b/app/Support/Metrics/PressureLevel.php @@ -0,0 +1,39 @@ += $error => 'error', + $percent >= $warn => 'warning', + default => 'ok', + }; + } + + public static function textClass(string $level): string + { + return match ($level) { + 'error' => 'text-red-500 dark:text-red-400', + 'warning' => 'text-orange-500 dark:text-warning', + default => 'text-black dark:text-fg', + }; + } + + public static function barClass(string $level): string + { + return match ($level) { + 'error' => 'bg-red-500 dark:bg-red-400', + 'warning' => 'bg-orange-500 dark:bg-warning', + default => 'bg-neutral-400 dark:bg-white/30', + }; + } +} diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php index 712f09a48b..cdf5a9c2e9 100644 --- a/app/Traits/HasMetrics.php +++ b/app/Traits/HasMetrics.php @@ -27,7 +27,63 @@ trait HasMetrics return convertContainerMemoryBytesToMegabytes($metrics); } + /** + * Root filesystem usage percentage over time. Disk history interleaves every + * mount, so we keep only the root mount before mapping. + */ + public function getDiskMetrics(int $mins = 5): ?array + { + $rows = $this->fetchMetricRows('disk', $mins); + if ($rows === null) { + return null; + } + + if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) { + $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/')); + } + + return $this->mapRows($rows, 'usedPercent', $mins); + } + + /** + * Host load average (1 minute) over time. + */ + public function getLoadMetrics(int $mins = 5): ?array + { + return $this->getMetrics('load', $mins, 'load1'); + } + + /** + * Network throughput over time, as separate receive/transmit byte-rate series. + * + * @return array{rx: ?array, tx: ?array}|null + */ + public function getNetworkMetrics(int $mins = 5): ?array + { + $rows = $this->fetchMetricRows('network', $mins); + if ($rows === null) { + return null; + } + + return [ + 'rx' => $this->mapRows($rows, 'rxBytesPerSec', $mins), + 'tx' => $this->mapRows($rows, 'txBytesPerSec', $mins), + ]; + } + private function getMetrics(string $type, int $mins, string $valueField): ?array + { + return $this->mapRows($this->fetchMetricRows($type, $mins), $valueField, $mins); + } + + /** + * Fetch and decode the raw Sentinel history rows for a metric type, or null when + * metrics are disabled. Throws on a Sentinel error response (caller decides whether + * to treat that as fatal or optional). + * + * @return array>|null + */ + protected function fetchMetricRows(string $type, int $mins): ?array { $server = $this->getMetricsServer(); if (! $server->isMetricsEnabled()) { @@ -63,7 +119,21 @@ trait HasMetrics throw new \Exception($error); } - $metrics = collect(json_decode($response, true))->map(function ($metric) use ($valueField) { + return json_decode($response, true) ?: []; + } + + /** + * Map raw rows to [timestampMs, value] pairs, downsampling long ranges. + * + * @param array>|null $rows + */ + private function mapRows(?array $rows, string $valueField, int $mins): ?array + { + if ($rows === null) { + return null; + } + + $metrics = collect($rows)->map(function ($metric) use ($valueField) { return [(int) $metric['time'], (float) ($metric[$valueField] ?? 0.0)]; })->toArray(); diff --git a/config/app.php b/config/app.php index 59aa6f4c28..43ce289433 100644 --- a/config/app.php +++ b/config/app.php @@ -11,6 +11,7 @@ use Illuminate\Auth\Passwords\PasswordResetServiceProvider; use Illuminate\Broadcasting\BroadcastServiceProvider; use Illuminate\Bus\BusServiceProvider; use Illuminate\Cache\CacheServiceProvider; +use Illuminate\Concurrency\ConcurrencyServiceProvider; use Illuminate\Cookie\CookieServiceProvider; use Illuminate\Database\DatabaseServiceProvider; use Illuminate\Encryption\EncryptionServiceProvider; @@ -217,6 +218,7 @@ return [ BroadcastServiceProvider::class, BusServiceProvider::class, CacheServiceProvider::class, + ConcurrencyServiceProvider::class, ConsoleSupportServiceProvider::class, CookieServiceProvider::class, DatabaseServiceProvider::class, diff --git a/resources/css/app.css b/resources/css/app.css index c963d109fe..57d75d4b4d 100644 --- a/resources/css/app.css +++ b/resources/css/app.css @@ -3155,6 +3155,26 @@ input[type="search"]::-webkit-search-results-decoration { grid-template-columns: minmax(9rem, 1.15fr) minmax(14rem, 1.9fr) 7rem 6.5rem 6.5rem 5.5rem; } +.fleet-servers-table-grid { + grid-template-columns: minmax(9rem, 1.4fr) 8rem 8rem 8rem 5rem 8rem 5.5rem; +} + +.fleet-containers-table-grid { + grid-template-columns: 2.25rem minmax(9rem, 1.7fr) minmax(6rem, 0.9fr) minmax(8.5rem, 1fr); +} + +.scheduled-executions-table-grid { + grid-template-columns: 5.5rem minmax(9rem, 1fr) minmax(7rem, 0.7fr) 8.5rem 5rem minmax(12rem, 1.5fr); +} + +.scheduler-runs-table-grid { + grid-template-columns: 9rem minmax(12rem, 1.5fr) 6rem 6rem 6rem; +} + +.skipped-jobs-table-grid { + grid-template-columns: 9rem 7rem minmax(10rem, 1fr) minmax(12rem, 1.4fr); +} + .server-resources-managed-table-grid { grid-template-columns: minmax(10rem, 1.3fr) minmax(8rem, 1fr) minmax(8rem, 1fr) 8rem 9.5rem; } diff --git a/resources/views/components/navbar.blade.php b/resources/views/components/navbar.blade.php index 361b7fc701..0ca9120973 100644 --- a/resources/views/components/navbar.blade.php +++ b/resources/views/components/navbar.blade.php @@ -101,6 +101,14 @@ Analytics +
  • + + + Metrics + +
    • @can('canAccessTerminal')
  • + + {{ $percent === null ? '-' : round($percent).'%' }} + + + diff --git a/resources/views/livewire/fleet/metrics-placeholder.blade.php b/resources/views/livewire/fleet/metrics-placeholder.blade.php new file mode 100644 index 0000000000..0ff339a199 --- /dev/null +++ b/resources/views/livewire/fleet/metrics-placeholder.blade.php @@ -0,0 +1,38 @@ +
    + {{-- Header (real chrome; only the data below is a skeleton) --}} +
    +
    +

    Metrics

    +

    + Resource usage across every server, reported by Sentinel. +

    +
    + + {{-- Filter bar --}} +
    + + +
    +
    + + + + + +
    + @foreach (['CPU', 'Memory used', 'Network throughput', 'Load average', 'Disk usage'] as $label) + $label === 'CPU'])> + + + @endforeach +
    + + + + + + + + +
    diff --git a/resources/views/livewire/fleet/metrics.blade.php b/resources/views/livewire/fleet/metrics.blade.php new file mode 100644 index 0000000000..4ff5b14ddb --- /dev/null +++ b/resources/views/livewire/fleet/metrics.blade.php @@ -0,0 +1,431 @@ + '', 'label' => 'All servers']], + collect($serverOptions)->map(fn ($name, $uuid) => ['value' => $uuid, 'label' => $name])->values()->all(), +); +?> +
    + + Metrics | Coolify + + + {{-- Header --}} +
    +
    +

    Metrics

    +

    + Resource usage across every server, reported by Sentinel. +

    +
    + + @if ($servers->isNotEmpty()) +
    +
    + + +
    + +
    + @if ($range === '24h') +
    + +
    + @endif +
    + @foreach (['24h' => '24 hours', '7d' => '7 days', '30d' => '30 days'] as $value => $label) + + @endforeach +
    +
    +
    + @endif +
    + + @if ($this->isLivePollable()) + + @endif + + @if ($servers->isEmpty()) + + +     + View servers + + + + @else + {{-- Overview KPIs --}} + +
    +
    + Servers online + {{ $kpis['serversOnline'] ?? 0 }}/{{ $kpis['serversTotal'] ?? 0 }} + @php + $sTotal = max(1, (int) ($kpis['serversTotal'] ?? 0)); + $sOnline = (int) ($kpis['serversOnline'] ?? 0); + $sAttention = min($sOnline, (int) ($kpis['needsAttention'] ?? 0)); + $sHealthy = max(0, $sOnline - $sAttention); + $sOffline = max(0, $sTotal - $sOnline); + @endphp +
    +
    +
    + @if ($sHealthy > 0)
    @endif + @if ($sAttention > 0)
    @endif + @if ($sOffline > 0)
    @endif +
    +
    + {{ $sHealthy }} healthy + @if ($sAttention > 0){{ $sAttention }} attention@endif + @if ($sOffline > 0){{ $sOffline }} offline@endif +
    +
    +
    +
    +
    + + {{ $scoped ? 'CPU' : 'Fleet CPU' }} + @if ($kpis['cpuApproximate'] ?? false) + ~ + @endif + + {{ round($kpis['cpuAvg'] ?? 0) }}% +
    + @include('livewire.traffic._sparkline', [ + 'id' => $chartId.'-spark-cpu', + 'initial' => $chartData['cpuSpark'] ?? [], + 'color' => '#3b82f6', + 'categories' => $chartData['sparkCategories'] ?? [], + 'event' => 'refreshChartData-'.$chartId, + 'key' => 'cpuSpark', + 'label' => $scoped ? 'CPU' : 'Fleet CPU', + 'format' => 'percent', + ]) +
    +
    +
    + {{ $scoped ? 'Memory' : 'Fleet memory' }} + {{ round($kpis['memPercent'] ?? 0) }}% +
    + @include('livewire.traffic._sparkline', [ + 'id' => $chartId.'-spark-memory', + 'initial' => $chartData['memSpark'] ?? [], + 'color' => '#8b5cf6', + 'categories' => $chartData['sparkCategories'] ?? [], + 'event' => 'refreshChartData-'.$chartId, + 'key' => 'memSpark', + 'label' => 'Memory used', + 'format' => 'bytes', + ]) +
    +
    +
    + {{ $scoped ? 'Disk' : 'Fleet disk' }} + {{ round($kpis['diskPercent'] ?? 0) }}% +
    + @include('livewire.traffic._sparkline', [ + 'id' => $chartId.'-spark-disk', + 'initial' => $chartData['diskSpark'] ?? [], + 'color' => '#ef4444', + 'categories' => $chartData['sparkCategories'] ?? [], + 'event' => 'refreshChartData-'.$chartId, + 'key' => 'diskSpark', + 'label' => $scoped ? 'Disk' : 'Fleet disk', + 'format' => 'percent', + ]) +
    +
    +
    + {{ $scoped ? 'Network' : 'Fleet network' }} + {{ formatBytes(($kpis['netRx'] ?? 0) + ($kpis['netTx'] ?? 0)) }}/s +
    + @include('livewire.traffic._sparkline', [ + 'id' => $chartId.'-spark-network', + 'initial' => $chartData['netSpark'] ?? [], + 'color' => '#10b981', + 'categories' => $chartData['sparkCategories'] ?? [], + 'event' => 'refreshChartData-'.$chartId, + 'key' => 'netSpark', + 'label' => 'Network', + 'format' => 'bytes', + ]) +
    +
    +
    + Containers + {{ compactNumber($kpis['containers'] ?? 0) }} +
    +
    + @if (! empty($topContainers)) + @php + $hottest = $this->containerMeta($topContainers[0]['id']); + $hottestValue = match ($containerMetric) { + 'memory' => formatBytes($topContainers[0]['memUsed'] ?? 0), + 'disk' => formatBytes($topContainers[0]['diskBytes'] ?? 0), + 'network' => formatBytes($topContainers[0]['net'] ?? 0).'/s', + default => round($topContainers[0]['cpu'] ?? 0).'%', + }; + @endphp + {{ $hottest['name'] }} + hottest, {{ $hottestValue }} + @endif +
    +
    +
    +
    +     + + {{-- Trend charts --}} + {{-- CPU is the primary triage signal, so it spans full width; the other four + fleet series fall into a clean 2x2 below (no orphaned cell). --}} +
    + @foreach (['cpu' => 'CPU', 'memory' => 'Memory used', 'network' => 'Network throughput', 'load' => 'Load average', 'disk' => 'Disk usage'] as $key => $label) + $key === 'cpu'])> +
    +
    +
    +     + @endforeach +
    + + {{-- Servers table --}} + +
    +
    + Server + CPU + Memory + Disk + Load + Network + Containers +
    + @foreach (collect($serverRows)->sortByDesc(fn ($r) => $r['cpu'] ?? -1) as $row) +
    +
    + {{ $row['name'] }} + @unless ($row['online']) + Offline + @endunless +
    +
    @include('livewire.fleet._usage-cell', ['percent' => $row['cpu'], 'kind' => 'cpu'])
    +
    @include('livewire.fleet._usage-cell', ['percent' => ($row['memTotal'] ?? 0) ? round($row['memUsed'] / $row['memTotal'] * 100, 1) : null, 'kind' => 'memory'])
    +
    @include('livewire.fleet._usage-cell', ['percent' => $row['diskPercent'], 'kind' => 'disk'])
    +
    {{ $row['load1'] === null ? '-' : number_format($row['load1'], 2) }}
    +
    {{ $row['netRx'] === null ? '-' : formatBytes($row['netRx'] + $row['netTx']).'/s' }}
    +
    {{ $row['containers'] ?? '-' }}
    +
    + @endforeach +
    +     + + {{-- Hottest containers --}} + + +
    + +
    + @foreach (['cpu' => 'CPU', 'memory' => 'Memory', 'disk' => 'Disk', 'network' => 'Network'] as $m => $mlabel) + + @endforeach +
    +
    +     + @if (empty($topContainers)) +
    + +
    + @else + @php + $metricOf = fn ($c) => match ($containerMetric) { + 'memory' => $c['memUsed'] ?? 0, + 'disk' => $c['diskBytes'] ?? 0, + 'network' => $c['net'] ?? 0, + default => $c['cpu'] ?? 0, + }; + // Bars are scaled to the leader so the row length reads as "share of the hottest". + $maxMetric = max(1, collect($topContainers)->map($metricOf)->max()); + @endphp +
    +
    + # + Container + Server + {{ ['cpu' => 'CPU', 'memory' => 'Memory', 'disk' => 'Disk', 'network' => 'Network'][$containerMetric] }} +
    + @foreach ($topContainers as $c) + @php + $meta = $this->containerMeta($c['id']); + $pct = min(100, round($metricOf($c) / $maxMetric * 100, 1)); + $display = match ($containerMetric) { + 'memory' => formatBytes($c['memUsed'] ?? 0), + 'disk' => formatBytes($c['diskBytes'] ?? 0), + 'network' => formatBytes($c['net'] ?? 0).'/s', + default => round($c['cpu'] ?? 0).'%', + }; + @endphp +
    +
    {{ $loop->iteration }}
    +
    + @if ($meta['link']) + {{ $meta['name'] }} + @else + {{ $meta['name'] }} + @endif + @if ($meta['image']) + {{ $meta['image'] }} + @endif +
    +
    {{ $c['server'] }}
    +
    + + {{ $display }} +
    +
    + @endforeach +
    + @endif +     + @endif + + @script + + @endscript +
    diff --git a/resources/views/livewire/server/charts.blade.php b/resources/views/livewire/server/charts.blade.php index 442fc1979f..1d889a2ac6 100644 --- a/resources/views/livewire/server/charts.blade.php +++ b/resources/views/livewire/server/charts.blade.php @@ -59,15 +59,34 @@ - -
    -     + {{-- CPU is the primary signal, so it spans full width; the other four + charts fall into a 2x2 below (mirrors the fleet /metrics layout). --}} +
    + +
    +     - -
    -     + +
    +     + + +
    +     + + +
    +     + + +
    +     +
    @script diff --git a/resources/views/livewire/traffic/_sparkline.blade.php b/resources/views/livewire/traffic/_sparkline.blade.php index 4fb3ce8cc2..5999cc1b82 100644 --- a/resources/views/livewire/traffic/_sparkline.blade.php +++ b/resources/views/livewire/traffic/_sparkline.blade.php @@ -16,14 +16,16 @@ @param string $colorVar CSS custom property for the line color (e.g. --chart-status-3xx) @param ?string $event Livewire event to listen on for updates (optional) @param ?string $key payload key holding the numeric array (required with $event) + @param ?array $categories initial x-axis timestamps (defaults to $series buckets) --}} @php $initial = $initial ?? []; $colorVar = $colorVar ?? '--chart-status-3xx'; + $color = $color ?? ''; $event = $event ?? ''; $key = $key ?? ''; - $initialCategories = array_column($series ?? [], 'bucket'); - $label = match ($key) { + $initialCategories = $categories ?? array_column($series ?? [], 'bucket'); + $label = $label ?? match ($key) { 'requestsSpark' => 'Requests', 'uniquesSpark' => 'Visitors', 'bandwidthSpark' => 'Bandwidth', @@ -31,12 +33,17 @@ 'latencySpark' => 'p95 latency', default => 'Value', }; + $format = $format ?? match ($key) { + 'bandwidthSpark' => 'bytes', + 'latencySpark' => 'ms', + default => 'number', + }; @endphp
    team, $this->privateKey); + StubMetricsClient::$summaries = [$s->uuid => onlineSummary()]; + + Livewire::test(TestableMetrics::class) + ->assertSee('Fleet CPU') + ->set('serverUuid', $s->uuid) + ->assertDontSee('Fleet') + ->assertSee('for the selected server'); +}); + +it('renders container metric values without leaking a blade directive', function () { + $s = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$s->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$s->uuid => [ + ['id' => 'a', 'cpu' => 5.0, 'memUsed' => 900, 'memPercent' => 9.0, 'diskBytes' => 1, 'net' => 2048.0], + ]]; + + Livewire::test(TestableMetrics::class) + ->set('containerMetric', 'network') + ->assertDontSee('@break') + ->assertSee('/s'); +}); + +it('falls back to cpu for an unknown container metric', function () { + $s = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$s->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$s->uuid => [ + ['id' => 'a', 'cpu' => 5.0, 'memUsed' => 900, 'memPercent' => 9.0, 'diskBytes' => 1, 'net' => 0.0], + ]]; + + Livewire::withQueryParams(['cmetric' => 'foo']) + ->test(TestableMetrics::class) + ->assertOk() + ->assertSet('containerMetric', 'cpu') + ->set('containerMetric', 'bar') + ->assertSet('containerMetric', 'cpu'); +}); + +it('keys container rows by server so shared container ids do not collide', function () { + $one = metricsServer($this->team, $this->privateKey); + $two = metricsServer($this->team, $this->privateKey); + $proxy = ['id' => 'coolify-proxy', 'cpu' => 1.0, 'memUsed' => 10, 'memPercent' => 1.0, 'diskBytes' => 1, 'net' => 0.0]; + + StubMetricsClient::$summaries = [$one->uuid => onlineSummary(), $two->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$one->uuid => [$proxy], $two->uuid => [$proxy]]; + + Livewire::test(TestableMetrics::class) + ->assertSeeHtml('wire:key="fleet-container-'.$one->uuid.'-coolify-proxy"') + ->assertSeeHtml('wire:key="fleet-container-'.$two->uuid.'-coolify-proxy"'); +}); + +it('passes the spark time axis to the KPI sparklines on first paint', function () { + $s = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$s->uuid => onlineSummary()]; + + $component = Livewire::test(TestableMetrics::class); + $component->set('chartData', ['sparkCategories' => [1700000100000], 'cpuSpark' => [12.5]]); + + $component->assertSeeHtml('1700000100000'); +}); + +it('removes a transferred server on refresh', function () { + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + $component = Livewire::test(TestableMetrics::class); + expect($component->get('serverRows'))->toHaveCount(1); + + $server->update(['team_id' => Team::factory()->create()->id]); + $component->call('loadData'); + + expect($component->get('serverRows'))->toBe([]); + expect($component->get('containersRaw'))->toBe([]); + expect($component->get('kpis')['serversTotal'])->toBe(0); +}); + +it('permits team members to read metrics', function () { + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + $member = User::factory()->create(); + $member->teams()->attach($this->team, ['role' => 'member']); + $this->actingAs($member); + + expect(Livewire::test(TestableMetrics::class)->get('kpis')['serversOnline'])->toBe(1); +}); + +it('keeps host trends when an older Sentinel has no summary endpoint', function () { + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]], 'memory' => [[1000, 100.0]]]]; + + $component = Livewire::test(TestableMetrics::class); + + expect($component->get('chartData')['cpu'])->toBe([[1000, 25.0]]); + expect($component->get('chartData')['memory'])->toBe([[1000, 100.0]]); + expect($component->get('topContainers'))->toBe([]); +}); + +it('keeps other metrics when one independent operation fails', function () { + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]]]]; + StubMetricsClient::$failingMetrics = ['disk']; + + $component = Livewire::test(TestableMetrics::class); + + expect($component->get('chartData')['cpu'])->toBe([[1000, 25.0]]); + expect($component->get('chartData')['disk'])->toBe([]); + expect($component->get('kpis')['serversOnline'])->toBe(1); +}); + +it('resolves container names for the instance team with id zero', function () { + $rootTeam = Team::factory()->create(['id' => 0]); + auth()->user()->teams()->attach($rootTeam, ['role' => 'owner']); + auth()->user()->unsetRelation('teams'); + session(['currentTeam' => $rootTeam]); + $project = Project::factory()->create(['team_id' => 0]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create(['environment_id' => $environment->id]); + + $meta = Livewire::test(TestableMetrics::class)->instance()->containerMeta($application->uuid); + + expect($meta['name'])->toBe($application->name); + expect($meta['link'])->toContain($application->uuid); +}); + +it('denies metrics refresh after team membership is removed', function () { + $this->withoutVite(); + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + $component = Livewire::test(TestableMetrics::class); + + auth()->user()->teams()->detach($this->team); + auth()->user()->unsetRelation('teams'); + + $component->call('loadData')->assertForbidden(); +}); + +it('clears stale container rankings after a server transfer', function () { + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$server->uuid => [['id' => 'private-container', 'cpu' => 10]]]; + $component = Livewire::test(TestableMetrics::class); + + $server->update(['team_id' => Team::factory()->create()->id]); + $component->set('containerMetric', 'memory'); + + expect($component->get('topContainers'))->toBe([]); + expect($component->get('serverRows'))->toBe([]); +}); + +it('merges metrics from real concurrent workers and isolates endpoint failures', function () { + Concurrency::setDefaultInstance('process'); + metricsServer($this->team, $this->privateKey); + metricsServer($this->team, $this->privateKey); + + $component = Livewire::test(ProcessTestableMetrics::class); + + expect($component->get('kpis')['serversOnline'])->toBe(2); + expect($component->get('topContainers'))->toHaveCount(2); + expect($component->get('chartData')['cpu'])->toBe([[1000, 25.0]]); + expect($component->get('chartData')['memory'])->toBe([[1000, 200.0]]); + expect($component->get('chartData')['disk'])->toBe([]); + expect($component->get('chartData')['networkRx'])->toBe([[1000, 10.0]]); +}); + +it('clears stale metrics after the last server is deleted', function () { + $server = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$server->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$server->uuid => [['id' => 'deleted-container', 'cpu' => 10]]]; + $component = Livewire::test(TestableMetrics::class); + + $server->delete(); + $component->set('containerMetric', 'memory'); + + expect($component->get('topContainers'))->toBe([]); + expect($component->get('serverRows'))->toBe([]); +}); + +it('repairs the Sentinel token before creating worker clients', function () { + $server = metricsServer($this->team, $this->privateKey); + $server->settings->updateQuietly(['sentinel_token' => 'invalid token']); + app()->bind(SentinelMetricsClient::class, fn ($app, $parameters) => new ConcurrentMetricsClient($parameters['server'])); + + loadLazy(Livewire::test(Metrics::class)); + + $token = $server->settings->fresh()->sentinel_token; + expect(ServerSetting::isValidSentinelToken($token))->toBeTrue(); + expect($token)->not->toBe('invalid token'); +}); diff --git a/tests/Feature/Livewire/Fleet/MetricsPageTest.php b/tests/Feature/Livewire/Fleet/MetricsPageTest.php new file mode 100644 index 0000000000..ad44bf6d72 --- /dev/null +++ b/tests/Feature/Livewire/Fleet/MetricsPageTest.php @@ -0,0 +1,33 @@ + InstanceSettings::query()->create(['id' => 0])); + $user = User::factory()->create(); + $team = Team::factory()->create(); + $user->teams()->attach($team, ['role' => 'owner']); + $this->actingAs($user); + session(['currentTeam' => $team]); + $this->team = $team; +}); + +it('renders the metrics page with an empty state when no server has metrics enabled', function () { + $component = loadLazy(Livewire::test(Metrics::class)); + + $component + ->assertOk() + ->assertSee('Metrics') + ->assertSee('Enable metrics on a server'); +}); + +it('registers the metrics route', function () { + expect(route('metrics'))->toEndWith('/metrics'); +}); diff --git a/tests/Support/ConcurrentMetricsClient.php b/tests/Support/ConcurrentMetricsClient.php new file mode 100644 index 0000000000..278d92b95d --- /dev/null +++ b/tests/Support/ConcurrentMetricsClient.php @@ -0,0 +1,36 @@ + true, 'cpu' => 25.0, 'memUsed' => 100, 'memTotal' => 200, + 'diskUsed' => 10, 'diskTotal' => 100, 'diskPercent' => 10.0, + 'load1' => 0.4, 'netRx' => 5.0, 'netTx' => 1.0, + ]; + } + + public function containersCurrent(): array + { + return [['id' => 'container-'.$this->server->uuid, 'cpu' => 25.0]]; + } + + public function history(string $metric, string $range): array + { + if ($metric === 'disk') { + throw new \RuntimeException('Disk endpoint unavailable.'); + } + + return [[1000, $metric === 'cpu' ? 25.0 : 100.0]]; + } + + public function networkHistory(string $range): array + { + return ['rx' => [[1000, 5.0]], 'tx' => [[1000, 1.0]]]; + } +} diff --git a/tests/Unit/Metrics/HasMetricsMappingTest.php b/tests/Unit/Metrics/HasMetricsMappingTest.php new file mode 100644 index 0000000000..d8b4b805bd --- /dev/null +++ b/tests/Unit/Metrics/HasMetricsMappingTest.php @@ -0,0 +1,75 @@ + type => rows */ + public array $rowsByType = []; + + protected function fetchMetricRows(string $type, int $mins): ?array + { + return $this->rowsByType[$type] ?? null; + } +} + +it('keeps only the root mount for disk metrics', function () { + $d = new MetricsMappingDouble; + $d->rowsByType['disk'] = [ + ['time' => 1000, 'mount' => '/boot', 'usedPercent' => 90.0], + ['time' => 1000, 'mount' => '/', 'usedPercent' => 40.0], + ['time' => 2000, 'mount' => '/', 'usedPercent' => 42.0], + ]; + + expect($d->getDiskMetrics(5))->toBe([[1000, 40.0], [2000, 42.0]]); +}); + +it('splits network metrics into rx and tx series', function () { + $d = new MetricsMappingDouble; + $d->rowsByType['network'] = [ + ['time' => 1000, 'rxBytesPerSec' => 100.0, 'txBytesPerSec' => 10.0], + ['time' => 2000, 'rxBytesPerSec' => 200.0, 'txBytesPerSec' => 20.0], + ]; + + expect($d->getNetworkMetrics(5))->toBe([ + 'rx' => [[1000, 100.0], [2000, 200.0]], + 'tx' => [[1000, 10.0], [2000, 20.0]], + ]); +}); + +it('maps the one minute load average', function () { + $d = new MetricsMappingDouble; + $d->rowsByType['load'] = [ + ['time' => 1000, 'load1' => 0.5, 'load5' => 1.0, 'load15' => 1.5], + ]; + + expect($d->getLoadMetrics(5))->toBe([[1000, 0.5]]); +}); + +it('returns null for a metric when metrics are unavailable', function () { + $d = new MetricsMappingDouble; // rowsByType empty -> fetch returns null + + expect($d->getDiskMetrics(5))->toBeNull(); + expect($d->getNetworkMetrics(5))->toBeNull(); + expect($d->getLoadMetrics(5))->toBeNull(); +}); + +it('does not use non-root mounts when root disk samples are missing', function () { + $double = new MetricsMappingDouble; + $double->rowsByType['disk'] = [['time' => 1000, 'mount' => '/boot', 'usedPercent' => 90]]; + + expect($double->getDiskMetrics(5))->toBe([]); +}); + +it('supports older disk rows without a mount field', function () { + $double = new MetricsMappingDouble; + $double->rowsByType['disk'] = [['time' => 1000, 'usedPercent' => 40]]; + + expect($double->getDiskMetrics(5))->toBe([[1000, 40.0]]); +}); diff --git a/tests/Unit/Metrics/PressureLevelTest.php b/tests/Unit/Metrics/PressureLevelTest.php new file mode 100644 index 0000000000..e2c922a216 --- /dev/null +++ b/tests/Unit/Metrics/PressureLevelTest.php @@ -0,0 +1,22 @@ +toBe('ok'); + expect(PressureLevel::for(70, 'cpu'))->toBe('warning'); + expect(PressureLevel::for(89.9, 'memory'))->toBe('warning'); + expect(PressureLevel::for(90, 'memory'))->toBe('error'); +}); + +it('classifies disk pressure at 80/90 cutoffs', function () { + expect(PressureLevel::for(75, 'disk'))->toBe('ok'); + expect(PressureLevel::for(80, 'disk'))->toBe('warning'); + expect(PressureLevel::for(95, 'disk'))->toBe('error'); +}); + +it('treats a null percent as ok and maps levels to classes', function () { + expect(PressureLevel::for(null, 'cpu'))->toBe('ok'); + expect(PressureLevel::textClass('error'))->toContain('text-red-500'); + expect(PressureLevel::barClass('warning'))->toContain('bg-orange-500'); +}); diff --git a/tests/Unit/Services/FleetMetricsAggregatorRankingTest.php b/tests/Unit/Services/FleetMetricsAggregatorRankingTest.php new file mode 100644 index 0000000000..6b6fe2ee1e --- /dev/null +++ b/tests/Unit/Services/FleetMetricsAggregatorRankingTest.php @@ -0,0 +1,33 @@ +toBe([[1000, 3.0], [2000, 5.0], [3000, 9.0]]); +}); + +it('averages series across servers per bucket', function () { + $merged = FleetMetricsAggregator::sumSeriesByBucket([ + [[1000, 20.0]], + [[1000, 40.0]], + ], 'avg'); + + expect($merged)->toBe([[1000, 30.0]]); +}); + +it('ranks containers by the selected metric and caps the list', function () { + $containers = [ + ['id' => 'a', 'cpu' => 10.0, 'memUsed' => 500, 'diskBytes' => 5, 'net' => 1.0], + ['id' => 'b', 'cpu' => 90.0, 'memUsed' => 100, 'diskBytes' => 9, 'net' => 2.0], + ['id' => 'c', 'cpu' => 50.0, 'memUsed' => 900, 'diskBytes' => 1, 'net' => 3.0], + ]; + + expect(array_column(FleetMetricsAggregator::rankContainers($containers, 'cpu', 2), 'id'))->toBe(['b', 'c']); + expect(array_column(FleetMetricsAggregator::rankContainers($containers, 'memory'), 'id'))->toBe(['c', 'a', 'b']); + expect(array_column(FleetMetricsAggregator::rankContainers($containers, 'network'), 'id'))->toBe(['c', 'b', 'a']); +}); diff --git a/tests/Unit/Services/FleetMetricsAggregatorTest.php b/tests/Unit/Services/FleetMetricsAggregatorTest.php new file mode 100644 index 0000000000..2de55616f7 --- /dev/null +++ b/tests/Unit/Services/FleetMetricsAggregatorTest.php @@ -0,0 +1,69 @@ + 'u', 'name' => 'srv', 'online' => true, + 'cpu' => 10.0, 'memUsed' => 1_000, 'memTotal' => 10_000, + 'diskUsed' => 1_000, 'diskTotal' => 10_000, 'diskPercent' => 10.0, + 'load1' => 0.5, 'netRx' => 100.0, 'netTx' => 50.0, 'containers' => 2, + ], $overrides); +} + +it('weights memory and disk by totals, not mean of percents', function () { + $rows = [ + row(['memUsed' => 100, 'memTotal' => 100, 'diskUsed' => 100, 'diskTotal' => 100, 'diskPercent' => 100.0]), + row(['memUsed' => 10, 'memTotal' => 1_000, 'diskUsed' => 10, 'diskTotal' => 1_000, 'diskPercent' => 1.0]), + ]; + + $kpis = FleetMetricsAggregator::fleetKpis($rows); + + // Σused/Σtotal = 110/1100 = 10%, not the (100+1)/2 = 50.5% a naive mean gives. + expect(round($kpis['memPercent'], 1))->toBe(10.0); + expect(round($kpis['diskPercent'], 1))->toBe(10.0); +}); + +it('means cpu, flags approximate for multiple servers, and finds the busiest', function () { + $kpis = FleetMetricsAggregator::fleetKpis([ + row(['name' => 'a', 'cpu' => 20.0]), + row(['name' => 'b', 'cpu' => 80.0]), + ]); + + expect($kpis['cpuAvg'])->toBe(50.0); + expect($kpis['cpuApproximate'])->toBeTrue(); + expect($kpis['cpuBusiest'])->toBe(['name' => 'b', 'percent' => 80.0]); +}); + +it('sums network rates and reports busiest and average load', function () { + $kpis = FleetMetricsAggregator::fleetKpis([ + row(['netRx' => 100.0, 'netTx' => 10.0, 'load1' => 0.5]), + row(['netRx' => 200.0, 'netTx' => 20.0, 'load1' => 2.5]), + ]); + + expect($kpis['netRx'])->toBe(300.0); + expect($kpis['netTx'])->toBe(30.0); + expect($kpis['loadBusiest'])->toBe(2.5); + expect($kpis['loadAvg'])->toBe(1.5); +}); + +it('ignores offline rows and counts servers needing attention', function () { + $kpis = FleetMetricsAggregator::fleetKpis([ + row(['cpu' => 95.0]), // over cpu error cutoff + row(['online' => false, 'cpu' => 99.0]), + ]); + + expect($kpis['serversOnline'])->toBe(1); + expect($kpis['serversTotal'])->toBe(2); + expect($kpis['needsAttention'])->toBe(1); + expect($kpis['cpuAvg'])->toBe(95.0); // offline row excluded from the mean +}); + +it('returns zeroed kpis for no rows', function () { + $kpis = FleetMetricsAggregator::fleetKpis([]); + + expect($kpis['serversOnline'])->toBe(0); + expect($kpis['cpuAvg'])->toBe(0.0); + expect($kpis['cpuBusiest'])->toBeNull(); +}); diff --git a/tests/Unit/Services/SentinelMetricsClientTest.php b/tests/Unit/Services/SentinelMetricsClientTest.php new file mode 100644 index 0000000000..3efbdf42b9 --- /dev/null +++ b/tests/Unit/Services/SentinelMetricsClientTest.php @@ -0,0 +1,181 @@ + substring => body */ + public array $bodies = []; + + /** @var array */ + public array $failing = []; + + protected function raw(string $url): string + { + foreach ($this->failing as $needle) { + if (str_contains($url, $needle)) { + throw new RuntimeException('boom'); + } + } + foreach ($this->bodies as $needle => $body) { + if (str_contains($url, $needle)) { + return $body; + } + } + + return '[]'; + } +} + +function fakeMetricsClient(): FakeMetricsClient +{ + $server = Server::factory()->make(['uuid' => 'srv-uuid']); + + return new FakeMetricsClient($server); +} + +it('normalizes the host summary, choosing the root disk mount', function () { + $client = fakeMetricsClient(); + $client->bodies = [ + '/summary' => json_encode([ + 'cpu' => ['time' => 't', 'percent' => 42.5], + 'memory' => ['time' => 't', 'total' => 1000, 'used' => 250, 'usedPercent' => 25.0], + 'disk' => [ + ['time' => 't', 'mount' => '/boot', 'total' => 10, 'used' => 9, 'usedPercent' => 90.0], + ['time' => 't', 'mount' => '/', 'total' => 100, 'used' => 40, 'usedPercent' => 40.0], + ], + 'network' => ['time' => 't', 'rxBytesPerSec' => 100.0, 'txBytesPerSec' => 20.0], + 'load' => ['time' => 't', 'load1' => 1.5, 'load5' => 1.0, 'load15' => 0.5], + ]), + ]; + + $summary = $client->summary(); + + expect($summary['cpu'])->toBe(42.5); + expect($summary['memUsed'])->toBe(250); + expect($summary['diskPercent'])->toBe(40.0); // root mount, not /boot + expect($summary['netRx'])->toBe(100.0); + expect($summary['load1'])->toBe(1.5); +}); + +it('returns null summary when the endpoint fails', function () { + $client = fakeMetricsClient(); + $client->failing = ['/summary']; + + expect($client->summary())->toBeNull(); +}); + +it('normalizes containers, parsing the stringified cpu percent', function () { + $client = fakeMetricsClient(); + $client->bodies = [ + '/containers/current' => json_encode([ + ['id' => 'app-a', 'cpu' => ['time' => 't', 'percent' => '12.5'], + 'memory' => ['used' => 500, 'usedPercent' => 5.0], + 'disk' => ['writableLayer' => 3, 'volumesTotal' => 7], + 'network' => ['rxBytesPerSec' => 4.0, 'txBytesPerSec' => 1.0]], + ]), + ]; + + $rows = $client->containersCurrent(); + + expect($rows[0]['id'])->toBe('app-a'); + expect($rows[0]['cpu'])->toBe(12.5); + expect($rows[0]['memUsed'])->toBe(500); + expect($rows[0]['diskBytes'])->toBe(10); // writableLayer + volumesTotal + expect($rows[0]['net'])->toBe(5.0); // rx + tx +}); + +it('returns an empty container list when the endpoint 404s', function () { + $client = fakeMetricsClient(); + $client->failing = ['/containers/current']; + + expect($client->containersCurrent())->toBe([]); +}); + +it('averages cpu history into range-aligned buckets', function () { + $client = fakeMetricsClient(); + $client->bodies = [ + '/cpu/history' => json_encode([ + ['time' => '1700000000000', 'percent' => '10'], + ['time' => '1700000060000', 'percent' => '20'], + ['time' => '1700000300000', 'percent' => '40'], + ]), + ]; + + // 24h range → 5-minute buckets floored to absolute epoch boundaries. + expect($client->history('cpu', '24h'))->toBe([[1699999800000, 15.0], [1700000100000, 40.0]]); +}); + +it('aligns buckets across servers whose samples have different timestamps', function () { + $a = fakeMetricsClient(); + $a->bodies = ['/cpu/history' => json_encode([['time' => 1700000001234, 'percent' => '10']])]; + $b = fakeMetricsClient(); + $b->bodies = ['/cpu/history' => json_encode([['time' => 1700000047890, 'percent' => '30']])]; + + expect(array_column($a->history('cpu', '24h'), 0))->toBe(array_column($b->history('cpu', '24h'), 0)); +}); + +it('keeps long history series small', function () { + $rows = []; + // 24h of raw 20-second samples. + for ($i = 0; $i < 4320; $i++) { + $rows[] = ['time' => 1700000000000 + $i * 20_000, 'percent' => (string) ($i % 100)]; + } + + $client = fakeMetricsClient(); + $client->bodies = ['/cpu/history' => json_encode($rows)]; + + expect(count($client->history('cpu', '24h')))->toBeLessThanOrEqual(300); +}); + +it('keeps only the root mount in disk history', function () { + $client = fakeMetricsClient(); + $client->bodies = [ + '/disk/history' => json_encode([ + ['time' => 1700000000000, 'mount' => '/', 'usedPercent' => 85], + ['time' => 1700000000000, 'mount' => '/boot', 'usedPercent' => 20], + ]), + ]; + + expect($client->history('disk', '24h'))->toBe([[1699999800000, 85.0]]); +}); + +it('rounds the range start to the minute so history responses are cacheable', function () { + Carbon::setTestNow('2026-01-01 12:30:05'); + $first = SentinelMetricsClient::rangeFrom('24h'); + + Carbon::setTestNow('2026-01-01 12:30:45'); + + expect(SentinelMetricsClient::rangeFrom('24h'))->toBe($first)->toBe('2025-12-31T12:30:00Z'); + + Carbon::setTestNow(); +}); + +it('does not substitute another mount when root disk history is absent', function () { + $client = fakeMetricsClient(); + $client->bodies = ['/disk/history' => json_encode([ + ['time' => 1700000000000, 'mount' => '/boot', 'usedPercent' => 90], + ])]; + + expect($client->history('disk', '24h'))->toBe([]); +}); + +it('supports disk history without mount fields', function () { + $client = fakeMetricsClient(); + $client->bodies = ['/disk/history' => json_encode([ + ['time' => 1700000000000, 'usedPercent' => 40], + ])]; + + expect($client->history('disk', '24h'))->toBe([[1699999800000, 40.0]]); +}); + +it('does not substitute another disk mount in the summary', function () { + $client = fakeMetricsClient(); + $client->bodies = ['/summary' => json_encode(['disk' => [ + ['mount' => '/boot', 'used' => 90, 'total' => 100, 'usedPercent' => 90], + ]])]; + + expect($client->summary()['diskPercent'])->toBeNull(); +});