From faca466cd60bfca552f48658fbf3c4e470a87fbc Mon Sep 17 00:00:00 2001 From: Aditya Tripathi Date: Fri, 18 Sep 2026 07:44:22 +0000 Subject: [PATCH] fix(metrics): align fleet series by time bucket, harden fleet page inputs Address review findings on the fleet metrics page. Data correctness: - bucket raw Sentinel samples into epoch-aligned windows (5m/30m/2h per range) and average them, replacing per-server LTTB. Every server now reports the same bucket timestamps, so the fleet sums add up point by point instead of zigzagging between servers (and RX+TX really combine) - keep only the root mount in disk history, matching HasMetrics::getDiskMetrics - leave sparkline gaps as null instead of 0, so absent samples read as gaps Robustness: - normalize containerMetric in mount()/updatedContainerMetric(), so /metrics?cmetric=foo no longer throws an undefined array key - include the server uuid in the container row wire:key; system containers share one Sentinel id across servers - round the range start to the minute so the 60s history cache can hit - cap the Sentinel curl with --max-time 10, so one hung request cannot stall the serial per-server loop UI: - pass the spark time axis on first paint, so tooltips no longer show 1970 - show the active series name (RX/TX) in the server network chart tooltip --- .../Concerns/BuildsMetricsChartPayload.php | 10 +-- app/Livewire/Fleet/Metrics.php | 26 ++++++-- app/Services/SentinelMetricsClient.php | 65 ++++++++++++------- .../views/livewire/fleet/metrics.blade.php | 6 +- .../views/livewire/server/charts.blade.php | 2 +- .../livewire/traffic/_sparkline.blade.php | 4 +- .../Fleet/MetricsChartPayloadTest.php | 10 +-- .../Livewire/Fleet/MetricsDataTest.php | 42 +++++++++++- .../Services/SentinelMetricsClientTest.php | 52 ++++++++++++--- 9 files changed, 163 insertions(+), 54 deletions(-) diff --git a/app/Livewire/Concerns/BuildsMetricsChartPayload.php b/app/Livewire/Concerns/BuildsMetricsChartPayload.php index 8ec43ee2b8..12796c81e6 100644 --- a/app/Livewire/Concerns/BuildsMetricsChartPayload.php +++ b/app/Livewire/Concerns/BuildsMetricsChartPayload.php @@ -55,8 +55,8 @@ trait BuildsMetricsChartPayload * evenly-spaced indices (first and last always kept), so the series stay aligned. * * @param array $categories - * @param array> $sparks - * @return array{0: array, 1: array>} + * @param array> $sparks + * @return array{0: array, 1: array>} */ private function capSparks(array $categories, array $sparks, int $max): array { @@ -117,11 +117,11 @@ trait BuildsMetricsChartPayload } /** - * Project a [ts, value] series onto the shared category axis, filling gaps with 0. + * Project a [ts, value] series onto the shared category axis, leaving gaps as null. * * @param array $series * @param array $categories - * @return array + * @return array */ private function alignSpark(array $series, array $categories): array { @@ -130,6 +130,6 @@ trait BuildsMetricsChartPayload $map[(int) $ts] = $value; } - return array_map(fn ($ts) => $map[$ts] ?? 0, $categories); + return array_map(fn ($ts) => $map[$ts] ?? null, $categories); } } diff --git a/app/Livewire/Fleet/Metrics.php b/app/Livewire/Fleet/Metrics.php index 901fc7cdcd..99c596d716 100644 --- a/app/Livewire/Fleet/Metrics.php +++ b/app/Livewire/Fleet/Metrics.php @@ -35,6 +35,8 @@ class Metrics extends Component #[Url(as: 'cmetric')] public string $containerMetric = 'cpu'; + private const CONTAINER_METRICS = ['cpu', 'memory', 'disk', 'network']; + public bool $live = false; /** @var array */ @@ -71,6 +73,8 @@ class Metrics extends Component $this->serverUuid = ''; } + $this->normalizeContainerMetric(); + if ($this->servers->isNotEmpty()) { $this->loadData(); } @@ -83,10 +87,19 @@ class Metrics extends Component public function updatedContainerMetric(): void { + $this->normalizeContainerMetric(); + // Re-rank the already-fetched containers; no need to re-hit Sentinel. $this->topContainers = FleetMetricsAggregator::rankContainers($this->containersRaw, $this->containerMetric); } + private function normalizeContainerMetric(): void + { + if (! in_array($this->containerMetric, self::CONTAINER_METRICS, true)) { + $this->containerMetric = 'cpu'; + } + } + public function setRange(string $range): void { $this->range = in_array($range, ['24h', '7d', '30d'], true) ? $range : '24h'; @@ -119,7 +132,6 @@ class Metrics extends Component $rows = []; $containers = []; $series = ['cpu' => [], 'memory' => [], 'disk' => [], 'load' => [], 'networkRx' => [], 'networkTx' => []]; - $from = SentinelMetricsClient::rangeFrom($this->range); foreach ($this->targetServers() as $server) { $client = $this->metricsClient($server); @@ -145,14 +157,14 @@ class Metrics extends Component foreach ($current as $c) { // Names/images are resolved lazily for the displayed rows only (containerMeta), // so a server with many containers doesn't trigger a resolve storm here. - $containers[] = array_merge($c, ['server' => $server->name]); + $containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]); } - $series['cpu'][] = $client->history('cpu', $from); - $series['memory'][] = $client->history('memory', $from); - $series['disk'][] = $client->history('disk', $from); - $series['load'][] = $client->history('load', $from); - $net = $client->networkHistory($from); + $series['cpu'][] = $client->history('cpu', $this->range); + $series['memory'][] = $client->history('memory', $this->range); + $series['disk'][] = $client->history('disk', $this->range); + $series['load'][] = $client->history('load', $this->range); + $net = $client->networkHistory($this->range); $series['networkRx'][] = $net['rx']; $series['networkTx'][] = $net['tx']; } diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php index 820fd9c0f8..ab942e6e06 100644 --- a/app/Services/SentinelMetricsClient.php +++ b/app/Services/SentinelMetricsClient.php @@ -9,22 +9,32 @@ class SentinelMetricsClient { private string $base = 'http://localhost:8888/api'; - /** Cap on points per trend series after downsampling raw Sentinel samples. */ - private const MAX_SERIES_POINTS = 300; - public function __construct(protected Server $server) {} + /** Rounded to the minute so the history URL (and its cache key) is stable within a minute. */ public static function rangeFrom(string $range): string { + $now = now()->startOfMinute(); + $from = match ($range) { - '7d' => now()->subDays(7), - '30d' => now()->subDays(30), - default => now()->subDay(), + '7d' => $now->subDays(7), + '30d' => $now->subDays(30), + default => $now->subDay(), }; return $from->toIso8601ZuluString(); } + /** Bucket width per range, kept at roughly 300 points per series. */ + public static function bucketMs(string $range): int + { + return match ($range) { + '7d' => 30 * 60_000, + '30d' => 2 * 60 * 60_000, + default => 5 * 60_000, + }; + } + /** * @return array|null */ @@ -89,7 +99,7 @@ class SentinelMetricsClient /** * @return array */ - public function history(string $metric, string $from): array + public function history(string $metric, string $range): array { [$path, $field] = match ($metric) { 'memory' => ['/memory/history', 'used'], @@ -98,26 +108,26 @@ class SentinelMetricsClient default => ['/cpu/history', 'percent'], }; - return $this->seriesRows($this->url($path, ['from' => $from]), $field); + return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range); } /** * @return array{rx: array, tx: array} */ - public function networkHistory(string $from): array + public function networkHistory(string $range): array { - $url = $this->url('/network/history', ['from' => $from]); + $url = $this->url('/network/history', ['from' => self::rangeFrom($range)]); return [ - 'rx' => $this->seriesRows($url, 'rxBytesPerSec'), - 'tx' => $this->seriesRows($url, 'txBytesPerSec'), + 'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range), + 'tx' => $this->seriesRows($url, 'txBytesPerSec', $range), ]; } /** * @return array */ - private function seriesRows(string $url, string $field): array + private function seriesRows(string $url, string $field, string $range): array { try { $rows = json_decode($this->raw($url), true); @@ -129,18 +139,25 @@ class SentinelMetricsClient return []; } - $series = array_values(array_map( - fn ($r) => [(int) ($r['time'] ?? 0), (float) ($r[$field] ?? 0)], - $rows - )); - - // Sentinel returns raw samples (thousands of points over 24h). Cap the series so - // the chart payload stays small enough to embed and morph cheaply. - if (count($series) > self::MAX_SERIES_POINTS) { - $series = downsampleLTTB($series, self::MAX_SERIES_POINTS); + // Disk history interleaves every mount; keep the root filesystem only. + $root = array_filter($rows, fn ($r) => ($r['mount'] ?? null) === '/'); + if ($root !== []) { + $rows = $root; } - return $series; + // Average raw samples into epoch-aligned buckets so every server shares the same + // timestamps and the fleet aggregator can sum them point by point. + $bucketMs = self::bucketMs($range); + $sums = []; + $counts = []; + foreach ($rows as $r) { + $bucket = intdiv((int) ($r['time'] ?? 0), $bucketMs) * $bucketMs; + $sums[$bucket] = ($sums[$bucket] ?? 0.0) + (float) ($r[$field] ?? 0); + $counts[$bucket] = ($counts[$bucket] ?? 0) + 1; + } + ksort($sums); + + return array_map(fn ($bucket) => [$bucket, round($sums[$bucket] / $counts[$bucket], 2)], array_keys($sums)); } /** @@ -187,7 +204,7 @@ class SentinelMetricsClient $token = $this->server->settings->ensureValidSentinelToken(); return instant_remote_process( - ["docker exec coolify-sentinel sh -c 'curl -s -H \"Authorization: Bearer {$token}\" \"{$url}\"'"], + ["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"], $this->server, false ); diff --git a/resources/views/livewire/fleet/metrics.blade.php b/resources/views/livewire/fleet/metrics.blade.php index b5b0856811..6a1e7355f5 100644 --- a/resources/views/livewire/fleet/metrics.blade.php +++ b/resources/views/livewire/fleet/metrics.blade.php @@ -131,6 +131,7 @@ $serverListboxOptions = array_merge( 'id' => $chartId.'-spark-cpu', 'initial' => $chartData['cpuSpark'] ?? [], 'color' => '#3b82f6', + 'categories' => $chartData['sparkCategories'] ?? [], 'event' => 'refreshChartData-'.$chartId, 'key' => 'cpuSpark', 'label' => $scoped ? 'CPU' : 'Fleet CPU', @@ -146,6 +147,7 @@ $serverListboxOptions = array_merge( 'id' => $chartId.'-spark-memory', 'initial' => $chartData['memSpark'] ?? [], 'color' => '#8b5cf6', + 'categories' => $chartData['sparkCategories'] ?? [], 'event' => 'refreshChartData-'.$chartId, 'key' => 'memSpark', 'label' => 'Memory used', @@ -161,6 +163,7 @@ $serverListboxOptions = array_merge( 'id' => $chartId.'-spark-disk', 'initial' => $chartData['diskSpark'] ?? [], 'color' => '#ef4444', + 'categories' => $chartData['sparkCategories'] ?? [], 'event' => 'refreshChartData-'.$chartId, 'key' => 'diskSpark', 'label' => $scoped ? 'Disk' : 'Fleet disk', @@ -176,6 +179,7 @@ $serverListboxOptions = array_merge( 'id' => $chartId.'-spark-network', 'initial' => $chartData['netSpark'] ?? [], 'color' => '#10b981', + 'categories' => $chartData['sparkCategories'] ?? [], 'event' => 'refreshChartData-'.$chartId, 'key' => 'netSpark', 'label' => 'Network', @@ -307,7 +311,7 @@ $serverListboxOptions = array_merge( default => round($c['cpu'] ?? 0).'%', }; @endphp -
{{ $loop->iteration }}
diff --git a/resources/views/livewire/server/charts.blade.php b/resources/views/livewire/server/charts.blade.php index edf970ffae..1d889a2ac6 100644 --- a/resources/views/livewire/server/charts.blade.php +++ b/resources/views/livewire/server/charts.blade.php @@ -209,7 +209,7 @@ const timestamp = w.globals.seriesX[seriesIndex][dataPointIndex]; return `
-
${name}: ${formatter(value)}
+
${w.globals.seriesNames?.[seriesIndex] ?? name}: ${formatter(value)}
Your time: ${formatLocalTimestamp(timestamp)}
UTC: ${formatUtcTimestamp(timestamp)}
`; diff --git a/resources/views/livewire/traffic/_sparkline.blade.php b/resources/views/livewire/traffic/_sparkline.blade.php index c170902a79..5999cc1b82 100644 --- a/resources/views/livewire/traffic/_sparkline.blade.php +++ b/resources/views/livewire/traffic/_sparkline.blade.php @@ -16,6 +16,7 @@ @param string $colorVar CSS custom property for the line color (e.g. --chart-status-3xx) @param ?string $event Livewire event to listen on for updates (optional) @param ?string $key payload key holding the numeric array (required with $event) + @param ?array $categories initial x-axis timestamps (defaults to $series buckets) --}} @php $initial = $initial ?? []; @@ -23,7 +24,7 @@ $color = $color ?? ''; $event = $event ?? ''; $key = $key ?? ''; - $initialCategories = array_column($series ?? [], 'bucket'); + $initialCategories = $categories ?? array_column($series ?? [], 'bucket'); $label = $label ?? match ($key) { 'requestsSpark' => 'Requests', 'uniquesSpark' => 'Visitors', @@ -74,6 +75,7 @@ timeZoneName: 'short', }); const formatValue = value => { + if (value === null || value === undefined) { return '—'; } const fmt = @js($format); if (fmt === 'bytes') { const units = ['B', 'KB', 'MB', 'GB', 'TB']; diff --git a/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php b/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php index 83206bbfb5..9739b201a0 100644 --- a/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php +++ b/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php @@ -25,12 +25,12 @@ class ChartStubClient extends SentinelMetricsClient return []; } - public function history(string $metric, string $from): array + public function history(string $metric, string $range): array { return [[1000, 10.0], [2000, 20.0]]; } - public function networkHistory(string $from): array + public function networkHistory(string $range): array { return ['rx' => [[1000, 5.0]], 'tx' => [[1000, 1.0]]]; } @@ -46,7 +46,7 @@ class ChartTestableMetrics extends Metrics class DenseChartStubClient extends ChartStubClient { - public function history(string $metric, string $from): array + public function history(string $metric, string $range): array { // 300 points, like a downsampled 24h host series. return array_map(fn ($i) => [$i * 1000, (float) $i], range(1, 300)); @@ -99,12 +99,12 @@ it('includes KPI-tile spark series aligned to one shared time axis', function () $payload = $params[0]; // Union of every series' buckets, sorted; each spark projects onto it, - // filling gaps with 0 (network only has the 1000 bucket → tail is 0). + // leaving gaps as null (network only has the 1000 bucket → tail is null). return $payload['sparkCategories'] == [1000, 2000] && $payload['cpuSpark'] == [10, 20] && $payload['memSpark'] == [10, 20] && $payload['diskSpark'] == [10, 20] - && $payload['netSpark'] == [6, 0]; + && $payload['netSpark'] === [6, null]; }); }); diff --git a/tests/Feature/Livewire/Fleet/MetricsDataTest.php b/tests/Feature/Livewire/Fleet/MetricsDataTest.php index 023b007b90..430b224ded 100644 --- a/tests/Feature/Livewire/Fleet/MetricsDataTest.php +++ b/tests/Feature/Livewire/Fleet/MetricsDataTest.php @@ -30,12 +30,12 @@ class StubMetricsClient extends SentinelMetricsClient return self::$containers[$this->server->uuid] ?? []; } - public function history(string $metric, string $from): array + public function history(string $metric, string $range): array { return []; } - public function networkHistory(string $from): array + public function networkHistory(string $range): array { return ['rx' => [], 'tx' => []]; } @@ -189,3 +189,41 @@ it('renders container metric values without leaking a blade directive', function ->assertDontSee('@break') ->assertSee('/s'); }); + +it('falls back to cpu for an unknown container metric', function () { + $s = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$s->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$s->uuid => [ + ['id' => 'a', 'cpu' => 5.0, 'memUsed' => 900, 'memPercent' => 9.0, 'diskBytes' => 1, 'net' => 0.0], + ]]; + + Livewire::withQueryParams(['cmetric' => 'foo']) + ->test(TestableMetrics::class) + ->assertOk() + ->assertSet('containerMetric', 'cpu') + ->set('containerMetric', 'bar') + ->assertSet('containerMetric', 'cpu'); +}); + +it('keys container rows by server so shared container ids do not collide', function () { + $one = metricsServer($this->team, $this->privateKey); + $two = metricsServer($this->team, $this->privateKey); + $proxy = ['id' => 'coolify-proxy', 'cpu' => 1.0, 'memUsed' => 10, 'memPercent' => 1.0, 'diskBytes' => 1, 'net' => 0.0]; + + StubMetricsClient::$summaries = [$one->uuid => onlineSummary(), $two->uuid => onlineSummary()]; + StubMetricsClient::$containers = [$one->uuid => [$proxy], $two->uuid => [$proxy]]; + + Livewire::test(TestableMetrics::class) + ->assertSeeHtml('wire:key="fleet-container-'.$one->uuid.'-coolify-proxy"') + ->assertSeeHtml('wire:key="fleet-container-'.$two->uuid.'-coolify-proxy"'); +}); + +it('passes the spark time axis to the KPI sparklines on first paint', function () { + $s = metricsServer($this->team, $this->privateKey); + StubMetricsClient::$summaries = [$s->uuid => onlineSummary()]; + + $component = Livewire::test(TestableMetrics::class); + $component->set('chartData', ['sparkCategories' => [1700000100000], 'cpuSpark' => [12.5]]); + + $component->assertSeeHtml('1700000100000'); +}); diff --git a/tests/Unit/Services/SentinelMetricsClientTest.php b/tests/Unit/Services/SentinelMetricsClientTest.php index 388f1e780d..8b48aeabb3 100644 --- a/tests/Unit/Services/SentinelMetricsClientTest.php +++ b/tests/Unit/Services/SentinelMetricsClientTest.php @@ -2,6 +2,7 @@ use App\Models\Server; use App\Services\SentinelMetricsClient; +use Illuminate\Support\Carbon; class FakeMetricsClient extends SentinelMetricsClient { @@ -93,26 +94,61 @@ it('returns an empty container list when the endpoint 404s', function () { expect($client->containersCurrent())->toBe([]); }); -it('maps cpu history to [time, value] pairs', function () { +it('averages cpu history into range-aligned buckets', function () { $client = fakeMetricsClient(); $client->bodies = [ '/cpu/history' => json_encode([ - ['time' => 1000, 'percent' => '10'], - ['time' => 2000, 'percent' => '20'], + ['time' => '1700000000000', 'percent' => '10'], + ['time' => '1700000060000', 'percent' => '20'], + ['time' => '1700000300000', 'percent' => '40'], ]), ]; - expect($client->history('cpu', 'from'))->toBe([[1000, 10.0], [2000, 20.0]]); + // 24h range → 5-minute buckets floored to absolute epoch boundaries. + expect($client->history('cpu', '24h'))->toBe([[1699999800000, 15.0], [1700000100000, 40.0]]); }); -it('downsamples long history series so the chart payload stays small', function () { +it('aligns buckets across servers whose samples have different timestamps', function () { + $a = fakeMetricsClient(); + $a->bodies = ['/cpu/history' => json_encode([['time' => 1700000001234, 'percent' => '10']])]; + $b = fakeMetricsClient(); + $b->bodies = ['/cpu/history' => json_encode([['time' => 1700000047890, 'percent' => '30']])]; + + expect(array_column($a->history('cpu', '24h'), 0))->toBe(array_column($b->history('cpu', '24h'), 0)); +}); + +it('keeps long history series small', function () { $rows = []; - for ($i = 0; $i < 5000; $i++) { - $rows[] = ['time' => $i, 'percent' => (string) ($i % 100)]; + // 24h of raw 20-second samples. + for ($i = 0; $i < 4320; $i++) { + $rows[] = ['time' => 1700000000000 + $i * 20_000, 'percent' => (string) ($i % 100)]; } $client = fakeMetricsClient(); $client->bodies = ['/cpu/history' => json_encode($rows)]; - expect(count($client->history('cpu', 'from')))->toBe(300); + expect(count($client->history('cpu', '24h')))->toBeLessThanOrEqual(300); +}); + +it('keeps only the root mount in disk history', function () { + $client = fakeMetricsClient(); + $client->bodies = [ + '/disk/history' => json_encode([ + ['time' => 1700000000000, 'mount' => '/', 'usedPercent' => 85], + ['time' => 1700000000000, 'mount' => '/boot', 'usedPercent' => 20], + ]), + ]; + + expect($client->history('disk', '24h'))->toBe([[1699999800000, 85.0]]); +}); + +it('rounds the range start to the minute so history responses are cacheable', function () { + Carbon::setTestNow('2026-01-01 12:30:05'); + $first = SentinelMetricsClient::rangeFrom('24h'); + + Carbon::setTestNow('2026-01-01 12:30:45'); + + expect(SentinelMetricsClient::rangeFrom('24h'))->toBe($first)->toBe('2025-12-31T12:30:00Z'); + + Carbon::setTestNow(); });