starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-06 13:10:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(metrics): align fleet series by time bucket, harden fleet page inputs

Browse Source 
Address review findings on the fleet metrics page.

Data correctness:
- bucket raw Sentinel samples into epoch-aligned windows (5m/30m/2h per range)
  and average them, replacing per-server LTTB. Every server now reports the
  same bucket timestamps, so the fleet sums add up point by point instead of
  zigzagging between servers (and RX+TX really combine)
- keep only the root mount in disk history, matching HasMetrics::getDiskMetrics
- leave sparkline gaps as null instead of 0, so absent samples read as gaps

Robustness:
- normalize containerMetric in mount()/updatedContainerMetric(), so
  /metrics?cmetric=foo no longer throws an undefined array key
- include the server uuid in the container row wire:key; system containers
  share one Sentinel id across servers
- round the range start to the minute so the 60s history cache can hit
- cap the Sentinel curl with --max-time 10, so one hung request cannot stall
  the serial per-server loop

UI:
- pass the spark time axis on first paint, so tooltips no longer show 1970
- show the active series name (RX/TX) in the server network chart tooltip
This commit is contained in:
Aditya Tripathi
2026-09-18 07:44:22 +00:00
parent ab436942c9
commit faca466cd6
9 changed files with 163 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Livewire/Concerns/BuildsMetricsChartPayload.php
+5 -5
View File
@@ -55,8 +55,8 @@ trait BuildsMetricsChartPayload
* evenly-spaced indices (first and last always kept), so the series stay aligned.
*
* @param array<int, int> $categories
* @param array<string, array<int, float|int>> $sparks
* @return array{0: array<int, int>, 1: array<string, array<int, float|int>>}
* @param array<string, array<int, float|int|null>> $sparks
* @return array{0: array<int, int>, 1: array<string, array<int, float|int|null>>}
*/
private function capSparks(array $categories, array $sparks, int $max): array
{
@@ -117,11 +117,11 @@ trait BuildsMetricsChartPayload
}
/**
* Project a [ts, value] series onto the shared category axis, filling gaps with 0.
* Project a [ts, value] series onto the shared category axis, leaving gaps as null.
*
* @param array<int, array{0: int, 1: float}> $series
* @param array<int, int> $categories
* @return array<int, float|int>
* @return array<int, float|int|null>
*/
private function alignSpark(array $series, array $categories): array
{
@@ -130,6 +130,6 @@ trait BuildsMetricsChartPayload
$map[(int) $ts] = $value;
}
return array_map(fn ($ts) => $map[$ts] ?? 0, $categories);
return array_map(fn ($ts) => $map[$ts] ?? null, $categories);
}
}
app/Livewire/Fleet/Metrics.php
+19 -7
View File
@@ -35,6 +35,8 @@ class Metrics extends Component
#[Url(as: 'cmetric')]
public string $containerMetric = 'cpu';
private const CONTAINER_METRICS = ['cpu', 'memory', 'disk', 'network'];
public bool $live = false;
/** @var array<string, mixed> */
@@ -71,6 +73,8 @@ class Metrics extends Component
$this->serverUuid = '';
}
$this->normalizeContainerMetric();
if ($this->servers->isNotEmpty()) {
$this->loadData();
}
@@ -83,10 +87,19 @@ class Metrics extends Component
public function updatedContainerMetric(): void
{
$this->normalizeContainerMetric();
// Re-rank the already-fetched containers; no need to re-hit Sentinel.
$this->topContainers = FleetMetricsAggregator::rankContainers($this->containersRaw, $this->containerMetric);
}
private function normalizeContainerMetric(): void
{
if (! in_array($this->containerMetric, self::CONTAINER_METRICS, true)) {
$this->containerMetric = 'cpu';
}
}
public function setRange(string $range): void
{
$this->range = in_array($range, ['24h', '7d', '30d'], true) ? $range : '24h';
@@ -119,7 +132,6 @@ class Metrics extends Component
$rows = [];
$containers = [];
$series = ['cpu' => [], 'memory' => [], 'disk' => [], 'load' => [], 'networkRx' => [], 'networkTx' => []];
$from = SentinelMetricsClient::rangeFrom($this->range);
foreach ($this->targetServers() as $server) {
$client = $this->metricsClient($server);
@@ -145,14 +157,14 @@ class Metrics extends Component
foreach ($current as $c) {
// Names/images are resolved lazily for the displayed rows only (containerMeta),
// so a server with many containers doesn't trigger a resolve storm here.
$containers[] = array_merge($c, ['server' => $server->name]);
$containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]);
}
$series['cpu'][] = $client->history('cpu', $from);
$series['memory'][] = $client->history('memory', $from);
$series['disk'][] = $client->history('disk', $from);
$series['load'][] = $client->history('load', $from);
$net = $client->networkHistory($from);
$series['cpu'][] = $client->history('cpu', $this->range);
$series['memory'][] = $client->history('memory', $this->range);
$series['disk'][] = $client->history('disk', $this->range);
$series['load'][] = $client->history('load', $this->range);
$net = $client->networkHistory($this->range);
$series['networkRx'][] = $net['rx'];
$series['networkTx'][] = $net['tx'];
}
app/Services/SentinelMetricsClient.php
+41 -24
View File
@@ -9,22 +9,32 @@ class SentinelMetricsClient
{
private string $base = 'http://localhost:8888/api';
/** Cap on points per trend series after downsampling raw Sentinel samples. */
private const MAX_SERIES_POINTS = 300;
public function __construct(protected Server $server) {}
/** Rounded to the minute so the history URL (and its cache key) is stable within a minute. */
public static function rangeFrom(string $range): string
{
$now = now()->startOfMinute();
$from = match ($range) {
'7d' => now()->subDays(7),
'30d' => now()->subDays(30),
default => now()->subDay(),
'7d' => $now->subDays(7),
'30d' => $now->subDays(30),
default => $now->subDay(),
};
return $from->toIso8601ZuluString();
}
/** Bucket width per range, kept at roughly 300 points per series. */
public static function bucketMs(string $range): int
{
return match ($range) {
'7d' => 30 * 60_000,
'30d' => 2 * 60 * 60_000,
default => 5 * 60_000,
};
}
/**
* @return array<string, mixed>|null
*/
@@ -89,7 +99,7 @@ class SentinelMetricsClient
/**
* @return array<int, array{0: int, 1: float}>
*/
public function history(string $metric, string $from): array
public function history(string $metric, string $range): array
{
[$path, $field] = match ($metric) {
'memory' => ['/memory/history', 'used'],
@@ -98,26 +108,26 @@ class SentinelMetricsClient
default => ['/cpu/history', 'percent'],
};
return $this->seriesRows($this->url($path, ['from' => $from]), $field);
return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range);
}
/**
* @return array{rx: array<int, array{0: int, 1: float}>, tx: array<int, array{0: int, 1: float}>}
*/
public function networkHistory(string $from): array
public function networkHistory(string $range): array
{
$url = $this->url('/network/history', ['from' => $from]);
$url = $this->url('/network/history', ['from' => self::rangeFrom($range)]);
return [
'rx' => $this->seriesRows($url, 'rxBytesPerSec'),
'tx' => $this->seriesRows($url, 'txBytesPerSec'),
'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range),
'tx' => $this->seriesRows($url, 'txBytesPerSec', $range),
];
}
/**
* @return array<int, array{0: int, 1: float}>
*/
private function seriesRows(string $url, string $field): array
private function seriesRows(string $url, string $field, string $range): array
{
try {
$rows = json_decode($this->raw($url), true);
@@ -129,18 +139,25 @@ class SentinelMetricsClient
return [];
}
$series = array_values(array_map(
fn ($r) => [(int) ($r['time'] ?? 0), (float) ($r[$field] ?? 0)],
$rows
));
// Sentinel returns raw samples (thousands of points over 24h). Cap the series so
// the chart payload stays small enough to embed and morph cheaply.
if (count($series) > self::MAX_SERIES_POINTS) {
$series = downsampleLTTB($series, self::MAX_SERIES_POINTS);
// Disk history interleaves every mount; keep the root filesystem only.
$root = array_filter($rows, fn ($r) => ($r['mount'] ?? null) === '/');
if ($root !== []) {
$rows = $root;
}
return $series;
// Average raw samples into epoch-aligned buckets so every server shares the same
// timestamps and the fleet aggregator can sum them point by point.
$bucketMs = self::bucketMs($range);
$sums = [];
$counts = [];
foreach ($rows as $r) {
$bucket = intdiv((int) ($r['time'] ?? 0), $bucketMs) * $bucketMs;
$sums[$bucket] = ($sums[$bucket] ?? 0.0) + (float) ($r[$field] ?? 0);
$counts[$bucket] = ($counts[$bucket] ?? 0) + 1;
}
ksort($sums);
return array_map(fn ($bucket) => [$bucket, round($sums[$bucket] / $counts[$bucket], 2)], array_keys($sums));
}
/**
@@ -187,7 +204,7 @@ class SentinelMetricsClient
$token = $this->server->settings->ensureValidSentinelToken();
return instant_remote_process(
["docker exec coolify-sentinel sh -c 'curl -s -H \"Authorization: Bearer {$token}\" \"{$url}\"'"],
["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"],
$this->server,
false
);
resources/views/livewire/fleet/metrics.blade.php
+5 -1
View File
@@ -131,6 +131,7 @@ $serverListboxOptions = array_merge(
'id' => $chartId.'-spark-cpu',
'initial' => $chartData['cpuSpark'] ?? [],
'color' => '#3b82f6',
'categories' => $chartData['sparkCategories'] ?? [],
'event' => 'refreshChartData-'.$chartId,
'key' => 'cpuSpark',
'label' => $scoped ? 'CPU' : 'Fleet CPU',
@@ -146,6 +147,7 @@ $serverListboxOptions = array_merge(
'id' => $chartId.'-spark-memory',
'initial' => $chartData['memSpark'] ?? [],
'color' => '#8b5cf6',
'categories' => $chartData['sparkCategories'] ?? [],
'event' => 'refreshChartData-'.$chartId,
'key' => 'memSpark',
'label' => 'Memory used',
@@ -161,6 +163,7 @@ $serverListboxOptions = array_merge(
'id' => $chartId.'-spark-disk',
'initial' => $chartData['diskSpark'] ?? [],
'color' => '#ef4444',
'categories' => $chartData['sparkCategories'] ?? [],
'event' => 'refreshChartData-'.$chartId,
'key' => 'diskSpark',
'label' => $scoped ? 'Disk' : 'Fleet disk',
@@ -176,6 +179,7 @@ $serverListboxOptions = array_merge(
'id' => $chartId.'-spark-network',
'initial' => $chartData['netSpark'] ?? [],
'color' => '#10b981',
'categories' => $chartData['sparkCategories'] ?? [],
'event' => 'refreshChartData-'.$chartId,
'key' => 'netSpark',
'label' => 'Network',
@@ -307,7 +311,7 @@ $serverListboxOptions = array_merge(
default => round($c['cpu'] ?? 0).'%',
};
@endphp
<div wire:key="fleet-container-{{ $c['id'] }}"
<div wire:key="fleet-container-{{ $c['serverUuid'] ?? $c['server'] }}-{{ $c['id'] }}"
class="data-table-row fleet-containers-table-grid border-b border-neutral-200 last:border-b-0 dark:border-white/[0.07]">
<div class="self-center text-center text-[12px] tabular-nums text-neutral-400 dark:text-fg-faint">{{ $loop->iteration }}</div>
<div class="flex min-w-0 flex-col">
resources/views/livewire/server/charts.blade.php
+1 -1
View File
@@ -209,7 +209,7 @@
const timestamp = w.globals.seriesX[seriesIndex][dataPointIndex];
return `<div class="apexcharts-tooltip-custom">
<div class="apexcharts-tooltip-custom-value">${name}: <span class="apexcharts-tooltip-value-bold">${formatter(value)}</span></div>
<div class="apexcharts-tooltip-custom-value">${w.globals.seriesNames?.[seriesIndex] ?? name}: <span class="apexcharts-tooltip-value-bold">${formatter(value)}</span></div>
<div class="apexcharts-tooltip-custom-title">Your time: ${formatLocalTimestamp(timestamp)}</div>
<div class="apexcharts-tooltip-custom-title">UTC: ${formatUtcTimestamp(timestamp)}</div>
</div>`;
resources/views/livewire/traffic/_sparkline.blade.php
+3 -1
View File
@@ -16,6 +16,7 @@
@param string $colorVar CSS custom property for the line color (e.g. --chart-status-3xx)
@param ?string $event Livewire event to listen on for updates (optional)
@param ?string $key payload key holding the numeric array (required with $event)
@param ?array $categories initial x-axis timestamps (defaults to $series buckets)
--}}
@php
$initial = $initial ?? [];
@@ -23,7 +24,7 @@
$color = $color ?? '';
$event = $event ?? '';
$key = $key ?? '';
$initialCategories = array_column($series ?? [], 'bucket');
$initialCategories = $categories ?? array_column($series ?? [], 'bucket');
$label = $label ?? match ($key) {
'requestsSpark' => 'Requests',
'uniquesSpark' => 'Visitors',
@@ -74,6 +75,7 @@
timeZoneName: 'short',
});
const formatValue = value => {
if (value === null || value === undefined) { return '—'; }
const fmt = @js($format);
if (fmt === 'bytes') {
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php
+5 -5
View File
@@ -25,12 +25,12 @@ class ChartStubClient extends SentinelMetricsClient
return [];
}
public function history(string $metric, string $from): array
public function history(string $metric, string $range): array
{
return [[1000, 10.0], [2000, 20.0]];
}
public function networkHistory(string $from): array
public function networkHistory(string $range): array
{
return ['rx' => [[1000, 5.0]], 'tx' => [[1000, 1.0]]];
}
@@ -46,7 +46,7 @@ class ChartTestableMetrics extends Metrics
class DenseChartStubClient extends ChartStubClient
{
public function history(string $metric, string $from): array
public function history(string $metric, string $range): array
{
// 300 points, like a downsampled 24h host series.
return array_map(fn ($i) => [$i * 1000, (float) $i], range(1, 300));
@@ -99,12 +99,12 @@ it('includes KPI-tile spark series aligned to one shared time axis', function ()
$payload = $params[0];
// Union of every series' buckets, sorted; each spark projects onto it,
// filling gaps with 0 (network only has the 1000 bucket → tail is 0).
// leaving gaps as null (network only has the 1000 bucket → tail is null).
return $payload['sparkCategories'] == [1000, 2000]
&& $payload['cpuSpark'] == [10, 20]
&& $payload['memSpark'] == [10, 20]
&& $payload['diskSpark'] == [10, 20]
&& $payload['netSpark'] == [6, 0];
&& $payload['netSpark'] === [6, null];
});
});
tests/Feature/Livewire/Fleet/MetricsDataTest.php
+40 -2
View File
@@ -30,12 +30,12 @@ class StubMetricsClient extends SentinelMetricsClient
return self::$containers[$this->server->uuid] ?? [];
}
public function history(string $metric, string $from): array
public function history(string $metric, string $range): array
{
return [];
}
public function networkHistory(string $from): array
public function networkHistory(string $range): array
{
return ['rx' => [], 'tx' => []];
}
@@ -189,3 +189,41 @@ it('renders container metric values without leaking a blade directive', function
->assertDontSee('@break')
->assertSee('/s');
});
it('falls back to cpu for an unknown container metric', function () {
$s = metricsServer($this->team, $this->privateKey);
StubMetricsClient::$summaries = [$s->uuid => onlineSummary()];
StubMetricsClient::$containers = [$s->uuid => [
['id' => 'a', 'cpu' => 5.0, 'memUsed' => 900, 'memPercent' => 9.0, 'diskBytes' => 1, 'net' => 0.0],
]];
Livewire::withQueryParams(['cmetric' => 'foo'])
->test(TestableMetrics::class)
->assertOk()
->assertSet('containerMetric', 'cpu')
->set('containerMetric', 'bar')
->assertSet('containerMetric', 'cpu');
});
it('keys container rows by server so shared container ids do not collide', function () {
$one = metricsServer($this->team, $this->privateKey);
$two = metricsServer($this->team, $this->privateKey);
$proxy = ['id' => 'coolify-proxy', 'cpu' => 1.0, 'memUsed' => 10, 'memPercent' => 1.0, 'diskBytes' => 1, 'net' => 0.0];
StubMetricsClient::$summaries = [$one->uuid => onlineSummary(), $two->uuid => onlineSummary()];
StubMetricsClient::$containers = [$one->uuid => [$proxy], $two->uuid => [$proxy]];
Livewire::test(TestableMetrics::class)
->assertSeeHtml('wire:key="fleet-container-'.$one->uuid.'-coolify-proxy"')
->assertSeeHtml('wire:key="fleet-container-'.$two->uuid.'-coolify-proxy"');
});
it('passes the spark time axis to the KPI sparklines on first paint', function () {
$s = metricsServer($this->team, $this->privateKey);
StubMetricsClient::$summaries = [$s->uuid => onlineSummary()];
$component = Livewire::test(TestableMetrics::class);
$component->set('chartData', ['sparkCategories' => [1700000100000], 'cpuSpark' => [12.5]]);
$component->assertSeeHtml('1700000100000');
});
tests/Unit/Services/SentinelMetricsClientTest.php
+44 -8
View File
@@ -2,6 +2,7 @@
use App\Models\Server;
use App\Services\SentinelMetricsClient;
use Illuminate\Support\Carbon;
class FakeMetricsClient extends SentinelMetricsClient
{
@@ -93,26 +94,61 @@ it('returns an empty container list when the endpoint 404s', function () {
expect($client->containersCurrent())->toBe([]);
});
it('maps cpu history to [time, value] pairs', function () {
it('averages cpu history into range-aligned buckets', function () {
$client = fakeMetricsClient();
$client->bodies = [
'/cpu/history' => json_encode([
['time' => 1000, 'percent' => '10'],
['time' => 2000, 'percent' => '20'],
['time' => '1700000000000', 'percent' => '10'],
['time' => '1700000060000', 'percent' => '20'],
['time' => '1700000300000', 'percent' => '40'],
]),
];
expect($client->history('cpu', 'from'))->toBe([[1000, 10.0], [2000, 20.0]]);
// 24h range → 5-minute buckets floored to absolute epoch boundaries.
expect($client->history('cpu', '24h'))->toBe([[1699999800000, 15.0], [1700000100000, 40.0]]);
});
it('downsamples long history series so the chart payload stays small', function () {
it('aligns buckets across servers whose samples have different timestamps', function () {
$a = fakeMetricsClient();
$a->bodies = ['/cpu/history' => json_encode([['time' => 1700000001234, 'percent' => '10']])];
$b = fakeMetricsClient();
$b->bodies = ['/cpu/history' => json_encode([['time' => 1700000047890, 'percent' => '30']])];
expect(array_column($a->history('cpu', '24h'), 0))->toBe(array_column($b->history('cpu', '24h'), 0));
});
it('keeps long history series small', function () {
$rows = [];
for ($i = 0; $i < 5000; $i++) {
$rows[] = ['time' => $i, 'percent' => (string) ($i % 100)];
// 24h of raw 20-second samples.
for ($i = 0; $i < 4320; $i++) {
$rows[] = ['time' => 1700000000000 + $i * 20_000, 'percent' => (string) ($i % 100)];
}
$client = fakeMetricsClient();
$client->bodies = ['/cpu/history' => json_encode($rows)];
expect(count($client->history('cpu', 'from')))->toBe(300);
expect(count($client->history('cpu', '24h')))->toBeLessThanOrEqual(300);
});
it('keeps only the root mount in disk history', function () {
$client = fakeMetricsClient();
$client->bodies = [
'/disk/history' => json_encode([
['time' => 1700000000000, 'mount' => '/', 'usedPercent' => 85],
['time' => 1700000000000, 'mount' => '/boot', 'usedPercent' => 20],
]),
];
expect($client->history('disk', '24h'))->toBe([[1699999800000, 85.0]]);
});
it('rounds the range start to the minute so history responses are cacheable', function () {
Carbon::setTestNow('2026-01-01 12:30:05');
$first = SentinelMetricsClient::rangeFrom('24h');
Carbon::setTestNow('2026-01-01 12:30:45');
expect(SentinelMetricsClient::rangeFrom('24h'))->toBe($first)->toBe('2025-12-31T12:30:00Z');
Carbon::setTestNow();
});