From 5312ea4dbcb5bd9f0903ef74331cc04614e2a14a Mon Sep 17 00:00:00 2001 From: Aditya Tripathi Date: Sat, 24 Oct 2026 06:23:51 +0000 Subject: [PATCH 1/2] perf(metrics): ask Sentinel to bucket metric history rows Send bucket_ms on the Sentinel metric history reads so Sentinel averages the rows and sends far fewer of them over SSH. Older Sentinels ignore the param and return raw rows, which Coolify still buckets and downsamples. Also quote the history URL in the single-fetch curl command, because the URL now has an & in it. --- app/Services/SentinelMetricsClient.php | 20 ++++++++++++---- app/Traits/HasMetrics.php | 24 +++++++++++++------ tests/Feature/ServerMetricsPrefetchTest.php | 15 ++++++++++++ tests/Unit/Metrics/HasMetricsMappingTest.php | 16 +++++++++++++ .../Services/SentinelMetricsClientTest.php | 17 +++++++++++++ 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php index b9ad06a164..184c4bc4e7 100644 --- a/app/Services/SentinelMetricsClient.php +++ b/app/Services/SentinelMetricsClient.php @@ -35,6 +35,17 @@ class SentinelMetricsClient }; } + /** + * Sentinel averages rows into `bucket_ms` windows. Older Sentinels ignore the param + * and return raw rows, which seriesRows() buckets the same way. + * + * @return array{from: string, bucket_ms: int} + */ + private static function historyQuery(string $range): array + { + return ['from' => self::rangeFrom($range), 'bucket_ms' => self::bucketMs($range)]; + } + /** * @return array|null */ @@ -112,7 +123,7 @@ class SentinelMetricsClient default => ['/cpu/history', 'percent'], }; - return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range, $metric === 'disk'); + return $this->seriesRows($this->url($path, self::historyQuery($range)), $field, $range, $metric === 'disk'); } /** @@ -120,7 +131,7 @@ class SentinelMetricsClient */ public function networkHistory(string $range): array { - $url = $this->url('/network/history', ['from' => self::rangeFrom($range)]); + $url = $this->url('/network/history', self::historyQuery($range)); return [ 'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range), @@ -149,8 +160,9 @@ class SentinelMetricsClient $rows = array_filter($rows, fn ($row) => ($row['mount'] ?? null) === $mount); } - // Average raw samples into epoch-aligned buckets so every server shares the same - // timestamps and the fleet aggregator can sum them point by point. + // Average samples into epoch-aligned buckets so every server shares the same + // timestamps and the fleet aggregator can sum them point by point. A no-op for + // rows Sentinel already bucketed. $bucketMs = self::bucketMs($range); $sums = []; $counts = []; diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php index d39675276d..a9fe454e3a 100644 --- a/app/Traits/HasMetrics.php +++ b/app/Traits/HasMetrics.php @@ -35,7 +35,7 @@ trait HasMetrics // only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it // cannot end the heredoc. Responses are separated by a 0x1E record separator. $script = implode("\nprintf '\\036'\n", array_map( - fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH", + fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from, $mins)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH", $types )); @@ -147,11 +147,11 @@ trait HasMetrics unset($this->prefetchedMetricResponses[$key]); } else { $from = now()->subMinutes($mins)->toIso8601ZuluString(); - $endpoint = $this->getMetricsEndpoint($type, $from); + $endpoint = $this->getMetricsEndpoint($type, $from, $mins); $token = $this->metricsToken($server); $response = instant_remote_process( - ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"], + ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" \"{$endpoint}\"'"], $server, false ); @@ -248,13 +248,23 @@ trait HasMetrics return $this->isServerMetrics() ? $this : $this->destination->server; } - private function getMetricsEndpoint(string $type, string $from): string + /** + * Ranges over an hour ask Sentinel to average rows into whole-minute windows, at most + * 1000 per range, so it sends far fewer rows. Older Sentinels ignore `bucket_ms`; + * mapRows() still downsamples their raw rows. + */ + private function getMetricsEndpoint(string $type, string $from, int $mins): string { $base = 'http://localhost:8888/api'; - if ($this->isServerMetrics()) { - return "{$base}/{$type}/history?from={$from}"; + $query = "from={$from}"; + if ($mins > 60) { + $query .= '&bucket_ms='.((int) ceil($mins / 1000) * 60_000); } - return "{$base}/container/{$this->uuid}/{$type}/history?from={$from}"; + if ($this->isServerMetrics()) { + return "{$base}/{$type}/history?{$query}"; + } + + return "{$base}/container/{$this->uuid}/{$type}/history?{$query}"; } } diff --git a/tests/Feature/ServerMetricsPrefetchTest.php b/tests/Feature/ServerMetricsPrefetchTest.php index 85a51e1ac4..578e39c726 100644 --- a/tests/Feature/ServerMetricsPrefetchTest.php +++ b/tests/Feature/ServerMetricsPrefetchTest.php @@ -68,6 +68,21 @@ it('falls back to one fetch per metric when the batch fails', function () { ->and($commands)->toHaveCount(2); }); +it('quotes the bucketed history URL in a single fetch for a non-root user', function () { + $commands = []; + Process::fake(function (PendingProcess $process) use (&$commands) { + $commands[] = $process->command; + + return Process::result('[{"time":60000,"percent":5}]'); + }); + $this->server->forceFill(['user' => 'deploy'])->saveQuietly(); + + expect($this->server->getCpuMetrics(720))->toBe([[60000, 5.0]]) + ->and($commands)->toHaveCount(1) + ->and($commands[0])->toContain('sudo bash -c') + ->and($commands[0])->toMatch('/"http:\/\/localhost:8888\/api\/cpu\/history\?from=[^"]+&bucket_ms=60000"/'); +}); + it('runs the batched metrics script with sudo for a non-root user', function () { $commands = []; Process::fake(function (PendingProcess $process) use (&$commands) { diff --git a/tests/Unit/Metrics/HasMetricsMappingTest.php b/tests/Unit/Metrics/HasMetricsMappingTest.php index d8b4b805bd..e1f2425343 100644 --- a/tests/Unit/Metrics/HasMetricsMappingTest.php +++ b/tests/Unit/Metrics/HasMetricsMappingTest.php @@ -13,10 +13,17 @@ class MetricsMappingDouble /** @var array type => rows */ public array $rowsByType = []; + public string $uuid = 'app-uuid'; + protected function fetchMetricRows(string $type, int $mins): ?array { return $this->rowsByType[$type] ?? null; } + + public function endpoint(string $type, int $mins): string + { + return $this->getMetricsEndpoint($type, '2026-01-01T00:00:00Z', $mins); + } } it('keeps only the root mount for disk metrics', function () { @@ -73,3 +80,12 @@ it('supports older disk rows without a mount field', function () { expect($double->getDiskMetrics(5))->toBe([[1000, 40.0]]); }); + +it('asks sentinel to bucket ranges over an hour into at most 1000 whole-minute rows', function () { + $double = new MetricsMappingDouble; + + expect($double->endpoint('cpu', 60))->not->toContain('bucket_ms') + ->and($double->endpoint('cpu', 720))->toBe('http://localhost:8888/api/container/app-uuid/cpu/history?from=2026-01-01T00:00:00Z&bucket_ms=60000') + ->and($double->endpoint('cpu', 10080))->toEndWith('&bucket_ms=660000') + ->and($double->endpoint('cpu', 43200))->toEndWith('&bucket_ms=2640000'); +}); diff --git a/tests/Unit/Services/SentinelMetricsClientTest.php b/tests/Unit/Services/SentinelMetricsClientTest.php index 4bf1793d4c..c8c42286b0 100644 --- a/tests/Unit/Services/SentinelMetricsClientTest.php +++ b/tests/Unit/Services/SentinelMetricsClientTest.php @@ -12,8 +12,12 @@ class FakeMetricsClient extends SentinelMetricsClient /** @var array */ public array $failing = []; + /** @var array */ + public array $urls = []; + protected function raw(string $url): string { + $this->urls[] = $url; foreach ($this->failing as $needle) { if (str_contains($url, $needle)) { throw new RuntimeException('boom'); @@ -247,3 +251,16 @@ it('prefers the root mount over the host mount of the Coolify data directory', f expect($client->history('disk', '24h'))->toBe([[1699999800000, 60.0]]); }); + +it('asks sentinel to bucket history rows by range', function (string $range, int $bucketMs) { + $client = fakeMetricsClient(); + + $client->history('memory', $range); + $client->networkHistory($range); + + expect($client->urls)->each->toContain("bucket_ms={$bucketMs}"); +})->with([ + ['24h', 300_000], + ['7d', 1_800_000], + ['30d', 7_200_000], +]); From f8ec00740334ffb620b0bffb3fc7039ac4f86c49 Mon Sep 17 00:00:00 2001 From: Aditya Tripathi Date: Sat, 24 Oct 2026 06:35:52 +0000 Subject: [PATCH 2/2] fix(metrics): pick the root disk on the server Metrics page like the Fleet page The server Metrics page now uses SentinelMetricsClient::rootMountName(): /, else /app/db (Sentinel 1.1.2 and older), else /data/coolify or /data (Sentinel 1.1.3 and newer). Before, it kept only /, so the disk chart was empty when Sentinel did not report a / mount. --- app/Services/SentinelMetricsClient.php | 2 +- app/Traits/HasMetrics.php | 6 ++++-- tests/Unit/Metrics/HasMetricsMappingTest.php | 10 ++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php index 184c4bc4e7..fe3d785c63 100644 --- a/app/Services/SentinelMetricsClient.php +++ b/app/Services/SentinelMetricsClient.php @@ -184,7 +184,7 @@ class SentinelMetricsClient * * @param array> $rows */ - private static function rootMountName(array $rows): ?string + public static function rootMountName(array $rows): ?string { $mounts = array_column($rows, 'mount'); diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php index a9fe454e3a..3a1648fc3f 100644 --- a/app/Traits/HasMetrics.php +++ b/app/Traits/HasMetrics.php @@ -3,6 +3,7 @@ namespace App\Traits; use App\Models\Server; +use App\Services\SentinelMetricsClient; use Illuminate\Contracts\Encryption\DecryptException; use Illuminate\Support\Facades\Log; @@ -80,7 +81,7 @@ trait HasMetrics /** * Root filesystem usage percentage over time. Disk history interleaves every - * mount, so we keep only the root mount before mapping. + * mount, so we keep only the root mount (see SentinelMetricsClient::rootMountName()). */ public function getDiskMetrics(int $mins = 5): ?array { @@ -90,7 +91,8 @@ trait HasMetrics } if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) { - $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/')); + $mount = SentinelMetricsClient::rootMountName($rows); + $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === $mount)); } return $this->mapRows($rows, 'usedPercent', $mins); diff --git a/tests/Unit/Metrics/HasMetricsMappingTest.php b/tests/Unit/Metrics/HasMetricsMappingTest.php index e1f2425343..e6c70be05b 100644 --- a/tests/Unit/Metrics/HasMetricsMappingTest.php +++ b/tests/Unit/Metrics/HasMetricsMappingTest.php @@ -74,6 +74,16 @@ it('does not use non-root mounts when root disk samples are missing', function ( expect($double->getDiskMetrics(5))->toBe([]); }); +it('uses the Sentinel data mount for disk metrics when no root mount is reported', function () { + $double = new MetricsMappingDouble; + $double->rowsByType['disk'] = [ + ['time' => 1000, 'mount' => '/boot', 'usedPercent' => 90], + ['time' => 1000, 'mount' => '/app/db', 'usedPercent' => 30], + ]; + + expect($double->getDiskMetrics(5))->toBe([[1000, 30.0]]); +}); + it('supports older disk rows without a mount field', function () { $double = new MetricsMappingDouble; $double->rowsByType['disk'] = [['time' => 1000, 'usedPercent' => 40]];