starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-06 13:10:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

perf(metrics): ask Sentinel to bucket metric history rows (#12300)

Browse Source
This commit is contained in:
Andras Bacsai
2026-10-27 09:56:41 +01:00
committed by GitHub
parent 1d286427ad f8ec007403
commit 740e5d12fa
5 changed files with 96 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Services/SentinelMetricsClient.php
+17 -5
View File
@@ -35,6 +35,17 @@ class SentinelMetricsClient
};
}
/**
* Sentinel averages rows into `bucket_ms` windows. Older Sentinels ignore the param
* and return raw rows, which seriesRows() buckets the same way.
*
* @return array{from: string, bucket_ms: int}
*/
private static function historyQuery(string $range): array
{
return ['from' => self::rangeFrom($range), 'bucket_ms' => self::bucketMs($range)];
}
/**
* @return array<string, mixed>|null
*/
@@ -112,7 +123,7 @@ class SentinelMetricsClient
default => ['/cpu/history', 'percent'],
};
return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range, $metric === 'disk');
return $this->seriesRows($this->url($path, self::historyQuery($range)), $field, $range, $metric === 'disk');
}
/**
@@ -120,7 +131,7 @@ class SentinelMetricsClient
*/
public function networkHistory(string $range): array
{
$url = $this->url('/network/history', ['from' => self::rangeFrom($range)]);
$url = $this->url('/network/history', self::historyQuery($range));
return [
'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range),
@@ -149,8 +160,9 @@ class SentinelMetricsClient
$rows = array_filter($rows, fn ($row) => ($row['mount'] ?? null) === $mount);
}
// Average raw samples into epoch-aligned buckets so every server shares the same
// timestamps and the fleet aggregator can sum them point by point.
// Average samples into epoch-aligned buckets so every server shares the same
// timestamps and the fleet aggregator can sum them point by point. A no-op for
// rows Sentinel already bucketed.
$bucketMs = self::bucketMs($range);
$sums = [];
$counts = [];
@@ -172,7 +184,7 @@ class SentinelMetricsClient
*
* @param array<int, array<string, mixed>> $rows
*/
private static function rootMountName(array $rows): ?string
public static function rootMountName(array $rows): ?string
{
$mounts = array_column($rows, 'mount');
app/Traits/HasMetrics.php
+21 -9
View File
@@ -3,6 +3,7 @@
namespace App\Traits;
use App\Models\Server;
use App\Services\SentinelMetricsClient;
use Illuminate\Contracts\Encryption\DecryptException;
use Illuminate\Support\Facades\Log;
@@ -35,7 +36,7 @@ trait HasMetrics
// only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it
// cannot end the heredoc. Responses are separated by a 0x1E record separator.
$script = implode("\nprintf '\\036'\n", array_map(
fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH",
fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from, $mins)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH",
$types
));
@@ -80,7 +81,7 @@ trait HasMetrics
/**
* Root filesystem usage percentage over time. Disk history interleaves every
* mount, so we keep only the root mount before mapping.
* mount, so we keep only the root mount (see SentinelMetricsClient::rootMountName()).
*/
public function getDiskMetrics(int $mins = 5): ?array
{
@@ -90,7 +91,8 @@ trait HasMetrics
}
if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) {
$rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/'));
$mount = SentinelMetricsClient::rootMountName($rows);
$rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === $mount));
}
return $this->mapRows($rows, 'usedPercent', $mins);
@@ -147,11 +149,11 @@ trait HasMetrics
unset($this->prefetchedMetricResponses[$key]);
} else {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$endpoint = $this->getMetricsEndpoint($type, $from);
$endpoint = $this->getMetricsEndpoint($type, $from, $mins);
$token = $this->metricsToken($server);
$response = instant_remote_process(
["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"],
["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" \"{$endpoint}\"'"],
$server,
false
);
@@ -248,13 +250,23 @@ trait HasMetrics
return $this->isServerMetrics() ? $this : $this->destination->server;
}
private function getMetricsEndpoint(string $type, string $from): string
/**
* Ranges over an hour ask Sentinel to average rows into whole-minute windows, at most
* 1000 per range, so it sends far fewer rows. Older Sentinels ignore `bucket_ms`;
* mapRows() still downsamples their raw rows.
*/
private function getMetricsEndpoint(string $type, string $from, int $mins): string
{
$base = 'http://localhost:8888/api';
if ($this->isServerMetrics()) {
return "{$base}/{$type}/history?from={$from}";
$query = "from={$from}";
if ($mins > 60) {
$query .= '&bucket_ms='.((int) ceil($mins / 1000) * 60_000);
}
return "{$base}/container/{$this->uuid}/{$type}/history?from={$from}";
if ($this->isServerMetrics()) {
return "{$base}/{$type}/history?{$query}";
}
return "{$base}/container/{$this->uuid}/{$type}/history?{$query}";
}
}
tests/Feature/ServerMetricsPrefetchTest.php
+15
View File
@@ -68,6 +68,21 @@ it('falls back to one fetch per metric when the batch fails', function () {
->and($commands)->toHaveCount(2);
});
it('quotes the bucketed history URL in a single fetch for a non-root user', function () {
$commands = [];
Process::fake(function (PendingProcess $process) use (&$commands) {
$commands[] = $process->command;
return Process::result('[{"time":60000,"percent":5}]');
});
$this->server->forceFill(['user' => 'deploy'])->saveQuietly();
expect($this->server->getCpuMetrics(720))->toBe([[60000, 5.0]])
->and($commands)->toHaveCount(1)
->and($commands[0])->toContain('sudo bash -c')
->and($commands[0])->toMatch('/"http:\/\/localhost:8888\/api\/cpu\/history\?from=[^"]+&bucket_ms=60000"/');
});
it('runs the batched metrics script with sudo for a non-root user', function () {
$commands = [];
Process::fake(function (PendingProcess $process) use (&$commands) {
tests/Unit/Metrics/HasMetricsMappingTest.php
+26
View File
@@ -13,10 +13,17 @@ class MetricsMappingDouble
/** @var array<string, ?array> type => rows */
public array $rowsByType = [];
public string $uuid = 'app-uuid';
protected function fetchMetricRows(string $type, int $mins): ?array
{
return $this->rowsByType[$type] ?? null;
}
public function endpoint(string $type, int $mins): string
{
return $this->getMetricsEndpoint($type, '2026-01-01T00:00:00Z', $mins);
}
}
it('keeps only the root mount for disk metrics', function () {
@@ -67,9 +74,28 @@ it('does not use non-root mounts when root disk samples are missing', function (
expect($double->getDiskMetrics(5))->toBe([]);
});
it('uses the Sentinel data mount for disk metrics when no root mount is reported', function () {
$double = new MetricsMappingDouble;
$double->rowsByType['disk'] = [
['time' => 1000, 'mount' => '/boot', 'usedPercent' => 90],
['time' => 1000, 'mount' => '/app/db', 'usedPercent' => 30],
];
expect($double->getDiskMetrics(5))->toBe([[1000, 30.0]]);
});
it('supports older disk rows without a mount field', function () {
$double = new MetricsMappingDouble;
$double->rowsByType['disk'] = [['time' => 1000, 'usedPercent' => 40]];
expect($double->getDiskMetrics(5))->toBe([[1000, 40.0]]);
});
it('asks sentinel to bucket ranges over an hour into at most 1000 whole-minute rows', function () {
$double = new MetricsMappingDouble;
expect($double->endpoint('cpu', 60))->not->toContain('bucket_ms')
->and($double->endpoint('cpu', 720))->toBe('http://localhost:8888/api/container/app-uuid/cpu/history?from=2026-01-01T00:00:00Z&bucket_ms=60000')
->and($double->endpoint('cpu', 10080))->toEndWith('&bucket_ms=660000')
->and($double->endpoint('cpu', 43200))->toEndWith('&bucket_ms=2640000');
});
tests/Unit/Services/SentinelMetricsClientTest.php
+17
View File
@@ -12,8 +12,12 @@ class FakeMetricsClient extends SentinelMetricsClient
/** @var array<int, string> */
public array $failing = [];
/** @var array<int, string> */
public array $urls = [];
protected function raw(string $url): string
{
$this->urls[] = $url;
foreach ($this->failing as $needle) {
if (str_contains($url, $needle)) {
throw new RuntimeException('boom');
@@ -247,3 +251,16 @@ it('prefers the root mount over the host mount of the Coolify data directory', f
expect($client->history('disk', '24h'))->toBe([[1699999800000, 60.0]]);
});
it('asks sentinel to bucket history rows by range', function (string $range, int $bucketMs) {
$client = fakeMetricsClient();
$client->history('memory', $range);
$client->networkHistory($range);
expect($client->urls)->each->toContain("bucket_ms={$bucketMs}");
})->with([
['24h', 300_000],
['7d', 1_800_000],
['30d', 7_200_000],
]);