feat(metrics): add SentinelMetricsClient with graceful degradation

2026-11-12 15:53:46 -04:00 · 2026-09-16 07:56:44 +00:00
parent bc3ec590d0
commit ba8ed4e8fb
2 changed files with 305 additions and 0 deletions
@@ -0,0 +1,199 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\Server;
+use Illuminate\Support\Facades\Cache;
+
+class SentinelMetricsClient
+{
+    private string $base = 'http://localhost:8888/api';
+
+    public function __construct(protected Server $server) {}
+
+    public static function rangeFrom(string $range): string
+    {
+        $from = match ($range) {
+            '7d' => now()->subDays(7),
+            '30d' => now()->subDays(30),
+            default => now()->subDay(),
+        };
+
+        return $from->toIso8601ZuluString();
+    }
+
+    /**
+     * @return array<string, mixed>|null
+     */
+    public function summary(): ?array
+    {
+        try {
+            $json = json_decode($this->raw($this->url('/summary')), true);
+        } catch (\Throwable) {
+            return null;
+        }
+
+        if (! is_array($json)) {
+            return null;
+        }
+
+        $disk = $this->rootMount($json['disk'] ?? []);
+
+        return [
+            'online' => true,
+            'cpu' => isset($json['cpu']['percent']) ? (float) $json['cpu']['percent'] : null,
+            'memUsed' => isset($json['memory']['used']) ? (int) $json['memory']['used'] : null,
+            'memTotal' => isset($json['memory']['total']) ? (int) $json['memory']['total'] : null,
+            'diskUsed' => $disk['used'],
+            'diskTotal' => $disk['total'],
+            'diskPercent' => $disk['percent'],
+            'load1' => isset($json['load']['load1']) ? (float) $json['load']['load1'] : null,
+            'netRx' => isset($json['network']['rxBytesPerSec']) ? (float) $json['network']['rxBytesPerSec'] : null,
+            'netTx' => isset($json['network']['txBytesPerSec']) ? (float) $json['network']['txBytesPerSec'] : null,
+        ];
+    }
+
+    /**
+     * @return array<int, array<string, mixed>>
+     */
+    public function containersCurrent(): array
+    {
+        try {
+            $rows = json_decode($this->raw($this->url('/containers/current')), true);
+        } catch (\Throwable) {
+            return [];
+        }
+
+        if (! is_array($rows)) {
+            return [];
+        }
+
+        return array_values(array_map(function ($c) {
+            $disk = ($c['disk']['writableLayer'] ?? 0) + ($c['disk']['volumesTotal'] ?? 0);
+            $net = ($c['network']['rxBytesPerSec'] ?? 0) + ($c['network']['txBytesPerSec'] ?? 0);
+
+            return [
+                'id' => (string) ($c['id'] ?? ''),
+                'cpu' => isset($c['cpu']['percent']) ? (float) $c['cpu']['percent'] : null,
+                'memUsed' => isset($c['memory']['used']) ? (int) $c['memory']['used'] : null,
+                'memPercent' => isset($c['memory']['usedPercent']) ? (float) $c['memory']['usedPercent'] : null,
+                'diskBytes' => isset($c['disk']) ? (int) $disk : null,
+                'net' => isset($c['network']) ? (float) $net : null,
+            ];
+        }, $rows));
+    }
+
+    /**
+     * @return array<int, array{0: int, 1: float}>
+     */
+    public function history(string $metric, string $from): array
+    {
+        [$path, $field] = match ($metric) {
+            'memory' => ['/memory/history', 'used'],
+            'disk' => ['/disk/history', 'usedPercent'],
+            'load' => ['/load/history', 'load1'],
+            default => ['/cpu/history', 'percent'],
+        };
+
+        return $this->seriesRows($this->url($path, ['from' => $from]), $field);
+    }
+
+    /**
+     * @return array{rx: array<int, array{0: int, 1: float}>, tx: array<int, array{0: int, 1: float}>}
+     */
+    public function networkHistory(string $from): array
+    {
+        $url = $this->url('/network/history', ['from' => $from]);
+
+        return [
+            'rx' => $this->seriesRows($url, 'rxBytesPerSec'),
+            'tx' => $this->seriesRows($url, 'txBytesPerSec'),
+        ];
+    }
+
+    /**
+     * @return array<int, array{0: int, 1: float}>
+     */
+    private function seriesRows(string $url, string $field): array
+    {
+        try {
+            $rows = json_decode($this->raw($url), true);
+        } catch (\Throwable) {
+            return [];
+        }
+
+        if (! is_array($rows)) {
+            return [];
+        }
+
+        return array_values(array_map(
+            fn ($r) => [(int) ($r['time'] ?? 0), (float) ($r[$field] ?? 0)],
+            $rows
+        ));
+    }
+
+    /**
+     * @param  array<int, array<string, mixed>>  $mounts
+     * @return array{used: ?int, total: ?int, percent: ?float}
+     */
+    private function rootMount(array $mounts): array
+    {
+        if ($mounts === []) {
+            return ['used' => null, 'total' => null, 'percent' => null];
+        }
+
+        $root = collect($mounts)->firstWhere('mount', '/') ?? $mounts[0];
+
+        return [
+            'used' => (int) ($root['used'] ?? 0),
+            'total' => (int) ($root['total'] ?? 0),
+            'percent' => (float) ($root['usedPercent'] ?? 0),
+        ];
+    }
+
+    /**
+     * @param  array<string, mixed>  $query
+     */
+    private function url(string $path, array $query = []): string
+    {
+        $qs = empty($query) ? '' : '?'.str_replace('%3A', ':', http_build_query($query));
+
+        return $this->base.$path.$qs;
+    }
+
+    private function cacheKey(string $url): string
+    {
+        return 'metrics:'.$this->server->uuid.':'.md5($url);
+    }
+
+    protected function raw(string $url): string
+    {
+        return Cache::remember($this->cacheKey($url), 60, fn () => $this->guard($this->remoteFetch($url)));
+    }
+
+    protected function remoteFetch(string $url): string
+    {
+        $token = $this->server->settings->ensureValidSentinelToken();
+
+        return instant_remote_process(
+            ["docker exec coolify-sentinel sh -c 'curl -s -H \"Authorization: Bearer {$token}\" \"{$url}\"'"],
+            $this->server,
+            false
+        );
+    }
+
+    private function guard(string $response): string
+    {
+        $payload = json_decode($response, true);
+
+        if ($payload === null) {
+            throw new \RuntimeException('Metrics returned an invalid response.');
+        }
+
+        if (is_array($payload) && array_key_exists('error', $payload)) {
+            throw new \RuntimeException('Metrics request failed.');
+        }
+
+        return $response;
+    }
+}
@@ -0,0 +1,106 @@
+<?php
+
+use App\Models\Server;
+use App\Services\SentinelMetricsClient;
+
+class FakeMetricsClient extends SentinelMetricsClient
+{
+    /** @var array<string, string> substring => body */
+    public array $bodies = [];
+
+    /** @var array<int, string> */
+    public array $failing = [];
+
+    protected function raw(string $url): string
+    {
+        foreach ($this->failing as $needle) {
+            if (str_contains($url, $needle)) {
+                throw new RuntimeException('boom');
+            }
+        }
+        foreach ($this->bodies as $needle => $body) {
+            if (str_contains($url, $needle)) {
+                return $body;
+            }
+        }
+
+        return '[]';
+    }
+}
+
+function fakeMetricsClient(): FakeMetricsClient
+{
+    $server = Server::factory()->make(['uuid' => 'srv-uuid']);
+
+    return new FakeMetricsClient($server);
+}
+
+it('normalizes the host summary, choosing the root disk mount', function () {
+    $client = fakeMetricsClient();
+    $client->bodies = [
+        '/summary' => json_encode([
+            'cpu' => ['time' => 't', 'percent' => 42.5],
+            'memory' => ['time' => 't', 'total' => 1000, 'used' => 250, 'usedPercent' => 25.0],
+            'disk' => [
+                ['time' => 't', 'mount' => '/boot', 'total' => 10, 'used' => 9, 'usedPercent' => 90.0],
+                ['time' => 't', 'mount' => '/', 'total' => 100, 'used' => 40, 'usedPercent' => 40.0],
+            ],
+            'network' => ['time' => 't', 'rxBytesPerSec' => 100.0, 'txBytesPerSec' => 20.0],
+            'load' => ['time' => 't', 'load1' => 1.5, 'load5' => 1.0, 'load15' => 0.5],
+        ]),
+    ];
+
+    $summary = $client->summary();
+
+    expect($summary['cpu'])->toBe(42.5);
+    expect($summary['memUsed'])->toBe(250);
+    expect($summary['diskPercent'])->toBe(40.0); // root mount, not /boot
+    expect($summary['netRx'])->toBe(100.0);
+    expect($summary['load1'])->toBe(1.5);
+});
+
+it('returns null summary when the endpoint fails', function () {
+    $client = fakeMetricsClient();
+    $client->failing = ['/summary'];
+
+    expect($client->summary())->toBeNull();
+});
+
+it('normalizes containers, parsing the stringified cpu percent', function () {
+    $client = fakeMetricsClient();
+    $client->bodies = [
+        '/containers/current' => json_encode([
+            ['id' => 'app-a', 'cpu' => ['time' => 't', 'percent' => '12.5'],
+                'memory' => ['used' => 500, 'usedPercent' => 5.0],
+                'disk' => ['writableLayer' => 3, 'volumesTotal' => 7],
+                'network' => ['rxBytesPerSec' => 4.0, 'txBytesPerSec' => 1.0]],
+        ]),
+    ];
+
+    $rows = $client->containersCurrent();
+
+    expect($rows[0]['id'])->toBe('app-a');
+    expect($rows[0]['cpu'])->toBe(12.5);
+    expect($rows[0]['memUsed'])->toBe(500);
+    expect($rows[0]['diskBytes'])->toBe(10); // writableLayer + volumesTotal
+    expect($rows[0]['net'])->toBe(5.0);      // rx + tx
+});
+
+it('returns an empty container list when the endpoint 404s', function () {
+    $client = fakeMetricsClient();
+    $client->failing = ['/containers/current'];
+
+    expect($client->containersCurrent())->toBe([]);
+});
+
+it('maps cpu history to [time, value] pairs', function () {
+    $client = fakeMetricsClient();
+    $client->bodies = [
+        '/cpu/history' => json_encode([
+            ['time' => 1000, 'percent' => '10'],
+            ['time' => 2000, 'percent' => '20'],
+        ]),
+    ];
+
+    expect($client->history('cpu', 'from'))->toBe([[1000, 10.0], [2000, 20.0]]);
+});