perf(metrics): fetch all Sentinel metric types in one SSH command

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/Livewire/Dashboard/ServerMetricsChart.php

@@ -22,6 +22,7 @@ class ServerMetricsChart extends Component
         }
 
         try {
+            $this->server->prefetchMetrics(['cpu', 'memory'], 10);
             $this->dispatch("dashboard-server-metrics-{$this->server->uuid}", [
                 'cpu' => $this->server->getCpuMetrics(10),
                 'memory' => $this->server->getMemoryMetrics(10),

app/Livewire/Project/Shared/Metrics.php

@@ -31,6 +31,7 @@ class Metrics extends Component
     public function loadData()
     {
         try {
+            $this->resource->prefetchMetrics(['cpu', 'memory'], $this->interval);
             $cpuMetrics = $this->resource->getCpuMetrics($this->interval);
             $memoryMetrics = $this->resource->getMemoryMetrics($this->interval);
             $this->dispatch("refreshChartData-{$this->chartId}-cpu", [

app/Livewire/Server/Charts.php

@@ -102,6 +102,7 @@ class Charts extends Component
     public function loadData()
     {
         try {
+            $this->server->prefetchMetrics(['cpu', 'memory', 'disk', 'load', 'network'], $this->interval);
             $cpuMetrics = $this->server->getCpuMetrics($this->interval);
             $memoryMetrics = $this->server->getMemoryMetrics($this->interval);
             // Disk/network/load arrived in a later Sentinel; fetch each optionally so an

app/Services/ServerCanvas.php

@@ -145,6 +145,8 @@ class ServerCanvas
                 }
             };
 
+            $server->prefetchMetrics(['cpu', 'memory', 'disk'], 1);
+
             return [
                 'cpu' => $latest(fn () => $server->getCpuMetrics(1)),
                 'memory' => $latest(fn () => $server->getMemoryMetrics(1)),

app/Traits/HasMetrics.php

@@ -8,6 +8,57 @@ use Illuminate\Support\Facades\Log;
 
 trait HasMetrics
 {
+    /**
+     * Raw Sentinel responses from prefetchMetrics(), keyed by "type:mins"; each is read once.
+     *
+     * @var array<string, string>
+     */
+    protected array $prefetchedMetricResponses = [];
+
+    /**
+     * Fetch several metric types in one SSH round-trip instead of one per type. The
+     * get*Metrics() getters then read these responses. On a failure nothing is stored,
+     * so each getter fetches on its own as before.
+     *
+     * @param  array<int, string>  $types
+     */
+    public function prefetchMetrics(array $types, int $mins): void
+    {
+        $server = $this->getMetricsServer();
+        if (! $server->isMetricsEnabled()) {
+            return;
+        }
+
+        $from = now()->subMinutes($mins)->toIso8601ZuluString();
+        $token = $this->metricsToken($server);
+        // The token reaches curl on stdin (`-H @-`), so it is not a process argument. The token
+        // only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it
+        // cannot end the heredoc. Responses are separated by a 0x1E record separator.
+        $script = implode("\nprintf '\\036'\n", array_map(
+            fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH",
+            $types
+        ));
+
+        try {
+            $output = instant_remote_process(
+                ["docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$script}\nCOOLIFY_SENTINEL_SCRIPT"],
+                $server,
+                maxAttempts: 1
+            );
+        } catch (\Throwable) {
+            return;
+        }
+
+        $responses = explode("\x1e", (string) $output);
+        if (count($responses) !== count($types)) {
+            return;
+        }
+
+        foreach ($types as $index => $type) {
+            $this->prefetchedMetricResponses["{$type}:{$mins}"] = trim($responses[$index]);
+        }
+    }
+
     public function getCpuMetrics(int $mins = 5): ?array
     {
         return $this->getMetrics('cpu', $mins, 'percent');
@@ -90,25 +141,21 @@ trait HasMetrics
             return null;
         }
 
-        $from = now()->subMinutes($mins)->toIso8601ZuluString();
-        $endpoint = $this->getMetricsEndpoint($type, $from);
+        $key = "{$type}:{$mins}";
+        if (array_key_exists($key, $this->prefetchedMetricResponses)) {
+            $response = $this->prefetchedMetricResponses[$key];
+            unset($this->prefetchedMetricResponses[$key]);
+        } else {
+            $from = now()->subMinutes($mins)->toIso8601ZuluString();
+            $endpoint = $this->getMetricsEndpoint($type, $from);
+            $token = $this->metricsToken($server);
 
-        $previousToken = null;
-        try {
-            $previousToken = $server->settings->sentinel_token;
-        } catch (DecryptException) {
-            // fall through to ensureValidSentinelToken which will regenerate
+            $response = instant_remote_process(
+                ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"],
+                $server,
+                false
+            );
         }
-        $token = $server->settings->ensureValidSentinelToken();
-        if ($token !== $previousToken) {
-            Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]);
-        }
-
-        $response = instant_remote_process(
-            ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"],
-            $server,
-            false
-        );
 
         if (str($response)->contains('error')) {
             $error = json_decode($response, true);
@@ -144,6 +191,22 @@ trait HasMetrics
         return $metrics;
     }
 
+    private function metricsToken(Server $server): string
+    {
+        $previousToken = null;
+        try {
+            $previousToken = $server->settings->sentinel_token;
+        } catch (DecryptException) {
+            // fall through to ensureValidSentinelToken which will regenerate
+        }
+        $token = $server->settings->ensureValidSentinelToken();
+        if ($token !== $previousToken) {
+            Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]);
+        }
+
+        return $token;
+    }
+
     private function isServerMetrics(): bool
     {
         return $this instanceof Server;

tests/Feature/ServerCanvasTest.php

@@ -386,6 +386,7 @@ test('server zones show host facts, the proxy and a usage endpoint only when met
 
 test('the usage endpoint returns the latest Sentinel values and never another team server', function () {
     $server = Mockery::mock($this->web)->makePartial();
+    $server->shouldReceive('prefetchMetrics')->once()->with(['cpu', 'memory', 'disk'], 1);
     $server->shouldReceive('getCpuMetrics')->andReturn([[1, 10.0], [2, 12.34]]);
     $server->shouldReceive('getMemoryMetrics')->andReturn([[2, 40.0]]);
     $server->shouldReceive('getDiskMetrics')->andThrow(new RuntimeException('no disk data'));

tests/Feature/ServerMetricsPrefetchTest.php

@@ -0,0 +1,87 @@
+<?php
+
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\PendingProcess;
+use Illuminate\Support\Facades\Process;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    InstanceSettings::create(['id' => 0]);
+    config(['constants.ssh.mux_enabled' => false]);
+
+    $team = Team::factory()->create();
+    $key = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $this->server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $key->id]);
+    $this->server->settings->forceFill(['is_metrics_enabled' => true])->saveQuietly();
+    $this->server->refresh();
+});
+
+it('fetches several metric types in one SSH command', function () {
+    $commands = [];
+    Process::fake(function (PendingProcess $process) use (&$commands) {
+        $commands[] = $process->command;
+
+        return Process::result(implode("\x1e", [
+            '[{"time":1000,"percent":12.5}]',
+            '[{"time":1000,"usedPercent":40}]',
+            '[{"time":1000,"mount":"/","usedPercent":70}]',
+        ]));
+    });
+
+    $this->server->prefetchMetrics(['cpu', 'memory', 'disk'], 5);
+
+    expect($this->server->getCpuMetrics(5))->toBe([[1000, 12.5]])
+        ->and($this->server->getMemoryMetrics(5))->toBe([[1000, 40.0]])
+        ->and($this->server->getDiskMetrics(5))->toBe([[1000, 70.0]])
+        ->and($commands)->toHaveCount(1)
+        ->and($commands[0])->toContain('/api/cpu/history', '/api/memory/history', '/api/disk/history', '-H @-')
+        ->and($commands[0])->not->toContain('Bearer '.$this->server->settings->sentinel_token.'"');
+});
+
+it('keeps the Sentinel error of one prefetched metric', function () {
+    Process::fake(fn () => Process::result("[{\"time\":1000,\"percent\":1}]\x1e{\"error\":\"Unauthorized\"}"));
+
+    $this->server->prefetchMetrics(['cpu', 'memory'], 5);
+
+    expect($this->server->getCpuMetrics(5))->toBe([[1000, 1.0]])
+        ->and(fn () => $this->server->getMemoryMetrics(5))->toThrow(Exception::class, 'Unauthorized');
+});
+
+it('falls back to one fetch per metric when the batch fails', function () {
+    $commands = [];
+    Process::fake(function (PendingProcess $process) use (&$commands) {
+        $commands[] = $process->command;
+
+        return count($commands) === 1
+            ? Process::result('', 'ssh: connect to host failed', 1)
+            : Process::result('[{"time":2000,"percent":5}]');
+    });
+
+    $this->server->prefetchMetrics(['cpu', 'memory'], 5);
+
+    expect($this->server->getCpuMetrics(5))->toBe([[2000, 5.0]])
+        ->and($commands)->toHaveCount(2);
+});
+
+it('runs the batched metrics script with sudo for a non-root user', function () {
+    $commands = [];
+    Process::fake(function (PendingProcess $process) use (&$commands) {
+        $commands[] = $process->command;
+
+        return Process::result("[]\x1e[]");
+    });
+    $this->server->forceFill(['user' => 'deploy'])->saveQuietly();
+
+    $this->server->prefetchMetrics(['cpu', 'memory'], 5);
+
+    expect($commands)->toHaveCount(1)
+        ->and($commands[0])->toContain('sudo docker exec -i coolify-sentinel sh -s')
+        ->and($commands[0])->toContain("\ncurl -s -H @- ")
+        ->and($commands[0])->not->toContain('sudo curl')
+        ->and($commands[0])->not->toContain('sudo printf');
+});