diff --git a/app/Livewire/Dashboard/ServerMetricsChart.php b/app/Livewire/Dashboard/ServerMetricsChart.php index 83748052ce..f2ba0df17c 100644 --- a/app/Livewire/Dashboard/ServerMetricsChart.php +++ b/app/Livewire/Dashboard/ServerMetricsChart.php @@ -22,6 +22,7 @@ class ServerMetricsChart extends Component } try { + $this->server->prefetchMetrics(['cpu', 'memory'], 10); $this->dispatch("dashboard-server-metrics-{$this->server->uuid}", [ 'cpu' => $this->server->getCpuMetrics(10), 'memory' => $this->server->getMemoryMetrics(10), diff --git a/app/Livewire/Project/Shared/Metrics.php b/app/Livewire/Project/Shared/Metrics.php index e5b87b48cd..f4138fdd41 100644 --- a/app/Livewire/Project/Shared/Metrics.php +++ b/app/Livewire/Project/Shared/Metrics.php @@ -31,6 +31,7 @@ class Metrics extends Component public function loadData() { try { + $this->resource->prefetchMetrics(['cpu', 'memory'], $this->interval); $cpuMetrics = $this->resource->getCpuMetrics($this->interval); $memoryMetrics = $this->resource->getMemoryMetrics($this->interval); $this->dispatch("refreshChartData-{$this->chartId}-cpu", [ diff --git a/app/Livewire/Server/Charts.php b/app/Livewire/Server/Charts.php index 6ff1f682fa..c63cf47ed3 100644 --- a/app/Livewire/Server/Charts.php +++ b/app/Livewire/Server/Charts.php @@ -102,6 +102,7 @@ class Charts extends Component public function loadData() { try { + $this->server->prefetchMetrics(['cpu', 'memory', 'disk', 'load', 'network'], $this->interval); $cpuMetrics = $this->server->getCpuMetrics($this->interval); $memoryMetrics = $this->server->getMemoryMetrics($this->interval); // Disk/network/load arrived in a later Sentinel; fetch each optionally so an diff --git a/app/Services/ServerCanvas.php b/app/Services/ServerCanvas.php index 5539c481c3..9eaf12c39a 100644 --- a/app/Services/ServerCanvas.php +++ b/app/Services/ServerCanvas.php @@ -145,6 +145,8 @@ class ServerCanvas } }; + $server->prefetchMetrics(['cpu', 'memory', 'disk'], 1); + return [ 'cpu' => $latest(fn () => $server->getCpuMetrics(1)), 'memory' => $latest(fn () => $server->getMemoryMetrics(1)), diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php index cdf5a9c2e9..cc096fd1e3 100644 --- a/app/Traits/HasMetrics.php +++ b/app/Traits/HasMetrics.php @@ -8,6 +8,57 @@ use Illuminate\Support\Facades\Log; trait HasMetrics { + /** + * Raw Sentinel responses from prefetchMetrics(), keyed by "type:mins"; each is read once. + * + * @var array + */ + protected array $prefetchedMetricResponses = []; + + /** + * Fetch several metric types in one SSH round-trip instead of one per type. The + * get*Metrics() getters then read these responses. On a failure nothing is stored, + * so each getter fetches on its own as before. + * + * @param array $types + */ + public function prefetchMetrics(array $types, int $mins): void + { + $server = $this->getMetricsServer(); + if (! $server->isMetricsEnabled()) { + return; + } + + $from = now()->subMinutes($mins)->toIso8601ZuluString(); + $token = $this->metricsToken($server); + // The token reaches curl on stdin (`-H @-`), so it is not a process argument. The token + // only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it + // cannot end the heredoc. Responses are separated by a 0x1E record separator. + $script = implode("\nprintf '\\036'\n", array_map( + fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH", + $types + )); + + try { + $output = instant_remote_process( + ["docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$script}\nCOOLIFY_SENTINEL_SCRIPT"], + $server, + maxAttempts: 1 + ); + } catch (\Throwable) { + return; + } + + $responses = explode("\x1e", (string) $output); + if (count($responses) !== count($types)) { + return; + } + + foreach ($types as $index => $type) { + $this->prefetchedMetricResponses["{$type}:{$mins}"] = trim($responses[$index]); + } + } + public function getCpuMetrics(int $mins = 5): ?array { return $this->getMetrics('cpu', $mins, 'percent'); @@ -90,25 +141,21 @@ trait HasMetrics return null; } - $from = now()->subMinutes($mins)->toIso8601ZuluString(); - $endpoint = $this->getMetricsEndpoint($type, $from); + $key = "{$type}:{$mins}"; + if (array_key_exists($key, $this->prefetchedMetricResponses)) { + $response = $this->prefetchedMetricResponses[$key]; + unset($this->prefetchedMetricResponses[$key]); + } else { + $from = now()->subMinutes($mins)->toIso8601ZuluString(); + $endpoint = $this->getMetricsEndpoint($type, $from); + $token = $this->metricsToken($server); - $previousToken = null; - try { - $previousToken = $server->settings->sentinel_token; - } catch (DecryptException) { - // fall through to ensureValidSentinelToken which will regenerate + $response = instant_remote_process( + ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"], + $server, + false + ); } - $token = $server->settings->ensureValidSentinelToken(); - if ($token !== $previousToken) { - Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]); - } - - $response = instant_remote_process( - ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"], - $server, - false - ); if (str($response)->contains('error')) { $error = json_decode($response, true); @@ -144,6 +191,22 @@ trait HasMetrics return $metrics; } + private function metricsToken(Server $server): string + { + $previousToken = null; + try { + $previousToken = $server->settings->sentinel_token; + } catch (DecryptException) { + // fall through to ensureValidSentinelToken which will regenerate + } + $token = $server->settings->ensureValidSentinelToken(); + if ($token !== $previousToken) { + Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]); + } + + return $token; + } + private function isServerMetrics(): bool { return $this instanceof Server; diff --git a/tests/Feature/ServerCanvasTest.php b/tests/Feature/ServerCanvasTest.php index 58de97837d..e98e8a0dcb 100644 --- a/tests/Feature/ServerCanvasTest.php +++ b/tests/Feature/ServerCanvasTest.php @@ -386,6 +386,7 @@ test('server zones show host facts, the proxy and a usage endpoint only when met test('the usage endpoint returns the latest Sentinel values and never another team server', function () { $server = Mockery::mock($this->web)->makePartial(); + $server->shouldReceive('prefetchMetrics')->once()->with(['cpu', 'memory', 'disk'], 1); $server->shouldReceive('getCpuMetrics')->andReturn([[1, 10.0], [2, 12.34]]); $server->shouldReceive('getMemoryMetrics')->andReturn([[2, 40.0]]); $server->shouldReceive('getDiskMetrics')->andThrow(new RuntimeException('no disk data')); diff --git a/tests/Feature/ServerMetricsPrefetchTest.php b/tests/Feature/ServerMetricsPrefetchTest.php new file mode 100644 index 0000000000..ca1d6eeab8 --- /dev/null +++ b/tests/Feature/ServerMetricsPrefetchTest.php @@ -0,0 +1,87 @@ + 0]); + config(['constants.ssh.mux_enabled' => false]); + + $team = Team::factory()->create(); + $key = PrivateKey::factory()->create(['team_id' => $team->id]); + $this->server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $key->id]); + $this->server->settings->forceFill(['is_metrics_enabled' => true])->saveQuietly(); + $this->server->refresh(); +}); + +it('fetches several metric types in one SSH command', function () { + $commands = []; + Process::fake(function (PendingProcess $process) use (&$commands) { + $commands[] = $process->command; + + return Process::result(implode("\x1e", [ + '[{"time":1000,"percent":12.5}]', + '[{"time":1000,"usedPercent":40}]', + '[{"time":1000,"mount":"/","usedPercent":70}]', + ])); + }); + + $this->server->prefetchMetrics(['cpu', 'memory', 'disk'], 5); + + expect($this->server->getCpuMetrics(5))->toBe([[1000, 12.5]]) + ->and($this->server->getMemoryMetrics(5))->toBe([[1000, 40.0]]) + ->and($this->server->getDiskMetrics(5))->toBe([[1000, 70.0]]) + ->and($commands)->toHaveCount(1) + ->and($commands[0])->toContain('/api/cpu/history', '/api/memory/history', '/api/disk/history', '-H @-') + ->and($commands[0])->not->toContain('Bearer '.$this->server->settings->sentinel_token.'"'); +}); + +it('keeps the Sentinel error of one prefetched metric', function () { + Process::fake(fn () => Process::result("[{\"time\":1000,\"percent\":1}]\x1e{\"error\":\"Unauthorized\"}")); + + $this->server->prefetchMetrics(['cpu', 'memory'], 5); + + expect($this->server->getCpuMetrics(5))->toBe([[1000, 1.0]]) + ->and(fn () => $this->server->getMemoryMetrics(5))->toThrow(Exception::class, 'Unauthorized'); +}); + +it('falls back to one fetch per metric when the batch fails', function () { + $commands = []; + Process::fake(function (PendingProcess $process) use (&$commands) { + $commands[] = $process->command; + + return count($commands) === 1 + ? Process::result('', 'ssh: connect to host failed', 1) + : Process::result('[{"time":2000,"percent":5}]'); + }); + + $this->server->prefetchMetrics(['cpu', 'memory'], 5); + + expect($this->server->getCpuMetrics(5))->toBe([[2000, 5.0]]) + ->and($commands)->toHaveCount(2); +}); + +it('runs the batched metrics script with sudo for a non-root user', function () { + $commands = []; + Process::fake(function (PendingProcess $process) use (&$commands) { + $commands[] = $process->command; + + return Process::result("[]\x1e[]"); + }); + $this->server->forceFill(['user' => 'deploy'])->saveQuietly(); + + $this->server->prefetchMetrics(['cpu', 'memory'], 5); + + expect($commands)->toHaveCount(1) + ->and($commands[0])->toContain('sudo docker exec -i coolify-sentinel sh -s') + ->and($commands[0])->toContain("\ncurl -s -H @- ") + ->and($commands[0])->not->toContain('sudo curl') + ->and($commands[0])->not->toContain('sudo printf'); +});