*/ protected array $prefetchedMetricResponses = []; /** * Fetch several metric types in one SSH round-trip instead of one per type. The * get*Metrics() getters then read these responses. Failed prefetches are remembered * for this request so the getters do not retry the same unavailable server. * * @param array $types */ public function prefetchMetrics(array $types, int $mins): void { $server = $this->getMetricsServer(); if (! $server->isMetricsEnabled()) { return; } foreach ($types as $type) { $this->prefetchedMetricResponses["{$type}:{$mins}"] = null; } $from = now()->subMinutes($mins)->toIso8601ZuluString(); $token = $this->metricsToken($server); // The token reaches curl on stdin (`-H @-`), so it is not a process argument. The token // only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it // cannot end the heredoc. Responses are separated by a 0x1E record separator. $script = implode("\nprintf '\\036'\n", array_map( fn (string $type) => 'curl -sS --connect-timeout 2 --max-time 3 -H @- "'.$this->getMetricsEndpoint($type, $from, $mins)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH", $types )); $output = SentinelMetricsClient::fetchOverSsh( $server, ["docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$script}\nCOOLIFY_SENTINEL_SCRIPT"], static::class.':'.$this->uuid.':'.implode(',', $types).':'.$mins ); if ($output === null) { return; } $responses = explode("\x1e", (string) $output); if (count($responses) !== count($types)) { return; } foreach ($types as $index => $type) { $this->prefetchedMetricResponses["{$type}:{$mins}"] = trim($responses[$index]); } } public function getCpuMetrics(int $mins = 5): ?array { return $this->getMetrics('cpu', $mins, 'percent'); } public function getMemoryMetrics(int $mins = 5): ?array { if ($this->isServerMetrics()) { return $this->getMetrics('memory', $mins, 'usedPercent'); } $metrics = $this->getMetrics('memory', $mins, 'used'); if ($metrics === null) { return null; } return convertContainerMemoryBytesToMegabytes($metrics); } /** * Root filesystem usage percentage over time. Disk history interleaves every * mount, so we keep only the root mount (see SentinelMetricsClient::rootMountName()). */ public function getDiskMetrics(int $mins = 5): ?array { $rows = $this->fetchMetricRows('disk', $mins); if ($rows === null) { return null; } if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) { $mount = SentinelMetricsClient::rootMountName($rows); $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === $mount)); } return $this->mapRows($rows, 'usedPercent', $mins); } /** * Host load average (1 minute) over time. */ public function getLoadMetrics(int $mins = 5): ?array { return $this->getMetrics('load', $mins, 'load1'); } /** * Network throughput over time, as separate receive/transmit byte-rate series. * * @return array{rx: ?array, tx: ?array}|null */ public function getNetworkMetrics(int $mins = 5): ?array { $rows = $this->fetchMetricRows('network', $mins); if ($rows === null) { return null; } return [ 'rx' => $this->mapRows($rows, 'rxBytesPerSec', $mins), 'tx' => $this->mapRows($rows, 'txBytesPerSec', $mins), ]; } private function getMetrics(string $type, int $mins, string $valueField): ?array { return $this->mapRows($this->fetchMetricRows($type, $mins), $valueField, $mins); } /** * Fetch and decode the raw Sentinel history rows for a metric type, or null when * metrics are disabled. Throws on a Sentinel error response (caller decides whether * to treat that as fatal or optional). * * @return array>|null */ protected function fetchMetricRows(string $type, int $mins): ?array { $server = $this->getMetricsServer(); if (! $server->isMetricsEnabled()) { return null; } $key = "{$type}:{$mins}"; if (array_key_exists($key, $this->prefetchedMetricResponses)) { $response = $this->prefetchedMetricResponses[$key]; unset($this->prefetchedMetricResponses[$key]); } else { $from = now()->subMinutes($mins)->toIso8601ZuluString(); $endpoint = $this->getMetricsEndpoint($type, $from, $mins); $token = $this->metricsToken($server); $command = "curl -sS --connect-timeout 2 --max-time 3 -H @- \"{$endpoint}\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH"; $response = SentinelMetricsClient::fetchOverSsh( $server, ["docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$command}\nCOOLIFY_SENTINEL_SCRIPT"], static::class.':'.$this->uuid.':'.$type.':'.$mins ); } if ($response === null) { return null; } if (str($response)->contains('error')) { $error = json_decode($response, true); $error = data_get($error, 'error', 'Something is not okay, are you okay?'); if ($error === 'Unauthorized') { $error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.'; } throw new \Exception($error); } return json_decode($response, true) ?: []; } /** * Map raw rows to [timestampMs, value] pairs, downsampling long ranges. * * Sentinel keeps raw samples for the last 24 to 48 hours and one-minute averages * before that. LTTB buckets by point count, so on ranges that reach past the raw * window the raw part got most of the chart points. Those ranges are averaged per * minute first, the same as Sentinel does, so the density is even over time. * * @param array>|null $rows */ private function mapRows(?array $rows, string $valueField, int $mins): ?array { if ($rows === null) { return null; } $metrics = collect($rows)->map(function ($metric) use ($valueField) { return [(int) $metric['time'], (float) ($metric[$valueField] ?? 0.0)]; })->toArray(); if ($mins > 24 * 60) { $metrics = $this->averagePerMinute($metrics); } if ($mins > 60 && count($metrics) > 1000) { $metrics = downsampleLTTB($metrics, 1000); } return $metrics; } /** * Average [timestampMs, value] pairs into one-minute buckets, timestamped at the * bucket start like Sentinel's own downsampling. Samples that are a minute or more * apart keep their own value. * * @param array $metrics * @return array */ private function averagePerMinute(array $metrics): array { $sums = []; $counts = []; foreach ($metrics as [$time, $value]) { $bucket = intdiv($time, 60_000) * 60_000; $sums[$bucket] = ($sums[$bucket] ?? 0.0) + $value; $counts[$bucket] = ($counts[$bucket] ?? 0) + 1; } ksort($sums); return array_map(fn (int $bucket) => [$bucket, $sums[$bucket] / $counts[$bucket]], array_keys($sums)); } private function metricsToken(Server $server): string { $previousToken = null; try { $previousToken = $server->settings->sentinel_token; } catch (DecryptException) { // fall through to ensureValidSentinelToken which will regenerate } $token = $server->settings->ensureValidSentinelToken(); if ($token !== $previousToken) { Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]); } return $token; } private function isServerMetrics(): bool { return $this instanceof Server; } private function getMetricsServer(): Server { return $this->isServerMetrics() ? $this : $this->destination->server; } /** * Ranges over an hour ask Sentinel to average rows into whole-minute windows, at most * 1000 per range, so it sends far fewer rows. Older Sentinels ignore `bucket_ms`; * mapRows() still downsamples their raw rows. */ private function getMetricsEndpoint(string $type, string $from, int $mins): string { $base = 'http://localhost:8888/api'; $query = "from={$from}"; if ($mins > 60) { $query .= '&bucket_ms='.((int) ceil($mins / 1000) * 60_000); } if ($this->isServerMetrics()) { return "{$base}/{$type}/history?{$query}"; } return "{$base}/container/{$this->uuid}/{$type}/history?{$query}"; } }