mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
perf(metrics): fetch Fleet Metrics of all servers at the same time (#12389)
This commit is contained in:
@@ -5,13 +5,13 @@ namespace App\Livewire\Fleet;
|
||||
use App\Livewire\Concerns\BuildsMetricsChartPayload;
|
||||
use App\Models\Application;
|
||||
use App\Models\Server;
|
||||
use App\Models\Service;
|
||||
use App\Models\ServiceDatabase;
|
||||
use App\Services\FleetMetricsAggregator;
|
||||
use App\Services\SentinelMetricsClient;
|
||||
use Illuminate\Contracts\View\View;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\RateLimiter;
|
||||
use Livewire\Attributes\Lazy;
|
||||
use Livewire\Attributes\Url;
|
||||
@@ -257,34 +257,21 @@ class Metrics extends Component
|
||||
|
||||
$targets = $this->targetServers();
|
||||
$clients = [];
|
||||
$summaryTasks = [];
|
||||
foreach ($targets as $server) {
|
||||
// Coolify does not connect to an unreachable server, and its SSH retries would hold a worker past the process timeout.
|
||||
if (! $server->isFunctional()) {
|
||||
continue;
|
||||
}
|
||||
$client = $clients[$server->uuid] = $this->metricsClient($server);
|
||||
$summaryTasks[$server->uuid] = static fn () => $client->summary();
|
||||
}
|
||||
$summaries = $this->runMetricTasks($summaryTasks);
|
||||
$tasks = [];
|
||||
$range = $this->range;
|
||||
foreach ($clients as $uuid => $client) {
|
||||
$metrics = $summaries[$uuid] === null ? ['cpu', 'memory'] : ['cpu', 'memory', 'disk', 'load'];
|
||||
foreach ($metrics as $metric) {
|
||||
$tasks[$uuid.':'.$metric] = static fn () => $client->history($metric, $range);
|
||||
}
|
||||
if ($summaries[$uuid] !== null) {
|
||||
$tasks[$uuid.':containers'] = static fn () => $client->containersCurrent();
|
||||
$tasks[$uuid.':network'] = static fn () => $client->networkHistory($range);
|
||||
// Coolify does not connect to an unreachable server.
|
||||
if ($server->isFunctional()) {
|
||||
$clients[$server->uuid] = $this->metricsClient($server);
|
||||
}
|
||||
}
|
||||
$results = $this->runMetricTasks($tasks);
|
||||
SentinelMetricsClient::prefetchConcurrently($clients, $this->range);
|
||||
|
||||
foreach ($targets as $server) {
|
||||
$summary = $summaries[$server->uuid] ?? null;
|
||||
$client = $clients[$server->uuid] ?? null;
|
||||
$summary = $client === null ? null : $this->attempt(fn () => $client->summary());
|
||||
foreach (['cpu', 'memory', 'disk', 'load'] as $metric) {
|
||||
$series[$metric][] = $results[$server->uuid.':'.$metric] ?? [];
|
||||
// A Sentinel without a summary endpoint still has cpu and memory history.
|
||||
$read = $client !== null && ($summary !== null || in_array($metric, ['cpu', 'memory'], true));
|
||||
$series[$metric][] = $read ? ($this->attempt(fn () => $client->history($metric, $this->range)) ?? []) : [];
|
||||
}
|
||||
|
||||
if ($summary === null) {
|
||||
@@ -293,19 +280,17 @@ class Metrics extends Component
|
||||
continue;
|
||||
}
|
||||
|
||||
$current = $results[$server->uuid.':containers'] ?? [];
|
||||
$current = $this->attempt(fn () => $client->containersCurrent()) ?? [];
|
||||
$rows[] = array_merge(
|
||||
$summary,
|
||||
['uuid' => $server->uuid, 'name' => $server->name, 'online' => true, 'reporting' => true, 'containers' => count($current)]
|
||||
);
|
||||
|
||||
foreach ($current as $c) {
|
||||
// Names/images are resolved lazily for the displayed rows only (containerMeta),
|
||||
// so a server with many containers doesn't trigger a resolve storm here.
|
||||
$containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]);
|
||||
}
|
||||
|
||||
$net = $results[$server->uuid.':network'] ?? ['rx' => [], 'tx' => []];
|
||||
$net = $this->attempt(fn () => $client->networkHistory($this->range)) ?? ['rx' => [], 'tx' => []];
|
||||
$series['networkRx'][] = $net['rx'];
|
||||
$series['networkTx'][] = $net['tx'];
|
||||
}
|
||||
@@ -320,34 +305,15 @@ class Metrics extends Component
|
||||
}
|
||||
|
||||
/**
|
||||
* Bound the number of PHP workers and SSH connections for large fleets.
|
||||
*
|
||||
* @param array<string, \Closure> $tasks
|
||||
* @return array<string, mixed>
|
||||
* One failed metric must not fail the others.
|
||||
*/
|
||||
protected function runMetricTasks(array $tasks): array
|
||||
private function attempt(\Closure $read): mixed
|
||||
{
|
||||
foreach ($tasks as $key => $task) {
|
||||
$tasks[$key] = static function () use ($task) {
|
||||
try {
|
||||
return $task();
|
||||
} catch (\Throwable) {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
try {
|
||||
return $read();
|
||||
} catch (\Throwable) {
|
||||
return null;
|
||||
}
|
||||
$results = [];
|
||||
foreach (array_chunk($tasks, 8, true) as $batch) {
|
||||
try {
|
||||
$results += Concurrency::run($batch);
|
||||
} catch (\Throwable $e) {
|
||||
// A worker that runs past the process timeout fails its batch. Show those metrics as missing, not a server error.
|
||||
Log::warning('Fleet metrics batch failed', ['error' => $e->getMessage()]);
|
||||
$results += array_fill_keys(array_keys($batch), null);
|
||||
}
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -383,18 +349,27 @@ class Metrics extends Component
|
||||
*/
|
||||
public function containerMeta(string $id): array
|
||||
{
|
||||
if (isset($this->containerMetaCache[$id])) {
|
||||
return $this->containerMetaCache[$id];
|
||||
if (! isset($this->containerMetaCache[$id])) {
|
||||
// Resolve every listed container at once, not one lookup per table row.
|
||||
$this->resolveContainerMeta([...array_column($this->containersRaw, 'id'), $id]);
|
||||
}
|
||||
|
||||
return $this->containerMetaCache[$id];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $ids
|
||||
*/
|
||||
private function resolveContainerMeta(array $ids): void
|
||||
{
|
||||
$ids = array_values(array_diff(array_unique($ids), array_keys($this->containerMetaCache)));
|
||||
$teamId = currentTeam()?->id;
|
||||
$resource = $teamId !== null ? getResourceByUuid($id, $teamId) : null;
|
||||
|
||||
$image = null;
|
||||
$link = null;
|
||||
if ($resource) {
|
||||
$image = data_get($resource, 'docker_registry_image_name') ?: data_get($resource, 'image');
|
||||
$resources = $teamId === null ? [] : $this->teamResourcesByUuid($ids, $teamId);
|
||||
|
||||
foreach ($ids as $id) {
|
||||
$resource = $resources[$id] ?? null;
|
||||
$image = $resource ? (data_get($resource, 'docker_registry_image_name') ?: data_get($resource, 'image')) : null;
|
||||
$link = null;
|
||||
if ($resource instanceof Application && data_get($resource, 'environment.project.uuid')) {
|
||||
$link = route('project.application.metrics', [
|
||||
'project_uuid' => $resource->environment->project->uuid,
|
||||
@@ -402,13 +377,41 @@ class Metrics extends Component
|
||||
'application_uuid' => $resource->uuid,
|
||||
]);
|
||||
}
|
||||
|
||||
$this->containerMetaCache[$id] = [
|
||||
'name' => $resource?->name ?? $id,
|
||||
'image' => $image ?: null,
|
||||
'link' => $link,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The team's resources with these uuids, one query per resource type.
|
||||
*
|
||||
* @param array<int, string> $uuids
|
||||
* @return array<string, mixed> uuid => resource
|
||||
*/
|
||||
private function teamResourcesByUuid(array $uuids, int $teamId): array
|
||||
{
|
||||
$found = [];
|
||||
foreach ([Application::class, Service::class, ...array_values(STANDALONE_DATABASE_MODELS), ServiceDatabase::class] as $model) {
|
||||
$missing = array_values(array_diff($uuids, array_keys($found)));
|
||||
if ($missing === []) {
|
||||
break;
|
||||
}
|
||||
$teamPath = $model === ServiceDatabase::class ? 'service.environment.project' : 'environment.project';
|
||||
$model::query()
|
||||
->whereIn('uuid', $missing)
|
||||
->whereRelation($teamPath, 'team_id', $teamId)
|
||||
->with($teamPath)
|
||||
->get()
|
||||
->each(function ($resource) use (&$found) {
|
||||
$found[$resource->uuid] = $resource;
|
||||
});
|
||||
}
|
||||
|
||||
return $this->containerMetaCache[$id] = [
|
||||
'name' => $resource?->name ?? $id,
|
||||
'image' => $image ?: null,
|
||||
'link' => $link,
|
||||
];
|
||||
return $found;
|
||||
}
|
||||
|
||||
public function placeholder(): View
|
||||
|
||||
@@ -2,15 +2,89 @@
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Helpers\SshMultiplexingHelper;
|
||||
use App\Models\Server;
|
||||
use Illuminate\Contracts\Process\InvokedProcess;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
|
||||
class SentinelMetricsClient
|
||||
{
|
||||
private const REMOTE_TIMEOUT = 20;
|
||||
|
||||
/** Mounts that can hold the root disk, in the order of rootMountName(). */
|
||||
private const ROOT_MOUNTS = ['/', '/app/db', '/data/coolify', '/data'];
|
||||
|
||||
private string $base = 'http://localhost:8888/api';
|
||||
|
||||
/** @var array<string, ?string> url => response from prefetchConcurrently(); null when it failed */
|
||||
private array $prefetched = [];
|
||||
|
||||
public function __construct(protected Server $server) {}
|
||||
|
||||
/**
|
||||
* Fetch the Fleet Metrics responses of several servers at the same time, with one SSH
|
||||
* command per server, so a page waits for the slowest server only. raw() then reads the
|
||||
* responses; a failed fetch is not tried again per metric.
|
||||
*
|
||||
* @param iterable<int|string, self> $clients
|
||||
*/
|
||||
public static function prefetchConcurrently(iterable $clients, string $range): void
|
||||
{
|
||||
$started = [];
|
||||
foreach ($clients as $client) {
|
||||
// A subclass is a test fake with canned data; it must not reach real SSH.
|
||||
if ($client::class !== self::class) {
|
||||
continue;
|
||||
}
|
||||
$urls = array_values(array_filter($client->pageUrls($range), fn (string $url) => ! Cache::has($client->cacheKey($url))));
|
||||
if ($urls === []) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$started[] = [$client, $urls, $client->startBatch($urls)];
|
||||
} catch (\Throwable) {
|
||||
$client->prefetched += array_fill_keys($urls, null);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($started as [$client, $urls, $process]) {
|
||||
try {
|
||||
$responses = explode("\x1e", $client->awaitBatch($process));
|
||||
} catch (\Throwable) {
|
||||
$responses = [];
|
||||
}
|
||||
if (count($responses) !== count($urls)) {
|
||||
$responses = [];
|
||||
}
|
||||
|
||||
foreach ($urls as $index => $url) {
|
||||
try {
|
||||
$body = $client->guard(trim($responses[$index] ?? ''));
|
||||
Cache::put($client->cacheKey($url), $body, 60);
|
||||
$client->prefetched[$url] = $body;
|
||||
} catch (\Throwable) {
|
||||
$client->prefetched[$url] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every URL that the Fleet Metrics page reads for one server.
|
||||
*
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function pageUrls(string $range): array
|
||||
{
|
||||
return [
|
||||
$this->url('/summary'),
|
||||
$this->url('/containers/current'),
|
||||
...array_map(fn (string $metric) => $this->historyUrl($metric, $range), ['cpu', 'memory', 'disk', 'load', 'network']),
|
||||
];
|
||||
}
|
||||
|
||||
/** Rounded to the minute so the history URL (and its cache key) is stable within a minute. */
|
||||
public static function rangeFrom(string $range): string
|
||||
{
|
||||
@@ -116,14 +190,14 @@ class SentinelMetricsClient
|
||||
*/
|
||||
public function history(string $metric, string $range): array
|
||||
{
|
||||
[$path, $field] = match ($metric) {
|
||||
'memory' => ['/memory/history', 'used'],
|
||||
'disk' => ['/disk/history', 'usedPercent'],
|
||||
'load' => ['/load/history', 'load1'],
|
||||
default => ['/cpu/history', 'percent'],
|
||||
$field = match ($metric) {
|
||||
'memory' => 'used',
|
||||
'disk' => 'usedPercent',
|
||||
'load' => 'load1',
|
||||
default => 'percent',
|
||||
};
|
||||
|
||||
return $this->seriesRows($this->url($path, self::historyQuery($range)), $field, $range, $metric === 'disk');
|
||||
return $this->seriesRows($this->historyUrl($metric, $range), $field, $range, $metric === 'disk');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -131,7 +205,7 @@ class SentinelMetricsClient
|
||||
*/
|
||||
public function networkHistory(string $range): array
|
||||
{
|
||||
$url = $this->url('/network/history', self::historyQuery($range));
|
||||
$url = $this->historyUrl('network', $range);
|
||||
|
||||
return [
|
||||
'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range),
|
||||
@@ -188,7 +262,7 @@ class SentinelMetricsClient
|
||||
{
|
||||
$mounts = array_column($rows, 'mount');
|
||||
|
||||
return collect(['/', '/app/db', '/data/coolify', '/data'])->first(fn ($mount) => in_array($mount, $mounts, true));
|
||||
return collect(self::ROOT_MOUNTS)->first(fn ($mount) => in_array($mount, $mounts, true));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -214,6 +288,21 @@ class SentinelMetricsClient
|
||||
];
|
||||
}
|
||||
|
||||
private function historyUrl(string $metric, string $range): string
|
||||
{
|
||||
$path = match ($metric) {
|
||||
'memory', 'disk', 'load', 'network' => "/{$metric}/history",
|
||||
default => '/cpu/history',
|
||||
};
|
||||
$query = self::historyQuery($range);
|
||||
if ($metric === 'disk') {
|
||||
// Sentinel 1.1.4+ returns only these mounts; older versions ignore the param.
|
||||
$query['mount'] = implode(',', self::ROOT_MOUNTS);
|
||||
}
|
||||
|
||||
return $this->url($path, $query);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $query
|
||||
*/
|
||||
@@ -231,6 +320,10 @@ class SentinelMetricsClient
|
||||
|
||||
protected function raw(string $url): string
|
||||
{
|
||||
if (array_key_exists($url, $this->prefetched)) {
|
||||
return $this->prefetched[$url] ?? throw new \RuntimeException('Metrics request failed.');
|
||||
}
|
||||
|
||||
return Cache::remember($this->cacheKey($url), 60, fn () => $this->guard($this->remoteFetch($url)));
|
||||
}
|
||||
|
||||
@@ -248,6 +341,41 @@ class SentinelMetricsClient
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start one `docker exec` that curls every URL in order, with the responses separated by
|
||||
* a 0x1E record separator. The token reaches curl on stdin, so it is not a process argument.
|
||||
* The token only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it
|
||||
* cannot end the heredoc. No pipe or `sh -c` is used, so the non-root sudo parser only puts
|
||||
* sudo in front of the line. Unlike instant_remote_process(), there are no SSH retries.
|
||||
*
|
||||
* @param array<int, string> $urls
|
||||
*/
|
||||
private function startBatch(array $urls): InvokedProcess
|
||||
{
|
||||
$token = $this->server->settings->ensureValidSentinelToken();
|
||||
$script = implode("\nprintf '\\036'\n", array_map(
|
||||
fn (string $url) => "curl -s --connect-timeout 3 --max-time 10 -H @- \"{$url}\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH",
|
||||
$urls
|
||||
));
|
||||
// `true` last: a failed curl empties only its own response, not the whole batch.
|
||||
$command = "docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$script}\ntrue\nCOOLIFY_SENTINEL_SCRIPT";
|
||||
$commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];
|
||||
|
||||
return Process::timeout(self::REMOTE_TIMEOUT)->start(
|
||||
SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT)
|
||||
);
|
||||
}
|
||||
|
||||
private function awaitBatch(InvokedProcess $invoked): string
|
||||
{
|
||||
$process = $invoked->wait();
|
||||
if ($process->exitCode() !== 0) {
|
||||
excludeCertainErrors($process->errorOutput(), $process->exitCode());
|
||||
}
|
||||
|
||||
return $process->output();
|
||||
}
|
||||
|
||||
private function guard(string $response): string
|
||||
{
|
||||
$payload = json_decode($response, true);
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
<?php
|
||||
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Services\SentinelMetricsClient;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Process\PendingProcess;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
InstanceSettings::create(['id' => 0]);
|
||||
config(['constants.ssh.mux_enabled' => false]);
|
||||
|
||||
$team = Team::factory()->create();
|
||||
$this->privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
||||
$this->team = $team;
|
||||
});
|
||||
|
||||
function prefetchServer($team, $privateKey): Server
|
||||
{
|
||||
$server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]);
|
||||
$server->settings->forceFill(['is_metrics_enabled' => true])->saveQuietly();
|
||||
|
||||
return $server->refresh();
|
||||
}
|
||||
|
||||
/** Sentinel responses in the order of the batch: summary, containers, cpu, memory, disk, load, network. */
|
||||
function fleetBatchOutput(?string $summary = null): string
|
||||
{
|
||||
return implode("\x1e", [
|
||||
$summary ?? '{"cpu":{"percent":12.5},"memory":{"used":100,"total":200}}',
|
||||
'[{"id":"app-1","cpu":{"percent":3},"status":{"state":"running"}}]',
|
||||
'[{"time":300000,"percent":12.5}]',
|
||||
'[{"time":300000,"used":100}]',
|
||||
'[{"time":300000,"mount":"/","usedPercent":40}]',
|
||||
'[{"time":300000,"load1":0.5}]',
|
||||
'[{"time":300000,"rxBytesPerSec":10,"txBytesPerSec":2}]',
|
||||
]);
|
||||
}
|
||||
|
||||
it('fetches the page metrics of every server in one SSH command each', function () {
|
||||
$commands = [];
|
||||
Process::fake(function (PendingProcess $process) use (&$commands) {
|
||||
$commands[] = remoteProcessScript($process);
|
||||
|
||||
return Process::result(fleetBatchOutput());
|
||||
});
|
||||
$clients = collect([prefetchServer($this->team, $this->privateKey), prefetchServer($this->team, $this->privateKey)])
|
||||
->map(fn (Server $server) => new SentinelMetricsClient($server));
|
||||
|
||||
SentinelMetricsClient::prefetchConcurrently($clients, '24h');
|
||||
|
||||
foreach ($clients as $client) {
|
||||
expect($client->summary()['cpu'])->toBe(12.5)
|
||||
->and($client->containersCurrent())->toHaveCount(1)
|
||||
->and($client->history('cpu', '24h'))->toBe([[300000, 12.5]])
|
||||
->and($client->history('disk', '24h'))->toBe([[300000, 40.0]])
|
||||
->and($client->networkHistory('24h')['tx'])->toBe([[300000, 2.0]]);
|
||||
}
|
||||
expect($commands)->toHaveCount(2)
|
||||
->and($commands[0])->toContain('/api/summary', '/api/containers/current', '/api/load/history', '/api/network/history', '-H @-')
|
||||
->and($commands[0])->toContain('/api/disk/history?from=')
|
||||
->and($commands[0])->toContain('mount=%2F%2C%2Fapp%2Fdb%2C%2Fdata%2Fcoolify%2C%2Fdata');
|
||||
});
|
||||
|
||||
it('does not fetch each metric again when the batch fails', function () {
|
||||
$commands = 0;
|
||||
Process::fake(function () use (&$commands) {
|
||||
$commands++;
|
||||
|
||||
return Process::result('', 'ssh: connect to host failed', 255);
|
||||
});
|
||||
$client = new SentinelMetricsClient(prefetchServer($this->team, $this->privateKey));
|
||||
|
||||
SentinelMetricsClient::prefetchConcurrently([$client], '24h');
|
||||
|
||||
expect($client->summary())->toBeNull()
|
||||
->and($client->containersCurrent())->toBe([])
|
||||
->and($client->history('cpu', '24h'))->toBe([])
|
||||
->and($client->networkHistory('24h'))->toBe(['rx' => [], 'tx' => []])
|
||||
->and($commands)->toBe(1);
|
||||
});
|
||||
|
||||
it('keeps the other metrics when Sentinel fails one endpoint', function () {
|
||||
$commands = 0;
|
||||
Process::fake(function () use (&$commands) {
|
||||
$commands++;
|
||||
|
||||
return Process::result(fleetBatchOutput(summary: '404 page not found'));
|
||||
});
|
||||
$client = new SentinelMetricsClient(prefetchServer($this->team, $this->privateKey));
|
||||
|
||||
SentinelMetricsClient::prefetchConcurrently([$client], '24h');
|
||||
|
||||
expect($client->summary())->toBeNull()
|
||||
->and($client->history('cpu', '24h'))->toBe([[300000, 12.5]])
|
||||
->and($commands)->toBe(1);
|
||||
});
|
||||
|
||||
it('reads cached responses on the next load without SSH', function () {
|
||||
$commands = 0;
|
||||
Process::fake(function () use (&$commands) {
|
||||
$commands++;
|
||||
|
||||
return Process::result(fleetBatchOutput());
|
||||
});
|
||||
$server = prefetchServer($this->team, $this->privateKey);
|
||||
SentinelMetricsClient::prefetchConcurrently([new SentinelMetricsClient($server)], '24h');
|
||||
|
||||
$next = new SentinelMetricsClient($server);
|
||||
SentinelMetricsClient::prefetchConcurrently([$next], '24h');
|
||||
|
||||
expect($next->history('memory', '24h'))->toBe([[300000, 100.0]])
|
||||
->and($commands)->toBe(1);
|
||||
});
|
||||
|
||||
it('runs the batched script with sudo for a non-root user', function () {
|
||||
$commands = [];
|
||||
Process::fake(function (PendingProcess $process) use (&$commands) {
|
||||
$commands[] = remoteProcessScript($process);
|
||||
|
||||
return Process::result(fleetBatchOutput());
|
||||
});
|
||||
$server = prefetchServer($this->team, $this->privateKey);
|
||||
$server->forceFill(['user' => 'deploy'])->saveQuietly();
|
||||
|
||||
SentinelMetricsClient::prefetchConcurrently([new SentinelMetricsClient($server)], '24h');
|
||||
|
||||
expect($commands)->toHaveCount(1)
|
||||
->and($commands[0])->toContain('sudo docker exec -i coolify-sentinel sh -s')
|
||||
->and($commands[0])->not->toContain('sudo curl');
|
||||
});
|
||||
|
||||
it('keeps the other metrics when the last curl of the batch fails', function () {
|
||||
$commands = [];
|
||||
Process::fake(function (PendingProcess $process) use (&$commands) {
|
||||
$commands[] = remoteProcessScript($process);
|
||||
|
||||
return Process::result(output: fleetBatchOutput(), exitCode: str_contains(end($commands), "\ntrue\nCOOLIFY_SENTINEL_SCRIPT") ? 0 : 28);
|
||||
});
|
||||
$client = new SentinelMetricsClient(prefetchServer($this->team, $this->privateKey));
|
||||
|
||||
SentinelMetricsClient::prefetchConcurrently([$client], '24h');
|
||||
|
||||
expect($client->summary()['cpu'])->toBe(12.5)
|
||||
->and($client->history('cpu', '24h'))->toBe([[300000, 12.5]])
|
||||
->and($commands)->toHaveCount(1);
|
||||
});
|
||||
@@ -8,7 +8,6 @@ use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Services\SentinelMetricsClient;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Livewire\Livewire;
|
||||
|
||||
@@ -66,7 +65,6 @@ class DenseTestableMetrics extends Metrics
|
||||
beforeEach(function () {
|
||||
// The settings hook starts Sentinel when metrics turn on. A sync queue would run it and make a real SSH call.
|
||||
Queue::fake();
|
||||
Concurrency::setDefaultInstance('sync');
|
||||
InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
|
||||
$user = User::factory()->create();
|
||||
$team = Team::factory()->create();
|
||||
|
||||
@@ -13,7 +13,6 @@ use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Services\SentinelMetricsClient;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Livewire\Livewire;
|
||||
@@ -35,6 +34,10 @@ class StubMetricsClient extends SentinelMetricsClient
|
||||
|
||||
public function summary(): ?array
|
||||
{
|
||||
if (in_array('summary', self::$failingMetrics, true)) {
|
||||
throw new RuntimeException('summary unavailable');
|
||||
}
|
||||
|
||||
return self::$summaries[$this->server->uuid] ?? null;
|
||||
}
|
||||
|
||||
@@ -77,7 +80,6 @@ class ProcessTestableMetrics extends Metrics
|
||||
beforeEach(function () {
|
||||
// The settings hook starts Sentinel when metrics turn on. A sync queue would run it and make a real SSH call.
|
||||
Queue::fake();
|
||||
Concurrency::setDefaultInstance('sync');
|
||||
StubMetricsClient::$summaries = [];
|
||||
StubMetricsClient::$containers = [];
|
||||
StubMetricsClient::$histories = [];
|
||||
@@ -312,9 +314,9 @@ it('does not query Sentinel on a server that Coolify cannot reach', function ()
|
||||
expect($component->get('chartData')['cpu'])->toBe([]);
|
||||
});
|
||||
|
||||
it('shows the page when a metrics worker runs past the process timeout', function () {
|
||||
it('shows the page when every Sentinel read fails', function () {
|
||||
$server = metricsServer($this->team, $this->privateKey);
|
||||
Concurrency::shouldReceive('run')->andThrow(new RuntimeException('The process exceeded the timeout of 60 seconds.'));
|
||||
StubMetricsClient::$failingMetrics = ['summary', 'cpu', 'memory', 'disk', 'load'];
|
||||
|
||||
$component = Livewire::test(TestableMetrics::class)->assertOk();
|
||||
|
||||
@@ -387,8 +389,7 @@ it('clears stale container rankings after a server transfer', function () {
|
||||
expect($component->get('serverRows'))->toBe([]);
|
||||
});
|
||||
|
||||
it('merges metrics from real concurrent workers and isolates endpoint failures', function () {
|
||||
Concurrency::setDefaultInstance('process');
|
||||
it('merges metrics of several servers and isolates endpoint failures', function () {
|
||||
metricsServer($this->team, $this->privateKey);
|
||||
metricsServer($this->team, $this->privateKey);
|
||||
|
||||
@@ -402,6 +403,32 @@ it('merges metrics from real concurrent workers and isolates endpoint failures',
|
||||
expect($component->get('chartData')['networkRx'])->toBe([[1000, 10.0]]);
|
||||
});
|
||||
|
||||
it('resolves container names with a fixed number of queries', function () {
|
||||
$project = Project::factory()->create(['team_id' => $this->team->id]);
|
||||
$environment = Environment::factory()->create(['project_id' => $project->id]);
|
||||
$apps = Application::factory()->count(3)->create(['environment_id' => $environment->id]);
|
||||
$component = Livewire::test(TestableMetrics::class)->instance();
|
||||
$component->containersRaw = $apps->map(fn ($app) => ['id' => $app->uuid])->push(['id' => 'coolify-proxy'])->all();
|
||||
|
||||
DB::enableQueryLog();
|
||||
$names = $apps->map(fn ($app) => $component->containerMeta($app->uuid)['name']);
|
||||
$proxy = $component->containerMeta('coolify-proxy');
|
||||
|
||||
expect($names->all())->toBe($apps->pluck('name')->all())
|
||||
->and($proxy)->toBe(['name' => 'coolify-proxy', 'image' => null, 'link' => null])
|
||||
->and(count(DB::getQueryLog()))->toBeLessThan(20);
|
||||
});
|
||||
|
||||
it('does not show the name of another team’s resource for a container', function () {
|
||||
$project = Project::factory()->create(['team_id' => Team::factory()->create()->id]);
|
||||
$environment = Environment::factory()->create(['project_id' => $project->id]);
|
||||
$foreign = Application::factory()->create(['environment_id' => $environment->id]);
|
||||
|
||||
$meta = Livewire::test(TestableMetrics::class)->instance()->containerMeta($foreign->uuid);
|
||||
|
||||
expect($meta)->toBe(['name' => $foreign->uuid, 'image' => null, 'link' => null]);
|
||||
});
|
||||
|
||||
it('clears stale metrics after the last server is deleted', function () {
|
||||
$server = metricsServer($this->team, $this->privateKey);
|
||||
StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
use App\Models\Server;
|
||||
use App\Services\SentinelMetricsClient;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
@@ -53,7 +52,6 @@ class FakeBrowserFleetMetricsClient extends SentinelMetricsClient
|
||||
|
||||
beforeEach(function () {
|
||||
$this->stack = seedBrowserResourceStack(['serverName' => 'alpha']);
|
||||
Concurrency::setDefaultInstance('sync');
|
||||
// The dashboard chart reads metrics over SSH for each server with metrics on.
|
||||
Process::fake();
|
||||
|
||||
@@ -121,11 +119,12 @@ it('sorts the servers and containers tables when a column header is selected', f
|
||||
$page->script('document.querySelector(\'#metrics-containers-section button[title="Sort by container"]\').click()');
|
||||
expect(visibleFleetMetricsRowNames($page, 'metrics-containers-section'))->toBe(['beta', 'zeta']);
|
||||
|
||||
// The CPU title is a tooltip, so select by the aria-label, which always starts with the column name.
|
||||
foreach (['memory' => ['beta', 'zeta'], 'disk' => ['beta', 'zeta'], 'network' => ['zeta', 'beta'], 'cpu' => ['zeta', 'beta']] as $metric => $expected) {
|
||||
$page->script("document.querySelector('#metrics-containers-section button[title=\"Sort by {$metric}\"]').click()");
|
||||
$page->script("document.querySelector('#metrics-containers-section button[aria-label^=\"Sort by {$metric}\"]').click()");
|
||||
expect(visibleFleetMetricsRowNames($page, 'metrics-containers-section'))->toBe($expected);
|
||||
|
||||
$page->script("document.querySelector('#metrics-containers-section button[title=\"Sort by {$metric}\"]').click()");
|
||||
$page->script("document.querySelector('#metrics-containers-section button[aria-label^=\"Sort by {$metric}\"]').click()");
|
||||
expect(visibleFleetMetricsRowNames($page, 'metrics-containers-section'))->toBe(array_reverse($expected));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user