perf(metrics): fetch Fleet Metrics of all servers at the same time (#12389)

This commit is contained in:
Aditya Tripathi
2026-10-30 17:53:15 +01:00
committed by GitHub
parent a0b56bc42f
commit 157f3e0fae
6 changed files with 393 additions and 86 deletions
+69 -66
View File
@@ -5,13 +5,13 @@ namespace App\Livewire\Fleet;
use App\Livewire\Concerns\BuildsMetricsChartPayload;
use App\Models\Application;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceDatabase;
use App\Services\FleetMetricsAggregator;
use App\Services\SentinelMetricsClient;
use Illuminate\Contracts\View\View;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Concurrency;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\RateLimiter;
use Livewire\Attributes\Lazy;
use Livewire\Attributes\Url;
@@ -257,34 +257,21 @@ class Metrics extends Component
$targets = $this->targetServers();
$clients = [];
$summaryTasks = [];
foreach ($targets as $server) {
// Coolify does not connect to an unreachable server, and its SSH retries would hold a worker past the process timeout.
if (! $server->isFunctional()) {
continue;
}
$client = $clients[$server->uuid] = $this->metricsClient($server);
$summaryTasks[$server->uuid] = static fn () => $client->summary();
}
$summaries = $this->runMetricTasks($summaryTasks);
$tasks = [];
$range = $this->range;
foreach ($clients as $uuid => $client) {
$metrics = $summaries[$uuid] === null ? ['cpu', 'memory'] : ['cpu', 'memory', 'disk', 'load'];
foreach ($metrics as $metric) {
$tasks[$uuid.':'.$metric] = static fn () => $client->history($metric, $range);
}
if ($summaries[$uuid] !== null) {
$tasks[$uuid.':containers'] = static fn () => $client->containersCurrent();
$tasks[$uuid.':network'] = static fn () => $client->networkHistory($range);
// Coolify does not connect to an unreachable server.
if ($server->isFunctional()) {
$clients[$server->uuid] = $this->metricsClient($server);
}
}
$results = $this->runMetricTasks($tasks);
SentinelMetricsClient::prefetchConcurrently($clients, $this->range);
foreach ($targets as $server) {
$summary = $summaries[$server->uuid] ?? null;
$client = $clients[$server->uuid] ?? null;
$summary = $client === null ? null : $this->attempt(fn () => $client->summary());
foreach (['cpu', 'memory', 'disk', 'load'] as $metric) {
$series[$metric][] = $results[$server->uuid.':'.$metric] ?? [];
// A Sentinel without a summary endpoint still has cpu and memory history.
$read = $client !== null && ($summary !== null || in_array($metric, ['cpu', 'memory'], true));
$series[$metric][] = $read ? ($this->attempt(fn () => $client->history($metric, $this->range)) ?? []) : [];
}
if ($summary === null) {
@@ -293,19 +280,17 @@ class Metrics extends Component
continue;
}
$current = $results[$server->uuid.':containers'] ?? [];
$current = $this->attempt(fn () => $client->containersCurrent()) ?? [];
$rows[] = array_merge(
$summary,
['uuid' => $server->uuid, 'name' => $server->name, 'online' => true, 'reporting' => true, 'containers' => count($current)]
);
foreach ($current as $c) {
// Names/images are resolved lazily for the displayed rows only (containerMeta),
// so a server with many containers doesn't trigger a resolve storm here.
$containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]);
}
$net = $results[$server->uuid.':network'] ?? ['rx' => [], 'tx' => []];
$net = $this->attempt(fn () => $client->networkHistory($this->range)) ?? ['rx' => [], 'tx' => []];
$series['networkRx'][] = $net['rx'];
$series['networkTx'][] = $net['tx'];
}
@@ -320,34 +305,15 @@ class Metrics extends Component
}
/**
* Bound the number of PHP workers and SSH connections for large fleets.
*
* @param array<string, \Closure> $tasks
* @return array<string, mixed>
* One failed metric must not fail the others.
*/
protected function runMetricTasks(array $tasks): array
private function attempt(\Closure $read): mixed
{
foreach ($tasks as $key => $task) {
$tasks[$key] = static function () use ($task) {
try {
return $task();
} catch (\Throwable) {
return null;
}
};
try {
return $read();
} catch (\Throwable) {
return null;
}
$results = [];
foreach (array_chunk($tasks, 8, true) as $batch) {
try {
$results += Concurrency::run($batch);
} catch (\Throwable $e) {
// A worker that runs past the process timeout fails its batch. Show those metrics as missing, not a server error.
Log::warning('Fleet metrics batch failed', ['error' => $e->getMessage()]);
$results += array_fill_keys(array_keys($batch), null);
}
}
return $results;
}
/**
@@ -383,18 +349,27 @@ class Metrics extends Component
*/
public function containerMeta(string $id): array
{
if (isset($this->containerMetaCache[$id])) {
return $this->containerMetaCache[$id];
if (! isset($this->containerMetaCache[$id])) {
// Resolve every listed container at once, not one lookup per table row.
$this->resolveContainerMeta([...array_column($this->containersRaw, 'id'), $id]);
}
return $this->containerMetaCache[$id];
}
/**
* @param array<int, string> $ids
*/
private function resolveContainerMeta(array $ids): void
{
$ids = array_values(array_diff(array_unique($ids), array_keys($this->containerMetaCache)));
$teamId = currentTeam()?->id;
$resource = $teamId !== null ? getResourceByUuid($id, $teamId) : null;
$image = null;
$link = null;
if ($resource) {
$image = data_get($resource, 'docker_registry_image_name') ?: data_get($resource, 'image');
$resources = $teamId === null ? [] : $this->teamResourcesByUuid($ids, $teamId);
foreach ($ids as $id) {
$resource = $resources[$id] ?? null;
$image = $resource ? (data_get($resource, 'docker_registry_image_name') ?: data_get($resource, 'image')) : null;
$link = null;
if ($resource instanceof Application && data_get($resource, 'environment.project.uuid')) {
$link = route('project.application.metrics', [
'project_uuid' => $resource->environment->project->uuid,
@@ -402,13 +377,41 @@ class Metrics extends Component
'application_uuid' => $resource->uuid,
]);
}
$this->containerMetaCache[$id] = [
'name' => $resource?->name ?? $id,
'image' => $image ?: null,
'link' => $link,
];
}
}
/**
* The team's resources with these uuids, one query per resource type.
*
* @param array<int, string> $uuids
* @return array<string, mixed> uuid => resource
*/
private function teamResourcesByUuid(array $uuids, int $teamId): array
{
$found = [];
foreach ([Application::class, Service::class, ...array_values(STANDALONE_DATABASE_MODELS), ServiceDatabase::class] as $model) {
$missing = array_values(array_diff($uuids, array_keys($found)));
if ($missing === []) {
break;
}
$teamPath = $model === ServiceDatabase::class ? 'service.environment.project' : 'environment.project';
$model::query()
->whereIn('uuid', $missing)
->whereRelation($teamPath, 'team_id', $teamId)
->with($teamPath)
->get()
->each(function ($resource) use (&$found) {
$found[$resource->uuid] = $resource;
});
}
return $this->containerMetaCache[$id] = [
'name' => $resource?->name ?? $id,
'image' => $image ?: null,
'link' => $link,
];
return $found;
}
public function placeholder(): View
+136 -8
View File
@@ -2,15 +2,89 @@
namespace App\Services;
use App\Helpers\SshMultiplexingHelper;
use App\Models\Server;
use Illuminate\Contracts\Process\InvokedProcess;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Process;
class SentinelMetricsClient
{
private const REMOTE_TIMEOUT = 20;
/** Mounts that can hold the root disk, in the order of rootMountName(). */
private const ROOT_MOUNTS = ['/', '/app/db', '/data/coolify', '/data'];
private string $base = 'http://localhost:8888/api';
/** @var array<string, ?string> url => response from prefetchConcurrently(); null when it failed */
private array $prefetched = [];
public function __construct(protected Server $server) {}
/**
* Fetch the Fleet Metrics responses of several servers at the same time, with one SSH
* command per server, so a page waits for the slowest server only. raw() then reads the
* responses; a failed fetch is not tried again per metric.
*
* @param iterable<int|string, self> $clients
*/
public static function prefetchConcurrently(iterable $clients, string $range): void
{
$started = [];
foreach ($clients as $client) {
// A subclass is a test fake with canned data; it must not reach real SSH.
if ($client::class !== self::class) {
continue;
}
$urls = array_values(array_filter($client->pageUrls($range), fn (string $url) => ! Cache::has($client->cacheKey($url))));
if ($urls === []) {
continue;
}
try {
$started[] = [$client, $urls, $client->startBatch($urls)];
} catch (\Throwable) {
$client->prefetched += array_fill_keys($urls, null);
}
}
foreach ($started as [$client, $urls, $process]) {
try {
$responses = explode("\x1e", $client->awaitBatch($process));
} catch (\Throwable) {
$responses = [];
}
if (count($responses) !== count($urls)) {
$responses = [];
}
foreach ($urls as $index => $url) {
try {
$body = $client->guard(trim($responses[$index] ?? ''));
Cache::put($client->cacheKey($url), $body, 60);
$client->prefetched[$url] = $body;
} catch (\Throwable) {
$client->prefetched[$url] = null;
}
}
}
}
/**
* Every URL that the Fleet Metrics page reads for one server.
*
* @return array<int, string>
*/
private function pageUrls(string $range): array
{
return [
$this->url('/summary'),
$this->url('/containers/current'),
...array_map(fn (string $metric) => $this->historyUrl($metric, $range), ['cpu', 'memory', 'disk', 'load', 'network']),
];
}
/** Rounded to the minute so the history URL (and its cache key) is stable within a minute. */
public static function rangeFrom(string $range): string
{
@@ -116,14 +190,14 @@ class SentinelMetricsClient
*/
public function history(string $metric, string $range): array
{
[$path, $field] = match ($metric) {
'memory' => ['/memory/history', 'used'],
'disk' => ['/disk/history', 'usedPercent'],
'load' => ['/load/history', 'load1'],
default => ['/cpu/history', 'percent'],
$field = match ($metric) {
'memory' => 'used',
'disk' => 'usedPercent',
'load' => 'load1',
default => 'percent',
};
return $this->seriesRows($this->url($path, self::historyQuery($range)), $field, $range, $metric === 'disk');
return $this->seriesRows($this->historyUrl($metric, $range), $field, $range, $metric === 'disk');
}
/**
@@ -131,7 +205,7 @@ class SentinelMetricsClient
*/
public function networkHistory(string $range): array
{
$url = $this->url('/network/history', self::historyQuery($range));
$url = $this->historyUrl('network', $range);
return [
'rx' => $this->seriesRows($url, 'rxBytesPerSec', $range),
@@ -188,7 +262,7 @@ class SentinelMetricsClient
{
$mounts = array_column($rows, 'mount');
return collect(['/', '/app/db', '/data/coolify', '/data'])->first(fn ($mount) => in_array($mount, $mounts, true));
return collect(self::ROOT_MOUNTS)->first(fn ($mount) => in_array($mount, $mounts, true));
}
/**
@@ -214,6 +288,21 @@ class SentinelMetricsClient
];
}
private function historyUrl(string $metric, string $range): string
{
$path = match ($metric) {
'memory', 'disk', 'load', 'network' => "/{$metric}/history",
default => '/cpu/history',
};
$query = self::historyQuery($range);
if ($metric === 'disk') {
// Sentinel 1.1.4+ returns only these mounts; older versions ignore the param.
$query['mount'] = implode(',', self::ROOT_MOUNTS);
}
return $this->url($path, $query);
}
/**
* @param array<string, mixed> $query
*/
@@ -231,6 +320,10 @@ class SentinelMetricsClient
protected function raw(string $url): string
{
if (array_key_exists($url, $this->prefetched)) {
return $this->prefetched[$url] ?? throw new \RuntimeException('Metrics request failed.');
}
return Cache::remember($this->cacheKey($url), 60, fn () => $this->guard($this->remoteFetch($url)));
}
@@ -248,6 +341,41 @@ class SentinelMetricsClient
);
}
/**
* Start one `docker exec` that curls every URL in order, with the responses separated by
* a 0x1E record separator. The token reaches curl on stdin, so it is not a process argument.
* The token only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it
* cannot end the heredoc. No pipe or `sh -c` is used, so the non-root sudo parser only puts
* sudo in front of the line. Unlike instant_remote_process(), there are no SSH retries.
*
* @param array<int, string> $urls
*/
private function startBatch(array $urls): InvokedProcess
{
$token = $this->server->settings->ensureValidSentinelToken();
$script = implode("\nprintf '\\036'\n", array_map(
fn (string $url) => "curl -s --connect-timeout 3 --max-time 10 -H @- \"{$url}\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH",
$urls
));
// `true` last: a failed curl empties only its own response, not the whole batch.
$command = "docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$script}\ntrue\nCOOLIFY_SENTINEL_SCRIPT";
$commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];
return Process::timeout(self::REMOTE_TIMEOUT)->start(
SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT)
);
}
private function awaitBatch(InvokedProcess $invoked): string
{
$process = $invoked->wait();
if ($process->exitCode() !== 0) {
excludeCertainErrors($process->errorOutput(), $process->exitCode());
}
return $process->output();
}
private function guard(string $response): string
{
$payload = json_decode($response, true);
+152
View File
@@ -0,0 +1,152 @@
<?php
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\Team;
use App\Services\SentinelMetricsClient;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Process\PendingProcess;
use Illuminate\Support\Facades\Process;
uses(RefreshDatabase::class);
beforeEach(function () {
InstanceSettings::create(['id' => 0]);
config(['constants.ssh.mux_enabled' => false]);
$team = Team::factory()->create();
$this->privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
$this->team = $team;
});
function prefetchServer($team, $privateKey): Server
{
$server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]);
$server->settings->forceFill(['is_metrics_enabled' => true])->saveQuietly();
return $server->refresh();
}
/** Sentinel responses in the order of the batch: summary, containers, cpu, memory, disk, load, network. */
function fleetBatchOutput(?string $summary = null): string
{
return implode("\x1e", [
$summary ?? '{"cpu":{"percent":12.5},"memory":{"used":100,"total":200}}',
'[{"id":"app-1","cpu":{"percent":3},"status":{"state":"running"}}]',
'[{"time":300000,"percent":12.5}]',
'[{"time":300000,"used":100}]',
'[{"time":300000,"mount":"/","usedPercent":40}]',
'[{"time":300000,"load1":0.5}]',
'[{"time":300000,"rxBytesPerSec":10,"txBytesPerSec":2}]',
]);
}
it('fetches the page metrics of every server in one SSH command each', function () {
$commands = [];
Process::fake(function (PendingProcess $process) use (&$commands) {
$commands[] = remoteProcessScript($process);
return Process::result(fleetBatchOutput());
});
$clients = collect([prefetchServer($this->team, $this->privateKey), prefetchServer($this->team, $this->privateKey)])
->map(fn (Server $server) => new SentinelMetricsClient($server));
SentinelMetricsClient::prefetchConcurrently($clients, '24h');
foreach ($clients as $client) {
expect($client->summary()['cpu'])->toBe(12.5)
->and($client->containersCurrent())->toHaveCount(1)
->and($client->history('cpu', '24h'))->toBe([[300000, 12.5]])
->and($client->history('disk', '24h'))->toBe([[300000, 40.0]])
->and($client->networkHistory('24h')['tx'])->toBe([[300000, 2.0]]);
}
expect($commands)->toHaveCount(2)
->and($commands[0])->toContain('/api/summary', '/api/containers/current', '/api/load/history', '/api/network/history', '-H @-')
->and($commands[0])->toContain('/api/disk/history?from=')
->and($commands[0])->toContain('mount=%2F%2C%2Fapp%2Fdb%2C%2Fdata%2Fcoolify%2C%2Fdata');
});
it('does not fetch each metric again when the batch fails', function () {
$commands = 0;
Process::fake(function () use (&$commands) {
$commands++;
return Process::result('', 'ssh: connect to host failed', 255);
});
$client = new SentinelMetricsClient(prefetchServer($this->team, $this->privateKey));
SentinelMetricsClient::prefetchConcurrently([$client], '24h');
expect($client->summary())->toBeNull()
->and($client->containersCurrent())->toBe([])
->and($client->history('cpu', '24h'))->toBe([])
->and($client->networkHistory('24h'))->toBe(['rx' => [], 'tx' => []])
->and($commands)->toBe(1);
});
it('keeps the other metrics when Sentinel fails one endpoint', function () {
$commands = 0;
Process::fake(function () use (&$commands) {
$commands++;
return Process::result(fleetBatchOutput(summary: '404 page not found'));
});
$client = new SentinelMetricsClient(prefetchServer($this->team, $this->privateKey));
SentinelMetricsClient::prefetchConcurrently([$client], '24h');
expect($client->summary())->toBeNull()
->and($client->history('cpu', '24h'))->toBe([[300000, 12.5]])
->and($commands)->toBe(1);
});
it('reads cached responses on the next load without SSH', function () {
$commands = 0;
Process::fake(function () use (&$commands) {
$commands++;
return Process::result(fleetBatchOutput());
});
$server = prefetchServer($this->team, $this->privateKey);
SentinelMetricsClient::prefetchConcurrently([new SentinelMetricsClient($server)], '24h');
$next = new SentinelMetricsClient($server);
SentinelMetricsClient::prefetchConcurrently([$next], '24h');
expect($next->history('memory', '24h'))->toBe([[300000, 100.0]])
->and($commands)->toBe(1);
});
it('runs the batched script with sudo for a non-root user', function () {
$commands = [];
Process::fake(function (PendingProcess $process) use (&$commands) {
$commands[] = remoteProcessScript($process);
return Process::result(fleetBatchOutput());
});
$server = prefetchServer($this->team, $this->privateKey);
$server->forceFill(['user' => 'deploy'])->saveQuietly();
SentinelMetricsClient::prefetchConcurrently([new SentinelMetricsClient($server)], '24h');
expect($commands)->toHaveCount(1)
->and($commands[0])->toContain('sudo docker exec -i coolify-sentinel sh -s')
->and($commands[0])->not->toContain('sudo curl');
});
it('keeps the other metrics when the last curl of the batch fails', function () {
$commands = [];
Process::fake(function (PendingProcess $process) use (&$commands) {
$commands[] = remoteProcessScript($process);
return Process::result(output: fleetBatchOutput(), exitCode: str_contains(end($commands), "\ntrue\nCOOLIFY_SENTINEL_SCRIPT") ? 0 : 28);
});
$client = new SentinelMetricsClient(prefetchServer($this->team, $this->privateKey));
SentinelMetricsClient::prefetchConcurrently([$client], '24h');
expect($client->summary()['cpu'])->toBe(12.5)
->and($client->history('cpu', '24h'))->toBe([[300000, 12.5]])
->and($commands)->toHaveCount(1);
});
@@ -8,7 +8,6 @@ use App\Models\Team;
use App\Models\User;
use App\Services\SentinelMetricsClient;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Concurrency;
use Illuminate\Support\Facades\Queue;
use Livewire\Livewire;
@@ -66,7 +65,6 @@ class DenseTestableMetrics extends Metrics
beforeEach(function () {
// The settings hook starts Sentinel when metrics turn on. A sync queue would run it and make a real SSH call.
Queue::fake();
Concurrency::setDefaultInstance('sync');
InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
$user = User::factory()->create();
$team = Team::factory()->create();
@@ -13,7 +13,6 @@ use App\Models\Team;
use App\Models\User;
use App\Services\SentinelMetricsClient;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Concurrency;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Queue;
use Livewire\Livewire;
@@ -35,6 +34,10 @@ class StubMetricsClient extends SentinelMetricsClient
public function summary(): ?array
{
if (in_array('summary', self::$failingMetrics, true)) {
throw new RuntimeException('summary unavailable');
}
return self::$summaries[$this->server->uuid] ?? null;
}
@@ -77,7 +80,6 @@ class ProcessTestableMetrics extends Metrics
beforeEach(function () {
// The settings hook starts Sentinel when metrics turn on. A sync queue would run it and make a real SSH call.
Queue::fake();
Concurrency::setDefaultInstance('sync');
StubMetricsClient::$summaries = [];
StubMetricsClient::$containers = [];
StubMetricsClient::$histories = [];
@@ -312,9 +314,9 @@ it('does not query Sentinel on a server that Coolify cannot reach', function ()
expect($component->get('chartData')['cpu'])->toBe([]);
});
it('shows the page when a metrics worker runs past the process timeout', function () {
it('shows the page when every Sentinel read fails', function () {
$server = metricsServer($this->team, $this->privateKey);
Concurrency::shouldReceive('run')->andThrow(new RuntimeException('The process exceeded the timeout of 60 seconds.'));
StubMetricsClient::$failingMetrics = ['summary', 'cpu', 'memory', 'disk', 'load'];
$component = Livewire::test(TestableMetrics::class)->assertOk();
@@ -387,8 +389,7 @@ it('clears stale container rankings after a server transfer', function () {
expect($component->get('serverRows'))->toBe([]);
});
it('merges metrics from real concurrent workers and isolates endpoint failures', function () {
Concurrency::setDefaultInstance('process');
it('merges metrics of several servers and isolates endpoint failures', function () {
metricsServer($this->team, $this->privateKey);
metricsServer($this->team, $this->privateKey);
@@ -402,6 +403,32 @@ it('merges metrics from real concurrent workers and isolates endpoint failures',
expect($component->get('chartData')['networkRx'])->toBe([[1000, 10.0]]);
});
it('resolves container names with a fixed number of queries', function () {
$project = Project::factory()->create(['team_id' => $this->team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$apps = Application::factory()->count(3)->create(['environment_id' => $environment->id]);
$component = Livewire::test(TestableMetrics::class)->instance();
$component->containersRaw = $apps->map(fn ($app) => ['id' => $app->uuid])->push(['id' => 'coolify-proxy'])->all();
DB::enableQueryLog();
$names = $apps->map(fn ($app) => $component->containerMeta($app->uuid)['name']);
$proxy = $component->containerMeta('coolify-proxy');
expect($names->all())->toBe($apps->pluck('name')->all())
->and($proxy)->toBe(['name' => 'coolify-proxy', 'image' => null, 'link' => null])
->and(count(DB::getQueryLog()))->toBeLessThan(20);
});
it('does not show the name of another team’s resource for a container', function () {
$project = Project::factory()->create(['team_id' => Team::factory()->create()->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$foreign = Application::factory()->create(['environment_id' => $environment->id]);
$meta = Livewire::test(TestableMetrics::class)->instance()->containerMeta($foreign->uuid);
expect($meta)->toBe(['name' => $foreign->uuid, 'image' => null, 'link' => null]);
});
it('clears stale metrics after the last server is deleted', function () {
$server = metricsServer($this->team, $this->privateKey);
StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
@@ -3,7 +3,6 @@
use App\Models\Server;
use App\Services\SentinelMetricsClient;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Concurrency;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
@@ -53,7 +52,6 @@ class FakeBrowserFleetMetricsClient extends SentinelMetricsClient
beforeEach(function () {
$this->stack = seedBrowserResourceStack(['serverName' => 'alpha']);
Concurrency::setDefaultInstance('sync');
// The dashboard chart reads metrics over SSH for each server with metrics on.
Process::fake();
@@ -121,11 +119,12 @@ it('sorts the servers and containers tables when a column header is selected', f
$page->script('document.querySelector(\'#metrics-containers-section button[title="Sort by container"]\').click()');
expect(visibleFleetMetricsRowNames($page, 'metrics-containers-section'))->toBe(['beta', 'zeta']);
// The CPU title is a tooltip, so select by the aria-label, which always starts with the column name.
foreach (['memory' => ['beta', 'zeta'], 'disk' => ['beta', 'zeta'], 'network' => ['zeta', 'beta'], 'cpu' => ['zeta', 'beta']] as $metric => $expected) {
$page->script("document.querySelector('#metrics-containers-section button[title=\"Sort by {$metric}\"]').click()");
$page->script("document.querySelector('#metrics-containers-section button[aria-label^=\"Sort by {$metric}\"]').click()");
expect(visibleFleetMetricsRowNames($page, 'metrics-containers-section'))->toBe($expected);
$page->script("document.querySelector('#metrics-containers-section button[title=\"Sort by {$metric}\"]').click()");
$page->script("document.querySelector('#metrics-containers-section button[aria-label^=\"Sort by {$metric}\"]').click()");
expect(visibleFleetMetricsRowNames($page, 'metrics-containers-section'))->toBe(array_reverse($expected));
}