diff --git a/app/Livewire/Analytics.php b/app/Livewire/Analytics.php index 62c3aab048..e4cc20741b 100644 --- a/app/Livewire/Analytics.php +++ b/app/Livewire/Analytics.php @@ -348,9 +348,14 @@ class Analytics extends Component $appRows = []; $hostTotals = []; + $clients = $servers->mapWithKeys(fn (Server $server) => [$server->uuid => $this->trafficClient($server)]); + if ($resource === null) { + SentinelTrafficClient::prefetchServerWideConcurrently($clients, $from, $to, $this->range, appsLimit: self::MAX_LEADERBOARD_APPS); + } + foreach ($servers as $server) { try { - $client = $this->trafficClient($server); + $client = $clients[$server->uuid]; if ($resource !== null) { // One server in Sentinel's resource scope is exact; a merge across servers diff --git a/app/Services/SentinelTrafficClient.php b/app/Services/SentinelTrafficClient.php index 1c5c28e2d8..54cb4758c6 100644 --- a/app/Services/SentinelTrafficClient.php +++ b/app/Services/SentinelTrafficClient.php @@ -8,6 +8,7 @@ use App\Data\Traffic\TrafficPathData; use App\Data\Traffic\TrafficSeriesBucketData; use App\Helpers\SshMultiplexingHelper; use App\Models\Server; +use Illuminate\Contracts\Process\InvokedProcess; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; @@ -354,6 +355,56 @@ class SentinelTrafficClient return $this->safeReportedKeys(array_map(fn ($app) => is_array($app) ? ($app['uuid'] ?? null) : null, $bundle['apps'] ?? [])); } + /** + * Fetch the server-wide dashboard bundle of several servers at the same time, so a page + * waits for the slowest server instead of for every server in turn. Each response warms + * the cache that prefetchServerWide() reads; a failed fetch is remembered like in + * fetchDashboard(), so prefetchServerWide() does not fetch it again. + * + * @param iterable $clients + */ + public static function prefetchServerWideConcurrently(iterable $clients, string $from, string $to, string $range, int $pathLimit = 50, int $breakdownLimit = 50, int $appsLimit = 200): void + { + $started = []; + foreach ($clients as $client) { + $url = $client->dashboardUrl(null, $from, $to, $range, $pathLimit, $breakdownLimit, $appsLimit); + if (! $client->fetchesThroughStartRemoteFetch() + || Cache::has($client->cacheKey($url)) + || Cache::get($client->dashboardAbsenceKey()) === true) { + continue; + } + + try { + $started[] = [$client, $url, $client->startRemoteFetch($url)]; + } catch (\Throwable) { + // An unavailable server fails again at once in prefetchServerWide(). + } + } + + foreach ($started as [$client, $url, $process]) { + try { + Cache::put($client->cacheKey($url), $client->guard($client->awaitRemote($process)), 60); + } catch (\Throwable) { + Cache::put($client->dashboardAbsenceKey(), true, 60); + } + } + } + + /** + * False for a fake that serves canned bodies from raw() or remoteFetch(): the concurrent + * prefetch would otherwise reach real SSH. + */ + private function fetchesThroughStartRemoteFetch(): bool + { + foreach (['raw', 'remoteFetch'] as $method) { + if ((new \ReflectionMethod($this, $method))->getDeclaringClass()->getName() !== self::class) { + return false; + } + } + + return true; + } + /** * The usable keys of a Sentinel-reported key list. A key that would be unsafe in a request * url is dropped (and logged) on its own, so it doesn't fail the whole server's analytics. @@ -685,13 +736,21 @@ class SentinelTrafficClient } protected function remoteFetch(string $url): string + { + // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing + // container). A literal "null" body comes back empty; guard() rejects the empty string. + return $this->awaitRemote($this->startRemoteFetch($url)); + } + + /** + * Start the fetch of one URL without waiting for it, so several servers can be fetched at the same time. + */ + protected function startRemoteFetch(string $url): InvokedProcess { $this->ensureRemoteAvailable(); $token = $this->server->settings->ensureValidSentinelToken(); - // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing - // container). A literal "null" body comes back empty; guard() rejects the empty string. - return $this->runRemote($this->buildFetchCommand($token, $url)); + return $this->startRemote($this->buildFetchCommand($token, $url)); } /** @@ -702,7 +761,7 @@ class SentinelTrafficClient $this->ensureRemoteAvailable(); $token = $this->server->settings->ensureValidSentinelToken(); - return $this->runRemote($this->buildBatchCommand($token, $urls)); + return $this->awaitRemote($this->startRemote($this->buildBatchCommand($token, $urls))); } /** @@ -722,14 +781,25 @@ class SentinelTrafficClient /** * Unlike instant_remote_process(), there are no SSH retries; a failure marks the server unavailable. */ - private function runRemote(string $command): string + private function startRemote(string $command): InvokedProcess { $commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command]; try { - $process = Process::timeout(self::REMOTE_TIMEOUT)->run( + return Process::timeout(self::REMOTE_TIMEOUT)->start( SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT) ); + } catch (\Throwable $e) { + Cache::put($this->unavailableKey(), true, self::UNAVAILABLE_TTL); + + throw $e; + } + } + + private function awaitRemote(InvokedProcess $invoked): string + { + try { + $process = $invoked->wait(); if ($process->exitCode() !== 0) { excludeCertainErrors($process->errorOutput(), $process->exitCode()); diff --git a/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php b/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php index 870befc0f5..1bf1b554e1 100644 --- a/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php +++ b/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php @@ -12,8 +12,11 @@ use App\Models\StandaloneDocker; use App\Models\Team; use App\Models\User; use App\Services\SentinelTrafficClient; +use Illuminate\Contracts\Process\InvokedProcess; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Process; use Livewire\Livewire; uses(RefreshDatabase::class); @@ -34,6 +37,20 @@ class FakeGlobalAnalyticsTrafficClient extends SentinelTrafficClient } } +/** + * Answers every dashboard fetch with a real process that takes one second, so a test can + * tell concurrent fetches from serial ones. + */ +class SlowDashboardTrafficClient extends SentinelTrafficClient +{ + protected function startRemoteFetch(string $url): InvokedProcess + { + $bundle = json_encode(['overview' => ['requests' => 1000], 'apps' => []]); + + return Process::start('sleep 1 && printf %s '.escapeshellarg($bundle)); + } +} + function fakeGlobalAnalyticsResponses(array $appUuids = []): array { return [ @@ -519,3 +536,17 @@ it('combines the client IP mode of several servers', function (array $modes, str 'anonymized and off' => [['anonymized', 'off'], 'anonymized'], 'full and anonymized' => [['full', 'anonymized'], 'full'], ]); + +it('fetches the dashboard of every server at the same time', function () { + Cache::flush(); + collect(range(1, 3))->each(fn () => bootEnabledGlobalServer()); + app()->bind(SentinelTrafficClient::class, fn ($app, array $parameters) => new SlowDashboardTrafficClient($parameters['server'])); + + $started = microtime(true); + $component = loadLazy(Livewire::test(Analytics::class)); + $elapsed = microtime(true) - $started; + + // Three servers fetched one after another take at least 3 seconds. + expect($elapsed)->toBeLessThan(2.5) + ->and($component->get('overview')['requests'])->toBe(3000); +}); diff --git a/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php b/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php index ddaf9c8a58..886da64ed6 100644 --- a/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php +++ b/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php @@ -116,5 +116,6 @@ it('shows no data without an error when Sentinel has no traffic routes (0.0.x an ->assertSet('series', []) ->assertNotDispatched('error'); - Process::assertRan(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard')); + // The concurrent prefetch remembers the failure, so the per-server pass does not fetch it again. + Process::assertRanTimes(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'), 1); });