fix(analytics): fetch Sentinel dashboards of all servers concurrently

The analytics page fetched each server's dashboard bundle over SSH one
after another, so the skeleton stayed until every server answered. Start
all fetches at the same time and remember failures, so the page waits
only for the slowest server.

Closes #12347

app/Livewire/Analytics.php

@@ -348,9 +348,14 @@ class Analytics extends Component
         $appRows = [];
         $hostTotals = [];
 
+        $clients = $servers->mapWithKeys(fn (Server $server) => [$server->uuid => $this->trafficClient($server)]);
+        if ($resource === null) {
+            SentinelTrafficClient::prefetchServerWideConcurrently($clients, $from, $to, $this->range, appsLimit: self::MAX_LEADERBOARD_APPS);
+        }
+
         foreach ($servers as $server) {
             try {
-                $client = $this->trafficClient($server);
+                $client = $clients[$server->uuid];
 
                 if ($resource !== null) {
                     // One server in Sentinel's resource scope is exact; a merge across servers

app/Services/SentinelTrafficClient.php

@@ -8,6 +8,7 @@ use App\Data\Traffic\TrafficPathData;
 use App\Data\Traffic\TrafficSeriesBucketData;
 use App\Helpers\SshMultiplexingHelper;
 use App\Models\Server;
+use Illuminate\Contracts\Process\InvokedProcess;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
@@ -354,6 +355,56 @@ class SentinelTrafficClient
         return $this->safeReportedKeys(array_map(fn ($app) => is_array($app) ? ($app['uuid'] ?? null) : null, $bundle['apps'] ?? []));
     }
 
+    /**
+     * Fetch the server-wide dashboard bundle of several servers at the same time, so a page
+     * waits for the slowest server instead of for every server in turn. Each response warms
+     * the cache that prefetchServerWide() reads; a failed fetch is remembered like in
+     * fetchDashboard(), so prefetchServerWide() does not fetch it again.
+     *
+     * @param  iterable<int, self>  $clients
+     */
+    public static function prefetchServerWideConcurrently(iterable $clients, string $from, string $to, string $range, int $pathLimit = 50, int $breakdownLimit = 50, int $appsLimit = 200): void
+    {
+        $started = [];
+        foreach ($clients as $client) {
+            $url = $client->dashboardUrl(null, $from, $to, $range, $pathLimit, $breakdownLimit, $appsLimit);
+            if (! $client->fetchesThroughStartRemoteFetch()
+                || Cache::has($client->cacheKey($url))
+                || Cache::get($client->dashboardAbsenceKey()) === true) {
+                continue;
+            }
+
+            try {
+                $started[] = [$client, $url, $client->startRemoteFetch($url)];
+            } catch (\Throwable) {
+                // An unavailable server fails again at once in prefetchServerWide().
+            }
+        }
+
+        foreach ($started as [$client, $url, $process]) {
+            try {
+                Cache::put($client->cacheKey($url), $client->guard($client->awaitRemote($process)), 60);
+            } catch (\Throwable) {
+                Cache::put($client->dashboardAbsenceKey(), true, 60);
+            }
+        }
+    }
+
+    /**
+     * False for a fake that serves canned bodies from raw() or remoteFetch(): the concurrent
+     * prefetch would otherwise reach real SSH.
+     */
+    private function fetchesThroughStartRemoteFetch(): bool
+    {
+        foreach (['raw', 'remoteFetch'] as $method) {
+            if ((new \ReflectionMethod($this, $method))->getDeclaringClass()->getName() !== self::class) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     /**
      * The usable keys of a Sentinel-reported key list. A key that would be unsafe in a request
      * url is dropped (and logged) on its own, so it doesn't fail the whole server's analytics.
@@ -685,13 +736,21 @@ class SentinelTrafficClient
     }
 
     protected function remoteFetch(string $url): string
+    {
+        // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing
+        // container). A literal "null" body comes back empty; guard() rejects the empty string.
+        return $this->awaitRemote($this->startRemoteFetch($url));
+    }
+
+    /**
+     * Start the fetch of one URL without waiting for it, so several servers can be fetched at the same time.
+     */
+    protected function startRemoteFetch(string $url): InvokedProcess
     {
         $this->ensureRemoteAvailable();
         $token = $this->server->settings->ensureValidSentinelToken();
 
-        // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing
-        // container). A literal "null" body comes back empty; guard() rejects the empty string.
-        return $this->runRemote($this->buildFetchCommand($token, $url));
+        return $this->startRemote($this->buildFetchCommand($token, $url));
     }
 
     /**
@@ -702,7 +761,7 @@ class SentinelTrafficClient
         $this->ensureRemoteAvailable();
         $token = $this->server->settings->ensureValidSentinelToken();
 
-        return $this->runRemote($this->buildBatchCommand($token, $urls));
+        return $this->awaitRemote($this->startRemote($this->buildBatchCommand($token, $urls)));
     }
 
     /**
@@ -722,14 +781,25 @@ class SentinelTrafficClient
     /**
      * Unlike instant_remote_process(), there are no SSH retries; a failure marks the server unavailable.
      */
-    private function runRemote(string $command): string
+    private function startRemote(string $command): InvokedProcess
     {
         $commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];
 
         try {
-            $process = Process::timeout(self::REMOTE_TIMEOUT)->run(
+            return Process::timeout(self::REMOTE_TIMEOUT)->start(
                 SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT)
             );
+        } catch (\Throwable $e) {
+            Cache::put($this->unavailableKey(), true, self::UNAVAILABLE_TTL);
+
+            throw $e;
+        }
+    }
+
+    private function awaitRemote(InvokedProcess $invoked): string
+    {
+        try {
+            $process = $invoked->wait();
 
             if ($process->exitCode() !== 0) {
                 excludeCertainErrors($process->errorOutput(), $process->exitCode());

tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php

@@ -12,8 +12,11 @@ use App\Models\StandaloneDocker;
 use App\Models\Team;
 use App\Models\User;
 use App\Services\SentinelTrafficClient;
+use Illuminate\Contracts\Process\InvokedProcess;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Process;
 use Livewire\Livewire;
 
 uses(RefreshDatabase::class);
@@ -34,6 +37,20 @@ class FakeGlobalAnalyticsTrafficClient extends SentinelTrafficClient
     }
 }
 
+/**
+ * Answers every dashboard fetch with a real process that takes one second, so a test can
+ * tell concurrent fetches from serial ones.
+ */
+class SlowDashboardTrafficClient extends SentinelTrafficClient
+{
+    protected function startRemoteFetch(string $url): InvokedProcess
+    {
+        $bundle = json_encode(['overview' => ['requests' => 1000], 'apps' => []]);
+
+        return Process::start('sleep 1 && printf %s '.escapeshellarg($bundle));
+    }
+}
+
 function fakeGlobalAnalyticsResponses(array $appUuids = []): array
 {
     return [
@@ -519,3 +536,17 @@ it('combines the client IP mode of several servers', function (array $modes, str
     'anonymized and off' => [['anonymized', 'off'], 'anonymized'],
     'full and anonymized' => [['full', 'anonymized'], 'full'],
 ]);
+
+it('fetches the dashboard of every server at the same time', function () {
+    Cache::flush();
+    collect(range(1, 3))->each(fn () => bootEnabledGlobalServer());
+    app()->bind(SentinelTrafficClient::class, fn ($app, array $parameters) => new SlowDashboardTrafficClient($parameters['server']));
+
+    $started = microtime(true);
+    $component = loadLazy(Livewire::test(Analytics::class));
+    $elapsed = microtime(true) - $started;
+
+    // Three servers fetched one after another take at least 3 seconds.
+    expect($elapsed)->toBeLessThan(2.5)
+        ->and($component->get('overview')['requests'])->toBe(3000);
+});

tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php

@@ -116,5 +116,6 @@ it('shows no data without an error when Sentinel has no traffic routes (0.0.x an
         ->assertSet('series', [])
         ->assertNotDispatched('error');
 
-    Process::assertRan(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'));
+    // The concurrent prefetch remembers the failure, so the per-server pass does not fetch it again.
+    Process::assertRanTimes(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'), 1);
 });