fix(server): stagger Sentinel version and patch checks per server

Servers no longer all connect at minute 0. The hourly Sentinel check uses
a per-server minute, and the weekly patch check uses a per-server time on
Sunday between 04:00 and 23:59.

app/Console/Commands/ScheduledJobDiagnostics.php

@@ -2,6 +2,7 @@
 
 namespace App\Console\Commands;
 
+use App\Jobs\ServerManagerJob;
 use App\Models\DockerCleanupExecution;
 use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledJobDelivery;
@@ -210,7 +211,8 @@ class ScheduledJobDiagnostics extends Command
             }
 
             $dedupKeys = [
-                "server-patch-check:{$server->id}" => '0 0 * * 0',
+                "server-patch-check:{$server->id}" => ServerManagerJob::patchCheckCron($server),
+                "sentinel-version-check:{$server->id}" => ServerManagerJob::sentinelVersionCheckCron($server),
                 "server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',
                 "server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),
             ];

app/Jobs/ServerManagerJob.php

@@ -173,7 +173,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
         // Sentinel versions that report their version on push are updated by SentinelController.
         if ($server->isSentinelEnabled()
             && ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id))
-            && shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)
+            && shouldRunCronNow(self::sentinelVersionCheckCron($server), $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)
         ) {
             CheckAndStartSentinelJob::dispatch($server);
         }
@@ -192,16 +192,39 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
             }
         }
 
-        // Dispatch ServerPatchCheckJob if due (weekly)
-        $shouldRunPatchCheck = shouldRunCronNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);
+        // Dispatch ServerPatchCheckJob if due (weekly, staggered per server on Sunday)
+        $shouldRunPatchCheck = shouldRunCronNow(self::patchCheckCron($server), $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);
 
-        if ($shouldRunPatchCheck) { // Weekly on Sunday at midnight
+        if ($shouldRunPatchCheck) {
             ServerPatchCheckJob::dispatch($server);
         }
 
         // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
     }
 
+    /**
+     * Hourly Sentinel version check at a stable per-server minute, so servers do not all SSH at minute 0.
+     *
+     * ServerManagerJob evaluates this every minute and shouldRunCronNow() catches up a missed minute
+     * on the next run, so every minute of the hour is a safe slot.
+     */
+    public static function sentinelVersionCheckCron(Server $server): string
+    {
+        return sprintf('%d * * * *', $server->id % 60);
+    }
+
+    /**
+     * Weekly patch check at a stable per-server time on Sunday between 04:00 and 23:59.
+     *
+     * The early hours are skipped because most daylight saving transitions happen there.
+     */
+    public static function patchCheckCron(Server $server): string
+    {
+        $slot = $server->id % (20 * 60);
+
+        return sprintf('%d %d * * 0', $slot % 60, 4 + intdiv($slot, 60));
+    }
+
     /**
      * Determine the backoff cycle interval based on how many consecutive times a server has been unreachable.
      * Higher counts → less frequent checks (based on 5-min cloud cycle):

tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php

@@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Queue;
 uses(RefreshDatabase::class);
 
 beforeEach(function () {
+    Server::flushIdentityMap();
     Queue::fake();
     InstanceSettings::forceCreate(['id' => 0]);
 
@@ -32,6 +33,9 @@ beforeEach(function () {
     ]);
 
     $this->server->refresh();
+
+    // The hourly Sentinel check runs at a stable per-server minute.
+    $this->minute = sprintf('%02d', $this->server->id % 60);
 });
 
 afterEach(function () {
@@ -46,8 +50,8 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen
         'instance_timezone' => 'UTC',
     ]);
 
-    // Set time to top of any hour (sentinel should check every hour)
-    Carbon::setTestNow('2025-06-15 14:00:00'); // Random hour, not January 1st
+    // Set time to the server minute of any hour (sentinel should check every hour)
+    Carbon::setTestNow("2025-06-15 14:{$this->minute}:00"); // Random hour, not January 1st
 
     // Run ServerManagerJob
     $job = new ServerManagerJob;
@@ -59,7 +63,7 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen
     });
 });
 
-it('does not dispatch sentinel check when not at top of hour', function () {
+it('does not dispatch sentinel check when not at the server minute', function () {
     // Set instance update_check_frequency to hourly (most frequent)
     $instanceSettings = InstanceSettings::first();
     $instanceSettings->update([
@@ -67,14 +71,15 @@ it('does not dispatch sentinel check when not at top of hour', function () {
         'instance_timezone' => 'UTC',
     ]);
 
-    // Set time to middle of the hour (sentinel check cron won't match)
-    Carbon::setTestNow('2025-06-15 14:30:00'); // 30 minutes past the hour
+    // Set time to another minute of the hour (sentinel check cron won't match)
+    $otherMinute = sprintf('%02d', ($this->server->id + 30) % 60);
+    Carbon::setTestNow("2025-06-15 14:{$otherMinute}:00");
 
     // Run ServerManagerJob
     $job = new ServerManagerJob;
     $job->handle();
 
-    // Assert that CheckAndStartSentinelJob was NOT dispatched (not top of hour)
+    // Assert that CheckAndStartSentinelJob was NOT dispatched (not the server minute)
     Queue::assertNotPushed(CheckAndStartSentinelJob::class);
 });
 
@@ -91,7 +96,7 @@ it('dispatches sentinel check at every hour mark throughout the day', function (
     foreach ($hoursToTest as $hour) {
         Queue::fake(); // Reset queue for each test
 
-        Carbon::setTestNow("2025-06-15 {$hour}:00:00");
+        Carbon::setTestNow("2025-06-15 {$hour}:{$this->minute}:00");
 
         $job = new ServerManagerJob;
         $job->handle();
@@ -113,13 +118,13 @@ it('respects server timezone when checking sentinel updates', function () {
         'instance_timezone' => 'UTC',
     ]);
 
-    // Set time to 17:00 UTC which is 12:00 PM EST (top of hour in server's timezone)
-    Carbon::setTestNow('2025-01-15 17:00:00');
+    // Set time to 17:xx UTC which is 12:xx PM EST (server minute in server's timezone)
+    Carbon::setTestNow("2025-01-15 17:{$this->minute}:00");
 
     $job = new ServerManagerJob;
     $job->handle();
 
-    // Should dispatch because it's top of hour in server's timezone (America/New_York)
+    // Should dispatch because it's the server minute in server's timezone (America/New_York)
     Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) {
         return $job->server->id === $this->server->id;
     });
@@ -137,7 +142,7 @@ it('does not dispatch sentinel check for build servers', function () {
         'instance_timezone' => 'UTC',
     ]);
 
-    Carbon::setTestNow('2025-06-15 14:00:00');
+    Carbon::setTestNow("2025-06-15 14:{$this->minute}:00");
 
     $job = new ServerManagerJob;
     $job->handle();
@@ -171,13 +176,17 @@ it('handles multiple servers with different sentinel eligibility', function () {
         'instance_timezone' => 'UTC',
     ]);
 
-    Carbon::setTestNow('2025-06-15 14:00:00');
+    // Each server is checked at its own minute
+    Carbon::setTestNow("2025-06-15 14:{$this->minute}:00");
+    (new ServerManagerJob)->handle();
 
-    $job = new ServerManagerJob;
-    $job->handle();
+    Carbon::setTestNow(sprintf('2025-06-15 15:%02d:00', $server3->id % 60));
+    (new ServerManagerJob)->handle();
 
-    // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled)
-    Queue::assertPushed(CheckAndStartSentinelJob::class, 2);
+    // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled), never for server2
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class, function ($job) use ($server2) {
+        return $job->server->id === $server2->id;
+    });
 
     // Verify it was dispatched for the correct servers
     Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) {

tests/Feature/Server/ServerManagerJobStaggerTest.php

@@ -0,0 +1,115 @@
+<?php
+
+use App\Enums\ServerRole;
+use App\Jobs\CheckAndStartSentinelJob;
+use App\Jobs\ServerManagerJob;
+use App\Jobs\ServerPatchCheckJob;
+use App\Models\InstanceSettings;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    Queue::fake();
+    config(['constants.coolify.self_hosted' => true]);
+    Carbon::setTestNow('2025-01-15 12:00:00');
+    InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']);
+    $this->team = Team::factory()->create();
+});
+
+afterEach(function () {
+    Carbon::setTestNow();
+});
+
+function createStaggerTestServer(Team $team): Server
+{
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'sentinel_updated_at' => Carbon::now(),
+    ]);
+    $server->settings->update([
+        'server_timezone' => 'UTC',
+        'server_role' => ServerRole::BOTH,
+        'is_reachable' => true,
+        'is_usable' => true,
+    ]);
+
+    return $server->refresh();
+}
+
+function runStaggerTestManagerAt(Carbon $time): void
+{
+    Carbon::setTestNow($time);
+    Server::query()->update(['sentinel_updated_at' => $time]);
+    Server::flushIdentityMap();
+    (new ServerManagerJob)->handle();
+}
+
+it('gives each server a stable slot and spreads servers across minutes', function () {
+    $servers = collect(range(1, 120))->map(fn (int $id) => (new Server)->forceFill(['id' => $id]));
+
+    $hourlyMinutes = $servers->map(fn (Server $server) => ServerManagerJob::sentinelVersionCheckCron($server));
+    $weeklySlots = $servers->map(fn (Server $server) => ServerManagerJob::patchCheckCron($server));
+
+    expect($hourlyMinutes->unique())->toHaveCount(60)
+        ->and($weeklySlots->unique())->toHaveCount(120)
+        ->and(ServerManagerJob::sentinelVersionCheckCron($servers[6]))->toBe(ServerManagerJob::sentinelVersionCheckCron($servers[6]))
+        ->and(ServerManagerJob::patchCheckCron($servers[6]))->toBe(ServerManagerJob::patchCheckCron($servers[6]));
+
+    $weeklySlots->each(function (string $cron) {
+        [$minute, $hour, , , $dayOfWeek] = explode(' ', $cron);
+        expect((int) $minute)->toBeBetween(0, 59)
+            ->and((int) $hour)->toBeBetween(4, 23)
+            ->and($dayOfWeek)->toBe('0');
+    });
+});
+
+it('dispatches the Sentinel version check once per hour at the server minute', function () {
+    $server = createStaggerTestServer($this->team);
+    $minute = $server->id % 60;
+    $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($minute);
+
+    runStaggerTestManagerAt($slot->copy()->addMinute());
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+
+    runStaggerTestManagerAt($slot->copy()->addHour());
+    runStaggerTestManagerAt($slot->copy()->addHour()->addSeconds(30));
+    runStaggerTestManagerAt($slot->copy()->addHour()->addMinute());
+    Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+
+    runStaggerTestManagerAt($slot->copy()->addHours(2));
+    Queue::assertPushed(CheckAndStartSentinelJob::class, 2);
+});
+
+it('catches up a Sentinel version check when the server minute was not evaluated', function () {
+    $server = createStaggerTestServer($this->team);
+    $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($server->id % 60);
+
+    runStaggerTestManagerAt($slot->copy()->subMinute());
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+
+    runStaggerTestManagerAt($slot->copy()->addMinutes(3));
+    Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+});
+
+it('dispatches the patch check once per week at the server slot on Sunday', function () {
+    $server = createStaggerTestServer($this->team);
+    [$minute, $hour] = array_map('intval', explode(' ', ServerManagerJob::patchCheckCron($server)));
+    $sunday = Carbon::parse('2025-01-19 00:00:00');
+    $slot = $sunday->copy()->setTime($hour, $minute);
+
+    runStaggerTestManagerAt($sunday);
+    Queue::assertNotPushed(ServerPatchCheckJob::class);
+
+    runStaggerTestManagerAt($slot);
+    runStaggerTestManagerAt($slot->copy()->addMinute());
+    Queue::assertPushed(ServerPatchCheckJob::class, 1);
+
+    runStaggerTestManagerAt($slot->copy()->addWeek());
+    Queue::assertPushed(ServerPatchCheckJob::class, 2);
+});

tests/Unit/ServerManagerJobSentinelCheckTest.php

@@ -16,6 +16,7 @@ use Tests\TestCase;
 uses(TestCase::class, RefreshDatabase::class);
 
 beforeEach(function () {
+    Server::flushIdentityMap();
     Queue::fake();
     Carbon::setTestNow('2025-01-15 12:00:00');
     InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']);
@@ -45,6 +46,8 @@ function createSentinelCheckServer(Team $team, Carbon $sentinelUpdatedAt, Server
 
 it('dispatches an hourly Sentinel version check for a healthy Sentinel', function () {
     $server = createSentinelCheckServer($this->team, Carbon::now());
+    Carbon::setTestNow(Carbon::now()->setMinute($server->id % 60));
+    $server->update(['sentinel_updated_at' => Carbon::now()]);
 
     expect($server->isSentinelEnabled())->toBeTrue()
         ->and($server->isSentinelLive())->toBeTrue();