diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.php index 0b4f944a1d..a99184cc96 100644 --- a/app/Console/Commands/ScheduledJobDiagnostics.php +++ b/app/Console/Commands/ScheduledJobDiagnostics.php @@ -2,6 +2,7 @@ namespace App\Console\Commands; +use App\Jobs\ServerManagerJob; use App\Models\DockerCleanupExecution; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledJobDelivery; @@ -210,7 +211,8 @@ class ScheduledJobDiagnostics extends Command } $dedupKeys = [ - "server-patch-check:{$server->id}" => '0 0 * * 0', + "server-patch-check:{$server->id}" => ServerManagerJob::patchCheckCron($server), + "sentinel-version-check:{$server->id}" => ServerManagerJob::sentinelVersionCheckCron($server), "server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *', "server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'), ]; diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php index fdaa8b54a5..e33e639c61 100644 --- a/app/Jobs/ServerManagerJob.php +++ b/app/Jobs/ServerManagerJob.php @@ -173,7 +173,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue // Sentinel versions that report their version on push are updated by SentinelController. if ($server->isSentinelEnabled() && ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id)) - && shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime) + && shouldRunCronNow(self::sentinelVersionCheckCron($server), $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime) ) { CheckAndStartSentinelJob::dispatch($server); } @@ -192,16 +192,39 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue } } - // Dispatch ServerPatchCheckJob if due (weekly) - $shouldRunPatchCheck = shouldRunCronNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime); + // Dispatch ServerPatchCheckJob if due (weekly, staggered per server on Sunday) + $shouldRunPatchCheck = shouldRunCronNow(self::patchCheckCron($server), $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime); - if ($shouldRunPatchCheck) { // Weekly on Sunday at midnight + if ($shouldRunPatchCheck) { ServerPatchCheckJob::dispatch($server); } // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob. } + /** + * Hourly Sentinel version check at a stable per-server minute, so servers do not all SSH at minute 0. + * + * ServerManagerJob evaluates this every minute and shouldRunCronNow() catches up a missed minute + * on the next run, so every minute of the hour is a safe slot. + */ + public static function sentinelVersionCheckCron(Server $server): string + { + return sprintf('%d * * * *', $server->id % 60); + } + + /** + * Weekly patch check at a stable per-server time on Sunday between 04:00 and 23:59. + * + * The early hours are skipped because most daylight saving transitions happen there. + */ + public static function patchCheckCron(Server $server): string + { + $slot = $server->id % (20 * 60); + + return sprintf('%d %d * * 0', $slot % 60, 4 + intdiv($slot, 60)); + } + /** * Determine the backoff cycle interval based on how many consecutive times a server has been unreachable. * Higher counts → less frequent checks (based on 5-min cloud cycle): diff --git a/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php b/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php index ca1494840c..65a8157dcd 100644 --- a/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php +++ b/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php @@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Queue; uses(RefreshDatabase::class); beforeEach(function () { + Server::flushIdentityMap(); Queue::fake(); InstanceSettings::forceCreate(['id' => 0]); @@ -32,6 +33,9 @@ beforeEach(function () { ]); $this->server->refresh(); + + // The hourly Sentinel check runs at a stable per-server minute. + $this->minute = sprintf('%02d', $this->server->id % 60); }); afterEach(function () { @@ -46,8 +50,8 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen 'instance_timezone' => 'UTC', ]); - // Set time to top of any hour (sentinel should check every hour) - Carbon::setTestNow('2025-06-15 14:00:00'); // Random hour, not January 1st + // Set time to the server minute of any hour (sentinel should check every hour) + Carbon::setTestNow("2025-06-15 14:{$this->minute}:00"); // Random hour, not January 1st // Run ServerManagerJob $job = new ServerManagerJob; @@ -59,7 +63,7 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen }); }); -it('does not dispatch sentinel check when not at top of hour', function () { +it('does not dispatch sentinel check when not at the server minute', function () { // Set instance update_check_frequency to hourly (most frequent) $instanceSettings = InstanceSettings::first(); $instanceSettings->update([ @@ -67,14 +71,15 @@ it('does not dispatch sentinel check when not at top of hour', function () { 'instance_timezone' => 'UTC', ]); - // Set time to middle of the hour (sentinel check cron won't match) - Carbon::setTestNow('2025-06-15 14:30:00'); // 30 minutes past the hour + // Set time to another minute of the hour (sentinel check cron won't match) + $otherMinute = sprintf('%02d', ($this->server->id + 30) % 60); + Carbon::setTestNow("2025-06-15 14:{$otherMinute}:00"); // Run ServerManagerJob $job = new ServerManagerJob; $job->handle(); - // Assert that CheckAndStartSentinelJob was NOT dispatched (not top of hour) + // Assert that CheckAndStartSentinelJob was NOT dispatched (not the server minute) Queue::assertNotPushed(CheckAndStartSentinelJob::class); }); @@ -91,7 +96,7 @@ it('dispatches sentinel check at every hour mark throughout the day', function ( foreach ($hoursToTest as $hour) { Queue::fake(); // Reset queue for each test - Carbon::setTestNow("2025-06-15 {$hour}:00:00"); + Carbon::setTestNow("2025-06-15 {$hour}:{$this->minute}:00"); $job = new ServerManagerJob; $job->handle(); @@ -113,13 +118,13 @@ it('respects server timezone when checking sentinel updates', function () { 'instance_timezone' => 'UTC', ]); - // Set time to 17:00 UTC which is 12:00 PM EST (top of hour in server's timezone) - Carbon::setTestNow('2025-01-15 17:00:00'); + // Set time to 17:xx UTC which is 12:xx PM EST (server minute in server's timezone) + Carbon::setTestNow("2025-01-15 17:{$this->minute}:00"); $job = new ServerManagerJob; $job->handle(); - // Should dispatch because it's top of hour in server's timezone (America/New_York) + // Should dispatch because it's the server minute in server's timezone (America/New_York) Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) { return $job->server->id === $this->server->id; }); @@ -137,7 +142,7 @@ it('does not dispatch sentinel check for build servers', function () { 'instance_timezone' => 'UTC', ]); - Carbon::setTestNow('2025-06-15 14:00:00'); + Carbon::setTestNow("2025-06-15 14:{$this->minute}:00"); $job = new ServerManagerJob; $job->handle(); @@ -171,13 +176,17 @@ it('handles multiple servers with different sentinel eligibility', function () { 'instance_timezone' => 'UTC', ]); - Carbon::setTestNow('2025-06-15 14:00:00'); + // Each server is checked at its own minute + Carbon::setTestNow("2025-06-15 14:{$this->minute}:00"); + (new ServerManagerJob)->handle(); - $job = new ServerManagerJob; - $job->handle(); + Carbon::setTestNow(sprintf('2025-06-15 15:%02d:00', $server3->id % 60)); + (new ServerManagerJob)->handle(); - // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled) - Queue::assertPushed(CheckAndStartSentinelJob::class, 2); + // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled), never for server2 + Queue::assertNotPushed(CheckAndStartSentinelJob::class, function ($job) use ($server2) { + return $job->server->id === $server2->id; + }); // Verify it was dispatched for the correct servers Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) { diff --git a/tests/Feature/Server/ServerManagerJobStaggerTest.php b/tests/Feature/Server/ServerManagerJobStaggerTest.php new file mode 100644 index 0000000000..9911c2e8fc --- /dev/null +++ b/tests/Feature/Server/ServerManagerJobStaggerTest.php @@ -0,0 +1,115 @@ + true]); + Carbon::setTestNow('2025-01-15 12:00:00'); + InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']); + $this->team = Team::factory()->create(); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +function createStaggerTestServer(Team $team): Server +{ + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => Carbon::now(), + ]); + $server->settings->update([ + 'server_timezone' => 'UTC', + 'server_role' => ServerRole::BOTH, + 'is_reachable' => true, + 'is_usable' => true, + ]); + + return $server->refresh(); +} + +function runStaggerTestManagerAt(Carbon $time): void +{ + Carbon::setTestNow($time); + Server::query()->update(['sentinel_updated_at' => $time]); + Server::flushIdentityMap(); + (new ServerManagerJob)->handle(); +} + +it('gives each server a stable slot and spreads servers across minutes', function () { + $servers = collect(range(1, 120))->map(fn (int $id) => (new Server)->forceFill(['id' => $id])); + + $hourlyMinutes = $servers->map(fn (Server $server) => ServerManagerJob::sentinelVersionCheckCron($server)); + $weeklySlots = $servers->map(fn (Server $server) => ServerManagerJob::patchCheckCron($server)); + + expect($hourlyMinutes->unique())->toHaveCount(60) + ->and($weeklySlots->unique())->toHaveCount(120) + ->and(ServerManagerJob::sentinelVersionCheckCron($servers[6]))->toBe(ServerManagerJob::sentinelVersionCheckCron($servers[6])) + ->and(ServerManagerJob::patchCheckCron($servers[6]))->toBe(ServerManagerJob::patchCheckCron($servers[6])); + + $weeklySlots->each(function (string $cron) { + [$minute, $hour, , , $dayOfWeek] = explode(' ', $cron); + expect((int) $minute)->toBeBetween(0, 59) + ->and((int) $hour)->toBeBetween(4, 23) + ->and($dayOfWeek)->toBe('0'); + }); +}); + +it('dispatches the Sentinel version check once per hour at the server minute', function () { + $server = createStaggerTestServer($this->team); + $minute = $server->id % 60; + $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($minute); + + runStaggerTestManagerAt($slot->copy()->addMinute()); + Queue::assertNotPushed(CheckAndStartSentinelJob::class); + + runStaggerTestManagerAt($slot->copy()->addHour()); + runStaggerTestManagerAt($slot->copy()->addHour()->addSeconds(30)); + runStaggerTestManagerAt($slot->copy()->addHour()->addMinute()); + Queue::assertPushed(CheckAndStartSentinelJob::class, 1); + + runStaggerTestManagerAt($slot->copy()->addHours(2)); + Queue::assertPushed(CheckAndStartSentinelJob::class, 2); +}); + +it('catches up a Sentinel version check when the server minute was not evaluated', function () { + $server = createStaggerTestServer($this->team); + $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($server->id % 60); + + runStaggerTestManagerAt($slot->copy()->subMinute()); + Queue::assertNotPushed(CheckAndStartSentinelJob::class); + + runStaggerTestManagerAt($slot->copy()->addMinutes(3)); + Queue::assertPushed(CheckAndStartSentinelJob::class, 1); +}); + +it('dispatches the patch check once per week at the server slot on Sunday', function () { + $server = createStaggerTestServer($this->team); + [$minute, $hour] = array_map('intval', explode(' ', ServerManagerJob::patchCheckCron($server))); + $sunday = Carbon::parse('2025-01-19 00:00:00'); + $slot = $sunday->copy()->setTime($hour, $minute); + + runStaggerTestManagerAt($sunday); + Queue::assertNotPushed(ServerPatchCheckJob::class); + + runStaggerTestManagerAt($slot); + runStaggerTestManagerAt($slot->copy()->addMinute()); + Queue::assertPushed(ServerPatchCheckJob::class, 1); + + runStaggerTestManagerAt($slot->copy()->addWeek()); + Queue::assertPushed(ServerPatchCheckJob::class, 2); +}); diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.php index dbbe800ad8..f17b3957be 100644 --- a/tests/Unit/ServerManagerJobSentinelCheckTest.php +++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php @@ -16,6 +16,7 @@ use Tests\TestCase; uses(TestCase::class, RefreshDatabase::class); beforeEach(function () { + Server::flushIdentityMap(); Queue::fake(); Carbon::setTestNow('2025-01-15 12:00:00'); InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']); @@ -45,6 +46,8 @@ function createSentinelCheckServer(Team $team, Carbon $sentinelUpdatedAt, Server it('dispatches an hourly Sentinel version check for a healthy Sentinel', function () { $server = createSentinelCheckServer($this->team, Carbon::now()); + Carbon::setTestNow(Carbon::now()->setMinute($server->id % 60)); + $server->update(['sentinel_updated_at' => Carbon::now()]); expect($server->isSentinelEnabled())->toBeTrue() ->and($server->isSentinelLive())->toBeTrue();