fix(scheduler): spread CDN update checks over a random delay

All instances share the same default update cron, so they request the
CDN at the same second and Cloudflare rejects the burst (error 1200).
Queue the CDN fetching jobs with a random 0-600s delay to spread the load.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2026-10-16 21:38:39 +02:00
co-authored by Claude Opus 5.5
parent 154b50647e
commit 21dc34ae61
2 changed files with 38 additions and 5 deletions
+16 -5
View File
@@ -25,6 +25,8 @@ use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
class Kernel extends ConsoleKernel
{
private const CDN_JITTER_SECONDS = 600;
private Schedule $scheduleInstance;
private InstanceSettings $settings;
@@ -91,8 +93,8 @@ class Kernel extends ConsoleKernel
$this->scheduleInstance->command('horizon:snapshot')->everyFiveMinutes();
$this->scheduleInstance->command('cleanup:unreachable-servers')->daily()->onOneServer();
$this->scheduleInstance->job(new PullTemplatesFromCDN)->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job(new PullChangelog)->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job($this->withCdnJitter(new PullTemplatesFromCDN))->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job($this->withCdnJitter(new PullChangelog))->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer();
$this->scheduleUpdates();
@@ -117,7 +119,7 @@ class Kernel extends ConsoleKernel
private function pullImages(): void
{
$this->scheduleInstance->job(new CheckHelperImageJob)
$this->scheduleInstance->job($this->withCdnJitter(new CheckHelperImageJob))
->cron($this->updateCheckFrequency)
->timezone($this->instanceTimezone)
->onOneServer();
@@ -162,20 +164,29 @@ class Kernel extends ConsoleKernel
private function scheduleUpdates(): void
{
$this->scheduleInstance->job(new CheckForUpdatesJob)
$this->scheduleInstance->job($this->withCdnJitter(new CheckForUpdatesJob))
->cron($this->updateCheckFrequency)
->timezone($this->instanceTimezone)
->onOneServer();
if ($this->settings->is_auto_update_enabled) {
$autoUpdateFrequency = $this->settings->auto_update_frequency;
$this->scheduleInstance->job(new UpdateCoolifyJob)
$this->scheduleInstance->job($this->withCdnJitter(new UpdateCoolifyJob))
->cron($autoUpdateFrequency)
->timezone($this->instanceTimezone)
->onOneServer();
}
}
/**
* All instances share the same default cron, so without a random delay they all request the CDN
* at the same second and Cloudflare rejects the burst (error 1200).
*/
private function withCdnJitter(object $job): object
{
return $job->delay(random_int(0, self::CDN_JITTER_SECONDS));
}
protected function commands(): void
{
$this->load(__DIR__.'/Commands');
+22
View File
@@ -1,10 +1,16 @@
<?php
use App\Jobs\CheckForUpdatesJob;
use App\Jobs\CheckHelperImageJob;
use App\Jobs\PullChangelog;
use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\UpdateCoolifyJob;
use App\Models\InstanceSettings;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
@@ -51,6 +57,22 @@ it('schedules every production job with onOneServer', function () {
});
});
it('dispatches CDN fetching jobs with a random delay so instances do not hit the CDN at the same second', function () {
InstanceSettings::query()->whereKey(0)->update(['is_auto_update_enabled' => true]);
Queue::fake();
$jobs = [CheckHelperImageJob::class, CheckForUpdatesJob::class, PullTemplatesFromCDN::class, PullChangelog::class, UpdateCoolifyJob::class];
$schedule = app(Schedule::class);
foreach ($jobs as $job) {
$event = collect($schedule->events())->first(fn ($e) => $e->description === $job);
expect($event)->not->toBeNull("[{$job}] is not scheduled");
$event->run(app());
Queue::assertPushed($job, fn ($pushed) => is_int($pushed->delay) && $pushed->delay >= 0 && $pushed->delay <= 600);
}
});
it('does not schedule Stripe subscription reconciliation automatically', function () {
$schedule = app(Schedule::class);