fix(cdn): fetch versions.json once per hour, spread evenly across instances

- Retry CDN requests only for transient errors and honor Retry-After
  (Cloudflare error 1200 sends 60 s), with jitter.
- Read helper and Sentinel versions from the local versions.json, which
  the hourly update check refreshes, instead of fetching per job/server.
- Replace the random 0-600 s delay with a stable per-instance offset
  inside the cron interval (max 1 h).
- Manual update checks and upgrades make one CDN attempt only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2026-10-21 18:10:48 +02:00
co-authored by Claude Opus 5.5
parent 44c6322259
commit dbc3fb91bf
12 changed files with 214 additions and 62 deletions
+27 -19
View File
@@ -20,12 +20,13 @@ use App\Jobs\ServerManagerJob;
use App\Jobs\UpdateCoolifyJob;
use App\Models\InstanceSettings;
use App\Services\ScheduledJobDeliveryService;
use Cron\CronExpression;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
class Kernel extends ConsoleKernel
{
private const CDN_JITTER_SECONDS = 600;
private const MAX_CDN_DELAY_SECONDS = 3600;
private Schedule $scheduleInstance;
@@ -98,8 +99,8 @@ class Kernel extends ConsoleKernel
$this->scheduleInstance->command('horizon:snapshot')->everyFiveMinutes();
$this->scheduleInstance->command('cleanup:unreachable-servers')->daily()->onOneServer();
$this->scheduleInstance->job($this->withCdnJitter(new PullTemplatesFromCDN))->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job($this->withCdnJitter(new PullChangelog))->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job($this->withCdnDelay(new PullTemplatesFromCDN, $this->updateCheckFrequency))->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job($this->withCdnDelay(new PullChangelog, $this->updateCheckFrequency))->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
$this->scheduleInstance->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer();
$this->scheduleUpdates();
@@ -107,8 +108,6 @@ class Kernel extends ConsoleKernel
// Server Jobs
$this->scheduleInstance->job(new ServerManagerJob)->everyMinute()->onOneServer();
$this->pullImages();
// Scheduled Jobs (Backups & Tasks)
$this->scheduleScheduledJobManager();
@@ -122,14 +121,6 @@ class Kernel extends ConsoleKernel
}
}
private function pullImages(): void
{
$this->scheduleInstance->job($this->withCdnJitter(new CheckHelperImageJob))
->cron($this->updateCheckFrequency)
->timezone($this->instanceTimezone)
->onOneServer();
}
/**
* Run the manager from the scheduler, not from a queue worker. A busy queue could delay it
* past the catch-up window, and then due backups and tasks would be skipped.
@@ -169,14 +160,14 @@ class Kernel extends ConsoleKernel
private function scheduleUpdates(): void
{
$this->scheduleInstance->job($this->withCdnJitter(new CheckForUpdatesJob))
$this->scheduleInstance->job($this->withCdnDelay(new CheckForUpdatesJob, $this->updateCheckFrequency))
->cron($this->updateCheckFrequency)
->timezone($this->instanceTimezone)
->onOneServer();
if ($this->settings->is_auto_update_enabled) {
$autoUpdateFrequency = $this->settings->auto_update_frequency;
$this->scheduleInstance->job($this->withCdnJitter(new UpdateCoolifyJob))
$this->scheduleInstance->job($this->withCdnDelay(new UpdateCoolifyJob, $autoUpdateFrequency))
->cron($autoUpdateFrequency)
->timezone($this->instanceTimezone)
->onOneServer();
@@ -184,12 +175,29 @@ class Kernel extends ConsoleKernel
}
/**
* All instances share the same default cron, so without a random delay they all request the CDN
* at the same second and Cloudflare rejects the burst (error 1200).
* All instances share the same default cron, so without a delay they all request the CDN at the
* same second and Cloudflare rejects the burst (error 1200). Each instance gets a stable offset
* inside the cron interval (at most one hour), so the requests spread evenly and each instance
* still runs at a regular interval.
*/
private function withCdnJitter(object $job): object
private function withCdnDelay(object $job, string $cron): object
{
return $job->delay(random_int(0, self::CDN_JITTER_SECONDS));
return $job->delay(self::cdnDelaySeconds($cron));
}
public static function cdnDelaySeconds(string $cron): int
{
try {
$expression = new CronExpression($cron);
$nextRun = $expression->getNextRunDate();
$interval = $expression->getNextRunDate($nextRun)->getTimestamp() - $nextRun->getTimestamp();
} catch (\Throwable) {
$interval = self::MAX_CDN_DELAY_SECONDS;
}
$window = max(1, min($interval, self::MAX_CDN_DELAY_SECONDS));
$instanceId = (string) (config('app.id') ?: config('app.key'));
return crc32('cdn-delay:'.$instanceId) % $window;
}
protected function commands(): void