diff --git a/app/Jobs/UpdateCoolifyJob.php b/app/Jobs/UpdateCoolifyJob.php index f0e43cbc0a..e2310f0bee 100644 --- a/app/Jobs/UpdateCoolifyJob.php +++ b/app/Jobs/UpdateCoolifyJob.php @@ -44,8 +44,7 @@ class UpdateCoolifyJob implements ShouldBeEncrypted, ShouldQueue Log::info('Starting Coolify update process...'); UpdateCoolify::run(false); // false means it's not a manual update - $settings->update(['new_version_available' => false]); - Log::info('Coolify update completed successfully.'); + Log::info('Coolify upgrade script started. Result: /data/coolify/source/.upgrade-status, log: /data/coolify/source/upgrade-*.log.'); } catch (\Throwable $e) { Log::error('UpdateCoolifyJob failed: '.$e->getMessage()); // Consider implementing a notification to administrators diff --git a/app/Livewire/Settings/Updates.php b/app/Livewire/Settings/Updates.php index 10b3180640..aba7521dd9 100644 --- a/app/Livewire/Settings/Updates.php +++ b/app/Livewire/Settings/Updates.php @@ -130,7 +130,7 @@ class Updates extends Component if ($this->is_auto_update_enabled && ! validate_cron_expression($this->auto_update_frequency)) { $this->dispatch('error', 'Invalid Cron / Human expression for Auto Update Frequency.'); if (empty($this->auto_update_frequency)) { - $this->auto_update_frequency = '0 0 * * *'; + $this->auto_update_frequency = InstanceSettings::randomAutoUpdateFrequency(); } return; diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php index 16a1d7ea30..79d921fcf8 100644 --- a/app/Models/InstanceSettings.php +++ b/app/Models/InstanceSettings.php @@ -104,6 +104,12 @@ class InstanceSettings extends Model protected static function booted(): void { + static::creating(function ($settings) { + if (blank($settings->getAttributes()['auto_update_frequency'] ?? null)) { + $settings->auto_update_frequency = self::randomAutoUpdateFrequency(); + } + }); + static::created(function () { Once::flush(); }); @@ -121,6 +127,14 @@ class InstanceSettings extends Model }); } + /** + * Daily auto update at a random minute after midnight, so instances do not all pull images from the registry in the same second. + */ + public static function randomAutoUpdateFrequency(): string + { + return random_int(0, 59).' 0 * * *'; + } + public function isPasswordRegistrationAllowed(): bool { if (! $this->is_registration_enabled) { diff --git a/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php b/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php new file mode 100644 index 0000000000..9648ab50ec --- /dev/null +++ b/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php @@ -0,0 +1,24 @@ +where('auto_update_frequency', '0 0 * * *') + ->update(['auto_update_frequency' => InstanceSettings::randomAutoUpdateFrequency()]); + } + + public function down(): void + { + // + } +}; diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.sh index ce8fdb6589..88dd7ea093 100644 --- a/other/nightly/upgrade.sh +++ b/other/nightly/upgrade.sh @@ -38,6 +38,24 @@ write_status() { echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE" } +# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time. +pull_image() { + local image="$1" + local attempt=1 + local max_attempts=5 + local delay=15 + until docker pull "$image" >>"$LOGFILE" 2>&1; do + if [ "$attempt" -ge "$max_attempts" ]; then + return 1 + fi + log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..." + echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..." + sleep "$((delay + RANDOM % 10))" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done +} + echo "" echo "==========================================" echo " Coolify Upgrade - ${DATE}" @@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection." HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}" echo " - Pulling $HELPER_IMAGE..." log "Pulling image: $HELPER_IMAGE" -if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then +if pull_image "$HELPER_IMAGE"; then log "Successfully pulled $HELPER_IMAGE" else log "ERROR: Failed to pull $HELPER_IMAGE" @@ -220,7 +238,7 @@ for IMAGE in $IMAGES; do if [ -n "$IMAGE" ]; then echo " - Pulling $IMAGE..." log "Pulling image: $IMAGE" - if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then + if pull_image "$IMAGE"; then log "Successfully pulled $IMAGE" else log "ERROR: Failed to pull $IMAGE" diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index c2bcabc3f7..987ce636a0 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -38,6 +38,24 @@ write_status() { echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE" } +# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time. +pull_image() { + local image="$1" + local attempt=1 + local max_attempts=5 + local delay=15 + until docker pull "$image" >>"$LOGFILE" 2>&1; do + if [ "$attempt" -ge "$max_attempts" ]; then + return 1 + fi + log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..." + echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..." + sleep "$((delay + RANDOM % 10))" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done +} + echo "" echo "==========================================" echo " Coolify Upgrade - ${DATE}" @@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection." HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}" echo " - Pulling $HELPER_IMAGE..." log "Pulling image: $HELPER_IMAGE" -if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then +if pull_image "$HELPER_IMAGE"; then log "Successfully pulled $HELPER_IMAGE" else log "ERROR: Failed to pull $HELPER_IMAGE" @@ -220,7 +238,7 @@ for IMAGE in $IMAGES; do if [ -n "$IMAGE" ]; then echo " - Pulling $IMAGE..." log "Pulling image: $IMAGE" - if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then + if pull_image "$IMAGE"; then log "Successfully pulled $IMAGE" else log "ERROR: Failed to pull $IMAGE" diff --git a/tests/Feature/AutoUpdateFrequencyDefaultTest.php b/tests/Feature/AutoUpdateFrequencyDefaultTest.php new file mode 100644 index 0000000000..8d64e26b09 --- /dev/null +++ b/tests/Feature/AutoUpdateFrequencyDefaultTest.php @@ -0,0 +1,29 @@ + 0]); + + expect($settings->fresh()->auto_update_frequency)->toMatch('/^([0-9]|[1-5][0-9]) 0 \* \* \*$/'); +}); + +it('keeps an explicit auto update frequency on create', function () { + $settings = InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => '23 3 * * *']); + + expect($settings->fresh()->auto_update_frequency)->toBe('23 3 * * *'); +}); + +it('moves instances off the old midnight default and keeps custom schedules', function (string $frequency, string $expected) { + InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => $frequency]); + + (require database_path('migrations/2026_10_15_213357_spread_default_auto_update_frequency.php'))->up(); + + expect(InstanceSettings::find(0)->auto_update_frequency)->toMatch($expected); +})->with([ + 'old default' => ['0 0 * * *', '/^([0-9]|[1-5][0-9]) 0 \* \* \*$/'], + 'custom' => ['23 3 * * *', '/^23 3 \* \* \*$/'], +]); diff --git a/tests/Unit/UpgradeScriptPullRetryTest.php b/tests/Unit/UpgradeScriptPullRetryTest.php new file mode 100644 index 0000000000..d30514a965 --- /dev/null +++ b/tests/Unit/UpgradeScriptPullRetryTest.php @@ -0,0 +1,42 @@ +/dev/null || echo 0) + echo \$((count + 1)) > "{$dir}/pulls" + [ "\$count" -ge {$failures} ] + } + sleep() { :; } + {$log[0]} + {$pullImage[0]} + pull_image ghcr.io/coollabsio/coolify-helper:1.0.14 + BASH; + + exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode); + $pulls = (int) @file_get_contents("{$dir}/pulls"); + exec('rm -rf '.escapeshellarg($dir)); + + return ['exitCode' => $exitCode, 'pulls' => $pulls]; +} + +it('retries a rate limited image pull until it succeeds', function (string $script) { + expect(runUpgradeScriptPullImage($script, failures: 2))->toBe(['exitCode' => 0, 'pulls' => 3]); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); + +it('fails the image pull after five attempts', function (string $script) { + expect(runUpgradeScriptPullImage($script, failures: 10))->toBe(['exitCode' => 1, 'pulls' => 5]); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);