starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-06 13:10:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(upgrade): retry rate limited image pulls and spread auto updates

Browse Source 
Closes #11890
This commit is contained in:
Andras Bacsai
2026-10-15 23:36:47 +02:00
parent 3414b88e0d
commit 12fd9e448b
8 changed files with 151 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Jobs/UpdateCoolifyJob.php
+1 -2
View File
@@ -44,8 +44,7 @@ class UpdateCoolifyJob implements ShouldBeEncrypted, ShouldQueue
Log::info('Starting Coolify update process...');
UpdateCoolify::run(false); // false means it's not a manual update
$settings->update(['new_version_available' => false]);
Log::info('Coolify update completed successfully.');
Log::info('Coolify upgrade script started. Result: /data/coolify/source/.upgrade-status, log: /data/coolify/source/upgrade-*.log.');
} catch (\Throwable $e) {
Log::error('UpdateCoolifyJob failed: '.$e->getMessage());
// Consider implementing a notification to administrators
app/Livewire/Settings/Updates.php
+1 -1
View File
@@ -130,7 +130,7 @@ class Updates extends Component
if ($this->is_auto_update_enabled && ! validate_cron_expression($this->auto_update_frequency)) {
$this->dispatch('error', 'Invalid Cron / Human expression for Auto Update Frequency.');
if (empty($this->auto_update_frequency)) {
$this->auto_update_frequency = '0 0 * * *';
$this->auto_update_frequency = InstanceSettings::randomAutoUpdateFrequency();
}
return;
app/Models/InstanceSettings.php
+14
View File
@@ -104,6 +104,12 @@ class InstanceSettings extends Model
protected static function booted(): void
{
static::creating(function ($settings) {
if (blank($settings->getAttributes()['auto_update_frequency'] ?? null)) {
$settings->auto_update_frequency = self::randomAutoUpdateFrequency();
}
});
static::created(function () {
Once::flush();
});
@@ -121,6 +127,14 @@ class InstanceSettings extends Model
});
}
/**
* Daily auto update at a random minute after midnight, so instances do not all pull images from the registry in the same second.
*/
public static function randomAutoUpdateFrequency(): string
{
return random_int(0, 59).' 0 * * *';
}
public function isPasswordRegistrationAllowed(): bool
{
if (! $this->is_registration_enabled) {
database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php
+24
View File
@@ -0,0 +1,24 @@
<?php
use App\Models\InstanceSettings;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\DB;
return new class extends Migration
{
/**
* Move instances that use the old midnight default to a random minute after midnight,
* so they do not all pull images from the registry at 00:00.
*/
public function up(): void
{
DB::table('instance_settings')
->where('auto_update_frequency', '0 0 * * *')
->update(['auto_update_frequency' => InstanceSettings::randomAutoUpdateFrequency()]);
}
public function down(): void
{
//
}
};
other/nightly/upgrade.sh
+20 -2
View File
@@ -38,6 +38,24 @@ write_status() {
echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
}
# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.
pull_image() {
local image="$1"
local attempt=1
local max_attempts=5
local delay=15
until docker pull "$image" >>"$LOGFILE" 2>&1; do
if [ "$attempt" -ge "$max_attempts" ]; then
return 1
fi
log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
sleep "$((delay + RANDOM % 10))"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
echo ""
echo "=========================================="
echo " Coolify Upgrade - ${DATE}"
@@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection."
HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"
echo " - Pulling $HELPER_IMAGE..."
log "Pulling image: $HELPER_IMAGE"
if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then
if pull_image "$HELPER_IMAGE"; then
log "Successfully pulled $HELPER_IMAGE"
else
log "ERROR: Failed to pull $HELPER_IMAGE"
@@ -220,7 +238,7 @@ for IMAGE in $IMAGES; do
if [ -n "$IMAGE" ]; then
echo " - Pulling $IMAGE..."
log "Pulling image: $IMAGE"
if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then
if pull_image "$IMAGE"; then
log "Successfully pulled $IMAGE"
else
log "ERROR: Failed to pull $IMAGE"
scripts/upgrade.sh
+20 -2
View File
@@ -38,6 +38,24 @@ write_status() {
echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
}
# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.
pull_image() {
local image="$1"
local attempt=1
local max_attempts=5
local delay=15
until docker pull "$image" >>"$LOGFILE" 2>&1; do
if [ "$attempt" -ge "$max_attempts" ]; then
return 1
fi
log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
sleep "$((delay + RANDOM % 10))"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
echo ""
echo "=========================================="
echo " Coolify Upgrade - ${DATE}"
@@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection."
HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"
echo " - Pulling $HELPER_IMAGE..."
log "Pulling image: $HELPER_IMAGE"
if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then
if pull_image "$HELPER_IMAGE"; then
log "Successfully pulled $HELPER_IMAGE"
else
log "ERROR: Failed to pull $HELPER_IMAGE"
@@ -220,7 +238,7 @@ for IMAGE in $IMAGES; do
if [ -n "$IMAGE" ]; then
echo " - Pulling $IMAGE..."
log "Pulling image: $IMAGE"
if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then
if pull_image "$IMAGE"; then
log "Successfully pulled $IMAGE"
else
log "ERROR: Failed to pull $IMAGE"
tests/Feature/AutoUpdateFrequencyDefaultTest.php
+29
View File
@@ -0,0 +1,29 @@
<?php
use App\Models\InstanceSettings;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
it('gives a new instance a random minute after midnight for auto updates', function () {
$settings = InstanceSettings::forceCreate(['id' => 0]);
expect($settings->fresh()->auto_update_frequency)->toMatch('/^([0-9]|[1-5][0-9]) 0 \* \* \*$/');
});
it('keeps an explicit auto update frequency on create', function () {
$settings = InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => '23 3 * * *']);
expect($settings->fresh()->auto_update_frequency)->toBe('23 3 * * *');
});
it('moves instances off the old midnight default and keeps custom schedules', function (string $frequency, string $expected) {
InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => $frequency]);
(require database_path('migrations/2026_10_15_213357_spread_default_auto_update_frequency.php'))->up();
expect(InstanceSettings::find(0)->auto_update_frequency)->toMatch($expected);
})->with([
'old default' => ['0 0 * * *', '/^([0-9]|[1-5][0-9]) 0 \* \* \*$/'],
'custom' => ['23 3 * * *', '/^23 3 \* \* \*$/'],
]);
tests/Unit/UpgradeScriptPullRetryTest.php
+42
View File
@@ -0,0 +1,42 @@
<?php
/**
* Runs pull_image() from the upgrade script with a fake docker that fails the first $failures pulls.
*
* @return array{exitCode: int, pulls: int}
*/
function runUpgradeScriptPullImage(string $script, int $failures): array
{
$dir = sys_get_temp_dir().'/upgrade-pull-'.bin2hex(random_bytes(4));
mkdir($dir);
$source = file_get_contents(getcwd().'/'.$script);
preg_match('/^log\(\) \{.*?^\}\n/ms', $source, $log);
preg_match('/^pull_image\(\) \{.*?^\}\n/ms', $source, $pullImage);
$harness = <<<BASH
LOGFILE="{$dir}/upgrade.log"
docker() {
local count=\$(cat "{$dir}/pulls" 2>/dev/null || echo 0)
echo \$((count + 1)) > "{$dir}/pulls"
[ "\$count" -ge {$failures} ]
}
sleep() { :; }
{$log[0]}
{$pullImage[0]}
pull_image ghcr.io/coollabsio/coolify-helper:1.0.14
BASH;
exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);
$pulls = (int) @file_get_contents("{$dir}/pulls");
exec('rm -rf '.escapeshellarg($dir));
return ['exitCode' => $exitCode, 'pulls' => $pulls];
}
it('retries a rate limited image pull until it succeeds', function (string $script) {
expect(runUpgradeScriptPullImage($script, failures: 2))->toBe(['exitCode' => 0, 'pulls' => 3]);
})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
it('fails the image pull after five attempts', function (string $script) {
expect(runUpgradeScriptPullImage($script, failures: 10))->toBe(['exitCode' => 1, 'pulls' => 5]);
})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);