From a5c3ca0b73116ad128b437613ce33415a153154f Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:52:44 +0100 Subject: [PATCH] feat(updates): notify on automatic update start, failure, and finish (#12472) --- app/Actions/Server/UpdateCoolify.php | 18 +- app/Console/Commands/Init.php | 7 + .../UpgradeAlreadyRunningException.php | 21 +++ app/Jobs/CheckForUpdatesJob.php | 2 + app/Jobs/UpdateCoolifyJob.php | 13 +- app/Livewire/Upgrade.php | 5 + app/Models/InstanceSettings.php | 3 + app/Services/AutoUpdateNotifier.php | 175 ++++++++++++++++++ app/Services/CoolifyUpgradeStatus.php | 39 ++++ ...ate_attempt_to_instance_settings_table.php | 29 +++ scripts/upgrade.sh | 14 +- tests/Unit/UpdateCoolifyTest.php | 23 +++ 12 files changed, 339 insertions(+), 10 deletions(-) create mode 100644 app/Exceptions/UpgradeAlreadyRunningException.php create mode 100644 app/Services/AutoUpdateNotifier.php create mode 100644 database/migrations/2026_11_04_080000_add_auto_update_attempt_to_instance_settings_table.php diff --git a/app/Actions/Server/UpdateCoolify.php b/app/Actions/Server/UpdateCoolify.php index 0ee8424ce1..877b2325e9 100644 --- a/app/Actions/Server/UpdateCoolify.php +++ b/app/Actions/Server/UpdateCoolify.php @@ -2,9 +2,11 @@ namespace App\Actions\Server; +use App\Exceptions\UpgradeAlreadyRunningException; use App\Jobs\DatabaseBackupJob; use App\Models\Server; use App\Models\StandalonePostgresql; +use App\Services\AutoUpdateNotifier; use App\Services\CoolifyUpgradeStatus; use Illuminate\Support\Facades\Log; use Illuminate\Support\Sleep; @@ -40,6 +42,7 @@ class UpdateCoolify if ($manual_update && filled($settings->update_custom_tag)) { $this->latestVersion = $settings->update_custom_tag; $this->backupDatabase(); + $this->ensureNoUpgradeIsRunning(); $this->update(registryUrl: 'ghcr.io'); } @@ -127,7 +130,15 @@ class UpdateCoolify } $this->backupDatabase(); + // Check the lock before the attempt is saved, so a running upgrade keeps its own saved attempt. + $this->ensureNoUpgradeIsRunning(); + if (! $manual_update) { + AutoUpdateNotifier::saveAttempt((string) $this->latestVersion); + } $this->update(); + if (! $manual_update) { + AutoUpdateNotifier::started((string) $this->currentVersion, (string) $this->latestVersion); + } $settings->new_version_available = false; $settings->save(); } @@ -151,8 +162,6 @@ class UpdateCoolify private function update(?string $registryUrl = null) { - $this->ensureNoUpgradeIsRunning(); - $latestHelperImageVersion = getHelperVersion(); $upgradeScriptUrl = $this->upgradeScriptUrl ?? config('constants.coolify.upgrade_script_url'); $registryUrl ??= coolifyRegistryUrl(); @@ -175,10 +184,7 @@ class UpdateCoolify 'target_version' => $this->latestVersion, 'status' => $status, ]); - throw new \Exception( - 'Another Coolify upgrade is already running. Wait for it to finish. '. - 'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.' - ); + throw new UpgradeAlreadyRunningException; } } diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index 6cd914fd9c..01f372b90f 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -15,6 +15,7 @@ use App\Models\ScheduledTaskExecution; use App\Models\Server; use App\Models\StandalonePostgresql; use App\Models\User; +use App\Services\AutoUpdateNotifier; use App\Support\ResourceStartActivity; use App\Support\RootSetupWindow; use Carbon\Carbon; @@ -74,6 +75,12 @@ class Init extends Command } get_public_ips(); + try { + AutoUpdateNotifier::checkResult(); + } catch (\Throwable $e) { + echo "Could not check the automatic update result: {$e->getMessage()}\n"; + } + // Backward compatibility $this->replaceSlashInEnvironmentName(); $this->restoreCoolifyDbBackup(); diff --git a/app/Exceptions/UpgradeAlreadyRunningException.php b/app/Exceptions/UpgradeAlreadyRunningException.php new file mode 100644 index 0000000000..67059277aa --- /dev/null +++ b/app/Exceptions/UpgradeAlreadyRunningException.php @@ -0,0 +1,21 @@ +cdnTries); if ($versions === null) { return false; diff --git a/app/Jobs/UpdateCoolifyJob.php b/app/Jobs/UpdateCoolifyJob.php index e2310f0bee..c8976ba08a 100644 --- a/app/Jobs/UpdateCoolifyJob.php +++ b/app/Jobs/UpdateCoolifyJob.php @@ -3,7 +3,10 @@ namespace App\Jobs; use App\Actions\Server\UpdateCoolify; +use App\Exceptions\UpgradeAlreadyRunningException; use App\Models\Server; +use App\Services\AutoUpdateNotifier; +use App\Services\CoolifyUpgradeStatus; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -45,9 +48,17 @@ class UpdateCoolifyJob implements ShouldBeEncrypted, ShouldQueue UpdateCoolify::run(false); // false means it's not a manual update Log::info('Coolify upgrade script started. Result: /data/coolify/source/.upgrade-status, log: /data/coolify/source/upgrade-*.log.'); + } catch (UpgradeAlreadyRunningException $e) { + // The running upgrade keeps its saved attempt, so its result is still reported. + Log::warning('Automatic update skipped: '.$e->getMessage()); } catch (\Throwable $e) { Log::error('UpdateCoolifyJob failed: '.$e->getMessage()); - // Consider implementing a notification to administrators + try { + CoolifyUpgradeStatus::writeError(Server::findOrFail(0), $e->getMessage()); + } catch (\Throwable $statusException) { + report($statusException); + } + AutoUpdateNotifier::failed($e->getMessage()); } } } diff --git a/app/Livewire/Upgrade.php b/app/Livewire/Upgrade.php index cfcab71f56..b4a8daee4a 100644 --- a/app/Livewire/Upgrade.php +++ b/app/Livewire/Upgrade.php @@ -84,6 +84,11 @@ class Upgrade extends Component UpdateCoolify::run(manual_update: true); } catch (\Throwable $e) { report($e); + try { + CoolifyUpgradeStatus::writeError(Server::findOrFail(0), $e->getMessage()); + } catch (\Throwable $statusException) { + report($statusException); + } } })->afterResponse(); diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php index 55c51880ab..4eb56e12da 100644 --- a/app/Models/InstanceSettings.php +++ b/app/Models/InstanceSettings.php @@ -28,6 +28,8 @@ class InstanceSettings extends Model 'do_not_track', 'is_auto_update_enabled', 'is_backup_before_update_enabled', + 'auto_update_target_version', + 'auto_update_started_at', 'is_registration_enabled', 'disable_registration_when_oauth_enabled', 'next_channel', @@ -115,6 +117,7 @@ class InstanceSettings extends Model 'allowed_ip_ranges' => 'array', 'is_auto_update_enabled' => 'boolean', 'is_backup_before_update_enabled' => 'boolean', + 'auto_update_started_at' => 'datetime', 'is_registration_enabled' => 'boolean', 'disable_registration_when_oauth_enabled' => 'boolean', 'auto_update_frequency' => 'string', diff --git a/app/Services/AutoUpdateNotifier.php b/app/Services/AutoUpdateNotifier.php new file mode 100644 index 0000000000..e077bc87e9 --- /dev/null +++ b/app/Services/AutoUpdateNotifier.php @@ -0,0 +1,175 @@ +update([ + 'auto_update_target_version' => $targetVersion, + 'auto_update_started_at' => now(), + ]); + } + + public static function started(string $currentVersion, string $targetVersion): void + { + self::notify("Automatic update started: Coolify {$currentVersion} to {$targetVersion}.", true); + } + + public static function failed(string $reason): void + { + self::clearAttempt(); + self::notifyFailure($reason); + } + + /** + * Reports the result of the saved attempt: success when this version is the target, + * failure when the upgrade script wrote an error or the attempt timed out. + */ + public static function checkResult(): void + { + $settings = instanceSettings(); + $targetVersion = $settings->auto_update_target_version; + if (blank($targetVersion)) { + return; + } + + $startedAt = $settings->auto_update_started_at; + $runningVersion = (string) config('constants.coolify.version'); + if (CoolifyUpgradeStatus::hasReachedTargetVersion($runningVersion, $targetVersion)) { + if (self::claimAttempt($settings, $targetVersion) + && ! self::notify("Automatic update finished: Coolify is now on {$runningVersion}.", true)) { + self::restoreAttempt($targetVersion, $startedAt); + } + + return; + } + + $status = self::readUpgradeStatus(); + [$step, $message, $writtenAt] = array_pad(explode('|', trim($status)), 3, ''); + if ($step === 'error' && self::isWrittenDuringAttempt($writtenAt, $startedAt)) { + if (self::claimAttempt($settings, $targetVersion) + && ! self::notifyFailure("the upgrade to {$targetVersion} stopped with this error: {$message}. Log: /data/coolify/source/upgrade-*.log")) { + self::restoreAttempt($targetVersion, $startedAt); + } + + return; + } + + $timedOut = ! $startedAt || $startedAt->lt(now()->subMinutes(self::RESULT_TIMEOUT_MINUTES)); + if ($timedOut && ! CoolifyUpgradeStatus::isRunning($status) && self::claimAttempt($settings, $targetVersion) + && ! self::notifyFailure("Coolify {$targetVersion} did not start within ".self::RESULT_TIMEOUT_MINUTES." minutes. Coolify {$runningVersion} is still running. Log: /data/coolify/source/upgrade-*.log")) { + self::restoreAttempt($targetVersion, $startedAt); + } + } + + /** + * An error from an earlier attempt stays in the status file until the new upgrade script writes its first step. + */ + private static function isWrittenDuringAttempt(string $writtenAt, ?CarbonInterface $startedAt): bool + { + if (! $startedAt) { + return true; + } + + try { + return Carbon::parse(trim($writtenAt))->gte($startedAt->copy()->startOfSecond()); + } catch (\Throwable) { + return false; + } + } + + private static function readUpgradeStatus(): string + { + try { + $server = Server::find(0); + + return $server ? (string) instant_remote_process(['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'], $server, false) : ''; + } catch (\Throwable $e) { + Log::warning('Could not read the upgrade status', ['error' => $e->getMessage()]); + + return ''; + } + } + + /** + * Clears the attempt only if it is still saved for this target version. + * Returns false when another run already reported it, so each result is sent one time. + */ + private static function claimAttempt(InstanceSettings $settings, string $targetVersion): bool + { + $claimed = InstanceSettings::query() + ->whereKey($settings->getKey()) + ->where('auto_update_target_version', $targetVersion) + ->update([ + 'auto_update_target_version' => null, + 'auto_update_started_at' => null, + ]) === 1; + + $settings->refresh(); + + return $claimed; + } + + /** + * Saves a claimed attempt again after its result could not be queued, so the next check sends it again. + * Does nothing when a newer attempt was saved in the meantime. + */ + private static function restoreAttempt(string $targetVersion, ?CarbonInterface $startedAt): void + { + InstanceSettings::query() + ->whereKey(0) + ->whereNull('auto_update_target_version') + ->update([ + 'auto_update_target_version' => $targetVersion, + 'auto_update_started_at' => $startedAt, + ]); + } + + private static function clearAttempt(): void + { + instanceSettings()->update([ + 'auto_update_target_version' => null, + 'auto_update_started_at' => null, + ]); + } + + private static function notifyFailure(string $reason): bool + { + return self::notify("Automatic update failed: {$reason}\n\nFix the cause, then start the update again: ".base_url().'/settings/updates', false); + } + + /** Returns false when the notification could not be queued. */ + private static function notify(string $message, bool $success): bool + { + try { + Team::find(0)?->notify(new GeneralNotification($message, $success)); + + return true; + } catch (\Throwable $e) { + Log::warning('Could not send the automatic update notification', ['error' => $e->getMessage()]); + + return false; + } + } +} diff --git a/app/Services/CoolifyUpgradeStatus.php b/app/Services/CoolifyUpgradeStatus.php index fb27017083..9075d0b07e 100644 --- a/app/Services/CoolifyUpgradeStatus.php +++ b/app/Services/CoolifyUpgradeStatus.php @@ -2,12 +2,18 @@ namespace App\Services; +use App\Models\Server; use DateTimeInterface; class CoolifyUpgradeStatus { public const FILE = '/data/coolify/source/.upgrade-status'; + /** + * scripts/upgrade.sh holds this lock while it writes a status, so an error write cannot interleave with it. + */ + public const LOCK_FILE = '/data/coolify/source/.upgrade-status.lock'; + public const STALE_AFTER_MINUTES = 10; /** @@ -85,6 +91,39 @@ class CoolifyUpgradeStatus ]; } + /** + * Writes an error that stopped an upgrade before the upgrade script started, so the upgrade modal shows it. + * Does nothing while an upgrade script runs, because the error then belongs to this attempt only. + * The write happens only if the status file did not change after the read, so a status that an + * upgrade script wrote in the meantime is kept. + */ + public static function writeError(Server $server, string $message): void + { + $current = (string) instant_remote_process(['cat '.self::FILE.' 2>/dev/null || true'], $server, false); + if (self::isRunning($current)) { + return; + } + + $line = 'error|'.trim((string) preg_replace('/[|\r\n]+/', ' ', $message)).'|'.now()->toIso8601String(); + instant_remote_process([self::compareAndWriteCommand($current, $line)], $server, false); + } + + /** + * Shell command that writes $line to the status file only if the file still contains $expected. + * It holds the lock that scripts/upgrade.sh takes for its status writes, if flock is available. + */ + public static function compareAndWriteCommand(string $expected, string $line): string + { + $script = 'if command -v flock >/dev/null 2>&1; then exec 9>"$2"; flock -w 10 9 || exit 1; fi; ' + .'if [ "$(cat "$1" 2>/dev/null)" = "$(printf %s "$3" | base64 -d)" ]; then printf %s "$4" | base64 -d > "$1"; echo >> "$1"; fi'; + + return 'sh -c '.escapeshellarg($script).' _ ' + .escapeshellarg(self::FILE).' ' + .escapeshellarg(self::LOCK_FILE).' ' + .escapeshellarg(base64_encode(trim($expected))).' ' + .escapeshellarg(base64_encode($line)); + } + public static function isRunning(string $content, ?DateTimeInterface $now = null): bool { $parts = explode('|', trim($content)); diff --git a/database/migrations/2026_11_04_080000_add_auto_update_attempt_to_instance_settings_table.php b/database/migrations/2026_11_04_080000_add_auto_update_attempt_to_instance_settings_table.php new file mode 100644 index 0000000000..8b65d1e675 --- /dev/null +++ b/database/migrations/2026_11_04_080000_add_auto_update_attempt_to_instance_settings_table.php @@ -0,0 +1,29 @@ +string('auto_update_target_version')->nullable(); + $table->timestamp('auto_update_started_at')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('instance_settings', function (Blueprint $table) { + $table->dropColumn(['auto_update_target_version', 'auto_update_started_at']); + }); + } +}; diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index af8d12c67a..68e0713c82 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -14,6 +14,7 @@ else fi SKIP_BACKUP=${4:-false} STATUS_FILE="/data/coolify/source/.upgrade-status" +STATUS_LOCK_FILE="/data/coolify/source/.upgrade-status.lock" DATE=$(date +%Y-%m-%d-%H-%M-%S) LOGFILE="/data/coolify/source/upgrade-${DATE}.log" @@ -32,10 +33,17 @@ log_section() { } # Helper function to write upgrade status for API polling +# Holds the lock that Coolify takes when it writes an error before an upgrade starts, so neither write is lost. write_status() { - local step="$1" - local message="$2" - echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE" + local line="${1}|${2}|$(date -Iseconds)" + if command -v flock >/dev/null 2>&1; then + ( + flock -w 10 9 || true + echo "$line" > "$STATUS_FILE" + ) 9>"$STATUS_LOCK_FILE" + else + echo "$line" > "$STATUS_FILE" + fi } # Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64). diff --git a/tests/Unit/UpdateCoolifyTest.php b/tests/Unit/UpdateCoolifyTest.php index dc6fe4d247..182c336453 100644 --- a/tests/Unit/UpdateCoolifyTest.php +++ b/tests/Unit/UpdateCoolifyTest.php @@ -451,3 +451,26 @@ it('does not start the update when the instance backup fails', function () { ->toThrow(Exception::class, 'The database backup failed, so the update was not started.') ->and(Activity::query()->count())->toBe(0); }); + +it('saves the automatic update attempt before it queues the upgrade script', function () { + Notification::fake(); + updateCoolifyTestPrepareUpgrade(); + $savedTargetVersion = null; + Activity::created(function () use (&$savedTargetVersion) { + $savedTargetVersion = InstanceSettings::findOrFail(0)->auto_update_target_version; + }); + + updateCoolifyTestActionWithStatus('')->handle(); + + expect($savedTargetVersion)->toBe('4.0.10'); +}); + +it('keeps the saved attempt of a running upgrade when an automatic update is skipped', function () { + updateCoolifyTestPrepareUpgrade(); + InstanceSettings::findOrFail(0)->update(['auto_update_target_version' => '4.0.8', 'auto_update_started_at' => now()->subMinutes(5)]); + $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String()); + + expect(fn () => $action->handle())->toThrow(Exception::class, 'Another Coolify upgrade is already running'); + + expect(InstanceSettings::findOrFail(0)->auto_update_target_version)->toBe('4.0.8'); +});