From df953bdcd187bc26bbc307f0271931e24e002245 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:01:20 +0200 Subject: [PATCH] fix(upgrade): prevent duplicate upgrades and stalled progress polling Block new upgrades while a recent upgrade is running, allowing retries after 15 minutes without a status update. Add request timeouts and polling guards to keep upgrade progress responsive during restarts. --- app/Actions/Server/UpdateCoolify.php | 28 +++++++++++ app/Livewire/Upgrade.php | 2 +- app/Services/CoolifyUpgradeStatus.php | 32 ++++++++++++ resources/views/livewire/upgrade.blade.php | 58 ++++++++++++++++++++-- tests/Unit/CoolifyUpgradeStatusTest.php | 21 ++++++++ tests/Unit/UpdateCoolifyTest.php | 47 ++++++++++++++++++ 6 files changed, 184 insertions(+), 4 deletions(-) diff --git a/app/Actions/Server/UpdateCoolify.php b/app/Actions/Server/UpdateCoolify.php index a8c9d21e8f..3c60c34379 100644 --- a/app/Actions/Server/UpdateCoolify.php +++ b/app/Actions/Server/UpdateCoolify.php @@ -3,6 +3,7 @@ namespace App\Actions\Server; use App\Models\Server; +use App\Services\CoolifyUpgradeStatus; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Illuminate\Support\Sleep; @@ -116,6 +117,8 @@ class UpdateCoolify private function update() { + $this->ensureNoUpgradeIsRunning(); + $latestHelperImageVersion = getHelperVersion(); $upgradeScriptUrl = config('constants.coolify.upgrade_script_url'); $registryUrl = coolifyRegistryUrl(); @@ -128,4 +131,29 @@ class UpdateCoolify escapeshellarg($registryUrl), ], $this->server); } + + private function ensureNoUpgradeIsRunning(): void + { + $status = $this->readUpgradeStatus(); + + if (CoolifyUpgradeStatus::isRunning((string) $status)) { + Log::warning('Upgrade skipped because another upgrade is running', [ + 'target_version' => $this->latestVersion, + 'status' => $status, + ]); + throw new \Exception( + 'Another Coolify upgrade is already running. Wait for it to finish. '. + 'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.' + ); + } + } + + protected function readUpgradeStatus(): ?string + { + return instant_remote_process( + ['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'], + $this->server, + false + ); + } } diff --git a/app/Livewire/Upgrade.php b/app/Livewire/Upgrade.php index e2ab094944..75d9b32b3e 100644 --- a/app/Livewire/Upgrade.php +++ b/app/Livewire/Upgrade.php @@ -103,7 +103,7 @@ class Upgrade extends Component return ['status' => 'none']; } - $statusFile = '/data/coolify/source/.upgrade-status'; + $statusFile = CoolifyUpgradeStatus::FILE; try { $content = instant_remote_process( diff --git a/app/Services/CoolifyUpgradeStatus.php b/app/Services/CoolifyUpgradeStatus.php index 15f6c22ef2..3f274bb616 100644 --- a/app/Services/CoolifyUpgradeStatus.php +++ b/app/Services/CoolifyUpgradeStatus.php @@ -6,8 +6,16 @@ use DateTimeInterface; class CoolifyUpgradeStatus { + public const FILE = '/data/coolify/source/.upgrade-status'; + public const STALE_AFTER_MINUTES = 10; + /** + * An upgrade that has not written a status for this long is treated as stopped, + * so a crashed upgrade does not block new ones forever. + */ + public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15; + /** * @return array{status: string, step?: int, message?: string, running_version: string, target_version: string} */ @@ -77,6 +85,30 @@ class CoolifyUpgradeStatus ]; } + public static function isRunning(string $content, ?DateTimeInterface $now = null): bool + { + $parts = explode('|', trim($content)); + if (count($parts) < 3) { + return false; + } + + [$step, , $timestamp] = $parts; + if (! in_array($step, ['1', '2', '3', '4', '5'], true)) { + return false; + } + + try { + $statusTime = new \DateTime($timestamp); + } catch (\Throwable) { + return false; + } + + $now = $now ?? new \DateTime; + $diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60; + + return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES; + } + public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool { if ($runningVersion === '' || $targetVersion === '') { diff --git a/resources/views/livewire/upgrade.blade.php b/resources/views/livewire/upgrade.blade.php index b1ad048e1c..2af8873207 100644 --- a/resources/views/livewire/upgrade.blade.php +++ b/resources/views/livewire/upgrade.blade.php @@ -186,6 +186,9 @@ checkUpgradeStatusInterval: null, elapsedInterval: null, healthCheckAttempts: 0, + healthProbePending: false, + healthTimeouts: 0, + statusRequestPending: false, livewireFailures: 0, startTime: null, elapsedTime: 0, @@ -261,6 +264,19 @@ return `${minutes}:${seconds.toString().padStart(2, '0')}`; }, + // Livewire never settles a $wire call when its request fails, and a request to a removed + // container can hang. Time out both so the progress view does not freeze. + withTimeout(promise, ms) { + return Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error('Request timed out')), ms)), + ]); + }, + + fetchUpgradeStatus() { + return this.withTimeout(this.$wire.getUpgradeStatus(), 15000); + }, + mapStepToUI(apiStep) { // Map backend steps (1-6) to UI steps (1-4) // Backend: 1=config, 2=env, 3=pull, 4=stop, 5=start, 6=complete @@ -281,11 +297,30 @@ }, async probeHealth() { + if (this.healthProbePending) { + return; + } + this.healthProbePending = true; + try { + await this.runHealthProbe(); + } finally { + this.healthProbePending = false; + } + }, + + async runHealthProbe() { + if (this.upgradeComplete || this.upgradeError) { + return; + } this.healthCheckAttempts++; const elapsedMinutes = Math.floor((Date.now() - this.startTime) / 60000); try { - const response = await fetch('/api/health'); + const response = await fetch('/api/health', { + cache: 'no-store', + signal: AbortSignal.timeout(10000), + }); + this.healthTimeouts = 0; if (!response.ok) { this.instanceWentDown = true; this.currentStep = 4; @@ -298,7 +333,7 @@ let data; try { - data = await this.$wire.getUpgradeStatus(); + data = await this.fetchUpgradeStatus(); } catch (error) { if (this.instanceWentDown) { this.showSuccess(); @@ -328,6 +363,10 @@ this.currentStatus = data.message ?? this.getReviveStatusMessage(elapsedMinutes, this.healthCheckAttempts); } } catch (error) { + // A busy server can answer slowly, so one timeout does not prove that it is down. + if (error?.name === 'TimeoutError' && ++this.healthTimeouts < 2) { + return; + } console.error('Health check failed:', error); this.instanceWentDown = true; this.currentStep = 4; @@ -356,6 +395,9 @@ }, showSuccess() { + if (this.upgradeComplete) { + return; + } if (this.checkHealthInterval) { clearInterval(this.checkHealthInterval); this.checkHealthInterval = null; @@ -431,12 +473,17 @@ this.serviceDown = false; this.instanceWentDown = false; this.livewireFailures = 0; + this.healthTimeouts = 0; this.startHealthWatch(); // Poll upgrade status via Livewire this.checkUpgradeStatusInterval = setInterval(async () => { + if (this.statusRequestPending) { + return; + } + this.statusRequestPending = true; try { - const data = await this.$wire.getUpgradeStatus(); + const data = await this.fetchUpgradeStatus(); this.livewireFailures = 0; if (data.status === 'in_progress') { this.currentStep = this.mapStepToUI(data.step); @@ -450,6 +497,9 @@ await this.probeHealth(); } } catch (error) { + if (this.upgradeComplete || this.upgradeError) { + return; + } this.livewireFailures++; if (this.livewireFailures < 3) { this.currentStatus = 'Reconnecting. This is expected during an upgrade...'; @@ -468,6 +518,8 @@ } this.revive(); } + } finally { + this.statusRequestPending = false; } }, 2000); } diff --git a/tests/Unit/CoolifyUpgradeStatusTest.php b/tests/Unit/CoolifyUpgradeStatusTest.php index f701a8a6d3..1302e97370 100644 --- a/tests/Unit/CoolifyUpgradeStatusTest.php +++ b/tests/Unit/CoolifyUpgradeStatusTest.php @@ -128,3 +128,24 @@ it('reports whether the running version has reached the target', function (strin 'missing running version' => ['', '4.3.1', false], 'missing target version' => ['4.3.1', '', false], ]); + +it('detects a running upgrade only for recent intermediate steps', function (string $step, int $minutesAgo, bool $expected) { + $now = Carbon::parse('2026-08-13T12:00:00+00:00'); + $content = "{$step}|Some message|".$now->copy()->subMinutes($minutesAgo)->toIso8601String(); + + expect(CoolifyUpgradeStatus::isRunning($content, $now))->toBe($expected); +})->with([ + 'recent step 1' => ['1', 0, true], + 'step 5 at lock expiry' => ['5', 15, true], + 'step 3 after lock expiry' => ['3', 16, false], + 'complete' => ['6', 1, false], + 'error' => ['error', 1, false], +]); + +it('does not detect a running upgrade for empty or malformed status', function (string $content) { + expect(CoolifyUpgradeStatus::isRunning($content))->toBeFalse(); +})->with([ + 'empty' => '', + 'missing fields' => '3|Pulling Docker images', + 'invalid timestamp' => '3|Pulling Docker images|not-a-date', +]); diff --git a/tests/Unit/UpdateCoolifyTest.php b/tests/Unit/UpdateCoolifyTest.php index d27a634018..ddc8c2546b 100644 --- a/tests/Unit/UpdateCoolifyTest.php +++ b/tests/Unit/UpdateCoolifyTest.php @@ -40,6 +40,19 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi Once::flush(); } +function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify +{ + return new class($status) extends UpdateCoolify + { + public function __construct(private string $status) {} + + protected function readUpgradeStatus(): ?string + { + return $this->status; + } + }; +} + afterEach(function () { Mockery::close(); }); @@ -316,3 +329,37 @@ it('prevents downgrade even with manual update', function () { expect($e->getMessage())->toContain('4.0.0'); } }); + +it('does not start a second upgrade while a recent upgrade status is in progress', function () { + Queue::fake(); + config([ + 'app.env' => 'testing', + 'constants.coolify.version' => '4.0.9', + 'constants.ssh.mux_enabled' => false, + ]); + updateCoolifyTestCreateRootServerAndSettings(); + Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]); + $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String()); + + expect(fn () => $action->handle(manual_update: true)) + ->toThrow(Exception::class, 'Another Coolify upgrade is already running'); + + expect(Activity::query()->count())->toBe(0) + ->and(InstanceSettings::findOrFail(0)->new_version_available)->toBeFalsy(); +}); + +it('starts an upgrade when the previous upgrade status is older than the lock expiry', function () { + Queue::fake(); + config([ + 'app.env' => 'testing', + 'constants.coolify.version' => '4.0.9', + 'constants.ssh.mux_enabled' => false, + ]); + updateCoolifyTestCreateRootServerAndSettings(); + Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]); + updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(16)->toIso8601String()) + ->handle(manual_update: true); + + expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first())) + ->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'"); +});