fix(upgrade): prevent duplicate upgrades and stalled progress polling

Block new upgrades while a recent upgrade is running, allowing retries after 15 minutes without a status update.

Add request timeouts and polling guards to keep upgrade progress responsive during restarts.

app/Actions/Server/UpdateCoolify.php

@@ -3,6 +3,7 @@
 namespace App\Actions\Server;
 
 use App\Models\Server;
+use App\Services\CoolifyUpgradeStatus;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Sleep;
@@ -116,6 +117,8 @@ class UpdateCoolify
 
     private function update()
     {
+        $this->ensureNoUpgradeIsRunning();
+
         $latestHelperImageVersion = getHelperVersion();
         $upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
         $registryUrl = coolifyRegistryUrl();
@@ -128,4 +131,29 @@ class UpdateCoolify
                 escapeshellarg($registryUrl),
         ], $this->server);
     }
+
+    private function ensureNoUpgradeIsRunning(): void
+    {
+        $status = $this->readUpgradeStatus();
+
+        if (CoolifyUpgradeStatus::isRunning((string) $status)) {
+            Log::warning('Upgrade skipped because another upgrade is running', [
+                'target_version' => $this->latestVersion,
+                'status' => $status,
+            ]);
+            throw new \Exception(
+                'Another Coolify upgrade is already running. Wait for it to finish. '.
+                'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
+            );
+        }
+    }
+
+    protected function readUpgradeStatus(): ?string
+    {
+        return instant_remote_process(
+            ['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'],
+            $this->server,
+            false
+        );
+    }
 }

app/Livewire/Upgrade.php

@@ -103,7 +103,7 @@ class Upgrade extends Component
             return ['status' => 'none'];
         }
 
-        $statusFile = '/data/coolify/source/.upgrade-status';
+        $statusFile = CoolifyUpgradeStatus::FILE;
 
         try {
             $content = instant_remote_process(

app/Services/CoolifyUpgradeStatus.php

@@ -6,8 +6,16 @@ use DateTimeInterface;
 
 class CoolifyUpgradeStatus
 {
+    public const FILE = '/data/coolify/source/.upgrade-status';
+
     public const STALE_AFTER_MINUTES = 10;
 
+    /**
+     * An upgrade that has not written a status for this long is treated as stopped,
+     * so a crashed upgrade does not block new ones forever.
+     */
+    public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15;
+
     /**
      * @return array{status: string, step?: int, message?: string, running_version: string, target_version: string}
      */
@@ -77,6 +85,30 @@ class CoolifyUpgradeStatus
         ];
     }
 
+    public static function isRunning(string $content, ?DateTimeInterface $now = null): bool
+    {
+        $parts = explode('|', trim($content));
+        if (count($parts) < 3) {
+            return false;
+        }
+
+        [$step, , $timestamp] = $parts;
+        if (! in_array($step, ['1', '2', '3', '4', '5'], true)) {
+            return false;
+        }
+
+        try {
+            $statusTime = new \DateTime($timestamp);
+        } catch (\Throwable) {
+            return false;
+        }
+
+        $now = $now ?? new \DateTime;
+        $diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60;
+
+        return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES;
+    }
+
     public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool
     {
         if ($runningVersion === '' || $targetVersion === '') {

resources/views/livewire/upgrade.blade.php

@@ -186,6 +186,9 @@
             checkUpgradeStatusInterval: null,
             elapsedInterval: null,
             healthCheckAttempts: 0,
+            healthProbePending: false,
+            healthTimeouts: 0,
+            statusRequestPending: false,
             livewireFailures: 0,
             startTime: null,
             elapsedTime: 0,
@@ -261,6 +264,19 @@
                 return `${minutes}:${seconds.toString().padStart(2, '0')}`;
             },
 
+            // Livewire never settles a $wire call when its request fails, and a request to a removed
+            // container can hang. Time out both so the progress view does not freeze.
+            withTimeout(promise, ms) {
+                return Promise.race([
+                    promise,
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Request timed out')), ms)),
+                ]);
+            },
+
+            fetchUpgradeStatus() {
+                return this.withTimeout(this.$wire.getUpgradeStatus(), 15000);
+            },
+
             mapStepToUI(apiStep) {
                 // Map backend steps (1-6) to UI steps (1-4)
                 // Backend: 1=config, 2=env, 3=pull, 4=stop, 5=start, 6=complete
@@ -281,11 +297,30 @@
             },
 
             async probeHealth() {
+                if (this.healthProbePending) {
+                    return;
+                }
+                this.healthProbePending = true;
+                try {
+                    await this.runHealthProbe();
+                } finally {
+                    this.healthProbePending = false;
+                }
+            },
+
+            async runHealthProbe() {
+                if (this.upgradeComplete || this.upgradeError) {
+                    return;
+                }
                 this.healthCheckAttempts++;
                 const elapsedMinutes = Math.floor((Date.now() - this.startTime) / 60000);
 
                 try {
-                    const response = await fetch('/api/health');
+                    const response = await fetch('/api/health', {
+                        cache: 'no-store',
+                        signal: AbortSignal.timeout(10000),
+                    });
+                    this.healthTimeouts = 0;
                     if (!response.ok) {
                         this.instanceWentDown = true;
                         this.currentStep = 4;
@@ -298,7 +333,7 @@
 
                     let data;
                     try {
-                        data = await this.$wire.getUpgradeStatus();
+                        data = await this.fetchUpgradeStatus();
                     } catch (error) {
                         if (this.instanceWentDown) {
                             this.showSuccess();
@@ -328,6 +363,10 @@
                         this.currentStatus = data.message ?? this.getReviveStatusMessage(elapsedMinutes, this.healthCheckAttempts);
                     }
                 } catch (error) {
+                    // A busy server can answer slowly, so one timeout does not prove that it is down.
+                    if (error?.name === 'TimeoutError' && ++this.healthTimeouts < 2) {
+                        return;
+                    }
                     console.error('Health check failed:', error);
                     this.instanceWentDown = true;
                     this.currentStep = 4;
@@ -356,6 +395,9 @@
             },
 
             showSuccess() {
+                if (this.upgradeComplete) {
+                    return;
+                }
                 if (this.checkHealthInterval) {
                     clearInterval(this.checkHealthInterval);
                     this.checkHealthInterval = null;
@@ -431,12 +473,17 @@
                 this.serviceDown = false;
                 this.instanceWentDown = false;
                 this.livewireFailures = 0;
+                this.healthTimeouts = 0;
                 this.startHealthWatch();
 
                 // Poll upgrade status via Livewire
                 this.checkUpgradeStatusInterval = setInterval(async () => {
+                    if (this.statusRequestPending) {
+                        return;
+                    }
+                    this.statusRequestPending = true;
                     try {
-                        const data = await this.$wire.getUpgradeStatus();
+                        const data = await this.fetchUpgradeStatus();
                         this.livewireFailures = 0;
                         if (data.status === 'in_progress') {
                             this.currentStep = this.mapStepToUI(data.step);
@@ -450,6 +497,9 @@
                             await this.probeHealth();
                         }
                     } catch (error) {
+                        if (this.upgradeComplete || this.upgradeError) {
+                            return;
+                        }
                         this.livewireFailures++;
                         if (this.livewireFailures < 3) {
                             this.currentStatus = 'Reconnecting. This is expected during an upgrade...';
@@ -468,6 +518,8 @@
                             }
                             this.revive();
                         }
+                    } finally {
+                        this.statusRequestPending = false;
                     }
                 }, 2000);
             }

tests/Unit/CoolifyUpgradeStatusTest.php

@@ -128,3 +128,24 @@ it('reports whether the running version has reached the target', function (strin
     'missing running version' => ['', '4.3.1', false],
     'missing target version' => ['4.3.1', '', false],
 ]);
+
+it('detects a running upgrade only for recent intermediate steps', function (string $step, int $minutesAgo, bool $expected) {
+    $now = Carbon::parse('2026-08-13T12:00:00+00:00');
+    $content = "{$step}|Some message|".$now->copy()->subMinutes($minutesAgo)->toIso8601String();
+
+    expect(CoolifyUpgradeStatus::isRunning($content, $now))->toBe($expected);
+})->with([
+    'recent step 1' => ['1', 0, true],
+    'step 5 at lock expiry' => ['5', 15, true],
+    'step 3 after lock expiry' => ['3', 16, false],
+    'complete' => ['6', 1, false],
+    'error' => ['error', 1, false],
+]);
+
+it('does not detect a running upgrade for empty or malformed status', function (string $content) {
+    expect(CoolifyUpgradeStatus::isRunning($content))->toBeFalse();
+})->with([
+    'empty' => '',
+    'missing fields' => '3|Pulling Docker images',
+    'invalid timestamp' => '3|Pulling Docker images|not-a-date',
+]);

tests/Unit/UpdateCoolifyTest.php

@@ -40,6 +40,19 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi
     Once::flush();
 }
 
+function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify
+{
+    return new class($status) extends UpdateCoolify
+    {
+        public function __construct(private string $status) {}
+
+        protected function readUpgradeStatus(): ?string
+        {
+            return $this->status;
+        }
+    };
+}
+
 afterEach(function () {
     Mockery::close();
 });
@@ -316,3 +329,37 @@ it('prevents downgrade even with manual update', function () {
         expect($e->getMessage())->toContain('4.0.0');
     }
 });
+
+it('does not start a second upgrade while a recent upgrade status is in progress', function () {
+    Queue::fake();
+    config([
+        'app.env' => 'testing',
+        'constants.coolify.version' => '4.0.9',
+        'constants.ssh.mux_enabled' => false,
+    ]);
+    updateCoolifyTestCreateRootServerAndSettings();
+    Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
+    $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String());
+
+    expect(fn () => $action->handle(manual_update: true))
+        ->toThrow(Exception::class, 'Another Coolify upgrade is already running');
+
+    expect(Activity::query()->count())->toBe(0)
+        ->and(InstanceSettings::findOrFail(0)->new_version_available)->toBeFalsy();
+});
+
+it('starts an upgrade when the previous upgrade status is older than the lock expiry', function () {
+    Queue::fake();
+    config([
+        'app.env' => 'testing',
+        'constants.coolify.version' => '4.0.9',
+        'constants.ssh.mux_enabled' => false,
+    ]);
+    updateCoolifyTestCreateRootServerAndSettings();
+    Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
+    updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(16)->toIso8601String())
+        ->handle(manual_update: true);
+
+    expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
+        ->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'");
+});