chore: merge main into release/v4.5 (#12131)

app/Actions/Server/CleanupDocker.php

@@ -158,7 +158,7 @@ class CleanupDocker implements ShouldBeUnique
         );
 
         $commands = [
-            'docker container prune -f --filter "label=coolify.managed=true" --filter "label!=coolify.proxy=true" --filter "label!=coolify.type=database" --filter "label!=coolify.type=application" --filter "label!=coolify.type=service"',
+            $this->buildContainerPruneCommand(),
             $imagePruneCmd,
             'docker builder prune -af',
             // The prune fails on a stopped builder, so start it first.
@@ -205,6 +205,20 @@ class CleanupDocker implements ShouldBeUnique
         return instant_remote_process([$command], $server, false, timeout: $remaining);
     }
 
+    /**
+     * Build a command that removes stopped Coolify containers, except the proxy and the
+     * containers of databases, applications and services.
+     *
+     * docker container prune only excludes a container that matches all of its label!= filters
+     * together, so we list the stopped containers and filter them by label in the shell.
+     */
+    private function buildContainerPruneCommand(): string
+    {
+        return "docker ps -a --filter status=created --filter status=exited --filter status=dead --filter label=coolify.managed=true --format '{{.ID}} coolify.proxy={{.Label \"coolify.proxy\"}} coolify.type={{.Label \"coolify.type\"}}' | ".
+            "grep -v -E 'coolify\.proxy=true |coolify\.type=(database|application|service)$' | ".
+            "awk '{print $1}' | xargs -r docker rm";
+    }
+
     /**
      * Build a docker image prune command that excludes application image repositories.
      *

app/Actions/Server/UpdateCoolify.php

@@ -3,6 +3,7 @@
 namespace App\Actions\Server;
 
 use App\Models\Server;
+use App\Services\CoolifyUpgradeStatus;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Sleep;
@@ -116,6 +117,8 @@ class UpdateCoolify
 
     private function update()
     {
+        $this->ensureNoUpgradeIsRunning();
+
         $latestHelperImageVersion = getHelperVersion();
         $upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
         $registryUrl = coolifyRegistryUrl();
@@ -128,4 +131,29 @@ class UpdateCoolify
                 escapeshellarg($registryUrl),
         ], $this->server);
     }
+
+    private function ensureNoUpgradeIsRunning(): void
+    {
+        $status = $this->readUpgradeStatus();
+
+        if (CoolifyUpgradeStatus::isRunning((string) $status)) {
+            Log::warning('Upgrade skipped because another upgrade is running', [
+                'target_version' => $this->latestVersion,
+                'status' => $status,
+            ]);
+            throw new \Exception(
+                'Another Coolify upgrade is already running. Wait for it to finish. '.
+                'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
+            );
+        }
+    }
+
+    protected function readUpgradeStatus(): ?string
+    {
+        return instant_remote_process(
+            ['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'],
+            $this->server,
+            false
+        );
+    }
 }

app/Livewire/Upgrade.php

@@ -103,7 +103,7 @@ class Upgrade extends Component
             return ['status' => 'none'];
         }
 
-        $statusFile = '/data/coolify/source/.upgrade-status';
+        $statusFile = CoolifyUpgradeStatus::FILE;
 
         try {
             $content = instant_remote_process(

app/Services/CoolifyUpgradeStatus.php

@@ -6,8 +6,16 @@ use DateTimeInterface;
 
 class CoolifyUpgradeStatus
 {
+    public const FILE = '/data/coolify/source/.upgrade-status';
+
     public const STALE_AFTER_MINUTES = 10;
 
+    /**
+     * An upgrade that has not written a status for this long is treated as stopped,
+     * so a crashed upgrade does not block new ones forever.
+     */
+    public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15;
+
     /**
      * @return array{status: string, step?: int, message?: string, running_version: string, target_version: string}
      */
@@ -77,6 +85,30 @@ class CoolifyUpgradeStatus
         ];
     }
 
+    public static function isRunning(string $content, ?DateTimeInterface $now = null): bool
+    {
+        $parts = explode('|', trim($content));
+        if (count($parts) < 3) {
+            return false;
+        }
+
+        [$step, , $timestamp] = $parts;
+        if (! in_array($step, ['1', '2', '3', '4', '5'], true) || trim($timestamp) === '') {
+            return false;
+        }
+
+        try {
+            $statusTime = new \DateTime($timestamp);
+        } catch (\Throwable) {
+            return false;
+        }
+
+        $now = $now ?? new \DateTime;
+        $diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60;
+
+        return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES;
+    }
+
     public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool
     {
         if ($runningVersion === '' || $targetVersion === '') {

config/constants.php

@@ -5,7 +5,7 @@ $sshCommandTimeout = (int) env('SSH_COMMAND_TIMEOUT', 3600);
 
 return [
     'coolify' => [
-        'version' => env('COOLIFY_VERSION') ?: '4.4.3',
+        'version' => env('COOLIFY_VERSION') ?: '4.4.4',
         'helper_version' => '1.0.18',
         'railpack_version' => '0.40.1',
         'self_hosted' => env('SELF_HOSTED', true),

other/nightly/versions.json

@@ -1,7 +1,7 @@
 {
     "coolify": {
         "v4": {
-            "version": "4.4.3"
+            "version": "4.4.4"
         },
         "nightly": {
             "version": "4.5-rc.1"

resources/views/livewire/upgrade.blade.php

@@ -186,6 +186,9 @@
             checkUpgradeStatusInterval: null,
             elapsedInterval: null,
             healthCheckAttempts: 0,
+            healthProbePending: false,
+            healthTimeouts: 0,
+            statusRequestPending: false,
             livewireFailures: 0,
             startTime: null,
             elapsedTime: 0,
@@ -261,6 +264,19 @@
                 return `${minutes}:${seconds.toString().padStart(2, '0')}`;
             },
 
+            // Livewire never settles a $wire call when its request fails, and a request to a removed
+            // container can hang. Time out both so the progress view does not freeze.
+            withTimeout(promise, ms) {
+                return Promise.race([
+                    promise,
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Request timed out')), ms)),
+                ]);
+            },
+
+            fetchUpgradeStatus() {
+                return this.withTimeout(this.$wire.getUpgradeStatus(), 15000);
+            },
+
             mapStepToUI(apiStep) {
                 // Map backend steps (1-6) to UI steps (1-4)
                 // Backend: 1=config, 2=env, 3=pull, 4=stop, 5=start, 6=complete
@@ -281,11 +297,30 @@
             },
 
             async probeHealth() {
+                if (this.healthProbePending) {
+                    return;
+                }
+                this.healthProbePending = true;
+                try {
+                    await this.runHealthProbe();
+                } finally {
+                    this.healthProbePending = false;
+                }
+            },
+
+            async runHealthProbe() {
+                if (this.upgradeComplete || this.upgradeError) {
+                    return;
+                }
                 this.healthCheckAttempts++;
                 const elapsedMinutes = Math.floor((Date.now() - this.startTime) / 60000);
 
                 try {
-                    const response = await fetch('/api/health');
+                    const response = await fetch('/api/health', {
+                        cache: 'no-store',
+                        signal: AbortSignal.timeout(10000),
+                    });
+                    this.healthTimeouts = 0;
                     if (!response.ok) {
                         this.instanceWentDown = true;
                         this.currentStep = 4;
@@ -298,7 +333,7 @@
 
                     let data;
                     try {
-                        data = await this.$wire.getUpgradeStatus();
+                        data = await this.fetchUpgradeStatus();
                     } catch (error) {
                         if (this.instanceWentDown) {
                             this.showSuccess();
@@ -328,6 +363,10 @@
                         this.currentStatus = data.message ?? this.getReviveStatusMessage(elapsedMinutes, this.healthCheckAttempts);
                     }
                 } catch (error) {
+                    // A busy server can answer slowly, so one timeout does not prove that it is down.
+                    if (error?.name === 'TimeoutError' && ++this.healthTimeouts < 2) {
+                        return;
+                    }
                     console.error('Health check failed:', error);
                     this.instanceWentDown = true;
                     this.currentStep = 4;
@@ -356,6 +395,9 @@
             },
 
             showSuccess() {
+                if (this.upgradeComplete) {
+                    return;
+                }
                 if (this.checkHealthInterval) {
                     clearInterval(this.checkHealthInterval);
                     this.checkHealthInterval = null;
@@ -431,12 +473,17 @@
                 this.serviceDown = false;
                 this.instanceWentDown = false;
                 this.livewireFailures = 0;
+                this.healthTimeouts = 0;
                 this.startHealthWatch();
 
                 // Poll upgrade status via Livewire
                 this.checkUpgradeStatusInterval = setInterval(async () => {
+                    if (this.statusRequestPending) {
+                        return;
+                    }
+                    this.statusRequestPending = true;
                     try {
-                        const data = await this.$wire.getUpgradeStatus();
+                        const data = await this.fetchUpgradeStatus();
                         this.livewireFailures = 0;
                         if (data.status === 'in_progress') {
                             this.currentStep = this.mapStepToUI(data.step);
@@ -450,6 +497,9 @@
                             await this.probeHealth();
                         }
                     } catch (error) {
+                        if (this.upgradeComplete || this.upgradeError) {
+                            return;
+                        }
                         this.livewireFailures++;
                         if (this.livewireFailures < 3) {
                             this.currentStatus = 'Reconnecting. This is expected during an upgrade...';
@@ -468,6 +518,8 @@
                             }
                             this.revive();
                         }
+                    } finally {
+                        this.statusRequestPending = false;
                     }
                 }, 2000);
             }

tests/Feature/DockerCleanupJobTest.php

@@ -372,7 +372,7 @@ it('gives remote cleanup commands a local timeout and fails once the cleanup dea
     expect(fn () => CleanupDocker::run($server))
         ->toThrow(RuntimeException::class, 'Docker cleanup did not finish within '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' seconds');
 
-    $sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker container prune') || str_contains($command, 'docker image prune'))->values();
+    $sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'label=coolify.managed=true') || str_contains($command, 'docker image prune'))->values();
 
     expect(CleanupDocker::REMOTE_COMMANDS_DEADLINE)->toBeLessThan(CleanupDocker::JOB_TIMEOUT)
         ->and($sshCommands)->toHaveCount(2)
@@ -498,3 +498,38 @@ it('scans application images with a fixed number of remote commands and keeps th
         expect($removal)->not->toContain($kept);
     }
 });
+
+it('removes stopped Coolify containers but keeps the proxy, databases, applications and services', function (bool $nonRoot) {
+    $command = (new ReflectionMethod(CleanupDocker::class, 'buildContainerPruneCommand'))->invoke(new CleanupDocker);
+    preg_match("/--format '([^']+)'/", $command, $format);
+    $stoppedContainers = collect([
+        ['proxy', 'true', ''],
+        ['database', '', 'database'],
+        ['application', '', 'application'],
+        ['service', '', 'service'],
+        ['helper', '', ''],
+    ])->map(fn (array $container) => str_replace(['{{.ID}}', '{{.Label "coolify.proxy"}}', '{{.Label "coolify.type"}}'], $container, $format[1]));
+
+    // Fake docker on PATH: `docker ps` prints the stopped containers, `docker rm` prints what it is asked to remove
+    $bin = sys_get_temp_dir().'/docker-cleanup-'.bin2hex(random_bytes(4));
+    mkdir($bin);
+    file_put_contents("$bin/containers", $stoppedContainers->implode("\n")."\n");
+    file_put_contents("$bin/docker", "#!/bin/sh\nif [ \"\$1\" = ps ]; then cat $bin/containers; else echo \"\$@\"; fi\n");
+    chmod("$bin/docker", 0755);
+
+    if ($nonRoot) {
+        $server = new Server;
+        $server->user = 'coolify';
+        $command = parseCommandsByLineForSudo(collect([$command]), $server)[0];
+        file_put_contents("$bin/sudo", "#!/bin/sh\nexec \"\$@\"\n");
+        chmod("$bin/sudo", 0755);
+    }
+
+    $output = shell_exec('PATH='.escapeshellarg("$bin:".getenv('PATH')).' /bin/sh -c '.escapeshellarg($command));
+
+    array_map('unlink', glob("$bin/*"));
+    rmdir($bin);
+
+    expect($command)->toContain('--filter label=coolify.managed=true')
+        ->and(trim((string) $output))->toBe('rm helper');
+})->with(['root' => false, 'non-root sudo parser' => true]);

tests/Unit/CoolifyUpgradeStatusTest.php

@@ -128,3 +128,26 @@ it('reports whether the running version has reached the target', function (strin
     'missing running version' => ['', '4.3.1', false],
     'missing target version' => ['4.3.1', '', false],
 ]);
+
+it('detects a running upgrade only for recent intermediate steps', function (string $step, int $minutesAgo, bool $expected) {
+    $now = Carbon::parse('2026-08-13T12:00:00+00:00');
+    $content = "{$step}|Some message|".$now->copy()->subMinutes($minutesAgo)->toIso8601String();
+
+    expect(CoolifyUpgradeStatus::isRunning($content, $now))->toBe($expected);
+})->with([
+    'recent step 1' => ['1', 0, true],
+    'step 5 at lock expiry' => ['5', 15, true],
+    'step 3 after lock expiry' => ['3', 16, false],
+    'complete' => ['6', 1, false],
+    'error' => ['error', 1, false],
+]);
+
+it('does not detect a running upgrade for empty or malformed status', function (string $content) {
+    expect(CoolifyUpgradeStatus::isRunning($content))->toBeFalse();
+})->with([
+    'empty' => '',
+    'missing fields' => '3|Pulling Docker images',
+    'blank timestamp' => '3|Pulling Docker images|',
+    'whitespace timestamp' => "3|Pulling Docker images| \t",
+    'invalid timestamp' => '3|Pulling Docker images|not-a-date',
+]);

tests/Unit/UpdateCoolifyTest.php

@@ -40,6 +40,19 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi
     Once::flush();
 }
 
+function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify
+{
+    return new class($status) extends UpdateCoolify
+    {
+        public function __construct(private string $status) {}
+
+        protected function readUpgradeStatus(): ?string
+        {
+            return $this->status;
+        }
+    };
+}
+
 afterEach(function () {
     Mockery::close();
 });
@@ -316,3 +329,37 @@ it('prevents downgrade even with manual update', function () {
         expect($e->getMessage())->toContain('4.0.0');
     }
 });
+
+it('does not start a second upgrade while a recent upgrade status is in progress', function () {
+    Queue::fake();
+    config([
+        'app.env' => 'testing',
+        'constants.coolify.version' => '4.0.9',
+        'constants.ssh.mux_enabled' => false,
+    ]);
+    updateCoolifyTestCreateRootServerAndSettings();
+    Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
+    $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String());
+
+    expect(fn () => $action->handle(manual_update: true))
+        ->toThrow(Exception::class, 'Another Coolify upgrade is already running');
+
+    expect(Activity::query()->count())->toBe(0)
+        ->and(InstanceSettings::findOrFail(0)->new_version_available)->toBeFalsy();
+});
+
+it('starts an upgrade when the previous upgrade status is older than the lock expiry', function () {
+    Queue::fake();
+    config([
+        'app.env' => 'testing',
+        'constants.coolify.version' => '4.0.9',
+        'constants.ssh.mux_enabled' => false,
+    ]);
+    updateCoolifyTestCreateRootServerAndSettings();
+    Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
+    updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(16)->toIso8601String())
+        ->handle(manual_update: true);
+
+    expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
+        ->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'");
+});

versions.json

@@ -1,7 +1,7 @@
 {
     "coolify": {
         "v4": {
-            "version": "4.4.3"
+            "version": "4.4.4"
         },
         "nightly": {
             "version": "4.5-rc.1"