diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 3c0e2ba7e0..888169be5b 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -158,7 +158,7 @@ class CleanupDocker implements ShouldBeUnique ); $commands = [ - 'docker container prune -f --filter "label=coolify.managed=true" --filter "label!=coolify.proxy=true" --filter "label!=coolify.type=database" --filter "label!=coolify.type=application" --filter "label!=coolify.type=service"', + $this->buildContainerPruneCommand(), $imagePruneCmd, 'docker builder prune -af', // The prune fails on a stopped builder, so start it first. @@ -205,6 +205,20 @@ class CleanupDocker implements ShouldBeUnique return instant_remote_process([$command], $server, false, timeout: $remaining); } + /** + * Build a command that removes stopped Coolify containers, except the proxy and the + * containers of databases, applications and services. + * + * docker container prune only excludes a container that matches all of its label!= filters + * together, so we list the stopped containers and filter them by label in the shell. + */ + private function buildContainerPruneCommand(): string + { + return "docker ps -a --filter status=created --filter status=exited --filter status=dead --filter label=coolify.managed=true --format '{{.ID}} coolify.proxy={{.Label \"coolify.proxy\"}} coolify.type={{.Label \"coolify.type\"}}' | ". + "grep -v -E 'coolify\.proxy=true |coolify\.type=(database|application|service)$' | ". + "awk '{print $1}' | xargs -r docker rm"; + } + /** * Build a docker image prune command that excludes application image repositories. * diff --git a/app/Actions/Server/UpdateCoolify.php b/app/Actions/Server/UpdateCoolify.php index a8c9d21e8f..3c60c34379 100644 --- a/app/Actions/Server/UpdateCoolify.php +++ b/app/Actions/Server/UpdateCoolify.php @@ -3,6 +3,7 @@ namespace App\Actions\Server; use App\Models\Server; +use App\Services\CoolifyUpgradeStatus; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Illuminate\Support\Sleep; @@ -116,6 +117,8 @@ class UpdateCoolify private function update() { + $this->ensureNoUpgradeIsRunning(); + $latestHelperImageVersion = getHelperVersion(); $upgradeScriptUrl = config('constants.coolify.upgrade_script_url'); $registryUrl = coolifyRegistryUrl(); @@ -128,4 +131,29 @@ class UpdateCoolify escapeshellarg($registryUrl), ], $this->server); } + + private function ensureNoUpgradeIsRunning(): void + { + $status = $this->readUpgradeStatus(); + + if (CoolifyUpgradeStatus::isRunning((string) $status)) { + Log::warning('Upgrade skipped because another upgrade is running', [ + 'target_version' => $this->latestVersion, + 'status' => $status, + ]); + throw new \Exception( + 'Another Coolify upgrade is already running. Wait for it to finish. '. + 'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.' + ); + } + } + + protected function readUpgradeStatus(): ?string + { + return instant_remote_process( + ['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'], + $this->server, + false + ); + } } diff --git a/app/Livewire/Upgrade.php b/app/Livewire/Upgrade.php index e2ab094944..75d9b32b3e 100644 --- a/app/Livewire/Upgrade.php +++ b/app/Livewire/Upgrade.php @@ -103,7 +103,7 @@ class Upgrade extends Component return ['status' => 'none']; } - $statusFile = '/data/coolify/source/.upgrade-status'; + $statusFile = CoolifyUpgradeStatus::FILE; try { $content = instant_remote_process( diff --git a/app/Services/CoolifyUpgradeStatus.php b/app/Services/CoolifyUpgradeStatus.php index 15f6c22ef2..9027d5bcd9 100644 --- a/app/Services/CoolifyUpgradeStatus.php +++ b/app/Services/CoolifyUpgradeStatus.php @@ -6,8 +6,16 @@ use DateTimeInterface; class CoolifyUpgradeStatus { + public const FILE = '/data/coolify/source/.upgrade-status'; + public const STALE_AFTER_MINUTES = 10; + /** + * An upgrade that has not written a status for this long is treated as stopped, + * so a crashed upgrade does not block new ones forever. + */ + public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15; + /** * @return array{status: string, step?: int, message?: string, running_version: string, target_version: string} */ @@ -77,6 +85,30 @@ class CoolifyUpgradeStatus ]; } + public static function isRunning(string $content, ?DateTimeInterface $now = null): bool + { + $parts = explode('|', trim($content)); + if (count($parts) < 3) { + return false; + } + + [$step, , $timestamp] = $parts; + if (! in_array($step, ['1', '2', '3', '4', '5'], true) || trim($timestamp) === '') { + return false; + } + + try { + $statusTime = new \DateTime($timestamp); + } catch (\Throwable) { + return false; + } + + $now = $now ?? new \DateTime; + $diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60; + + return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES; + } + public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool { if ($runningVersion === '' || $targetVersion === '') { diff --git a/config/constants.php b/config/constants.php index 28b36469e2..58e709a213 100644 --- a/config/constants.php +++ b/config/constants.php @@ -5,7 +5,7 @@ $sshCommandTimeout = (int) env('SSH_COMMAND_TIMEOUT', 3600); return [ 'coolify' => [ - 'version' => env('COOLIFY_VERSION') ?: '4.4.3', + 'version' => env('COOLIFY_VERSION') ?: '4.4.4', 'helper_version' => '1.0.18', 'railpack_version' => '0.40.1', 'self_hosted' => env('SELF_HOSTED', true), diff --git a/other/nightly/versions.json b/other/nightly/versions.json index 40f297c2a1..33ac54fdec 100644 --- a/other/nightly/versions.json +++ b/other/nightly/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "4.4.3" + "version": "4.4.4" }, "nightly": { "version": "4.5-rc.1" diff --git a/resources/views/livewire/upgrade.blade.php b/resources/views/livewire/upgrade.blade.php index b1ad048e1c..2af8873207 100644 --- a/resources/views/livewire/upgrade.blade.php +++ b/resources/views/livewire/upgrade.blade.php @@ -186,6 +186,9 @@ checkUpgradeStatusInterval: null, elapsedInterval: null, healthCheckAttempts: 0, + healthProbePending: false, + healthTimeouts: 0, + statusRequestPending: false, livewireFailures: 0, startTime: null, elapsedTime: 0, @@ -261,6 +264,19 @@ return `${minutes}:${seconds.toString().padStart(2, '0')}`; }, + // Livewire never settles a $wire call when its request fails, and a request to a removed + // container can hang. Time out both so the progress view does not freeze. + withTimeout(promise, ms) { + return Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error('Request timed out')), ms)), + ]); + }, + + fetchUpgradeStatus() { + return this.withTimeout(this.$wire.getUpgradeStatus(), 15000); + }, + mapStepToUI(apiStep) { // Map backend steps (1-6) to UI steps (1-4) // Backend: 1=config, 2=env, 3=pull, 4=stop, 5=start, 6=complete @@ -281,11 +297,30 @@ }, async probeHealth() { + if (this.healthProbePending) { + return; + } + this.healthProbePending = true; + try { + await this.runHealthProbe(); + } finally { + this.healthProbePending = false; + } + }, + + async runHealthProbe() { + if (this.upgradeComplete || this.upgradeError) { + return; + } this.healthCheckAttempts++; const elapsedMinutes = Math.floor((Date.now() - this.startTime) / 60000); try { - const response = await fetch('/api/health'); + const response = await fetch('/api/health', { + cache: 'no-store', + signal: AbortSignal.timeout(10000), + }); + this.healthTimeouts = 0; if (!response.ok) { this.instanceWentDown = true; this.currentStep = 4; @@ -298,7 +333,7 @@ let data; try { - data = await this.$wire.getUpgradeStatus(); + data = await this.fetchUpgradeStatus(); } catch (error) { if (this.instanceWentDown) { this.showSuccess(); @@ -328,6 +363,10 @@ this.currentStatus = data.message ?? this.getReviveStatusMessage(elapsedMinutes, this.healthCheckAttempts); } } catch (error) { + // A busy server can answer slowly, so one timeout does not prove that it is down. + if (error?.name === 'TimeoutError' && ++this.healthTimeouts < 2) { + return; + } console.error('Health check failed:', error); this.instanceWentDown = true; this.currentStep = 4; @@ -356,6 +395,9 @@ }, showSuccess() { + if (this.upgradeComplete) { + return; + } if (this.checkHealthInterval) { clearInterval(this.checkHealthInterval); this.checkHealthInterval = null; @@ -431,12 +473,17 @@ this.serviceDown = false; this.instanceWentDown = false; this.livewireFailures = 0; + this.healthTimeouts = 0; this.startHealthWatch(); // Poll upgrade status via Livewire this.checkUpgradeStatusInterval = setInterval(async () => { + if (this.statusRequestPending) { + return; + } + this.statusRequestPending = true; try { - const data = await this.$wire.getUpgradeStatus(); + const data = await this.fetchUpgradeStatus(); this.livewireFailures = 0; if (data.status === 'in_progress') { this.currentStep = this.mapStepToUI(data.step); @@ -450,6 +497,9 @@ await this.probeHealth(); } } catch (error) { + if (this.upgradeComplete || this.upgradeError) { + return; + } this.livewireFailures++; if (this.livewireFailures < 3) { this.currentStatus = 'Reconnecting. This is expected during an upgrade...'; @@ -468,6 +518,8 @@ } this.revive(); } + } finally { + this.statusRequestPending = false; } }, 2000); } diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php index 31b12bb7ac..ff2bee897e 100644 --- a/tests/Feature/DockerCleanupJobTest.php +++ b/tests/Feature/DockerCleanupJobTest.php @@ -372,7 +372,7 @@ it('gives remote cleanup commands a local timeout and fails once the cleanup dea expect(fn () => CleanupDocker::run($server)) ->toThrow(RuntimeException::class, 'Docker cleanup did not finish within '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' seconds'); - $sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker container prune') || str_contains($command, 'docker image prune'))->values(); + $sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'label=coolify.managed=true') || str_contains($command, 'docker image prune'))->values(); expect(CleanupDocker::REMOTE_COMMANDS_DEADLINE)->toBeLessThan(CleanupDocker::JOB_TIMEOUT) ->and($sshCommands)->toHaveCount(2) @@ -498,3 +498,38 @@ it('scans application images with a fixed number of remote commands and keeps th expect($removal)->not->toContain($kept); } }); + +it('removes stopped Coolify containers but keeps the proxy, databases, applications and services', function (bool $nonRoot) { + $command = (new ReflectionMethod(CleanupDocker::class, 'buildContainerPruneCommand'))->invoke(new CleanupDocker); + preg_match("/--format '([^']+)'/", $command, $format); + $stoppedContainers = collect([ + ['proxy', 'true', ''], + ['database', '', 'database'], + ['application', '', 'application'], + ['service', '', 'service'], + ['helper', '', ''], + ])->map(fn (array $container) => str_replace(['{{.ID}}', '{{.Label "coolify.proxy"}}', '{{.Label "coolify.type"}}'], $container, $format[1])); + + // Fake docker on PATH: `docker ps` prints the stopped containers, `docker rm` prints what it is asked to remove + $bin = sys_get_temp_dir().'/docker-cleanup-'.bin2hex(random_bytes(4)); + mkdir($bin); + file_put_contents("$bin/containers", $stoppedContainers->implode("\n")."\n"); + file_put_contents("$bin/docker", "#!/bin/sh\nif [ \"\$1\" = ps ]; then cat $bin/containers; else echo \"\$@\"; fi\n"); + chmod("$bin/docker", 0755); + + if ($nonRoot) { + $server = new Server; + $server->user = 'coolify'; + $command = parseCommandsByLineForSudo(collect([$command]), $server)[0]; + file_put_contents("$bin/sudo", "#!/bin/sh\nexec \"\$@\"\n"); + chmod("$bin/sudo", 0755); + } + + $output = shell_exec('PATH='.escapeshellarg("$bin:".getenv('PATH')).' /bin/sh -c '.escapeshellarg($command)); + + array_map('unlink', glob("$bin/*")); + rmdir($bin); + + expect($command)->toContain('--filter label=coolify.managed=true') + ->and(trim((string) $output))->toBe('rm helper'); +})->with(['root' => false, 'non-root sudo parser' => true]); diff --git a/tests/Unit/CoolifyUpgradeStatusTest.php b/tests/Unit/CoolifyUpgradeStatusTest.php index f701a8a6d3..dcc074da7d 100644 --- a/tests/Unit/CoolifyUpgradeStatusTest.php +++ b/tests/Unit/CoolifyUpgradeStatusTest.php @@ -128,3 +128,26 @@ it('reports whether the running version has reached the target', function (strin 'missing running version' => ['', '4.3.1', false], 'missing target version' => ['4.3.1', '', false], ]); + +it('detects a running upgrade only for recent intermediate steps', function (string $step, int $minutesAgo, bool $expected) { + $now = Carbon::parse('2026-08-13T12:00:00+00:00'); + $content = "{$step}|Some message|".$now->copy()->subMinutes($minutesAgo)->toIso8601String(); + + expect(CoolifyUpgradeStatus::isRunning($content, $now))->toBe($expected); +})->with([ + 'recent step 1' => ['1', 0, true], + 'step 5 at lock expiry' => ['5', 15, true], + 'step 3 after lock expiry' => ['3', 16, false], + 'complete' => ['6', 1, false], + 'error' => ['error', 1, false], +]); + +it('does not detect a running upgrade for empty or malformed status', function (string $content) { + expect(CoolifyUpgradeStatus::isRunning($content))->toBeFalse(); +})->with([ + 'empty' => '', + 'missing fields' => '3|Pulling Docker images', + 'blank timestamp' => '3|Pulling Docker images|', + 'whitespace timestamp' => "3|Pulling Docker images| \t", + 'invalid timestamp' => '3|Pulling Docker images|not-a-date', +]); diff --git a/tests/Unit/UpdateCoolifyTest.php b/tests/Unit/UpdateCoolifyTest.php index d27a634018..ddc8c2546b 100644 --- a/tests/Unit/UpdateCoolifyTest.php +++ b/tests/Unit/UpdateCoolifyTest.php @@ -40,6 +40,19 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi Once::flush(); } +function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify +{ + return new class($status) extends UpdateCoolify + { + public function __construct(private string $status) {} + + protected function readUpgradeStatus(): ?string + { + return $this->status; + } + }; +} + afterEach(function () { Mockery::close(); }); @@ -316,3 +329,37 @@ it('prevents downgrade even with manual update', function () { expect($e->getMessage())->toContain('4.0.0'); } }); + +it('does not start a second upgrade while a recent upgrade status is in progress', function () { + Queue::fake(); + config([ + 'app.env' => 'testing', + 'constants.coolify.version' => '4.0.9', + 'constants.ssh.mux_enabled' => false, + ]); + updateCoolifyTestCreateRootServerAndSettings(); + Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]); + $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String()); + + expect(fn () => $action->handle(manual_update: true)) + ->toThrow(Exception::class, 'Another Coolify upgrade is already running'); + + expect(Activity::query()->count())->toBe(0) + ->and(InstanceSettings::findOrFail(0)->new_version_available)->toBeFalsy(); +}); + +it('starts an upgrade when the previous upgrade status is older than the lock expiry', function () { + Queue::fake(); + config([ + 'app.env' => 'testing', + 'constants.coolify.version' => '4.0.9', + 'constants.ssh.mux_enabled' => false, + ]); + updateCoolifyTestCreateRootServerAndSettings(); + Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]); + updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(16)->toIso8601String()) + ->handle(manual_update: true); + + expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first())) + ->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'"); +}); diff --git a/versions.json b/versions.json index 40f297c2a1..33ac54fdec 100644 --- a/versions.json +++ b/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "4.4.3" + "version": "4.4.4" }, "nightly": { "version": "4.5-rc.1"