From 376e7aab2449547ed3d8cd6b3d97b86dc7401d07 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:26:53 +0200 Subject: [PATCH] feat(upgrade): block upgrades when the disk is almost full An upgrade pulls new images and recreates the Coolify containers. A full disk during an upgrade can leave Coolify broken. - Measure free space on the Docker root dir and /data/coolify of localhost; require 5 GB. An unknown value does not block. - Manual upgrade: show the free and required space, with Run Docker cleanup, Check again, and a confirmed Upgrade anyway override. - Automatic upgrade: skip it and notify the team (disk usage channels, at most once per 24 hours per server). - upgrade.sh: stop before any change when the disk is low. A new 5th argument skips the check (used by the override). Co-Authored-By: Claude Opus 5.5 --- app/Actions/Server/UpdateCoolify.php | 59 ++++++++- app/Livewire/Upgrade.php | 64 ++++++++- .../Server/UpgradeSkippedLowDiskSpace.php | 121 ++++++++++++++++++ app/Services/CoolifyUpgradeDiskSpace.php | 63 +++++++++ .../upgrade-skipped-low-disk-space.blade.php | 5 + resources/views/livewire/upgrade.blade.php | 96 +++++++++++++- scripts/upgrade.sh | 20 +++ tests/Feature/UpgradeComponentTest.php | 85 ++++++++++++ tests/Unit/CoolifyUpgradeDiskSpaceTest.php | 30 +++++ tests/Unit/UpdateCoolifyTest.php | 91 +++++++++++++ 10 files changed, 617 insertions(+), 17 deletions(-) create mode 100644 app/Notifications/Server/UpgradeSkippedLowDiskSpace.php create mode 100644 app/Services/CoolifyUpgradeDiskSpace.php create mode 100644 resources/views/emails/upgrade-skipped-low-disk-space.blade.php create mode 100644 tests/Unit/CoolifyUpgradeDiskSpaceTest.php diff --git a/app/Actions/Server/UpdateCoolify.php b/app/Actions/Server/UpdateCoolify.php index 3c60c34379..7011ec5429 100644 --- a/app/Actions/Server/UpdateCoolify.php +++ b/app/Actions/Server/UpdateCoolify.php @@ -3,6 +3,8 @@ namespace App\Actions\Server; use App\Models\Server; +use App\Notifications\Server\UpgradeSkippedLowDiskSpace; +use App\Services\CoolifyUpgradeDiskSpace; use App\Services\CoolifyUpgradeStatus; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; @@ -19,7 +21,7 @@ class UpdateCoolify public ?string $currentVersion = null; - public function handle($manual_update = false) + public function handle($manual_update = false, bool $skipDiskSpaceCheck = false) { if (isDev()) { Sleep::for(10)->seconds(); @@ -110,12 +112,16 @@ class UpdateCoolify ); } - $this->update(); + if (! $skipDiskSpaceCheck && ! $this->ensureEnoughDiskSpace($manual_update)) { + return; + } + + $this->update($skipDiskSpaceCheck); $settings->new_version_available = false; $settings->save(); } - private function update() + private function update(bool $skipDiskSpaceCheck) { $this->ensureNoUpgradeIsRunning(); @@ -123,15 +129,54 @@ class UpdateCoolify $upgradeScriptUrl = config('constants.coolify.upgrade_script_url'); $registryUrl = coolifyRegistryUrl(); + $upgradeCommand = 'bash /data/coolify/source/upgrade.sh '. + escapeshellarg($this->latestVersion).' '. + escapeshellarg($latestHelperImageVersion).' '. + escapeshellarg($registryUrl); + if ($skipDiskSpaceCheck) { + // Arguments 4 and 5: do not skip the backup, skip the disk space check. + $upgradeCommand .= " 'false' 'true'"; + } + remote_process([ "curl -fsSL {$upgradeScriptUrl} -o /data/coolify/source/upgrade.sh", - 'bash /data/coolify/source/upgrade.sh '. - escapeshellarg($this->latestVersion).' '. - escapeshellarg($latestHelperImageVersion).' '. - escapeshellarg($registryUrl), + $upgradeCommand, ], $this->server); } + /** + * Automatic updates are skipped and the team is notified. Manual updates fail with an error. + */ + private function ensureEnoughDiskSpace(bool $manualUpdate): bool + { + $availableGb = $this->availableDiskSpaceGb(); + if (! CoolifyUpgradeDiskSpace::isLow($availableGb)) { + return true; + } + + Log::warning('Upgrade skipped because of low disk space', [ + 'target_version' => $this->latestVersion, + 'available_gb' => $availableGb, + 'required_gb' => CoolifyUpgradeDiskSpace::REQUIRED_GB, + 'manual_update' => $manualUpdate, + ]); + + if ($manualUpdate) { + throw new \Exception( + "Not enough free disk space to upgrade: {$availableGb} GB free, ".CoolifyUpgradeDiskSpace::REQUIRED_GB.' GB required.' + ); + } + + $this->server->team?->notify(new UpgradeSkippedLowDiskSpace($this->server, $this->latestVersion, $availableGb)); + + return false; + } + + protected function availableDiskSpaceGb(): ?float + { + return app(CoolifyUpgradeDiskSpace::class)->availableGb($this->server); + } + private function ensureNoUpgradeIsRunning(): void { $status = $this->readUpgradeStatus(); diff --git a/app/Livewire/Upgrade.php b/app/Livewire/Upgrade.php index 75d9b32b3e..62d922e4f6 100644 --- a/app/Livewire/Upgrade.php +++ b/app/Livewire/Upgrade.php @@ -3,8 +3,10 @@ namespace App\Livewire; use App\Actions\Server\UpdateCoolify; +use App\Jobs\DockerCleanupJob; use App\Models\InstanceSettings; use App\Models\Server; +use App\Services\CoolifyUpgradeDiskSpace; use App\Services\CoolifyUpgradeStatus; use Livewire\Component; @@ -62,19 +64,30 @@ class Upgrade extends Component $this->isUpgradeAvailable = $hasNewerVersion && $newVersionAvailable; } - public function upgrade() + /** + * Start the upgrade. Without $skipDiskSpaceCheck, the upgrade does not start when the disk is almost full. + * + * @return array{status: 'started'|'low_disk_space', available_gb?: float|null, required_gb?: int}|null + */ + public function upgrade(bool $skipDiskSpaceCheck = false) { try { if (! isInstanceAdmin()) { abort(403); } if ($this->updateInProgress) { - return; + return null; + } + if (! $skipDiskSpaceCheck) { + $diskSpace = $this->checkDiskSpace(); + if ($diskSpace['status'] === 'low_disk_space') { + return $diskSpace; + } } $this->updateInProgress = true; - dispatch(function () { + dispatch(function () use ($skipDiskSpaceCheck) { try { - UpdateCoolify::run(manual_update: true); + UpdateCoolify::run(manual_update: true, skipDiskSpaceCheck: $skipDiskSpaceCheck); } catch (\Throwable $e) { report($e); } @@ -85,7 +98,50 @@ class Upgrade extends Component 'resource' => 'instance', 'from_version' => config('constants.coolify.version'), 'to_version' => get_latest_version_of_coolify(), + 'skip_disk_space_check' => $skipDiskSpaceCheck, ]); + + return ['status' => 'started']; + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + + /** + * @return array{status: 'low_disk_space'|'ok', available_gb: float|null, required_gb: int} + */ + public function checkDiskSpace(): array + { + if (! isInstanceAdmin()) { + abort(403); + } + + $server = Server::find(0); + $availableGb = $server ? app(CoolifyUpgradeDiskSpace::class)->availableGb($server) : null; + + return [ + 'status' => CoolifyUpgradeDiskSpace::isLow($availableGb) ? 'low_disk_space' : 'ok', + 'available_gb' => $availableGb, + 'required_gb' => CoolifyUpgradeDiskSpace::REQUIRED_GB, + ]; + } + + public function runDockerCleanup() + { + try { + if (! isInstanceAdmin()) { + abort(403); + } + $server = Server::findOrFail(0); + DockerCleanupJob::dispatch($server, true); + auditLog('ui.server.docker_cleanup_started', [ + 'team_id' => $server->team_id, + 'server_uuid' => $server->uuid, + 'server_name' => $server->name, + 'delete_unused_volumes' => false, + 'delete_unused_networks' => false, + ]); + $this->dispatch('success', 'Docker cleanup started. When it is done, click "Check again".'); } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Notifications/Server/UpgradeSkippedLowDiskSpace.php b/app/Notifications/Server/UpgradeSkippedLowDiskSpace.php new file mode 100644 index 0000000000..4a015009b6 --- /dev/null +++ b/app/Notifications/Server/UpgradeSkippedLowDiskSpace.php @@ -0,0 +1,121 @@ +onQueue('high'); + } + + public function via(object $notifiable): array + { + return $notifiable->getEnabledChannels('server_disk_usage'); + } + + public function throttleSubject(): ?Model + { + return $this->server; + } + + public function throttleIntervalMinutes(): int + { + return 24 * 60; + } + + private function cleanupUrl(): string + { + return base_url().'/server/'.$this->server->uuid.'/docker-cleanup'; + } + + public function toMail(): MailMessage + { + $mail = new MailMessage; + $mail->subject("Coolify: Automatic update to {$this->version} skipped, not enough free disk space"); + $mail->view('emails.upgrade-skipped-low-disk-space', [ + 'version' => $this->version, + 'available_gb' => $this->availableGb, + 'required_gb' => $this->requiredGb, + 'cleanup_url' => $this->cleanupUrl(), + ]); + + return $mail; + } + + public function toDiscord(): DiscordMessage + { + $message = new DiscordMessage( + title: ':warning: Automatic update skipped', + description: "Coolify did not update to {$this->version} because the server does not have enough free disk space.", + color: DiscordMessage::warningColor(), + isCritical: true, + ); + + $message->addField('Free', "{$this->availableGb} GB", true); + $message->addField('Required', "{$this->requiredGb} GB", true); + $message->addField('What to do?', '[Docker cleanup]('.$this->cleanupUrl().')', true); + + return $message; + } + + public function toTelegram(): array + { + return [ + 'message' => "Coolify: Automatic update to {$this->version} skipped.\nFree disk space: {$this->availableGb} GB. Required: {$this->requiredGb} GB.\nFree up disk space (for example, run a Docker cleanup): {$this->cleanupUrl()}", + ]; + } + + public function toPushover(): PushoverMessage + { + return new PushoverMessage( + title: 'Automatic update skipped', + level: 'warning', + message: "Coolify did not update to {$this->version} because the server does not have enough free disk space.

Free: {$this->availableGb} GB.
Required: {$this->requiredGb} GB.", + buttons: [ + 'Docker cleanup' => $this->cleanupUrl(), + ], + ); + } + + public function toSlack(): SlackMessage + { + $description = "Coolify did not update to {$this->version} because the server does not have enough free disk space.\n"; + $description .= "Free: {$this->availableGb} GB\n"; + $description .= "Required: {$this->requiredGb} GB\n\n"; + $description .= 'Free up disk space (for example, run a Docker cleanup): '.$this->cleanupUrl(); + + return new SlackMessage( + title: 'Automatic update skipped', + description: $description, + color: SlackMessage::warningColor() + ); + } + + public function toWebhook(): array + { + return [ + 'success' => false, + 'message' => 'Automatic update skipped, not enough free disk space', + 'event' => 'upgrade_skipped_low_disk_space', + 'version' => $this->version, + 'server_name' => $this->server->name, + 'server_uuid' => $this->server->uuid, + 'available_gb' => $this->availableGb, + 'required_gb' => $this->requiredGb, + 'url' => $this->cleanupUrl(), + ]; + } +} diff --git a/app/Services/CoolifyUpgradeDiskSpace.php b/app/Services/CoolifyUpgradeDiskSpace.php new file mode 100644 index 0000000000..a0ec794f21 --- /dev/null +++ b/app/Services/CoolifyUpgradeDiskSpace.php @@ -0,0 +1,63 @@ +/dev/null || true"], + $server, + false + )); + + $paths = collect([...self::UPGRADE_PATHS, $dockerRootDir ?: self::DEFAULT_DOCKER_ROOT_DIR]) + ->map(fn (string $path) => escapeshellarg($path)) + ->implode(' '); + + $output = instant_remote_process(["df -Pk {$paths} 2>/dev/null || true"], $server, false); + + return self::parseDfOutput((string) $output); + } + + /** + * Read the smallest "Available" column (in 1K blocks) from `df -Pk` output and return it in GB. + */ + public static function parseDfOutput(string $output): ?float + { + $availableKb = collect(preg_split('/\R/', trim($output))) + ->skip(1) + ->map(fn (string $line) => preg_split('/\s+/', trim($line))[3] ?? null) + ->filter(fn (?string $value) => $value !== null && ctype_digit($value)) + ->map(fn (string $value) => (int) $value) + ->min(); + + if ($availableKb === null) { + return null; + } + + return round($availableKb / 1024 / 1024, 1); + } + + public static function isLow(?float $availableGb): bool + { + return $availableGb !== null && $availableGb < self::REQUIRED_GB; + } +} diff --git a/resources/views/emails/upgrade-skipped-low-disk-space.blade.php b/resources/views/emails/upgrade-skipped-low-disk-space.blade.php new file mode 100644 index 0000000000..fe9ecac38f --- /dev/null +++ b/resources/views/emails/upgrade-skipped-low-disk-space.blade.php @@ -0,0 +1,5 @@ + +Coolify did not update to {{ $version }} because the server does not have enough free disk space ({{ $available_gb }} GB free, {{ $required_gb }} GB required). + +Free up disk space, for example with a [Docker cleanup]({{ $cleanup_url }}). The next automatic update will try again. + diff --git a/resources/views/livewire/upgrade.blade.php b/resources/views/livewire/upgrade.blade.php index 2af8873207..aa4234474c 100644 --- a/resources/views/livewire/upgrade.blade.php +++ b/resources/views/livewire/upgrade.blade.php @@ -137,8 +137,53 @@ + {{-- Low Disk Space View --}} + + {{-- Confirmation View --}} -