feat(upgrade): block upgrades when the disk is almost full

An upgrade pulls new images and recreates the Coolify containers. A full
disk during an upgrade can leave Coolify broken.

- Measure free space on the Docker root dir and /data/coolify of localhost;
  require 5 GB. An unknown value does not block.
- Manual upgrade: show the free and required space, with Run Docker cleanup,
  Check again, and a confirmed Upgrade anyway override.
- Automatic upgrade: skip it and notify the team (disk usage channels, at
  most once per 24 hours per server).
- upgrade.sh: stop before any change when the disk is low. A new 5th
  argument skips the check (used by the override).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2026-10-09 13:26:53 +02:00
co-authored by Claude Opus 5.5
parent b850a06d26
commit 376e7aab24
10 changed files with 617 additions and 17 deletions
+91
View File
@@ -6,11 +6,13 @@ use App\Models\InstanceSettings;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use App\Notifications\Server\UpgradeSkippedLowDiskSpace;
use App\Support\RemoteProcessCommand;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Once;
@@ -53,6 +55,48 @@ function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify
};
}
function updateCoolifyTestActionWithDiskSpace(?float $availableGb): UpdateCoolify
{
return new class($availableGb) extends UpdateCoolify
{
public function __construct(private ?float $availableGb) {}
protected function readUpgradeStatus(): ?string
{
return '';
}
protected function availableDiskSpaceGb(): ?float
{
return $this->availableGb;
}
};
}
function updateCoolifyTestPrepareUpgrade(): void
{
Queue::fake();
config([
'app.env' => 'testing',
'constants.coolify.version' => '4.0.9',
'constants.coolify.helper_version' => '1.0.14',
'constants.coolify.upgrade_script_url' => 'https://cdn.example.com/upgrade.sh',
'constants.ssh.mux_enabled' => false,
]);
updateCoolifyTestCreateRootServerAndSettings([
'is_auto_update_enabled' => true,
'new_version_available' => true,
'docker_registry_url' => 'docker.io',
]);
Http::fake([
'*' => Http::response([
'coolify' => ['v4' => ['version' => '4.0.10']],
], 200),
]);
}
afterEach(function () {
Mockery::close();
});
@@ -363,3 +407,50 @@ it('starts an upgrade when the previous upgrade status is older than the lock ex
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'");
});
it('skips an automatic update and notifies the root team when the disk is almost full', function () {
updateCoolifyTestPrepareUpgrade();
Notification::fake();
Team::findOrFail(0)->discordNotificationSettings()->update(['discord_enabled' => true, 'server_disk_usage_discord_notifications' => true]);
updateCoolifyTestActionWithDiskSpace(3.2)->handle();
expect(Activity::query()->count())->toBe(0)
->and((bool) InstanceSettings::findOrFail(0)->new_version_available)->toBeTrue();
Notification::assertSentTo(
Team::findOrFail(0),
UpgradeSkippedLowDiskSpace::class,
fn (UpgradeSkippedLowDiskSpace $notification) => $notification->version === '4.0.10'
&& $notification->availableGb === 3.2
&& str_contains((string) $notification->toMail()->render(), '3.2 GB free'),
);
});
it('fails a manual update when the disk is almost full', function () {
updateCoolifyTestPrepareUpgrade();
Notification::fake();
expect(fn () => updateCoolifyTestActionWithDiskSpace(3.2)->handle(manual_update: true))
->toThrow(Exception::class, 'Not enough free disk space to upgrade: 3.2 GB free, 5 GB required.');
expect(Activity::query()->count())->toBe(0);
Notification::assertNothingSent();
});
it('upgrades when the free disk space cannot be measured', function () {
updateCoolifyTestPrepareUpgrade();
updateCoolifyTestActionWithDiskSpace(null)->handle();
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
->toEndWith("bash /data/coolify/source/upgrade.sh '4.0.10' '1.0.14' 'docker.io'");
});
it('upgrades and tells the upgrade script to skip its disk space check when the user overrides it', function () {
updateCoolifyTestPrepareUpgrade();
updateCoolifyTestActionWithDiskSpace(3.2)->handle(manual_update: true, skipDiskSpaceCheck: true);
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
->toEndWith("bash /data/coolify/source/upgrade.sh '4.0.10' '1.0.14' 'docker.io' 'false' 'true'");
});