mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
feat(upgrade): block upgrades when the disk is almost full
An upgrade pulls new images and recreates the Coolify containers. A full disk during an upgrade can leave Coolify broken. - Measure free space on the Docker root dir and /data/coolify of localhost; require 5 GB. An unknown value does not block. - Manual upgrade: show the free and required space, with Run Docker cleanup, Check again, and a confirmed Upgrade anyway override. - Automatic upgrade: skip it and notify the team (disk usage channels, at most once per 24 hours per server). - upgrade.sh: stop before any change when the disk is low. A new 5th argument skips the check (used by the override). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b850a06d26
commit
376e7aab24
@@ -6,11 +6,13 @@ use App\Models\InstanceSettings;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\Server\UpgradeSkippedLowDiskSpace;
|
||||
use App\Support\RemoteProcessCommand;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Illuminate\Support\Once;
|
||||
@@ -53,6 +55,48 @@ function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify
|
||||
};
|
||||
}
|
||||
|
||||
function updateCoolifyTestActionWithDiskSpace(?float $availableGb): UpdateCoolify
|
||||
{
|
||||
return new class($availableGb) extends UpdateCoolify
|
||||
{
|
||||
public function __construct(private ?float $availableGb) {}
|
||||
|
||||
protected function readUpgradeStatus(): ?string
|
||||
{
|
||||
return '';
|
||||
}
|
||||
|
||||
protected function availableDiskSpaceGb(): ?float
|
||||
{
|
||||
return $this->availableGb;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function updateCoolifyTestPrepareUpgrade(): void
|
||||
{
|
||||
Queue::fake();
|
||||
config([
|
||||
'app.env' => 'testing',
|
||||
'constants.coolify.version' => '4.0.9',
|
||||
'constants.coolify.helper_version' => '1.0.14',
|
||||
'constants.coolify.upgrade_script_url' => 'https://cdn.example.com/upgrade.sh',
|
||||
'constants.ssh.mux_enabled' => false,
|
||||
]);
|
||||
|
||||
updateCoolifyTestCreateRootServerAndSettings([
|
||||
'is_auto_update_enabled' => true,
|
||||
'new_version_available' => true,
|
||||
'docker_registry_url' => 'docker.io',
|
||||
]);
|
||||
|
||||
Http::fake([
|
||||
'*' => Http::response([
|
||||
'coolify' => ['v4' => ['version' => '4.0.10']],
|
||||
], 200),
|
||||
]);
|
||||
}
|
||||
|
||||
afterEach(function () {
|
||||
Mockery::close();
|
||||
});
|
||||
@@ -363,3 +407,50 @@ it('starts an upgrade when the previous upgrade status is older than the lock ex
|
||||
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
|
||||
->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'");
|
||||
});
|
||||
|
||||
it('skips an automatic update and notifies the root team when the disk is almost full', function () {
|
||||
updateCoolifyTestPrepareUpgrade();
|
||||
Notification::fake();
|
||||
Team::findOrFail(0)->discordNotificationSettings()->update(['discord_enabled' => true, 'server_disk_usage_discord_notifications' => true]);
|
||||
|
||||
updateCoolifyTestActionWithDiskSpace(3.2)->handle();
|
||||
|
||||
expect(Activity::query()->count())->toBe(0)
|
||||
->and((bool) InstanceSettings::findOrFail(0)->new_version_available)->toBeTrue();
|
||||
Notification::assertSentTo(
|
||||
Team::findOrFail(0),
|
||||
UpgradeSkippedLowDiskSpace::class,
|
||||
fn (UpgradeSkippedLowDiskSpace $notification) => $notification->version === '4.0.10'
|
||||
&& $notification->availableGb === 3.2
|
||||
&& str_contains((string) $notification->toMail()->render(), '3.2 GB free'),
|
||||
);
|
||||
});
|
||||
|
||||
it('fails a manual update when the disk is almost full', function () {
|
||||
updateCoolifyTestPrepareUpgrade();
|
||||
Notification::fake();
|
||||
|
||||
expect(fn () => updateCoolifyTestActionWithDiskSpace(3.2)->handle(manual_update: true))
|
||||
->toThrow(Exception::class, 'Not enough free disk space to upgrade: 3.2 GB free, 5 GB required.');
|
||||
|
||||
expect(Activity::query()->count())->toBe(0);
|
||||
Notification::assertNothingSent();
|
||||
});
|
||||
|
||||
it('upgrades when the free disk space cannot be measured', function () {
|
||||
updateCoolifyTestPrepareUpgrade();
|
||||
|
||||
updateCoolifyTestActionWithDiskSpace(null)->handle();
|
||||
|
||||
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
|
||||
->toEndWith("bash /data/coolify/source/upgrade.sh '4.0.10' '1.0.14' 'docker.io'");
|
||||
});
|
||||
|
||||
it('upgrades and tells the upgrade script to skip its disk space check when the user overrides it', function () {
|
||||
updateCoolifyTestPrepareUpgrade();
|
||||
|
||||
updateCoolifyTestActionWithDiskSpace(3.2)->handle(manual_update: true, skipDiskSpaceCheck: true);
|
||||
|
||||
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
|
||||
->toEndWith("bash /data/coolify/source/upgrade.sh '4.0.10' '1.0.14' 'docker.io' 'false' 'true'");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user