fix(upgrade): prevent duplicate upgrades and stalled progress polling

Block new upgrades while a recent upgrade is running, allowing retries after 15 minutes without a status update.

Add request timeouts and polling guards to keep upgrade progress responsive during restarts.
This commit is contained in:
Andras Bacsai
2026-10-09 08:01:20 +02:00
parent 3f953d1dce
commit df953bdcd1
6 changed files with 184 additions and 4 deletions
+21
View File
@@ -128,3 +128,24 @@ it('reports whether the running version has reached the target', function (strin
'missing running version' => ['', '4.3.1', false],
'missing target version' => ['4.3.1', '', false],
]);
it('detects a running upgrade only for recent intermediate steps', function (string $step, int $minutesAgo, bool $expected) {
$now = Carbon::parse('2026-08-13T12:00:00+00:00');
$content = "{$step}|Some message|".$now->copy()->subMinutes($minutesAgo)->toIso8601String();
expect(CoolifyUpgradeStatus::isRunning($content, $now))->toBe($expected);
})->with([
'recent step 1' => ['1', 0, true],
'step 5 at lock expiry' => ['5', 15, true],
'step 3 after lock expiry' => ['3', 16, false],
'complete' => ['6', 1, false],
'error' => ['error', 1, false],
]);
it('does not detect a running upgrade for empty or malformed status', function (string $content) {
expect(CoolifyUpgradeStatus::isRunning($content))->toBeFalse();
})->with([
'empty' => '',
'missing fields' => '3|Pulling Docker images',
'invalid timestamp' => '3|Pulling Docker images|not-a-date',
]);
+47
View File
@@ -40,6 +40,19 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi
Once::flush();
}
function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify
{
return new class($status) extends UpdateCoolify
{
public function __construct(private string $status) {}
protected function readUpgradeStatus(): ?string
{
return $this->status;
}
};
}
afterEach(function () {
Mockery::close();
});
@@ -316,3 +329,37 @@ it('prevents downgrade even with manual update', function () {
expect($e->getMessage())->toContain('4.0.0');
}
});
it('does not start a second upgrade while a recent upgrade status is in progress', function () {
Queue::fake();
config([
'app.env' => 'testing',
'constants.coolify.version' => '4.0.9',
'constants.ssh.mux_enabled' => false,
]);
updateCoolifyTestCreateRootServerAndSettings();
Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
$action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String());
expect(fn () => $action->handle(manual_update: true))
->toThrow(Exception::class, 'Another Coolify upgrade is already running');
expect(Activity::query()->count())->toBe(0)
->and(InstanceSettings::findOrFail(0)->new_version_available)->toBeFalsy();
});
it('starts an upgrade when the previous upgrade status is older than the lock expiry', function () {
Queue::fake();
config([
'app.env' => 'testing',
'constants.coolify.version' => '4.0.9',
'constants.ssh.mux_enabled' => false,
]);
updateCoolifyTestCreateRootServerAndSettings();
Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(16)->toIso8601String())
->handle(manual_update: true);
expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'");
});