mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
fix(upgrade): prevent duplicate upgrades and stalled progress polling
Block new upgrades while a recent upgrade is running, allowing retries after 15 minutes without a status update. Add request timeouts and polling guards to keep upgrade progress responsive during restarts.
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
namespace App\Actions\Server;
|
||||
|
||||
use App\Models\Server;
|
||||
use App\Services\CoolifyUpgradeStatus;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Sleep;
|
||||
@@ -116,6 +117,8 @@ class UpdateCoolify
|
||||
|
||||
private function update()
|
||||
{
|
||||
$this->ensureNoUpgradeIsRunning();
|
||||
|
||||
$latestHelperImageVersion = getHelperVersion();
|
||||
$upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
|
||||
$registryUrl = coolifyRegistryUrl();
|
||||
@@ -128,4 +131,29 @@ class UpdateCoolify
|
||||
escapeshellarg($registryUrl),
|
||||
], $this->server);
|
||||
}
|
||||
|
||||
private function ensureNoUpgradeIsRunning(): void
|
||||
{
|
||||
$status = $this->readUpgradeStatus();
|
||||
|
||||
if (CoolifyUpgradeStatus::isRunning((string) $status)) {
|
||||
Log::warning('Upgrade skipped because another upgrade is running', [
|
||||
'target_version' => $this->latestVersion,
|
||||
'status' => $status,
|
||||
]);
|
||||
throw new \Exception(
|
||||
'Another Coolify upgrade is already running. Wait for it to finish. '.
|
||||
'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
protected function readUpgradeStatus(): ?string
|
||||
{
|
||||
return instant_remote_process(
|
||||
['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'],
|
||||
$this->server,
|
||||
false
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ class Upgrade extends Component
|
||||
return ['status' => 'none'];
|
||||
}
|
||||
|
||||
$statusFile = '/data/coolify/source/.upgrade-status';
|
||||
$statusFile = CoolifyUpgradeStatus::FILE;
|
||||
|
||||
try {
|
||||
$content = instant_remote_process(
|
||||
|
||||
@@ -6,8 +6,16 @@ use DateTimeInterface;
|
||||
|
||||
class CoolifyUpgradeStatus
|
||||
{
|
||||
public const FILE = '/data/coolify/source/.upgrade-status';
|
||||
|
||||
public const STALE_AFTER_MINUTES = 10;
|
||||
|
||||
/**
|
||||
* An upgrade that has not written a status for this long is treated as stopped,
|
||||
* so a crashed upgrade does not block new ones forever.
|
||||
*/
|
||||
public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15;
|
||||
|
||||
/**
|
||||
* @return array{status: string, step?: int, message?: string, running_version: string, target_version: string}
|
||||
*/
|
||||
@@ -77,6 +85,30 @@ class CoolifyUpgradeStatus
|
||||
];
|
||||
}
|
||||
|
||||
public static function isRunning(string $content, ?DateTimeInterface $now = null): bool
|
||||
{
|
||||
$parts = explode('|', trim($content));
|
||||
if (count($parts) < 3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
[$step, , $timestamp] = $parts;
|
||||
if (! in_array($step, ['1', '2', '3', '4', '5'], true)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$statusTime = new \DateTime($timestamp);
|
||||
} catch (\Throwable) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$now = $now ?? new \DateTime;
|
||||
$diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60;
|
||||
|
||||
return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES;
|
||||
}
|
||||
|
||||
public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool
|
||||
{
|
||||
if ($runningVersion === '' || $targetVersion === '') {
|
||||
|
||||
Reference in New Issue
Block a user