fix(upgrade): prevent duplicate upgrades and stalled progress polling

Block new upgrades while a recent upgrade is running, allowing retries after 15 minutes without a status update.

Add request timeouts and polling guards to keep upgrade progress responsive during restarts.
This commit is contained in:
Andras Bacsai
2026-10-09 08:01:20 +02:00
parent 3f953d1dce
commit df953bdcd1
6 changed files with 184 additions and 4 deletions
+28
View File
@@ -3,6 +3,7 @@
namespace App\Actions\Server;
use App\Models\Server;
use App\Services\CoolifyUpgradeStatus;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Sleep;
@@ -116,6 +117,8 @@ class UpdateCoolify
private function update()
{
$this->ensureNoUpgradeIsRunning();
$latestHelperImageVersion = getHelperVersion();
$upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
$registryUrl = coolifyRegistryUrl();
@@ -128,4 +131,29 @@ class UpdateCoolify
escapeshellarg($registryUrl),
], $this->server);
}
private function ensureNoUpgradeIsRunning(): void
{
$status = $this->readUpgradeStatus();
if (CoolifyUpgradeStatus::isRunning((string) $status)) {
Log::warning('Upgrade skipped because another upgrade is running', [
'target_version' => $this->latestVersion,
'status' => $status,
]);
throw new \Exception(
'Another Coolify upgrade is already running. Wait for it to finish. '.
'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
);
}
}
protected function readUpgradeStatus(): ?string
{
return instant_remote_process(
['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'],
$this->server,
false
);
}
}
+1 -1
View File
@@ -103,7 +103,7 @@ class Upgrade extends Component
return ['status' => 'none'];
}
$statusFile = '/data/coolify/source/.upgrade-status';
$statusFile = CoolifyUpgradeStatus::FILE;
try {
$content = instant_remote_process(
+32
View File
@@ -6,8 +6,16 @@ use DateTimeInterface;
class CoolifyUpgradeStatus
{
public const FILE = '/data/coolify/source/.upgrade-status';
public const STALE_AFTER_MINUTES = 10;
/**
* An upgrade that has not written a status for this long is treated as stopped,
* so a crashed upgrade does not block new ones forever.
*/
public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15;
/**
* @return array{status: string, step?: int, message?: string, running_version: string, target_version: string}
*/
@@ -77,6 +85,30 @@ class CoolifyUpgradeStatus
];
}
public static function isRunning(string $content, ?DateTimeInterface $now = null): bool
{
$parts = explode('|', trim($content));
if (count($parts) < 3) {
return false;
}
[$step, , $timestamp] = $parts;
if (! in_array($step, ['1', '2', '3', '4', '5'], true)) {
return false;
}
try {
$statusTime = new \DateTime($timestamp);
} catch (\Throwable) {
return false;
}
$now = $now ?? new \DateTime;
$diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60;
return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES;
}
public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool
{
if ($runningVersion === '' || $targetVersion === '') {