feat(updates): notify on automatic update start, failure, and finish (#12472)

app/Actions/Server/UpdateCoolify.php

@@ -2,9 +2,11 @@
 
 namespace App\Actions\Server;
 
+use App\Exceptions\UpgradeAlreadyRunningException;
 use App\Jobs\DatabaseBackupJob;
 use App\Models\Server;
 use App\Models\StandalonePostgresql;
+use App\Services\AutoUpdateNotifier;
 use App\Services\CoolifyUpgradeStatus;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Sleep;
@@ -40,6 +42,7 @@ class UpdateCoolify
             if ($manual_update && filled($settings->update_custom_tag)) {
                 $this->latestVersion = $settings->update_custom_tag;
                 $this->backupDatabase();
+                $this->ensureNoUpgradeIsRunning();
                 $this->update(registryUrl: 'ghcr.io');
             }
 
@@ -127,7 +130,15 @@ class UpdateCoolify
         }
 
         $this->backupDatabase();
+        // Check the lock before the attempt is saved, so a running upgrade keeps its own saved attempt.
+        $this->ensureNoUpgradeIsRunning();
+        if (! $manual_update) {
+            AutoUpdateNotifier::saveAttempt((string) $this->latestVersion);
+        }
         $this->update();
+        if (! $manual_update) {
+            AutoUpdateNotifier::started((string) $this->currentVersion, (string) $this->latestVersion);
+        }
         $settings->new_version_available = false;
         $settings->save();
     }
@@ -151,8 +162,6 @@ class UpdateCoolify
 
     private function update(?string $registryUrl = null)
     {
-        $this->ensureNoUpgradeIsRunning();
-
         $latestHelperImageVersion = getHelperVersion();
         $upgradeScriptUrl = $this->upgradeScriptUrl ?? config('constants.coolify.upgrade_script_url');
         $registryUrl ??= coolifyRegistryUrl();
@@ -175,10 +184,7 @@ class UpdateCoolify
                 'target_version' => $this->latestVersion,
                 'status' => $status,
             ]);
-            throw new \Exception(
-                'Another Coolify upgrade is already running. Wait for it to finish. '.
-                'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
-            );
+            throw new UpgradeAlreadyRunningException;
         }
     }
 

app/Console/Commands/Init.php

@@ -15,6 +15,7 @@ use App\Models\ScheduledTaskExecution;
 use App\Models\Server;
 use App\Models\StandalonePostgresql;
 use App\Models\User;
+use App\Services\AutoUpdateNotifier;
 use App\Support\ResourceStartActivity;
 use App\Support\RootSetupWindow;
 use Carbon\Carbon;
@@ -74,6 +75,12 @@ class Init extends Command
         }
         get_public_ips();
 
+        try {
+            AutoUpdateNotifier::checkResult();
+        } catch (\Throwable $e) {
+            echo "Could not check the automatic update result: {$e->getMessage()}\n";
+        }
+
         // Backward compatibility
         $this->replaceSlashInEnvironmentName();
         $this->restoreCoolifyDbBackup();

app/Exceptions/UpgradeAlreadyRunningException.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Exceptions;
+
+use App\Services\CoolifyUpgradeStatus;
+use Exception;
+
+/**
+ * An upgrade was not started because another upgrade script is still running.
+ * This is not a failure of the running upgrade.
+ */
+class UpgradeAlreadyRunningException extends Exception
+{
+    public function __construct()
+    {
+        parent::__construct(
+            'Another Coolify upgrade is already running. Wait for it to finish. '.
+            'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
+        );
+    }
+}

app/Jobs/CheckForUpdatesJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Services\AutoUpdateNotifier;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -31,6 +32,7 @@ class CheckForUpdatesJob implements ShouldBeEncrypted, ShouldQueue
             if (isDev() || isCloud()) {
                 return false;
             }
+            AutoUpdateNotifier::checkResult();
             $versions = refresh_versions_file($this->cdnTries);
             if ($versions === null) {
                 return false;

app/Jobs/UpdateCoolifyJob.php

@@ -3,7 +3,10 @@
 namespace App\Jobs;
 
 use App\Actions\Server\UpdateCoolify;
+use App\Exceptions\UpgradeAlreadyRunningException;
 use App\Models\Server;
+use App\Services\AutoUpdateNotifier;
+use App\Services\CoolifyUpgradeStatus;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -45,9 +48,17 @@ class UpdateCoolifyJob implements ShouldBeEncrypted, ShouldQueue
             UpdateCoolify::run(false); // false means it's not a manual update
 
             Log::info('Coolify upgrade script started. Result: /data/coolify/source/.upgrade-status, log: /data/coolify/source/upgrade-*.log.');
+        } catch (UpgradeAlreadyRunningException $e) {
+            // The running upgrade keeps its saved attempt, so its result is still reported.
+            Log::warning('Automatic update skipped: '.$e->getMessage());
         } catch (\Throwable $e) {
             Log::error('UpdateCoolifyJob failed: '.$e->getMessage());
-            // Consider implementing a notification to administrators
+            try {
+                CoolifyUpgradeStatus::writeError(Server::findOrFail(0), $e->getMessage());
+            } catch (\Throwable $statusException) {
+                report($statusException);
+            }
+            AutoUpdateNotifier::failed($e->getMessage());
         }
     }
 }

app/Livewire/Upgrade.php

@@ -84,6 +84,11 @@ class Upgrade extends Component
                     UpdateCoolify::run(manual_update: true);
                 } catch (\Throwable $e) {
                     report($e);
+                    try {
+                        CoolifyUpgradeStatus::writeError(Server::findOrFail(0), $e->getMessage());
+                    } catch (\Throwable $statusException) {
+                        report($statusException);
+                    }
                 }
             })->afterResponse();
 

app/Models/InstanceSettings.php

@@ -28,6 +28,8 @@ class InstanceSettings extends Model
         'do_not_track',
         'is_auto_update_enabled',
         'is_backup_before_update_enabled',
+        'auto_update_target_version',
+        'auto_update_started_at',
         'is_registration_enabled',
         'disable_registration_when_oauth_enabled',
         'next_channel',
@@ -115,6 +117,7 @@ class InstanceSettings extends Model
         'allowed_ip_ranges' => 'array',
         'is_auto_update_enabled' => 'boolean',
         'is_backup_before_update_enabled' => 'boolean',
+        'auto_update_started_at' => 'datetime',
         'is_registration_enabled' => 'boolean',
         'disable_registration_when_oauth_enabled' => 'boolean',
         'auto_update_frequency' => 'string',

app/Services/AutoUpdateNotifier.php

@@ -0,0 +1,175 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\InstanceSettings;
+use App\Models\Server;
+use App\Models\Team;
+use App\Notifications\Internal\GeneralNotification;
+use Carbon\CarbonInterface;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Log;
+
+/**
+ * Tells the root team when an automatic update starts, fails, or finishes.
+ * The new version reports the result after it starts, so the attempt is saved in the instance settings.
+ */
+class AutoUpdateNotifier
+{
+    /** An attempt that has not reached the target version after this many minutes is reported as failed. */
+    public const RESULT_TIMEOUT_MINUTES = 30;
+
+    /**
+     * Saves the attempt before the upgrade script runs, so the new version can report the result
+     * also when the container restarts before started() is called.
+     */
+    public static function saveAttempt(string $targetVersion): void
+    {
+        instanceSettings()->update([
+            'auto_update_target_version' => $targetVersion,
+            'auto_update_started_at' => now(),
+        ]);
+    }
+
+    public static function started(string $currentVersion, string $targetVersion): void
+    {
+        self::notify("Automatic update started: Coolify {$currentVersion} to {$targetVersion}.", true);
+    }
+
+    public static function failed(string $reason): void
+    {
+        self::clearAttempt();
+        self::notifyFailure($reason);
+    }
+
+    /**
+     * Reports the result of the saved attempt: success when this version is the target,
+     * failure when the upgrade script wrote an error or the attempt timed out.
+     */
+    public static function checkResult(): void
+    {
+        $settings = instanceSettings();
+        $targetVersion = $settings->auto_update_target_version;
+        if (blank($targetVersion)) {
+            return;
+        }
+
+        $startedAt = $settings->auto_update_started_at;
+        $runningVersion = (string) config('constants.coolify.version');
+        if (CoolifyUpgradeStatus::hasReachedTargetVersion($runningVersion, $targetVersion)) {
+            if (self::claimAttempt($settings, $targetVersion)
+                && ! self::notify("Automatic update finished: Coolify is now on {$runningVersion}.", true)) {
+                self::restoreAttempt($targetVersion, $startedAt);
+            }
+
+            return;
+        }
+
+        $status = self::readUpgradeStatus();
+        [$step, $message, $writtenAt] = array_pad(explode('|', trim($status)), 3, '');
+        if ($step === 'error' && self::isWrittenDuringAttempt($writtenAt, $startedAt)) {
+            if (self::claimAttempt($settings, $targetVersion)
+                && ! self::notifyFailure("the upgrade to {$targetVersion} stopped with this error: {$message}. Log: /data/coolify/source/upgrade-*.log")) {
+                self::restoreAttempt($targetVersion, $startedAt);
+            }
+
+            return;
+        }
+
+        $timedOut = ! $startedAt || $startedAt->lt(now()->subMinutes(self::RESULT_TIMEOUT_MINUTES));
+        if ($timedOut && ! CoolifyUpgradeStatus::isRunning($status) && self::claimAttempt($settings, $targetVersion)
+            && ! self::notifyFailure("Coolify {$targetVersion} did not start within ".self::RESULT_TIMEOUT_MINUTES." minutes. Coolify {$runningVersion} is still running. Log: /data/coolify/source/upgrade-*.log")) {
+            self::restoreAttempt($targetVersion, $startedAt);
+        }
+    }
+
+    /**
+     * An error from an earlier attempt stays in the status file until the new upgrade script writes its first step.
+     */
+    private static function isWrittenDuringAttempt(string $writtenAt, ?CarbonInterface $startedAt): bool
+    {
+        if (! $startedAt) {
+            return true;
+        }
+
+        try {
+            return Carbon::parse(trim($writtenAt))->gte($startedAt->copy()->startOfSecond());
+        } catch (\Throwable) {
+            return false;
+        }
+    }
+
+    private static function readUpgradeStatus(): string
+    {
+        try {
+            $server = Server::find(0);
+
+            return $server ? (string) instant_remote_process(['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'], $server, false) : '';
+        } catch (\Throwable $e) {
+            Log::warning('Could not read the upgrade status', ['error' => $e->getMessage()]);
+
+            return '';
+        }
+    }
+
+    /**
+     * Clears the attempt only if it is still saved for this target version.
+     * Returns false when another run already reported it, so each result is sent one time.
+     */
+    private static function claimAttempt(InstanceSettings $settings, string $targetVersion): bool
+    {
+        $claimed = InstanceSettings::query()
+            ->whereKey($settings->getKey())
+            ->where('auto_update_target_version', $targetVersion)
+            ->update([
+                'auto_update_target_version' => null,
+                'auto_update_started_at' => null,
+            ]) === 1;
+
+        $settings->refresh();
+
+        return $claimed;
+    }
+
+    /**
+     * Saves a claimed attempt again after its result could not be queued, so the next check sends it again.
+     * Does nothing when a newer attempt was saved in the meantime.
+     */
+    private static function restoreAttempt(string $targetVersion, ?CarbonInterface $startedAt): void
+    {
+        InstanceSettings::query()
+            ->whereKey(0)
+            ->whereNull('auto_update_target_version')
+            ->update([
+                'auto_update_target_version' => $targetVersion,
+                'auto_update_started_at' => $startedAt,
+            ]);
+    }
+
+    private static function clearAttempt(): void
+    {
+        instanceSettings()->update([
+            'auto_update_target_version' => null,
+            'auto_update_started_at' => null,
+        ]);
+    }
+
+    private static function notifyFailure(string $reason): bool
+    {
+        return self::notify("Automatic update failed: {$reason}\n\nFix the cause, then start the update again: ".base_url().'/settings/updates', false);
+    }
+
+    /** Returns false when the notification could not be queued. */
+    private static function notify(string $message, bool $success): bool
+    {
+        try {
+            Team::find(0)?->notify(new GeneralNotification($message, $success));
+
+            return true;
+        } catch (\Throwable $e) {
+            Log::warning('Could not send the automatic update notification', ['error' => $e->getMessage()]);
+
+            return false;
+        }
+    }
+}

app/Services/CoolifyUpgradeStatus.php

@@ -2,12 +2,18 @@
 
 namespace App\Services;
 
+use App\Models\Server;
 use DateTimeInterface;
 
 class CoolifyUpgradeStatus
 {
     public const FILE = '/data/coolify/source/.upgrade-status';
 
+    /**
+     * scripts/upgrade.sh holds this lock while it writes a status, so an error write cannot interleave with it.
+     */
+    public const LOCK_FILE = '/data/coolify/source/.upgrade-status.lock';
+
     public const STALE_AFTER_MINUTES = 10;
 
     /**
@@ -85,6 +91,39 @@ class CoolifyUpgradeStatus
         ];
     }
 
+    /**
+     * Writes an error that stopped an upgrade before the upgrade script started, so the upgrade modal shows it.
+     * Does nothing while an upgrade script runs, because the error then belongs to this attempt only.
+     * The write happens only if the status file did not change after the read, so a status that an
+     * upgrade script wrote in the meantime is kept.
+     */
+    public static function writeError(Server $server, string $message): void
+    {
+        $current = (string) instant_remote_process(['cat '.self::FILE.' 2>/dev/null || true'], $server, false);
+        if (self::isRunning($current)) {
+            return;
+        }
+
+        $line = 'error|'.trim((string) preg_replace('/[|\r\n]+/', ' ', $message)).'|'.now()->toIso8601String();
+        instant_remote_process([self::compareAndWriteCommand($current, $line)], $server, false);
+    }
+
+    /**
+     * Shell command that writes $line to the status file only if the file still contains $expected.
+     * It holds the lock that scripts/upgrade.sh takes for its status writes, if flock is available.
+     */
+    public static function compareAndWriteCommand(string $expected, string $line): string
+    {
+        $script = 'if command -v flock >/dev/null 2>&1; then exec 9>"$2"; flock -w 10 9 || exit 1; fi; '
+            .'if [ "$(cat "$1" 2>/dev/null)" = "$(printf %s "$3" | base64 -d)" ]; then printf %s "$4" | base64 -d > "$1"; echo >> "$1"; fi';
+
+        return 'sh -c '.escapeshellarg($script).' _ '
+            .escapeshellarg(self::FILE).' '
+            .escapeshellarg(self::LOCK_FILE).' '
+            .escapeshellarg(base64_encode(trim($expected))).' '
+            .escapeshellarg(base64_encode($line));
+    }
+
     public static function isRunning(string $content, ?DateTimeInterface $now = null): bool
     {
         $parts = explode('|', trim($content));

database/migrations/2026_11_04_080000_add_auto_update_attempt_to_instance_settings_table.php

@@ -0,0 +1,29 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('instance_settings', function (Blueprint $table) {
+            $table->string('auto_update_target_version')->nullable();
+            $table->timestamp('auto_update_started_at')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('instance_settings', function (Blueprint $table) {
+            $table->dropColumn(['auto_update_target_version', 'auto_update_started_at']);
+        });
+    }
+};

scripts/upgrade.sh

@@ -14,6 +14,7 @@ else
 fi
 SKIP_BACKUP=${4:-false}
 STATUS_FILE="/data/coolify/source/.upgrade-status"
+STATUS_LOCK_FILE="/data/coolify/source/.upgrade-status.lock"
 
 DATE=$(date +%Y-%m-%d-%H-%M-%S)
 LOGFILE="/data/coolify/source/upgrade-${DATE}.log"
@@ -32,10 +33,17 @@ log_section() {
 }
 
 # Helper function to write upgrade status for API polling
+# Holds the lock that Coolify takes when it writes an error before an upgrade starts, so neither write is lost.
 write_status() {
-    local step="$1"
-    local message="$2"
-    echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
+    local line="${1}|${2}|$(date -Iseconds)"
+    if command -v flock >/dev/null 2>&1; then
+        (
+            flock -w 10 9 || true
+            echo "$line" > "$STATUS_FILE"
+        ) 9>"$STATUS_LOCK_FILE"
+    else
+        echo "$line" > "$STATUS_FILE"
+    fi
 }
 
 # Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64).

tests/Unit/UpdateCoolifyTest.php

@@ -451,3 +451,26 @@ it('does not start the update when the instance backup fails', function () {
         ->toThrow(Exception::class, 'The database backup failed, so the update was not started.')
         ->and(Activity::query()->count())->toBe(0);
 });
+
+it('saves the automatic update attempt before it queues the upgrade script', function () {
+    Notification::fake();
+    updateCoolifyTestPrepareUpgrade();
+    $savedTargetVersion = null;
+    Activity::created(function () use (&$savedTargetVersion) {
+        $savedTargetVersion = InstanceSettings::findOrFail(0)->auto_update_target_version;
+    });
+
+    updateCoolifyTestActionWithStatus('')->handle();
+
+    expect($savedTargetVersion)->toBe('4.0.10');
+});
+
+it('keeps the saved attempt of a running upgrade when an automatic update is skipped', function () {
+    updateCoolifyTestPrepareUpgrade();
+    InstanceSettings::findOrFail(0)->update(['auto_update_target_version' => '4.0.8', 'auto_update_started_at' => now()->subMinutes(5)]);
+    $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String());
+
+    expect(fn () => $action->handle())->toThrow(Exception::class, 'Another Coolify upgrade is already running');
+
+    expect(InstanceSettings::findOrFail(0)->auto_update_target_version)->toBe('4.0.8');
+});