fix(sentinel): repair stalled Sentinel and show why pushes fail

- Restart a running Sentinel that has not pushed for 10 minutes, at most
  once per hour. The restart applies the current URL and token.
- Restart an unhealthy Sentinel. Do not restart while its health check
  starts or while its API does not answer yet, which caused restart loops.
- Remember why pushes fail: the reason Coolify rejects a push, and the last
  push error from Sentinel's /api/push-status. Show it on the Sentinel page
  while Sentinel is out of sync.
- When the instance URL or public IP changes, move remote servers that use
  the generated Sentinel URL to the new URL. Custom URLs stay.

app/Http/Controllers/Api/SentinelController.php

@@ -60,6 +60,7 @@ class SentinelController extends Controller
                 'server_uuid' => $server->uuid,
                 'team_id' => $server->team_id,
             ]);
+            $server->rememberSentinelPushProblem('Coolify rejected the push: the subscription of this team is not active.');
 
             return response()->json(['message' => 'Unauthorized'], 401);
         }
@@ -69,6 +70,7 @@ class SentinelController extends Controller
                 'server_uuid' => $server->uuid,
                 'team_id' => $server->team_id,
             ]);
+            $server->rememberSentinelPushProblem('Coolify rejected the push: the server is marked unreachable, not usable, or disabled. Validate the server connection.');
 
             return response()->json(['message' => 'Server is not functional'], 401);
         }
@@ -78,6 +80,7 @@ class SentinelController extends Controller
                 'server_uuid' => $server->uuid,
                 'team_id' => $server->team_id,
             ]);
+            $server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match. Sync Sentinel to apply the current token.');
 
             return response()->json(['message' => 'Unauthorized'], 401);
         }
@@ -86,6 +89,8 @@ class SentinelController extends Controller
         ]);
 
         if ($validator->fails()) {
+            $server->rememberSentinelPushProblem('Coolify rejected the push: the payload is not valid.');
+
             return response()->json(serializeApiResponse([
                 'message' => 'Validation failed.',
                 'errors' => $validator->errors(),
@@ -101,6 +106,7 @@ class SentinelController extends Controller
         $server->sentinelHeartbeat();
 
         if (! $wasSentinelLive) {
+            $server->rememberSentinelPushProblem(null);
             SentinelSynchronized::dispatch($server);
         }
 

app/Jobs/CheckAndStartSentinelJob.php

@@ -11,11 +11,22 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
 
 class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * A running Sentinel that has not pushed for this long is restarted with the current settings.
+     * This repairs a changed Coolify URL or token, and a hung push loop.
+     */
+    public const STALL_RESTART_AFTER_SECONDS = 600;
+
+    /** A stalled Sentinel is restarted at most once in this period, because a firewall problem stays after a restart. */
+    public const STALL_RESTART_INTERVAL_SECONDS = 3600;
+
     public $timeout = 120;
 
     /**
@@ -40,28 +51,63 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, Sho
 
         // An SSH failure throws here, so it is never mistaken for a missing container.
         // A missing container gives empty output and exit code 0.
-        $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}}'"], $this->server);
-        if ($sentinelStatus !== 'running') {
+        $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}} {{.Status}}'"], $this->server);
+        if (! str_starts_with((string) $sentinelStatus, 'running')) {
             $this->startSentinel($latestVersion);
 
             return;
         }
-        // If sentinel is running, check if it needs an update
-        $runningVersion = instant_remote_process_with_timeout(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false);
-        if (empty($runningVersion)) {
-            $runningVersion = '0.0.0';
-        }
-        if ($latestVersion === '0.0.0' && $runningVersion === '0.0.0') {
-            $this->startSentinel('latest');
+        if (str_contains($sentinelStatus, '(unhealthy)')) {
+            $this->server->rememberSentinelPushProblem('The Sentinel container was unhealthy. Coolify restarted it.');
+            $this->startSentinel($latestVersion);
 
             return;
-        } else {
-            if (version_compare($runningVersion, $latestVersion, '<')) {
-                $this->startSentinel($latestVersion);
-
-                return;
-            }
         }
+
+        // One SSH call reads the version and the last push result. Sentinel versions without
+        // /api/push-status give an empty second line.
+        $output = instant_remote_process_with_timeout([
+            "docker exec coolify-sentinel sh -c 'curl -s http://127.0.0.1:8888/api/version; echo; curl -sf -H \"Authorization: Bearer \$TOKEN\" http://127.0.0.1:8888/api/push-status'",
+        ], $this->server, false);
+        [$runningVersion, $pushStatus] = array_pad(explode("\n", trim((string) $output), 2), 2, null);
+        $this->rememberPushError($pushStatus);
+
+        // An empty version means that the API does not answer yet, for example while Sentinel starts.
+        // A Sentinel whose API stays down fails its health check and is restarted above.
+        if (filled($runningVersion) && version_compare(trim($runningVersion), $latestVersion, '<')) {
+            $this->startSentinel($latestVersion);
+
+            return;
+        }
+
+        if ($this->isStalled() && Cache::add("sentinel:stall-restart:{$this->server->id}", true, self::STALL_RESTART_INTERVAL_SECONDS)) {
+            $this->startSentinel($latestVersion);
+        }
+    }
+
+    /**
+     * Sentinel is stalled when it has not pushed for STALL_RESTART_AFTER_SECONDS since its last push,
+     * or since its last start when it never pushed.
+     */
+    private function isStalled(): bool
+    {
+        $lastSignOfLife = $this->server->sentinel_waiting_since ?? $this->server->sentinel_updated_at;
+        if ($lastSignOfLife === null) {
+            return true;
+        }
+        $stallAfter = max(self::STALL_RESTART_AFTER_SECONDS, $this->server->waitBeforeDoingSshCheck());
+
+        return Carbon::parse($lastSignOfLife)->isBefore(now()->subSeconds($stallAfter));
+    }
+
+    private function rememberPushError(?string $pushStatus): void
+    {
+        $status = json_decode(trim((string) $pushStatus), true);
+        if (! is_array($status) || data_get($status, 'consecutive_failures', 0) < 1 || blank(data_get($status, 'last_error'))) {
+            return;
+        }
+
+        $this->server->rememberSentinelPushProblem('Sentinel cannot push to Coolify: '.data_get($status, 'last_error'));
     }
 
     private function sentinelIsEnabled(): bool

app/Models/InstanceSettings.php

@@ -111,6 +111,13 @@ class InstanceSettings extends Model
         static::updated(function ($settings) {
             // Clear once() cache so subsequent calls get fresh data
             Once::flush();
+
+            if ($settings->wasChanged(['fqdn', 'public_ipv4', 'public_ipv6'])) {
+                ServerSetting::followInstanceUrlChange(
+                    ServerSetting::instanceSentinelUrl($settings->getOriginal('fqdn'), $settings->getOriginal('public_ipv4'), $settings->getOriginal('public_ipv6')),
+                    ServerSetting::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6),
+                );
+            }
         });
     }
 

app/Models/Server.php

@@ -33,9 +33,11 @@ use Illuminate\Database\Eloquent\Relations\MorphMany;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Str;
 use Illuminate\Support\Stringable;
 use OpenApi\Attributes as OA;
 use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
@@ -1082,6 +1084,25 @@ $siteAddress {
         return "sentinel:reported-version:{$serverId}";
     }
 
+    /**
+     * The last known reason why Sentinel pushes do not arrive. It is shown while Sentinel is out of sync.
+     */
+    public function sentinelPushProblem(): ?string
+    {
+        return Cache::get("sentinel:push-problem:{$this->id}");
+    }
+
+    public function rememberSentinelPushProblem(?string $problem): void
+    {
+        if (blank($problem)) {
+            Cache::forget("sentinel:push-problem:{$this->id}");
+
+            return;
+        }
+
+        Cache::put("sentinel:push-problem:{$this->id}", Str::limit($problem, 500), now()->addDay());
+    }
+
     public function isSentinelLive()
     {
         return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));

app/Models/ServerSetting.php

@@ -315,6 +315,38 @@ class ServerSetting extends Model
         return $url;
     }
 
+    /**
+     * The Sentinel push URL that the instance URL or public IP gives to a remote server.
+     */
+    public static function instanceSentinelUrl(?string $fqdn, ?string $publicIpv4, ?string $publicIpv6): ?string
+    {
+        return match (true) {
+            filled($fqdn) => $fqdn,
+            filled($publicIpv4) => 'http://'.$publicIpv4.':8000',
+            filled($publicIpv6) => 'http://'.$publicIpv6.':8000',
+            default => null,
+        };
+    }
+
+    /**
+     * Move remote servers that use the generated Sentinel URL to the new instance URL. Saving the
+     * new URL restarts their Sentinel. Servers with a custom URL keep it.
+     */
+    public static function followInstanceUrlChange(?string $oldUrl, ?string $newUrl): void
+    {
+        if (blank($oldUrl) || blank($newUrl) || $oldUrl === $newUrl) {
+            return;
+        }
+
+        self::query()
+            ->where('sentinel_custom_url', $oldUrl)
+            ->where('server_id', '!=', 0)
+            ->each(function (ServerSetting $setting) use ($newUrl) {
+                $setting->sentinel_custom_url = $newUrl;
+                $setting->save();
+            });
+    }
+
     public function restoreDefaultSentinelConfiguration(): void
     {
         $this->generateSentinelUrl(save: false, ignoreEvent: true);
@@ -327,18 +359,12 @@ class ServerSetting extends Model
 
     public function generateSentinelUrl(bool $save = true, bool $ignoreEvent = false): ?string
     {
-        $domain = null;
         $settings = InstanceSettings::get();
         if ($this->server->isLocalhost()) {
             $domain = 'http://coolify:8080';
-        } elseif ($settings->fqdn) {
-            $domain = $settings->fqdn;
-        } elseif ($settings->public_ipv4) {
-            $domain = 'http://'.$settings->public_ipv4.':8000';
-        } elseif ($settings->public_ipv6) {
-            $domain = 'http://'.$settings->public_ipv6.':8000';
         } else {
-            $domain = $this->sentinelUrlFromCurrentRequest();
+            $domain = self::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6)
+                ?? $this->sentinelUrlFromCurrentRequest();
         }
         $this->sentinel_custom_url = $domain;
         if ($save) {

resources/views/livewire/server/sentinel.blade.php

@@ -38,6 +38,9 @@
                 <x-callout type="warning" title="Sentinel is out of sync">
                     <div class="space-y-3">
                         <p>Sentinel has not reported within the expected interval. Check these items before syncing again:</p>
+                        @if ($sentinelPushProblem = $server->sentinelPushProblem())
+                            <p><span class="font-medium">Last error:</span> <code class="break-all">{{ $sentinelPushProblem }}</code></p>
+                        @endif
                         <ul class="list-disc space-y-1 pl-4">
                             <li>Confirm that the <code>coolify-sentinel</code> container is running.</li>
                             <li>

tests/Feature/SentinelPushProblemTest.php

@@ -0,0 +1,76 @@
+<?php
+
+use App\Actions\Server\StartSentinel;
+use App\Models\InstanceSettings;
+use App\Models\Server;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    config(['app.maintenance.store' => 'array']);
+    Server::flushIdentityMap();
+    Queue::fake();
+    Cache::flush();
+    InstanceSettings::forceCreate(['id' => 0, 'fqdn' => 'https://old.example.com']);
+
+    $user = User::factory()->create();
+    $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]);
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+});
+
+function pushSentinelForProblemTest(Server $server, string $token)
+{
+    return test()->postJson('/api/v1/sentinel/push', ['containers' => []], ['Authorization' => 'Bearer '.$token]);
+}
+
+it('remembers when Coolify rejects a push from a server that is not functional', function () {
+    $this->server->settings->update(['is_reachable' => false]);
+
+    pushSentinelForProblemTest($this->server, $this->server->settings->sentinel_token)->assertUnauthorized();
+
+    expect($this->server->sentinelPushProblem())->toContain('marked unreachable');
+});
+
+it('remembers when Coolify rejects a push with an old token', function () {
+    $oldToken = $this->server->settings->sentinel_token;
+    $this->server->settings->generateSentinelToken(ignoreEvent: true);
+
+    pushSentinelForProblemTest($this->server, $oldToken)->assertUnauthorized();
+
+    expect($this->server->sentinelPushProblem())->toContain('token does not match');
+});
+
+it('forgets the problem after the next accepted push', function () {
+    $this->server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match.');
+    $this->server->forceFill(['sentinel_updated_at' => now()->subHour()])->save();
+
+    pushSentinelForProblemTest($this->server, $this->server->settings->sentinel_token)->assertOk();
+
+    expect($this->server->sentinelPushProblem())->toBeNull();
+});
+
+it('moves Sentinel to the new instance URL when the instance URL changes', function () {
+    $this->server->settings->update(['sentinel_custom_url' => 'https://old.example.com', 'is_sentinel_enabled' => true]);
+    Queue::fake();
+
+    $settings = InstanceSettings::get();
+    $settings->fqdn = 'https://new.example.com';
+    $settings->save();
+
+    expect($this->server->settings->refresh()->sentinel_custom_url)->toBe('https://new.example.com');
+    StartSentinel::assertPushed(fn (StartSentinel $action, array $arguments) => $arguments[0]->is($this->server));
+});
+
+it('keeps a custom Sentinel URL when the instance URL changes', function () {
+    $this->server->settings->update(['sentinel_custom_url' => 'https://custom.example.com']);
+
+    $settings = InstanceSettings::get();
+    $settings->fqdn = 'https://new.example.com';
+    $settings->save();
+
+    expect($this->server->settings->refresh()->sentinel_custom_url)->toBe('https://custom.example.com');
+});

tests/Feature/Server/CheckAndStartSentinelJobTest.php

@@ -7,6 +7,7 @@ use App\Models\PrivateKey;
 use App\Models\Server;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Process;
 use Illuminate\Support\Facades\Storage;
@@ -19,6 +20,7 @@ beforeEach(function () {
     Storage::fake('ssh-keys');
     Storage::fake('ssh-mux');
     config()->set('constants.ssh.max_retries', 1);
+    config()->set('constants.ssh.mux_enabled', false);
     Http::fake(['*' => Http::response(['coolify' => ['sentinel' => ['version' => '1.0.0']]])]);
 
     $user = User::factory()->create();
@@ -61,6 +63,7 @@ it('does not treat an SSH failure as a missing container', function () {
 });
 
 it('does not restart a running Sentinel that is up to date', function () {
+    $this->server->forceFill(['sentinel_updated_at' => now()])->save();
     Process::fake([
         '*docker ps*' => Process::result(output: 'running'),
         '*api/version*' => Process::result(output: '1.0.0'),
@@ -71,7 +74,7 @@ it('does not restart a running Sentinel that is up to date', function () {
 });
 
 it('uses sudo for the status check on a non-root server', function () {
-    $this->server->update(['user' => 'ubuntu']);
+    $this->server->forceFill(['user' => 'ubuntu', 'sentinel_updated_at' => now()])->save();
     Process::fake([
         '*docker ps*' => Process::result(output: 'running'),
         '*api/version*' => Process::result(output: '1.0.0'),
@@ -82,3 +85,93 @@ it('uses sudo for the status check on a non-root server', function () {
 
     Process::assertRan(fn ($process) => str_contains($process->command, 'sudo docker ps -a'));
 });
+
+it('restarts an unhealthy Sentinel and remembers why', function () {
+    $this->server->forceFill(['sentinel_updated_at' => now()])->save();
+    Process::fake(['*docker ps*' => Process::result(output: 'running Up 2 hours (unhealthy)')]);
+    StartSentinel::shouldRun()->once();
+
+    (new CheckAndStartSentinelJob($this->server))->handle();
+
+    expect($this->server->sentinelPushProblem())->toContain('unhealthy');
+});
+
+it('does not restart a Sentinel whose health check is still starting', function () {
+    $this->server->forceFill(['sentinel_updated_at' => now()])->save();
+    Process::fake([
+        '*docker ps*' => Process::result(output: 'running Up 30 seconds (health: starting)'),
+        '*api/version*' => Process::result(output: '1.0.0'),
+    ]);
+    StartSentinel::shouldNotRun();
+
+    (new CheckAndStartSentinelJob($this->server))->handle();
+});
+
+it('restarts a running Sentinel that stopped pushing, at most once per hour', function () {
+    Cache::flush();
+    $this->server->forceFill(['sentinel_updated_at' => now()->subMinutes(15), 'sentinel_waiting_since' => null])->save();
+    Process::fake([
+        '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),
+        '*api/version*' => Process::result(output: '1.0.0'),
+    ]);
+    StartSentinel::shouldRun()->once();
+
+    (new CheckAndStartSentinelJob($this->server))->handle();
+    (new CheckAndStartSentinelJob($this->server))->handle();
+});
+
+it('does not restart a Sentinel that started recently and has not pushed yet', function () {
+    Cache::flush();
+    $this->server->forceFill(['sentinel_updated_at' => now()->subDays(5), 'sentinel_waiting_since' => now()->subMinutes(3)])->save();
+    Process::fake([
+        '*docker ps*' => Process::result(output: 'running Up 3 minutes (healthy)'),
+        '*api/version*' => Process::result(output: '1.0.0'),
+    ]);
+    StartSentinel::shouldNotRun();
+
+    (new CheckAndStartSentinelJob($this->server))->handle();
+});
+
+it('remembers the push error that Sentinel reports', function () {
+    $this->server->forceFill(['sentinel_updated_at' => now()])->save();
+    $pushStatus = json_encode([
+        'last_attempt_at' => '2026-10-04T12:00:00Z',
+        'last_success_at' => null,
+        'last_error' => 'http: error sending request: Connection refused',
+        'last_status' => null,
+        'consecutive_failures' => 4,
+    ]);
+    Process::fake([
+        '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),
+        '*api/version*' => Process::result(output: "1.0.0\n{$pushStatus}"),
+    ]);
+    StartSentinel::shouldNotRun();
+
+    (new CheckAndStartSentinelJob($this->server))->handle();
+
+    expect($this->server->sentinelPushProblem())->toBe('Sentinel cannot push to Coolify: http: error sending request: Connection refused');
+});
+
+it('reads the push status with the token from inside the Sentinel container on a non-root server', function () {
+    $this->server->forceFill(['user' => 'ubuntu', 'sentinel_updated_at' => now()])->save();
+    Process::fake([
+        '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),
+        '*api/version*' => Process::result(output: '1.0.0'),
+    ]);
+
+    (new CheckAndStartSentinelJob($this->server->refresh()))->handle();
+
+    Process::assertRan(fn ($process) => str_contains($process->command, "sudo bash -c 'docker exec coolify-sentinel sh -c '\\''curl")
+        && str_contains($process->command, 'Bearer $TOKEN'));
+});
+
+it('does not restart a starting Sentinel whose API does not answer yet', function () {
+    $this->server->forceFill(['sentinel_updated_at' => now()->subDays(5), 'sentinel_waiting_since' => now()])->save();
+    Process::fake([
+        '*docker ps*' => Process::result(output: 'running Up 3 seconds (health: starting)'),
+        '*api/version*' => Process::result(output: ''),
+    ]);
+    StartSentinel::shouldNotRun();
+
+    (new CheckAndStartSentinelJob($this->server))->handle();
+});