fix(sentinel): stop hourly check job pile-up and self-heal Sentinel

- Start Sentinel with --restart unless-stopped so Docker restarts it
  after a crash or reboot.
- Make CheckAndStartSentinelJob unique per server so copies cannot
  collect in a slow queue.
- Read X-Sentinel-Version on push and update an outdated Sentinel
  without SSH; skip the hourly SSH version check for servers that
  report their version.

app/Actions/Server/StartSentinel.php

@@ -117,7 +117,7 @@ class StartSentinel
             ? '-v '.escapeshellarg("{$trafficLogDirectory}:{$trafficLogDirectory}:ro").' '
             : '';
         $network = $server->isLocalhost() ? ' --network coolify' : '';
-        $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";
+        $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel --restart unless-stopped -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";
 
         $server->sentinelHeartbeat(isReset: true);
         $server->forceFill(['sentinel_waiting_since' => now()])->save();

app/Http/Controllers/Api/SentinelController.php

@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;
 
 use App\Events\SentinelSynchronized;
 use App\Http\Controllers\Controller;
+use App\Jobs\CheckAndStartSentinelJob;
 use App\Jobs\PushServerUpdateJob;
 use App\Models\Server;
 use Exception;
@@ -103,6 +104,8 @@ class SentinelController extends Controller
             SentinelSynchronized::dispatch($server);
         }
 
+        $this->updateSentinelIfOutdated($server, $request->header('X-Sentinel-Version'));
+
         if ($this->shouldDispatchUpdate($server, $data)) {
             PushServerUpdateJob::dispatch($server, $data);
         }
@@ -110,6 +113,30 @@ class SentinelController extends Controller
         return response()->json(['message' => 'ok'], 200);
     }
 
+    /**
+     * Sentinel sends its version on every push, so an outdated Sentinel is found without SSH.
+     * The reported version also tells ServerManagerJob to skip its hourly SSH version check.
+     * Old Sentinel versions do not send the header and keep the hourly SSH check.
+     */
+    private function updateSentinelIfOutdated(Server $server, ?string $runningVersion): void
+    {
+        if (! is_string($runningVersion) || preg_match('/^\d+\.\d+\.\d+$/', $runningVersion) !== 1) {
+            return;
+        }
+
+        Cache::put(Server::sentinelReportedVersionCacheKey($server->id), $runningVersion, now()->addHours(2));
+
+        $latestVersion = data_get(get_versions_data(), 'coolify.sentinel.version');
+        if (blank($latestVersion) || version_compare($runningVersion, $latestVersion, '>=')) {
+            return;
+        }
+
+        // Retry at most hourly if an update fails.
+        if (Cache::add("sentinel:update-dispatched:{$server->id}", true, 3600)) {
+            CheckAndStartSentinelJob::dispatch($server);
+        }
+    }
+
     /**
      * Decide whether PushServerUpdateJob should be dispatched for this push.
      *

app/Jobs/CheckAndStartSentinelJob.php

@@ -6,19 +6,30 @@ use App\Actions\Server\StartSentinel;
 use App\Models\Server;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 
-class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue
+class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     public $timeout = 120;
 
+    /**
+     * Only one check per server can wait in the queue, so a slow queue cannot collect copies.
+     */
+    public $uniqueFor = 600;
+
     public function __construct(public Server $server) {}
 
+    public function uniqueId(): string
+    {
+        return $this->server->uuid;
+    }
+
     public function handle(): void
     {
         if (! $this->sentinelIsEnabled()) {

app/Jobs/ServerManagerJob.php

@@ -13,6 +13,7 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 
 class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
@@ -169,7 +170,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
             }
         }
 
+        // Sentinel versions that report their version on push are updated by SentinelController.
         if ($server->isSentinelEnabled()
+            && ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id))
             && shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)
         ) {
             CheckAndStartSentinelJob::dispatch($server);

app/Models/Server.php

@@ -1079,6 +1079,11 @@ $siteAddress {
         return max(30, $this->settings->sentinel_push_interval_seconds + 30);
     }
 
+    public static function sentinelReportedVersionCacheKey(int $serverId): string
+    {
+        return "sentinel:reported-version:{$serverId}";
+    }
+
     public function isSentinelLive()
     {
         return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));

tests/Feature/SentinelVersionReportTest.php

@@ -0,0 +1,61 @@
+<?php
+
+use App\Jobs\CheckAndStartSentinelJob;
+use App\Models\Server;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    config(['app.maintenance.store' => 'array']);
+
+    Queue::fake();
+    Cache::flush();
+    Cache::put('coolify:versions:all', ['coolify' => ['sentinel' => ['version' => '1.0.2']]], 3600);
+
+    $user = User::factory()->create();
+    $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]);
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+});
+
+function pushSentinelWithVersion(Server $server, ?string $version)
+{
+    $headers = ['Authorization' => 'Bearer '.$server->settings->sentinel_token];
+    if ($version !== null) {
+        $headers['X-Sentinel-Version'] = $version;
+    }
+
+    return test()->postJson('/api/v1/sentinel/push', ['containers' => []], $headers);
+}
+
+it('updates an outdated Sentinel once per hour', function () {
+    pushSentinelWithVersion($this->server, '1.0.1')->assertOk();
+    pushSentinelWithVersion($this->server, '1.0.1')->assertOk();
+
+    Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+    expect(Cache::get(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBe('1.0.1');
+});
+
+it('does not update a current Sentinel', function (string $version) {
+    pushSentinelWithVersion($this->server, $version)->assertOk();
+
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+    expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeTrue();
+})->with(['1.0.2', '1.1.0']);
+
+it('keeps the hourly SSH version check for Sentinel without a valid version header', function (?string $version) {
+    pushSentinelWithVersion($this->server, $version)->assertOk();
+
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+    expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeFalse();
+})->with(['no header' => null, 'not a version' => 'next']);
+
+it('queues only one Sentinel check per server while one is waiting', function () {
+    CheckAndStartSentinelJob::dispatch($this->server);
+    CheckAndStartSentinelJob::dispatch($this->server);
+
+    Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+});

tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php

@@ -164,6 +164,12 @@ it('creates the access log before starting sentinel with traffic analytics', fun
     'caddy' => ['CADDY', '/data/coolify/proxy/caddy'],
 ]);
 
+it('starts sentinel with a restart policy so Docker restarts it after a crash or reboot', function () {
+    $script = runStartSentinelAndCaptureScript(sentinelTrafficServer($this, 'TRAEFIK', analyticsEnabled: false));
+
+    expect($script)->toContain('--name coolify-sentinel --restart unless-stopped ');
+});
+
 it('does not touch the access log when traffic analytics cannot run', function (?string $proxyType, bool $analyticsEnabled) {
     $server = sentinelTrafficServer($this, $proxyType, $analyticsEnabled);
 

tests/Unit/ServerManagerJobSentinelCheckTest.php

@@ -9,6 +9,7 @@ use App\Models\Server;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Queue;
 use Tests\TestCase;
 
@@ -56,6 +57,15 @@ it('dispatches an hourly Sentinel version check for a healthy Sentinel', functio
     });
 });
 
+it('skips the hourly SSH version check when Sentinel reports its version on push', function () {
+    $server = createSentinelCheckServer($this->team, Carbon::now());
+    Cache::put(Server::sentinelReportedVersionCacheKey($server->id), '1.0.2', now()->addHours(2));
+
+    (new ServerManagerJob)->handle();
+
+    Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+});
+
 it('does not schedule periodic Sentinel restart checks', function () {
     $root = dirname(__DIR__, 2);
     $manager = file_get_contents($root.'/app/Jobs/ServerManagerJob.php');