From 51e6949c36547c301f5fa2a1bbb04ef2f8505157 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:41:04 +0200 Subject: [PATCH] fix(sentinel): stop hourly check job pile-up and self-heal Sentinel - Start Sentinel with --restart unless-stopped so Docker restarts it after a crash or reboot. - Make CheckAndStartSentinelJob unique per server so copies cannot collect in a slow queue. - Read X-Sentinel-Version on push and update an outdated Sentinel without SSH; skip the hourly SSH version check for servers that report their version. --- app/Actions/Server/StartSentinel.php | 2 +- .../Controllers/Api/SentinelController.php | 27 ++++++++ app/Jobs/CheckAndStartSentinelJob.php | 13 +++- app/Jobs/ServerManagerJob.php | 3 + app/Models/Server.php | 5 ++ tests/Feature/SentinelVersionReportTest.php | 61 +++++++++++++++++++ .../StartSentinelTrafficTest.php | 6 ++ .../ServerManagerJobSentinelCheckTest.php | 10 +++ 8 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/Feature/SentinelVersionReportTest.php diff --git a/app/Actions/Server/StartSentinel.php b/app/Actions/Server/StartSentinel.php index 8c9230c954..101f6b2d75 100644 --- a/app/Actions/Server/StartSentinel.php +++ b/app/Actions/Server/StartSentinel.php @@ -117,7 +117,7 @@ class StartSentinel ? '-v '.escapeshellarg("{$trafficLogDirectory}:{$trafficLogDirectory}:ro").' ' : ''; $network = $server->isLocalhost() ? ' --network coolify' : ''; - $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image"; + $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel --restart unless-stopped -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image"; $server->sentinelHeartbeat(isReset: true); $server->forceFill(['sentinel_waiting_since' => now()])->save(); diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php index 3d3978d1c8..23bcf4f13c 100644 --- a/app/Http/Controllers/Api/SentinelController.php +++ b/app/Http/Controllers/Api/SentinelController.php @@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api; use App\Events\SentinelSynchronized; use App\Http\Controllers\Controller; +use App\Jobs\CheckAndStartSentinelJob; use App\Jobs\PushServerUpdateJob; use App\Models\Server; use Exception; @@ -103,6 +104,8 @@ class SentinelController extends Controller SentinelSynchronized::dispatch($server); } + $this->updateSentinelIfOutdated($server, $request->header('X-Sentinel-Version')); + if ($this->shouldDispatchUpdate($server, $data)) { PushServerUpdateJob::dispatch($server, $data); } @@ -110,6 +113,30 @@ class SentinelController extends Controller return response()->json(['message' => 'ok'], 200); } + /** + * Sentinel sends its version on every push, so an outdated Sentinel is found without SSH. + * The reported version also tells ServerManagerJob to skip its hourly SSH version check. + * Old Sentinel versions do not send the header and keep the hourly SSH check. + */ + private function updateSentinelIfOutdated(Server $server, ?string $runningVersion): void + { + if (! is_string($runningVersion) || preg_match('/^\d+\.\d+\.\d+$/', $runningVersion) !== 1) { + return; + } + + Cache::put(Server::sentinelReportedVersionCacheKey($server->id), $runningVersion, now()->addHours(2)); + + $latestVersion = data_get(get_versions_data(), 'coolify.sentinel.version'); + if (blank($latestVersion) || version_compare($runningVersion, $latestVersion, '>=')) { + return; + } + + // Retry at most hourly if an update fails. + if (Cache::add("sentinel:update-dispatched:{$server->id}", true, 3600)) { + CheckAndStartSentinelJob::dispatch($server); + } + } + /** * Decide whether PushServerUpdateJob should be dispatched for this push. * diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.php index 5bfe5d504a..72213a9432 100644 --- a/app/Jobs/CheckAndStartSentinelJob.php +++ b/app/Jobs/CheckAndStartSentinelJob.php @@ -6,19 +6,30 @@ use App\Actions\Server\StartSentinel; use App\Models\Server; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; +use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue +class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $timeout = 120; + /** + * Only one check per server can wait in the queue, so a slow queue cannot collect copies. + */ + public $uniqueFor = 600; + public function __construct(public Server $server) {} + public function uniqueId(): string + { + return $this->server->uuid; + } + public function handle(): void { if (! $this->sentinelIsEnabled()) { diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php index a3710532b5..fdaa8b54a5 100644 --- a/app/Jobs/ServerManagerJob.php +++ b/app/Jobs/ServerManagerJob.php @@ -13,6 +13,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue @@ -169,7 +170,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue } } + // Sentinel versions that report their version on push are updated by SentinelController. if ($server->isSentinelEnabled() + && ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id)) && shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime) ) { CheckAndStartSentinelJob::dispatch($server); diff --git a/app/Models/Server.php b/app/Models/Server.php index 04683c8e2c..d80a924ae6 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -1079,6 +1079,11 @@ $siteAddress { return max(30, $this->settings->sentinel_push_interval_seconds + 30); } + public static function sentinelReportedVersionCacheKey(int $serverId): string + { + return "sentinel:reported-version:{$serverId}"; + } + public function isSentinelLive() { return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck())); diff --git a/tests/Feature/SentinelVersionReportTest.php b/tests/Feature/SentinelVersionReportTest.php new file mode 100644 index 0000000000..9b9227b6e7 --- /dev/null +++ b/tests/Feature/SentinelVersionReportTest.php @@ -0,0 +1,61 @@ + 'array']); + + Queue::fake(); + Cache::flush(); + Cache::put('coolify:versions:all', ['coolify' => ['sentinel' => ['version' => '1.0.2']]], 3600); + + $user = User::factory()->create(); + $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]); + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); +}); + +function pushSentinelWithVersion(Server $server, ?string $version) +{ + $headers = ['Authorization' => 'Bearer '.$server->settings->sentinel_token]; + if ($version !== null) { + $headers['X-Sentinel-Version'] = $version; + } + + return test()->postJson('/api/v1/sentinel/push', ['containers' => []], $headers); +} + +it('updates an outdated Sentinel once per hour', function () { + pushSentinelWithVersion($this->server, '1.0.1')->assertOk(); + pushSentinelWithVersion($this->server, '1.0.1')->assertOk(); + + Queue::assertPushed(CheckAndStartSentinelJob::class, 1); + expect(Cache::get(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBe('1.0.1'); +}); + +it('does not update a current Sentinel', function (string $version) { + pushSentinelWithVersion($this->server, $version)->assertOk(); + + Queue::assertNotPushed(CheckAndStartSentinelJob::class); + expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeTrue(); +})->with(['1.0.2', '1.1.0']); + +it('keeps the hourly SSH version check for Sentinel without a valid version header', function (?string $version) { + pushSentinelWithVersion($this->server, $version)->assertOk(); + + Queue::assertNotPushed(CheckAndStartSentinelJob::class); + expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeFalse(); +})->with(['no header' => null, 'not a version' => 'next']); + +it('queues only one Sentinel check per server while one is waiting', function () { + CheckAndStartSentinelJob::dispatch($this->server); + CheckAndStartSentinelJob::dispatch($this->server); + + Queue::assertPushed(CheckAndStartSentinelJob::class, 1); +}); diff --git a/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php b/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php index 52ff4ab656..0f23a64cbf 100644 --- a/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php +++ b/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php @@ -164,6 +164,12 @@ it('creates the access log before starting sentinel with traffic analytics', fun 'caddy' => ['CADDY', '/data/coolify/proxy/caddy'], ]); +it('starts sentinel with a restart policy so Docker restarts it after a crash or reboot', function () { + $script = runStartSentinelAndCaptureScript(sentinelTrafficServer($this, 'TRAEFIK', analyticsEnabled: false)); + + expect($script)->toContain('--name coolify-sentinel --restart unless-stopped '); +}); + it('does not touch the access log when traffic analytics cannot run', function (?string $proxyType, bool $analyticsEnabled) { $server = sentinelTrafficServer($this, $proxyType, $analyticsEnabled); diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.php index df1b1c3ee6..dbbe800ad8 100644 --- a/tests/Unit/ServerManagerJobSentinelCheckTest.php +++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php @@ -9,6 +9,7 @@ use App\Models\Server; use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Queue; use Tests\TestCase; @@ -56,6 +57,15 @@ it('dispatches an hourly Sentinel version check for a healthy Sentinel', functio }); }); +it('skips the hourly SSH version check when Sentinel reports its version on push', function () { + $server = createSentinelCheckServer($this->team, Carbon::now()); + Cache::put(Server::sentinelReportedVersionCacheKey($server->id), '1.0.2', now()->addHours(2)); + + (new ServerManagerJob)->handle(); + + Queue::assertNotPushed(CheckAndStartSentinelJob::class); +}); + it('does not schedule periodic Sentinel restart checks', function () { $root = dirname(__DIR__, 2); $manager = file_get_contents($root.'/app/Jobs/ServerManagerJob.php');