diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php index 23bcf4f13c..109b0db810 100644 --- a/app/Http/Controllers/Api/SentinelController.php +++ b/app/Http/Controllers/Api/SentinelController.php @@ -60,6 +60,7 @@ class SentinelController extends Controller 'server_uuid' => $server->uuid, 'team_id' => $server->team_id, ]); + $server->rememberSentinelPushProblem('Coolify rejected the push: the subscription of this team is not active.'); return response()->json(['message' => 'Unauthorized'], 401); } @@ -69,6 +70,7 @@ class SentinelController extends Controller 'server_uuid' => $server->uuid, 'team_id' => $server->team_id, ]); + $server->rememberSentinelPushProblem('Coolify rejected the push: the server is marked unreachable, not usable, or disabled. Validate the server connection.'); return response()->json(['message' => 'Server is not functional'], 401); } @@ -78,6 +80,7 @@ class SentinelController extends Controller 'server_uuid' => $server->uuid, 'team_id' => $server->team_id, ]); + $server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match. Sync Sentinel to apply the current token.'); return response()->json(['message' => 'Unauthorized'], 401); } @@ -86,6 +89,8 @@ class SentinelController extends Controller ]); if ($validator->fails()) { + $server->rememberSentinelPushProblem('Coolify rejected the push: the payload is not valid.'); + return response()->json(serializeApiResponse([ 'message' => 'Validation failed.', 'errors' => $validator->errors(), @@ -101,6 +106,7 @@ class SentinelController extends Controller $server->sentinelHeartbeat(); if (! $wasSentinelLive) { + $server->rememberSentinelPushProblem(null); SentinelSynchronized::dispatch($server); } diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.php index 49e08ab1a6..183063f09e 100644 --- a/app/Jobs/CheckAndStartSentinelJob.php +++ b/app/Jobs/CheckAndStartSentinelJob.php @@ -11,11 +11,22 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\Cache; class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + /** + * A running Sentinel that has not pushed for this long is restarted with the current settings. + * This repairs a changed Coolify URL or token, and a hung push loop. + */ + public const STALL_RESTART_AFTER_SECONDS = 600; + + /** A stalled Sentinel is restarted at most once in this period, because a firewall problem stays after a restart. */ + public const STALL_RESTART_INTERVAL_SECONDS = 3600; + public $timeout = 120; /** @@ -40,28 +51,63 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, Sho // An SSH failure throws here, so it is never mistaken for a missing container. // A missing container gives empty output and exit code 0. - $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}}'"], $this->server); - if ($sentinelStatus !== 'running') { + $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}} {{.Status}}'"], $this->server); + if (! str_starts_with((string) $sentinelStatus, 'running')) { $this->startSentinel($latestVersion); return; } - // If sentinel is running, check if it needs an update - $runningVersion = instant_remote_process_with_timeout(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false); - if (empty($runningVersion)) { - $runningVersion = '0.0.0'; - } - if ($latestVersion === '0.0.0' && $runningVersion === '0.0.0') { - $this->startSentinel('latest'); + if (str_contains($sentinelStatus, '(unhealthy)')) { + $this->server->rememberSentinelPushProblem('The Sentinel container was unhealthy. Coolify restarted it.'); + $this->startSentinel($latestVersion); return; - } else { - if (version_compare($runningVersion, $latestVersion, '<')) { - $this->startSentinel($latestVersion); - - return; - } } + + // One SSH call reads the version and the last push result. Sentinel versions without + // /api/push-status give an empty second line. + $output = instant_remote_process_with_timeout([ + "docker exec coolify-sentinel sh -c 'curl -s http://127.0.0.1:8888/api/version; echo; curl -sf -H \"Authorization: Bearer \$TOKEN\" http://127.0.0.1:8888/api/push-status'", + ], $this->server, false); + [$runningVersion, $pushStatus] = array_pad(explode("\n", trim((string) $output), 2), 2, null); + $this->rememberPushError($pushStatus); + + // An empty version means that the API does not answer yet, for example while Sentinel starts. + // A Sentinel whose API stays down fails its health check and is restarted above. + if (filled($runningVersion) && version_compare(trim($runningVersion), $latestVersion, '<')) { + $this->startSentinel($latestVersion); + + return; + } + + if ($this->isStalled() && Cache::add("sentinel:stall-restart:{$this->server->id}", true, self::STALL_RESTART_INTERVAL_SECONDS)) { + $this->startSentinel($latestVersion); + } + } + + /** + * Sentinel is stalled when it has not pushed for STALL_RESTART_AFTER_SECONDS since its last push, + * or since its last start when it never pushed. + */ + private function isStalled(): bool + { + $lastSignOfLife = $this->server->sentinel_waiting_since ?? $this->server->sentinel_updated_at; + if ($lastSignOfLife === null) { + return true; + } + $stallAfter = max(self::STALL_RESTART_AFTER_SECONDS, $this->server->waitBeforeDoingSshCheck()); + + return Carbon::parse($lastSignOfLife)->isBefore(now()->subSeconds($stallAfter)); + } + + private function rememberPushError(?string $pushStatus): void + { + $status = json_decode(trim((string) $pushStatus), true); + if (! is_array($status) || data_get($status, 'consecutive_failures', 0) < 1 || blank(data_get($status, 'last_error'))) { + return; + } + + $this->server->rememberSentinelPushProblem('Sentinel cannot push to Coolify: '.data_get($status, 'last_error')); } private function sentinelIsEnabled(): bool diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php index 9c9bc3b4ea..16a1d7ea30 100644 --- a/app/Models/InstanceSettings.php +++ b/app/Models/InstanceSettings.php @@ -111,6 +111,13 @@ class InstanceSettings extends Model static::updated(function ($settings) { // Clear once() cache so subsequent calls get fresh data Once::flush(); + + if ($settings->wasChanged(['fqdn', 'public_ipv4', 'public_ipv6'])) { + ServerSetting::followInstanceUrlChange( + ServerSetting::instanceSentinelUrl($settings->getOriginal('fqdn'), $settings->getOriginal('public_ipv4'), $settings->getOriginal('public_ipv6')), + ServerSetting::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6), + ); + } }); } diff --git a/app/Models/Server.php b/app/Models/Server.php index cf32ce702a..88f3ecc15a 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -33,9 +33,11 @@ use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; +use Illuminate\Support\Str; use Illuminate\Support\Stringable; use OpenApi\Attributes as OA; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; @@ -1082,6 +1084,25 @@ $siteAddress { return "sentinel:reported-version:{$serverId}"; } + /** + * The last known reason why Sentinel pushes do not arrive. It is shown while Sentinel is out of sync. + */ + public function sentinelPushProblem(): ?string + { + return Cache::get("sentinel:push-problem:{$this->id}"); + } + + public function rememberSentinelPushProblem(?string $problem): void + { + if (blank($problem)) { + Cache::forget("sentinel:push-problem:{$this->id}"); + + return; + } + + Cache::put("sentinel:push-problem:{$this->id}", Str::limit($problem, 500), now()->addDay()); + } + public function isSentinelLive() { return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck())); diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index 1fb3af6d6c..a4d51afdae 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -315,6 +315,38 @@ class ServerSetting extends Model return $url; } + /** + * The Sentinel push URL that the instance URL or public IP gives to a remote server. + */ + public static function instanceSentinelUrl(?string $fqdn, ?string $publicIpv4, ?string $publicIpv6): ?string + { + return match (true) { + filled($fqdn) => $fqdn, + filled($publicIpv4) => 'http://'.$publicIpv4.':8000', + filled($publicIpv6) => 'http://'.$publicIpv6.':8000', + default => null, + }; + } + + /** + * Move remote servers that use the generated Sentinel URL to the new instance URL. Saving the + * new URL restarts their Sentinel. Servers with a custom URL keep it. + */ + public static function followInstanceUrlChange(?string $oldUrl, ?string $newUrl): void + { + if (blank($oldUrl) || blank($newUrl) || $oldUrl === $newUrl) { + return; + } + + self::query() + ->where('sentinel_custom_url', $oldUrl) + ->where('server_id', '!=', 0) + ->each(function (ServerSetting $setting) use ($newUrl) { + $setting->sentinel_custom_url = $newUrl; + $setting->save(); + }); + } + public function restoreDefaultSentinelConfiguration(): void { $this->generateSentinelUrl(save: false, ignoreEvent: true); @@ -327,18 +359,12 @@ class ServerSetting extends Model public function generateSentinelUrl(bool $save = true, bool $ignoreEvent = false): ?string { - $domain = null; $settings = InstanceSettings::get(); if ($this->server->isLocalhost()) { $domain = 'http://coolify:8080'; - } elseif ($settings->fqdn) { - $domain = $settings->fqdn; - } elseif ($settings->public_ipv4) { - $domain = 'http://'.$settings->public_ipv4.':8000'; - } elseif ($settings->public_ipv6) { - $domain = 'http://'.$settings->public_ipv6.':8000'; } else { - $domain = $this->sentinelUrlFromCurrentRequest(); + $domain = self::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6) + ?? $this->sentinelUrlFromCurrentRequest(); } $this->sentinel_custom_url = $domain; if ($save) { diff --git a/resources/views/livewire/server/sentinel.blade.php b/resources/views/livewire/server/sentinel.blade.php index e3a72e8686..02b32b155e 100644 --- a/resources/views/livewire/server/sentinel.blade.php +++ b/resources/views/livewire/server/sentinel.blade.php @@ -38,6 +38,9 @@

Sentinel has not reported within the expected interval. Check these items before syncing again:

+ @if ($sentinelPushProblem = $server->sentinelPushProblem()) +

Last error: {{ $sentinelPushProblem }}

+ @endif