From 5c34a1b3d41aca560331229ecf9877ee62230e9a Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:19:14 +0100 Subject: [PATCH] fix(sentinel): stop rejected Sentinel pushes and reduce their audit rows - Reply 410 for deleted servers and 402 for unpaid teams, with Retry-After - Remove the Sentinel container of unpaid servers once a day, and before a server is deleted - Record server_not_found and subscription_unpaid push failures once per hour - Audit scheduled task API updates only for fields that really changed Co-Authored-By: Claude Opus 5.5 --- app/Actions/Server/DeleteServer.php | 21 +++++++++++++++++++ .../Api/ScheduledTasksController.php | 10 +++++++-- .../Controllers/Api/SentinelController.php | 10 +++++++-- bootstrap/helpers/audit.php | 9 ++++++-- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/app/Actions/Server/DeleteServer.php b/app/Actions/Server/DeleteServer.php index 92e66e6dfc..f772b2fc12 100644 --- a/app/Actions/Server/DeleteServer.php +++ b/app/Actions/Server/DeleteServer.php @@ -24,6 +24,11 @@ class DeleteServer $deleteFromHetzner = $deleteFromVultr = $deleteFromDigitalOcean = $deleteFromHostinger = false; } + // A Sentinel that keeps running would push to Coolify for a server that no longer exists. + if ($server && ! $server->isTransferredAway() && ! $deleteFromHetzner && ! $deleteFromVultr && ! $deleteFromDigitalOcean) { + $this->stopSentinel($server); + } + // Delete from Hetzner even if server is already gone from Coolify if ($deleteFromHetzner && ($hetznerServerId || ($server && $server->hetzner_server_id))) { $this->deleteFromHetznerById( @@ -74,6 +79,22 @@ class DeleteServer } } + private function stopSentinel(Server $server): void + { + if (! $server->settings?->is_sentinel_enabled || ! $server->isFunctional()) { + return; + } + + try { + instant_remote_process(['docker rm -f coolify-sentinel'], $server, false, timeout: 30, maxAttempts: 1); + } catch (\Throwable $e) { + logger()->warning('Failed to stop Sentinel before deleting the server', [ + 'error' => $e->getMessage(), + 'server_id' => $server->id, + ]); + } + } + private function deleteFromHetznerById(int $hetznerServerId, ?int $cloudProviderTokenId, int $teamId): void { try { diff --git a/app/Http/Controllers/Api/ScheduledTasksController.php b/app/Http/Controllers/Api/ScheduledTasksController.php index db982fe3a1..afbf0a9707 100644 --- a/app/Http/Controllers/Api/ScheduledTasksController.php +++ b/app/Http/Controllers/Api/ScheduledTasksController.php @@ -205,7 +205,13 @@ class ScheduledTasksController extends Controller return $response; } - $task->update($request->only($allowedFields)); + $task->fill($request->only($allowedFields)); + $changedFields = auditChangedFields($task); + $task->save(); + + if ($changedFields === []) { + return response()->json($this->removeSensitiveData($task), 200); + } auditLog('api.scheduled_task.updated', [ 'team_id' => getTeamIdFromToken(), @@ -213,7 +219,7 @@ class ScheduledTasksController extends Controller 'task_name' => $task->name, 'resource_type' => $resource instanceof Application ? 'application' : 'service', 'resource_uuid' => $resource->uuid, - 'changed_fields' => array_values(array_intersect($allowedFields, array_keys($request->all()))), + 'changed_fields' => $changedFields, ]); return response()->json($this->removeSensitiveData($task), 200); diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php index 87e4899f21..1e1ca6292d 100644 --- a/app/Http/Controllers/Api/SentinelController.php +++ b/app/Http/Controllers/Api/SentinelController.php @@ -2,6 +2,7 @@ namespace App\Http\Controllers\Api; +use App\Actions\Server\StopSentinel; use App\Events\SentinelSynchronized; use App\Http\Controllers\Controller; use App\Jobs\CheckAndStartSentinelJob; @@ -52,7 +53,8 @@ class SentinelController extends Controller 'server_uuid' => $server_uuid, ]); - return response()->json(['message' => 'Server not found'], 404); + // 410 and Retry-After make Sentinel wait one day before it pushes again. + return response()->json(['message' => 'Server not found'], 410, ['Retry-After' => 86400]); } if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { @@ -61,8 +63,12 @@ class SentinelController extends Controller 'team_id' => $server->team_id, ]); $server->rememberSentinelPushProblem('Coolify rejected the push: the subscription of this team is not active.'); + // Remove the Sentinel container of the unpaid server once a day. Server validation starts it again after payment. + if (Cache::add("sentinel:stop-unpaid:{$server->id}", true, now()->addDay())) { + StopSentinel::dispatch($server); + } - return response()->json(['message' => 'Unauthorized'], 401); + return response()->json(['message' => 'Subscription is not active'], 402, ['Retry-After' => 3600]); } if ($server->isFunctional() === false) { diff --git a/bootstrap/helpers/audit.php b/bootstrap/helpers/audit.php index 482a535897..a83afc675f 100644 --- a/bootstrap/helpers/audit.php +++ b/bootstrap/helpers/audit.php @@ -77,11 +77,16 @@ if (! function_exists('auditFailureDeduplicationWindow')) { * * Failed logins use a short window so an ongoing brute force attempt stays visible * (one row per email and source IP each minute, on top of the login rate limit). + * Sentinel pushes for a deleted server or an unpaid team fail until someone acts, so one row each hour is enough. + * + * @param array $context */ - function auditFailureDeduplicationWindow(string $event): ?int + function auditFailureDeduplicationWindow(string $event, array $context = []): ?int { return match (true) { $event === 'auth.user.login_failed' => 60, + $event === 'webhook.sentinel.signature_failed' + && in_array(data_get($context, 'reason'), ['server_not_found', 'subscription_unpaid'], true) => 3600, $event === 'api.auth.unauthenticated', preg_match('/^webhook\.[a-z0-9_-]+\.signature_failed$/i', $event) === 1 => 300, default => null, @@ -98,7 +103,7 @@ if (! function_exists('shouldRecordAuditEvent')) { */ function shouldRecordAuditEvent(string $event, array $context): bool { - $window = auditFailureDeduplicationWindow($event); + $window = auditFailureDeduplicationWindow($event, $context); if ($window === null) { return true; }