fix(horizon): route Sentinel push jobs to a dedicated cloud sentinel queue

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.env.development.example

@@ -44,6 +44,8 @@ DB_PORT=5432
 # HORIZON_MAINTENANCE_PROCESSES=10
 # GitHub pull request webhooks and Stripe events.
 # HORIZON_WEBHOOKS_PROCESSES=10
+# Sentinel pushes (PushServerUpdateJob).
+# HORIZON_SENTINEL_PROCESSES=20
 
 # Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local).
 # Force on in any environment (including production): SERVER_TIMING_ENABLED=true

app/Jobs/PushServerUpdateJob.php

@@ -132,6 +132,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public function __construct(public Server $server, public $data)
     {
+        $this->onQueue(sentinel_queue());
         $this->containers = collect();
         $this->foundApplicationIds = collect();
         $this->foundDatabaseUuids = collect();

bootstrap/helpers/shared.php

@@ -1133,6 +1133,18 @@ function webhooks_queue(): string
     return isCloud() ? 'webhooks' : 'high';
 }
 
+/**
+ * Resolve the queue for PushServerUpdateJob, which processes Sentinel pushes.
+ *
+ * On cloud these jobs run on a dedicated `sentinel` queue, so a burst of pushes from many
+ * servers cannot use the workers of the other queues. Self-hosted returns null, so the jobs
+ * stay on the default queue of the connection, as before.
+ */
+function sentinel_queue(): ?string
+{
+    return isCloud() ? 'sentinel' : null;
+}
+
 function translate_cron_expression($expression_to_validate): string
 {
     if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {

config/horizon.php

@@ -46,6 +46,10 @@ $selfHostedSupervisor = $workerOptions + [
 | The `webhooks` pool (see webhooks_queue()) processes GitHub pull request
 | webhooks, Stripe events, and server limit checks, so other jobs cannot
 | delay them.
+|
+| The `sentinel` pool (see sentinel_queue()) processes Sentinel pushes
+| (PushServerUpdateJob), so a push burst from many servers cannot take the
+| workers of the other pools.
 */
 $cloudQueues = [
     'deployments' => ['HORIZON_DEPLOYMENTS_PROCESSES', 60],
@@ -54,6 +58,7 @@ $cloudQueues = [
     'default' => ['HORIZON_DEFAULT_PROCESSES', 40],
     'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 60],
     'webhooks' => ['HORIZON_WEBHOOKS_PROCESSES', 10],
+    'sentinel' => ['HORIZON_SENTINEL_PROCESSES', 20],
 ];
 
 $cloudSupervisors = [];

docs/cloud-horizon-workers.md

@@ -12,8 +12,9 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others.
 | `default`     | `default`     | `HORIZON_DEFAULT_PROCESSES`     | 40               |
 | `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 60               |
 | `webhooks`    | `webhooks`    | `HORIZON_WEBHOOKS_PROCESSES`    | 10               |
+| `sentinel`    | `sentinel`    | `HORIZON_SENTINEL_PROCESSES`    | 20               |
 
-- Counts apply to **each node**. Two nodes with the defaults run 580 workers.
+- Counts apply to **each node**. Two nodes with the defaults run 620 workers.
 - `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup
   that runs after a resource stops) and the weekly server patch checks.
   Remote prunes and package checks can be slow, so this pool limits how many
@@ -24,6 +25,9 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others.
   server limit checks that follow subscription changes, so a full `high`
   queue cannot delay them. These jobs are short (60 s timeout or
   less). Self-hosted instances keep them on `high`.
+- `sentinel` processes Sentinel pushes (`PushServerUpdateJob`), so a burst
+  of pushes from many servers cannot take the workers of the other pools.
+  Self-hosted instances keep them on the default queue.
 - Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon
   does not scale them.
 - A value that is not a positive integer falls back to the default.
@@ -51,6 +55,7 @@ HORIZON_HIGH_PROCESSES=60
 HORIZON_DEFAULT_PROCESSES=40
 HORIZON_MAINTENANCE_PROCESSES=60
 HORIZON_WEBHOOKS_PROCESSES=10
+HORIZON_SENTINEL_PROCESSES=20
 # Worker timeout. Must stay above the longest job timeout (36000 s) and below
 # retry_after (86400 s). The config clamps it to 36600..85800.
 HORIZON_TIMEOUT=36600
@@ -114,13 +119,13 @@ Do these steps on one node at a time. Do not flush Redis or clear queues.
    docker exec coolify php artisan horizon:terminate
    ```
 
-5. Make sure the six supervisors are running on this node:
+5. Make sure the seven supervisors are running on this node:
 
    ```bash
    docker exec coolify php artisan horizon:supervisors
    ```
 
-6. Make sure the `maintenance` and `webhooks` queues drain (its depth goes down) and that
+6. Make sure the `maintenance`, `webhooks`, and `sentinel` queues drain (their depth goes down) and that
    `php artisan scheduled:diagnostics` shows no growing count of stale
    occurrences.
 7. Do the same steps on the next node.
@@ -132,10 +137,10 @@ jobs) stay on `high`, and the `high` pool still processes them.
 
 1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides.
 2. Run `config:cache` and `horizon:terminate` on one node at a time.
-3. Jobs that wait on the `maintenance` or `webhooks` queue are not lost, but
+3. Jobs that wait on the `maintenance`, `webhooks`, or `sentinel` queue are not lost, but
    the old configuration has no pool for them. Keep one node with the new
-   configuration until both queues are empty, or start a temporary worker:
-   `php artisan queue:work redis --queue=maintenance,webhooks --tries=1 --timeout=600`.
+   configuration until these queues are empty, or start a temporary worker:
+   `php artisan queue:work redis --queue=maintenance,webhooks,sentinel --tries=1 --timeout=600`.
 4. Do not roll back the migration. The old code ignores the new columns.
 
 ## Metrics to collect before you change worker counts

tests/Feature/HorizonConfigurationTest.php

@@ -98,12 +98,13 @@ test('cloud production runs one fixed-size pool per queue with default process c
         'HORIZON_DEFAULT_PROCESSES' => null,
         'HORIZON_MAINTENANCE_PROCESSES' => null,
         'HORIZON_WEBHOOKS_PROCESSES' => null,
+        'HORIZON_SENTINEL_PROCESSES' => null,
     ]));
 
     $production = $parsed['production'];
-    expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks']);
+    expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks', 'sentinel']);
 
-    foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 60, 'webhooks' => 10] as $queue => $processes) {
+    foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 60, 'webhooks' => 10, 'sentinel' => 20] as $queue => $processes) {
         $pool = $production[$queue];
         expect($pool->queue)->toBe($queue)
             ->and($pool->balancing())->toBeFalse()
@@ -152,7 +153,7 @@ test('cloud queue helpers route to queues that have a dedicated pool', function
 
     $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production'];
 
-    expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()]);
+    expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue(), sentinel_queue()]);
 });
 
 test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () {

tests/Feature/PushServerUpdateJobOptimizationTest.php

@@ -221,11 +221,15 @@ it('ends the proxy restart when the proxy status listener runs', function () {
     expect($server->isProxyTransitionInProgress())->toBeFalse();
 });
 
-it('uses default queue for PushServerUpdateJob', function () {
+it('runs PushServerUpdateJob on the sentinel queue on cloud and on the default queue on self-hosted', function (bool $selfHosted, ?string $queue) {
+    config(['constants.coolify.self_hosted' => $selfHosted]);
     $team = Team::factory()->create();
     $server = Server::factory()->create(['team_id' => $team->id]);
 
     $job = new PushServerUpdateJob($server, ['containers' => []]);
 
-    expect($job->queue)->toBeNull();
-});
+    expect($job->queue)->toBe($queue);
+})->with([
+    'cloud' => [false, 'sentinel'],
+    'self-hosted' => [true, null],
+]);