diff --git a/.env.development.example b/.env.development.example index 2f474260a9..dab5833e4b 100644 --- a/.env.development.example +++ b/.env.development.example @@ -44,6 +44,8 @@ DB_PORT=5432 # HORIZON_MAINTENANCE_PROCESSES=10 # GitHub pull request webhooks and Stripe events. # HORIZON_WEBHOOKS_PROCESSES=10 +# Sentinel pushes (PushServerUpdateJob). +# HORIZON_SENTINEL_PROCESSES=20 # Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local). # Force on in any environment (including production): SERVER_TIMING_ENABLED=true diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 0e9c074f25..b7b1c0d714 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -132,6 +132,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced public function __construct(public Server $server, public $data) { + $this->onQueue(sentinel_queue()); $this->containers = collect(); $this->foundApplicationIds = collect(); $this->foundDatabaseUuids = collect(); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 215b2e2434..82b7c97e33 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1133,6 +1133,18 @@ function webhooks_queue(): string return isCloud() ? 'webhooks' : 'high'; } +/** + * Resolve the queue for PushServerUpdateJob, which processes Sentinel pushes. + * + * On cloud these jobs run on a dedicated `sentinel` queue, so a burst of pushes from many + * servers cannot use the workers of the other queues. Self-hosted returns null, so the jobs + * stay on the default queue of the connection, as before. + */ +function sentinel_queue(): ?string +{ + return isCloud() ? 'sentinel' : null; +} + function translate_cron_expression($expression_to_validate): string { if (isset(VALID_CRON_STRINGS[$expression_to_validate])) { diff --git a/config/horizon.php b/config/horizon.php index 6b780013dc..3b6162b058 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -46,6 +46,10 @@ $selfHostedSupervisor = $workerOptions + [ | The `webhooks` pool (see webhooks_queue()) processes GitHub pull request | webhooks, Stripe events, and server limit checks, so other jobs cannot | delay them. +| +| The `sentinel` pool (see sentinel_queue()) processes Sentinel pushes +| (PushServerUpdateJob), so a push burst from many servers cannot take the +| workers of the other pools. */ $cloudQueues = [ 'deployments' => ['HORIZON_DEPLOYMENTS_PROCESSES', 60], @@ -54,6 +58,7 @@ $cloudQueues = [ 'default' => ['HORIZON_DEFAULT_PROCESSES', 40], 'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 60], 'webhooks' => ['HORIZON_WEBHOOKS_PROCESSES', 10], + 'sentinel' => ['HORIZON_SENTINEL_PROCESSES', 20], ]; $cloudSupervisors = []; diff --git a/docs/cloud-horizon-workers.md b/docs/cloud-horizon-workers.md index 926e6a9f9c..051b0ef684 100644 --- a/docs/cloud-horizon-workers.md +++ b/docs/cloud-horizon-workers.md @@ -12,8 +12,9 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others. | `default` | `default` | `HORIZON_DEFAULT_PROCESSES` | 40 | | `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 60 | | `webhooks` | `webhooks` | `HORIZON_WEBHOOKS_PROCESSES` | 10 | +| `sentinel` | `sentinel` | `HORIZON_SENTINEL_PROCESSES` | 20 | -- Counts apply to **each node**. Two nodes with the defaults run 580 workers. +- Counts apply to **each node**. Two nodes with the defaults run 620 workers. - `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup that runs after a resource stops) and the weekly server patch checks. Remote prunes and package checks can be slow, so this pool limits how many @@ -24,6 +25,9 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others. server limit checks that follow subscription changes, so a full `high` queue cannot delay them. These jobs are short (60 s timeout or less). Self-hosted instances keep them on `high`. +- `sentinel` processes Sentinel pushes (`PushServerUpdateJob`), so a burst + of pushes from many servers cannot take the workers of the other pools. + Self-hosted instances keep them on the default queue. - Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon does not scale them. - A value that is not a positive integer falls back to the default. @@ -51,6 +55,7 @@ HORIZON_HIGH_PROCESSES=60 HORIZON_DEFAULT_PROCESSES=40 HORIZON_MAINTENANCE_PROCESSES=60 HORIZON_WEBHOOKS_PROCESSES=10 +HORIZON_SENTINEL_PROCESSES=20 # Worker timeout. Must stay above the longest job timeout (36000 s) and below # retry_after (86400 s). The config clamps it to 36600..85800. HORIZON_TIMEOUT=36600 @@ -114,13 +119,13 @@ Do these steps on one node at a time. Do not flush Redis or clear queues. docker exec coolify php artisan horizon:terminate ``` -5. Make sure the six supervisors are running on this node: +5. Make sure the seven supervisors are running on this node: ```bash docker exec coolify php artisan horizon:supervisors ``` -6. Make sure the `maintenance` and `webhooks` queues drain (its depth goes down) and that +6. Make sure the `maintenance`, `webhooks`, and `sentinel` queues drain (their depth goes down) and that `php artisan scheduled:diagnostics` shows no growing count of stale occurrences. 7. Do the same steps on the next node. @@ -132,10 +137,10 @@ jobs) stay on `high`, and the `high` pool still processes them. 1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides. 2. Run `config:cache` and `horizon:terminate` on one node at a time. -3. Jobs that wait on the `maintenance` or `webhooks` queue are not lost, but +3. Jobs that wait on the `maintenance`, `webhooks`, or `sentinel` queue are not lost, but the old configuration has no pool for them. Keep one node with the new - configuration until both queues are empty, or start a temporary worker: - `php artisan queue:work redis --queue=maintenance,webhooks --tries=1 --timeout=600`. + configuration until these queues are empty, or start a temporary worker: + `php artisan queue:work redis --queue=maintenance,webhooks,sentinel --tries=1 --timeout=600`. 4. Do not roll back the migration. The old code ignores the new columns. ## Metrics to collect before you change worker counts diff --git a/tests/Feature/HorizonConfigurationTest.php b/tests/Feature/HorizonConfigurationTest.php index 864a2f0657..b982ac4ed5 100644 --- a/tests/Feature/HorizonConfigurationTest.php +++ b/tests/Feature/HorizonConfigurationTest.php @@ -98,12 +98,13 @@ test('cloud production runs one fixed-size pool per queue with default process c 'HORIZON_DEFAULT_PROCESSES' => null, 'HORIZON_MAINTENANCE_PROCESSES' => null, 'HORIZON_WEBHOOKS_PROCESSES' => null, + 'HORIZON_SENTINEL_PROCESSES' => null, ])); $production = $parsed['production']; - expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks']); + expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks', 'sentinel']); - foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 60, 'webhooks' => 10] as $queue => $processes) { + foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 60, 'webhooks' => 10, 'sentinel' => 20] as $queue => $processes) { $pool = $production[$queue]; expect($pool->queue)->toBe($queue) ->and($pool->balancing())->toBeFalse() @@ -152,7 +153,7 @@ test('cloud queue helpers route to queues that have a dedicated pool', function $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production']; - expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()]); + expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue(), sentinel_queue()]); }); test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () { diff --git a/tests/Feature/PushServerUpdateJobOptimizationTest.php b/tests/Feature/PushServerUpdateJobOptimizationTest.php index 1083420121..1fe06813ed 100644 --- a/tests/Feature/PushServerUpdateJobOptimizationTest.php +++ b/tests/Feature/PushServerUpdateJobOptimizationTest.php @@ -221,11 +221,15 @@ it('ends the proxy restart when the proxy status listener runs', function () { expect($server->isProxyTransitionInProgress())->toBeFalse(); }); -it('uses default queue for PushServerUpdateJob', function () { +it('runs PushServerUpdateJob on the sentinel queue on cloud and on the default queue on self-hosted', function (bool $selfHosted, ?string $queue) { + config(['constants.coolify.self_hosted' => $selfHosted]); $team = Team::factory()->create(); $server = Server::factory()->create(['team_id' => $team->id]); $job = new PushServerUpdateJob($server, ['containers' => []]); - expect($job->queue)->toBeNull(); -}); + expect($job->queue)->toBe($queue); +})->with([ + 'cloud' => [false, 'sentinel'], + 'self-hosted' => [true, null], +]);