diff --git a/.env.development.example b/.env.development.example index 4f4198ddb2..f7a01180e3 100644 --- a/.env.development.example +++ b/.env.development.example @@ -39,6 +39,8 @@ DB_PORT=5432 # HORIZON_DEFAULT_PROCESSES=40 # Docker cleanups; small on purpose to bound concurrent slow remote prunes. # HORIZON_MAINTENANCE_PROCESSES=10 +# GitHub pull request webhooks and Stripe events. +# HORIZON_WEBHOOKS_PROCESSES=10 # Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local). # Force on in any environment (including production): SERVER_TIMING_ENABLED=true diff --git a/app/Jobs/ProcessGithubPullRequestWebhook.php b/app/Jobs/ProcessGithubPullRequestWebhook.php index 5186a58bbf..362ad0a2d8 100644 --- a/app/Jobs/ProcessGithubPullRequestWebhook.php +++ b/app/Jobs/ProcessGithubPullRequestWebhook.php @@ -41,7 +41,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue public string $fullName, public bool $isForkPullRequest = false, ) { - $this->onQueue('high'); + $this->onQueue(webhooks_queue()); } public function handle(): void diff --git a/app/Jobs/StripeProcessJob.php b/app/Jobs/StripeProcessJob.php index 0f56476e25..58e67de085 100644 --- a/app/Jobs/StripeProcessJob.php +++ b/app/Jobs/StripeProcessJob.php @@ -23,7 +23,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue public function __construct(public $event) { - $this->onQueue('high'); + $this->onQueue(webhooks_queue()); } public function handle(): void diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index da6faa43ee..a3be2667ba 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1070,6 +1070,23 @@ function maintenance_queue(): string return isCloud() ? 'maintenance' : 'high'; } +/** + * Resolve the queue used for incoming webhook processing — GitHub pull request webhooks and + * Stripe events. + * + * On cloud these jobs run on a dedicated `webhooks` queue so a busy `high` queue cannot delay + * them; self-hosted keeps them on the shared `high` queue, so a custom `HORIZON_QUEUES` does + * not need a new queue name. Routing is decided by `isCloud()` (config-based), so the + * dispatching process needs no special env — only the worker must be configured to drain `webhooks`. + * + * On cloud, config/horizon.php provisions a dedicated `webhooks` pool in production + * (see docs/cloud-horizon-workers.md). + */ +function webhooks_queue(): string +{ + return isCloud() ? 'webhooks' : 'high'; +} + function translate_cron_expression($expression_to_validate): string { if (isset(VALID_CRON_STRINGS[$expression_to_validate])) { diff --git a/config/horizon.php b/config/horizon.php index cd532f936c..c29ad15fc9 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -42,6 +42,9 @@ $selfHostedSupervisor = $workerOptions + [ | cleanups. It is small on purpose: it bounds how many cleanups run at once | per node, and it comes on top of the other pools, so cleanups never take | deployment, cron or high workers. +| +| The `webhooks` pool (see webhooks_queue()) processes GitHub pull request +| webhooks and Stripe events, so other jobs cannot delay them. */ $cloudSupervisors = []; @@ -51,6 +54,7 @@ foreach ([ 'high' => ['HORIZON_HIGH_PROCESSES', 60], 'default' => ['HORIZON_DEFAULT_PROCESSES', 40], 'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10], + 'webhooks' => ['HORIZON_WEBHOOKS_PROCESSES', 10], ] as $queue => [$processesEnv, $defaultProcesses]) { $processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, [ 'options' => ['min_range' => 1, 'default' => $defaultProcesses], diff --git a/docs/cloud-horizon-workers.md b/docs/cloud-horizon-workers.md index 11ee81ef5e..fa494c65aa 100644 --- a/docs/cloud-horizon-workers.md +++ b/docs/cloud-horizon-workers.md @@ -11,14 +11,18 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others. | `high` | `high` | `HORIZON_HIGH_PROCESSES` | 60 | | `default` | `default` | `HORIZON_DEFAULT_PROCESSES` | 40 | | `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10 | +| `webhooks` | `webhooks` | `HORIZON_WEBHOOKS_PROCESSES` | 10 | -- Counts apply to **each node**. Two nodes with the defaults run 460 workers. +- Counts apply to **each node**. Two nodes with the defaults run 480 workers. - `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup that runs after a resource stops). Remote prunes can be slow, so this pool is small on purpose: it limits how many cleanups run at the same time and keeps them away from the `high` workers. It is added on top of the other pools. Each cleanup has a 600 s timeout, and only one cleanup runs per server at a time. Self-hosted instances keep cleanups on `high`. +- `webhooks` processes GitHub pull request webhooks and Stripe events, so a + full `high` queue cannot delay them. These jobs are short (60 s timeout or + less). Self-hosted instances keep them on `high`. - Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon does not scale them. - A value that is not a positive integer falls back to the default. @@ -45,6 +49,7 @@ HORIZON_CRONS_PROCESSES=60 HORIZON_HIGH_PROCESSES=60 HORIZON_DEFAULT_PROCESSES=40 HORIZON_MAINTENANCE_PROCESSES=10 +HORIZON_WEBHOOKS_PROCESSES=10 # Worker timeout. Must stay above the longest job timeout (36000 s) and below # retry_after (86400 s). The config clamps it to 36600..85800. HORIZON_TIMEOUT=36600 @@ -108,28 +113,28 @@ Do these steps on one node at a time. Do not flush Redis or clear queues. docker exec coolify php artisan horizon:terminate ``` -5. Make sure the five supervisors are running on this node: +5. Make sure the six supervisors are running on this node: ```bash docker exec coolify php artisan horizon:supervisors ``` -6. Make sure the `maintenance` queue drains (its depth goes down) and that +6. Make sure the `maintenance` and `webhooks` queues drain (its depth goes down) and that `php artisan scheduled:diagnostics` shows no growing count of stale occurrences. 7. Do the same steps on the next node. -Jobs that the old code put on `high` (for example Docker cleanups) stay on -`high`, and the `high` pool still processes them. +Jobs that the old code put on `high` (for example Docker cleanups and webhook +jobs) stay on `high`, and the `high` pool still processes them. ## Rollback 1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides. 2. Run `config:cache` and `horizon:terminate` on one node at a time. -3. Jobs that wait on the `maintenance` queue are not lost, but the old - configuration has no `maintenance` pool. Keep one node with the new - configuration until `maintenance` is empty, or start a temporary worker: - `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`. +3. Jobs that wait on the `maintenance` or `webhooks` queue are not lost, but + the old configuration has no pool for them. Keep one node with the new + configuration until both queues are empty, or start a temporary worker: + `php artisan queue:work redis --queue=maintenance,webhooks --tries=1 --timeout=600`. 4. Do not roll back the migration. The old code ignores the new columns. ## Metrics to collect before you change worker counts diff --git a/tests/Feature/HorizonConfigurationTest.php b/tests/Feature/HorizonConfigurationTest.php index 4f8a8d6fc3..70a2d4d6ea 100644 --- a/tests/Feature/HorizonConfigurationTest.php +++ b/tests/Feature/HorizonConfigurationTest.php @@ -1,5 +1,7 @@ null, 'HORIZON_DEFAULT_PROCESSES' => null, 'HORIZON_MAINTENANCE_PROCESSES' => null, + 'HORIZON_WEBHOOKS_PROCESSES' => null, ])); $production = $parsed['production']; - expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']); + expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks']); - foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) { + foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10, 'webhooks' => 10] as $queue => $processes) { $pool = $production[$queue]; expect($pool->queue)->toBe($queue) ->and($pool->balancing())->toBeFalse() @@ -119,6 +122,7 @@ test('cloud pool sizes come from env and invalid values fall back to defaults', 'HORIZON_HIGH_PROCESSES' => 'abc', 'HORIZON_DEFAULT_PROCESSES' => '-5', 'HORIZON_MAINTENANCE_PROCESSES' => '4', + 'HORIZON_WEBHOOKS_PROCESSES' => '6', ]))['production']; expect($production['deployments']->maxProcesses)->toBe(25) @@ -127,7 +131,9 @@ test('cloud pool sizes come from env and invalid values fall back to defaults', ->and($production['high']->maxProcesses)->toBe(60) ->and($production['default']->maxProcesses)->toBe(40) ->and($production['maintenance']->minProcesses)->toBe(4) - ->and($production['maintenance']->maxProcesses)->toBe(4); + ->and($production['maintenance']->maxProcesses)->toBe(4) + ->and($production['webhooks']->minProcesses)->toBe(6) + ->and($production['webhooks']->maxProcesses)->toBe(6); $fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv([ 'SELF_HOSTED' => 'false', @@ -142,7 +148,7 @@ test('cloud queue helpers route to queues that have a dedicated pool', function $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production']; - expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]); + expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()]); }); test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () { @@ -157,6 +163,31 @@ test('self-hosted queue helpers route to a queue the s6 supervisor drains', func foreach ($parsed as $supervisors) { expect(array_keys($supervisors))->toBe(['s6']) ->and(explode(',', $supervisors['s6']->queue)) - ->toContain(deployment_queue(), crons_queue(), maintenance_queue()); + ->toContain(deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()); } }); + +test('webhook jobs run on the webhooks queue on cloud and on high on self-hosted', function (bool $selfHosted, string $queue) { + config(['constants.coolify.self_hosted' => $selfHosted]); + + $stripeJob = new StripeProcessJob(['type' => 'invoice.paid']); + $githubJob = new ProcessGithubPullRequestWebhook( + applicationId: 1, + githubAppId: null, + action: 'opened', + pullRequestId: 1, + pullRequestHtmlUrl: 'https://github.com/coollabsio/coolify/pull/1', + pullRequestTitle: null, + beforeSha: null, + afterSha: null, + commitSha: 'abc123', + authorAssociation: null, + fullName: 'coollabsio/coolify', + ); + + expect($stripeJob->queue)->toBe($queue) + ->and($githubJob->queue)->toBe($queue); +})->with([ + 'cloud' => [false, 'webhooks'], + 'self-hosted' => [true, 'high'], +]);