fix(horizon): route webhook jobs to a dedicated cloud webhooks queue

On cloud, GitHub pull request webhooks and Stripe events now run on a
dedicated `webhooks` queue so a busy `high` queue cannot delay them.
Self-hosted instances keep these jobs on `high`.

- Add webhooks_queue() helper and use it in ProcessGithubPullRequestWebhook
  and StripeProcessJob
- Provision a `webhooks` Horizon pool in cloud production, sized by
  HORIZON_WEBHOOKS_PROCESSES (default 10)
- Document the new pool, rollout and rollback steps
- Cover pool provisioning and job queue routing in tests

.env.development.example

@@ -39,6 +39,8 @@ DB_PORT=5432
 # HORIZON_DEFAULT_PROCESSES=40
 # Docker cleanups; small on purpose to bound concurrent slow remote prunes.
 # HORIZON_MAINTENANCE_PROCESSES=10
+# GitHub pull request webhooks and Stripe events.
+# HORIZON_WEBHOOKS_PROCESSES=10
 
 # Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local).
 # Force on in any environment (including production): SERVER_TIMING_ENABLED=true

app/Jobs/ProcessGithubPullRequestWebhook.php

@@ -41,7 +41,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
         public string $fullName,
         public bool $isForkPullRequest = false,
     ) {
-        $this->onQueue('high');
+        $this->onQueue(webhooks_queue());
     }
 
     public function handle(): void

app/Jobs/StripeProcessJob.php

@@ -23,7 +23,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
 
     public function __construct(public $event)
     {
-        $this->onQueue('high');
+        $this->onQueue(webhooks_queue());
     }
 
     public function handle(): void

bootstrap/helpers/shared.php

@@ -1070,6 +1070,23 @@ function maintenance_queue(): string
     return isCloud() ? 'maintenance' : 'high';
 }
 
+/**
+ * Resolve the queue used for incoming webhook processing — GitHub pull request webhooks and
+ * Stripe events.
+ *
+ * On cloud these jobs run on a dedicated `webhooks` queue so a busy `high` queue cannot delay
+ * them; self-hosted keeps them on the shared `high` queue, so a custom `HORIZON_QUEUES` does
+ * not need a new queue name. Routing is decided by `isCloud()` (config-based), so the
+ * dispatching process needs no special env — only the worker must be configured to drain `webhooks`.
+ *
+ * On cloud, config/horizon.php provisions a dedicated `webhooks` pool in production
+ * (see docs/cloud-horizon-workers.md).
+ */
+function webhooks_queue(): string
+{
+    return isCloud() ? 'webhooks' : 'high';
+}
+
 function translate_cron_expression($expression_to_validate): string
 {
     if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {

config/horizon.php

@@ -42,6 +42,9 @@ $selfHostedSupervisor = $workerOptions + [
 | cleanups. It is small on purpose: it bounds how many cleanups run at once
 | per node, and it comes on top of the other pools, so cleanups never take
 | deployment, cron or high workers.
+|
+| The `webhooks` pool (see webhooks_queue()) processes GitHub pull request
+| webhooks and Stripe events, so other jobs cannot delay them.
 */
 $cloudSupervisors = [];
 
@@ -51,6 +54,7 @@ foreach ([
     'high' => ['HORIZON_HIGH_PROCESSES', 60],
     'default' => ['HORIZON_DEFAULT_PROCESSES', 40],
     'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10],
+    'webhooks' => ['HORIZON_WEBHOOKS_PROCESSES', 10],
 ] as $queue => [$processesEnv, $defaultProcesses]) {
     $processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, [
         'options' => ['min_range' => 1, 'default' => $defaultProcesses],

docs/cloud-horizon-workers.md

@@ -11,14 +11,18 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others.
 | `high`        | `high`        | `HORIZON_HIGH_PROCESSES`        | 60               |
 | `default`     | `default`     | `HORIZON_DEFAULT_PROCESSES`     | 40               |
 | `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10               |
+| `webhooks`    | `webhooks`    | `HORIZON_WEBHOOKS_PROCESSES`    | 10               |
 
-- Counts apply to **each node**. Two nodes with the defaults run 460 workers.
+- Counts apply to **each node**. Two nodes with the defaults run 480 workers.
 - `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup
   that runs after a resource stops). Remote prunes can be slow, so this pool
   is small on purpose: it limits how many cleanups run at the same time and
   keeps them away from the `high` workers. It is added on top of the other
   pools. Each cleanup has a 600 s timeout, and only one cleanup runs per
   server at a time. Self-hosted instances keep cleanups on `high`.
+- `webhooks` processes GitHub pull request webhooks and Stripe events, so a
+  full `high` queue cannot delay them. These jobs are short (60 s timeout or
+  less). Self-hosted instances keep them on `high`.
 - Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon
   does not scale them.
 - A value that is not a positive integer falls back to the default.
@@ -45,6 +49,7 @@ HORIZON_CRONS_PROCESSES=60
 HORIZON_HIGH_PROCESSES=60
 HORIZON_DEFAULT_PROCESSES=40
 HORIZON_MAINTENANCE_PROCESSES=10
+HORIZON_WEBHOOKS_PROCESSES=10
 # Worker timeout. Must stay above the longest job timeout (36000 s) and below
 # retry_after (86400 s). The config clamps it to 36600..85800.
 HORIZON_TIMEOUT=36600
@@ -108,28 +113,28 @@ Do these steps on one node at a time. Do not flush Redis or clear queues.
    docker exec coolify php artisan horizon:terminate
    ```
 
-5. Make sure the five supervisors are running on this node:
+5. Make sure the six supervisors are running on this node:
 
    ```bash
    docker exec coolify php artisan horizon:supervisors
    ```
 
-6. Make sure the `maintenance` queue drains (its depth goes down) and that
+6. Make sure the `maintenance` and `webhooks` queues drain (its depth goes down) and that
    `php artisan scheduled:diagnostics` shows no growing count of stale
    occurrences.
 7. Do the same steps on the next node.
 
-Jobs that the old code put on `high` (for example Docker cleanups) stay on
-`high`, and the `high` pool still processes them.
+Jobs that the old code put on `high` (for example Docker cleanups and webhook
+jobs) stay on `high`, and the `high` pool still processes them.
 
 ## Rollback
 
 1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides.
 2. Run `config:cache` and `horizon:terminate` on one node at a time.
-3. Jobs that wait on the `maintenance` queue are not lost, but the old
-   configuration has no `maintenance` pool. Keep one node with the new
-   configuration until `maintenance` is empty, or start a temporary worker:
-   `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`.
+3. Jobs that wait on the `maintenance` or `webhooks` queue are not lost, but
+   the old configuration has no pool for them. Keep one node with the new
+   configuration until both queues are empty, or start a temporary worker:
+   `php artisan queue:work redis --queue=maintenance,webhooks --tries=1 --timeout=600`.
 4. Do not roll back the migration. The old code ignores the new columns.
 
 ## Metrics to collect before you change worker counts

tests/Feature/HorizonConfigurationTest.php

@@ -1,5 +1,7 @@
 <?php
 
+use App\Jobs\ProcessGithubPullRequestWebhook;
+use App\Jobs\StripeProcessJob;
 use Laravel\Horizon\ProvisioningPlan;
 use Laravel\Horizon\SupervisorOptions;
 
@@ -91,12 +93,13 @@ test('cloud production runs one fixed-size pool per queue with default process c
         'HORIZON_HIGH_PROCESSES' => null,
         'HORIZON_DEFAULT_PROCESSES' => null,
         'HORIZON_MAINTENANCE_PROCESSES' => null,
+        'HORIZON_WEBHOOKS_PROCESSES' => null,
     ]));
 
     $production = $parsed['production'];
-    expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']);
+    expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks']);
 
-    foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) {
+    foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10, 'webhooks' => 10] as $queue => $processes) {
         $pool = $production[$queue];
         expect($pool->queue)->toBe($queue)
             ->and($pool->balancing())->toBeFalse()
@@ -119,6 +122,7 @@ test('cloud pool sizes come from env and invalid values fall back to defaults',
         'HORIZON_HIGH_PROCESSES' => 'abc',
         'HORIZON_DEFAULT_PROCESSES' => '-5',
         'HORIZON_MAINTENANCE_PROCESSES' => '4',
+        'HORIZON_WEBHOOKS_PROCESSES' => '6',
     ]))['production'];
 
     expect($production['deployments']->maxProcesses)->toBe(25)
@@ -127,7 +131,9 @@ test('cloud pool sizes come from env and invalid values fall back to defaults',
         ->and($production['high']->maxProcesses)->toBe(60)
         ->and($production['default']->maxProcesses)->toBe(40)
         ->and($production['maintenance']->minProcesses)->toBe(4)
-        ->and($production['maintenance']->maxProcesses)->toBe(4);
+        ->and($production['maintenance']->maxProcesses)->toBe(4)
+        ->and($production['webhooks']->minProcesses)->toBe(6)
+        ->and($production['webhooks']->maxProcesses)->toBe(6);
 
     $fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv([
         'SELF_HOSTED' => 'false',
@@ -142,7 +148,7 @@ test('cloud queue helpers route to queues that have a dedicated pool', function
 
     $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production'];
 
-    expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]);
+    expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()]);
 });
 
 test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () {
@@ -157,6 +163,31 @@ test('self-hosted queue helpers route to a queue the s6 supervisor drains', func
     foreach ($parsed as $supervisors) {
         expect(array_keys($supervisors))->toBe(['s6'])
             ->and(explode(',', $supervisors['s6']->queue))
-            ->toContain(deployment_queue(), crons_queue(), maintenance_queue());
+            ->toContain(deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue());
     }
 });
+
+test('webhook jobs run on the webhooks queue on cloud and on high on self-hosted', function (bool $selfHosted, string $queue) {
+    config(['constants.coolify.self_hosted' => $selfHosted]);
+
+    $stripeJob = new StripeProcessJob(['type' => 'invoice.paid']);
+    $githubJob = new ProcessGithubPullRequestWebhook(
+        applicationId: 1,
+        githubAppId: null,
+        action: 'opened',
+        pullRequestId: 1,
+        pullRequestHtmlUrl: 'https://github.com/coollabsio/coolify/pull/1',
+        pullRequestTitle: null,
+        beforeSha: null,
+        afterSha: null,
+        commitSha: 'abc123',
+        authorAssociation: null,
+        fullName: 'coollabsio/coolify',
+    );
+
+    expect($stripeJob->queue)->toBe($queue)
+        ->and($githubJob->queue)->toBe($queue);
+})->with([
+    'cloud' => [false, 'webhooks'],
+    'self-hosted' => [true, 'high'],
+]);