fix(deployments): include scheduled jobs in progress checks

Count running scheduled tasks, database backups, and volume backups,
excluding runs older than their job timeout. Report scheduled job counts
and keep the progress check active while any scheduled jobs are running.

bootstrap/helpers/shared.php

@@ -2,6 +2,8 @@
 
 use App\Enums\ApplicationDeploymentStatus;
 use App\Enums\ProxyTypes;
+use App\Jobs\DatabaseBackupJob;
+use App\Jobs\ScheduledTaskJob;
 use App\Jobs\ServerFilesFromServerJob;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
@@ -14,6 +16,10 @@ use App\Models\LocalFileVolume;
 use App\Models\LocalPersistentVolume;
 use App\Models\Project;
 use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledTaskExecution;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
 use App\Models\Server;
 use App\Models\Service;
 use App\Models\ServiceApplication;
@@ -35,6 +41,7 @@ use App\Models\User;
 use App\Support\DnsRecordHints;
 use Carbon\CarbonImmutable;
 use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException;
+use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\ModelNotFoundException;
 use Illuminate\Database\UniqueConstraintViolationException;
 use Illuminate\Process\Pool;
@@ -2614,11 +2621,6 @@ function isAnyDeploymentInprogress(bool $showAll = false)
 {
     $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get();
 
-    if ($runningJobs->isEmpty()) {
-        echo "No deployments in progress.\n";
-        exit(0);
-    }
-
     $horizonJobIds = [];
     $deploymentDetails = [];
 
@@ -2654,20 +2656,43 @@ function isAnyDeploymentInprogress(bool $showAll = false)
         }
     }
 
-    if (count($horizonJobIds) === 0) {
-        echo "No active deployments in progress (all jobs completed or failed).\n";
+    $scheduledJobCount = runningScheduledJobCount();
+
+    if (count($horizonJobIds) === 0 && $scheduledJobCount === 0) {
+        echo "No active deployments or scheduled jobs in progress.\n";
         exit(0);
     }
 
-    echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll);
+    echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll, $scheduledJobCount);
 
     exit(1);
 }
 
-function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false): string
+/**
+ * Counts scheduled task, database backup, and volume backup runs that are still running.
+ * A run that started longer ago than its job timeout is stale: the worker has stopped it.
+ */
+function runningScheduledJobCount(): int
+{
+    $startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false;
+
+    $tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get()
+        ->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS));
+
+    $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get()
+        ->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS));
+
+    $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get()
+        ->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT));
+
+    return $tasks->count() + $databaseBackups->count() + $volumeBackups->count();
+}
+
+function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false, int $scheduledJobCount = 0): string
 {
     $output = "\n=== Running Deployments ===\n";
     $output .= 'Total active deployments: '.$activeDeploymentCount."\n";
+    $output .= 'Total running scheduled jobs: '.$scheduledJobCount."\n";
 
     if (! $showAll) {
         return $output;

tests/Feature/RunningScheduledJobCountTest.php

@@ -0,0 +1,66 @@
+<?php
+
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledTask;
+use App\Models\ScheduledTaskExecution;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    $team = Team::factory()->create();
+
+    $this->task = ScheduledTask::factory()->create(['team_id' => $team->id, 'timeout' => 300]);
+    $this->databaseBackup = ScheduledDatabaseBackup::create([
+        'frequency' => '0 0 * * *',
+        'save_s3' => false,
+        'database_type' => 'App\Models\StandalonePostgresql',
+        'database_id' => 1,
+        'team_id' => $team->id,
+        'timeout' => 3600,
+    ]);
+    $this->volumeBackup = ScheduledVolumeBackup::create([
+        'uuid' => fake()->uuid(),
+        'backupable_type' => 'App\Models\LocalPersistentVolume',
+        'backupable_id' => 1,
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'enabled' => true,
+        'save_s3' => false,
+        'timeout' => 7200,
+    ]);
+});
+
+it('counts running scheduled tasks, database backups, and volume backups', function () {
+    ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']);
+    ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']);
+    $this->volumeBackup->executions()->create();
+
+    expect(runningScheduledJobCount())->toBe(3);
+});
+
+it('does not count finished runs', function () {
+    ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'success']);
+    ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'failed']);
+    $this->volumeBackup->executions()->create(['status' => 'success']);
+
+    expect(runningScheduledJobCount())->toBe(0);
+});
+
+it('does not count running rows older than their job timeout', function () {
+    ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']);
+    ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']);
+    $this->volumeBackup->executions()->create();
+
+    $this->travel(30)->minutes();
+    expect(runningScheduledJobCount())->toBe(2);
+
+    $this->travel(40)->minutes();
+    expect(runningScheduledJobCount())->toBe(1);
+
+    $this->travel(60)->minutes();
+    expect(runningScheduledJobCount())->toBe(0);
+});

tests/Unit/RunningDeploymentsOutputTest.php

@@ -11,7 +11,7 @@ it('shows only the active deployment count by default', function () {
     ]]);
 
     expect($output)
-        ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\n")
+        ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\nTotal running scheduled jobs: 0\n")
         ->not->toContain('secret-application')
         ->not->toContain('https://example.com/secret-deployment')
         ->not->toContain('member@example.com');