From 56497d50cf5e9e3b58a3c6882f6e319477efda4b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:59:16 +0200 Subject: [PATCH] fix(deployments): include scheduled jobs in progress checks Count running scheduled tasks, database backups, and volume backups, excluding runs older than their job timeout. Report scheduled job counts and keep the progress check active while any scheduled jobs are running. --- bootstrap/helpers/shared.php | 43 +++++++++--- .../Feature/RunningScheduledJobCountTest.php | 66 +++++++++++++++++++ tests/Unit/RunningDeploymentsOutputTest.php | 2 +- 3 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 tests/Feature/RunningScheduledJobCountTest.php diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 81f2dbe726..90063c7c28 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2,6 +2,8 @@ use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProxyTypes; +use App\Jobs\DatabaseBackupJob; +use App\Jobs\ScheduledTaskJob; use App\Jobs\ServerFilesFromServerJob; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; @@ -14,6 +16,10 @@ use App\Models\LocalFileVolume; use App\Models\LocalPersistentVolume; use App\Models\Project; use App\Models\S3Storage; +use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledTaskExecution; +use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; @@ -35,6 +41,7 @@ use App\Models\User; use App\Support\DnsRecordHints; use Carbon\CarbonImmutable; use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException; +use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Process\Pool; @@ -2614,11 +2621,6 @@ function isAnyDeploymentInprogress(bool $showAll = false) { $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get(); - if ($runningJobs->isEmpty()) { - echo "No deployments in progress.\n"; - exit(0); - } - $horizonJobIds = []; $deploymentDetails = []; @@ -2654,20 +2656,43 @@ function isAnyDeploymentInprogress(bool $showAll = false) } } - if (count($horizonJobIds) === 0) { - echo "No active deployments in progress (all jobs completed or failed).\n"; + $scheduledJobCount = runningScheduledJobCount(); + + if (count($horizonJobIds) === 0 && $scheduledJobCount === 0) { + echo "No active deployments or scheduled jobs in progress.\n"; exit(0); } - echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll); + echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll, $scheduledJobCount); exit(1); } -function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false): string +/** + * Counts scheduled task, database backup, and volume backup runs that are still running. + * A run that started longer ago than its job timeout is stale: the worker has stopped it. + */ +function runningScheduledJobCount(): int +{ + $startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false; + + $tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get() + ->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS)); + + $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get() + ->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS)); + + $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get() + ->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT)); + + return $tasks->count() + $databaseBackups->count() + $volumeBackups->count(); +} + +function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false, int $scheduledJobCount = 0): string { $output = "\n=== Running Deployments ===\n"; $output .= 'Total active deployments: '.$activeDeploymentCount."\n"; + $output .= 'Total running scheduled jobs: '.$scheduledJobCount."\n"; if (! $showAll) { return $output; diff --git a/tests/Feature/RunningScheduledJobCountTest.php b/tests/Feature/RunningScheduledJobCountTest.php new file mode 100644 index 0000000000..37edc5d6bc --- /dev/null +++ b/tests/Feature/RunningScheduledJobCountTest.php @@ -0,0 +1,66 @@ +create(); + + $this->task = ScheduledTask::factory()->create(['team_id' => $team->id, 'timeout' => 300]); + $this->databaseBackup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'timeout' => 3600, + ]); + $this->volumeBackup = ScheduledVolumeBackup::create([ + 'uuid' => fake()->uuid(), + 'backupable_type' => 'App\Models\LocalPersistentVolume', + 'backupable_id' => 1, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + 'save_s3' => false, + 'timeout' => 7200, + ]); +}); + +it('counts running scheduled tasks, database backups, and volume backups', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']); + $this->volumeBackup->executions()->create(); + + expect(runningScheduledJobCount())->toBe(3); +}); + +it('does not count finished runs', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'success']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'failed']); + $this->volumeBackup->executions()->create(['status' => 'success']); + + expect(runningScheduledJobCount())->toBe(0); +}); + +it('does not count running rows older than their job timeout', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']); + $this->volumeBackup->executions()->create(); + + $this->travel(30)->minutes(); + expect(runningScheduledJobCount())->toBe(2); + + $this->travel(40)->minutes(); + expect(runningScheduledJobCount())->toBe(1); + + $this->travel(60)->minutes(); + expect(runningScheduledJobCount())->toBe(0); +}); diff --git a/tests/Unit/RunningDeploymentsOutputTest.php b/tests/Unit/RunningDeploymentsOutputTest.php index 62d367514c..308c67e108 100644 --- a/tests/Unit/RunningDeploymentsOutputTest.php +++ b/tests/Unit/RunningDeploymentsOutputTest.php @@ -11,7 +11,7 @@ it('shows only the active deployment count by default', function () { ]]); expect($output) - ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\n") + ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\nTotal running scheduled jobs: 0\n") ->not->toContain('secret-application') ->not->toContain('https://example.com/secret-deployment') ->not->toContain('member@example.com');