From be72005980ae3eed2a352c53086760516b62a524 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:51:58 +0200 Subject: [PATCH 1/3] fix(deployments): expire missing Horizon jobs after deployment timeout Keep reserved jobs active and bound missing Horizon records by the server's deployment timeout. Add regression coverage for active and finished jobs. Use UTC PostgreSQL sessions and bump the release version to 4.4.1. --- app/Models/ApplicationDeploymentQueue.php | 13 ++++ bootstrap/helpers/shared.php | 3 +- config/constants.php | 2 +- config/database.php | 2 + other/nightly/versions.json | 2 +- .../DeploymentHorizonJobActiveTest.php | 59 +++++++++++++++++++ versions.json | 2 +- 7 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 tests/Feature/DeploymentHorizonJobActiveTest.php diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 6ebdf255ae..5994f187de 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -177,6 +177,19 @@ class ApplicationDeploymentQueue extends Model return getJobStatus($this->horizon_job_id); } + /** + * Horizon drops its job record 'trim.pending' minutes after the push, also while the job runs, + * so a missing record counts as running only until the deployment timeout has passed. + */ + public function isHorizonJobActive(): bool + { + return match ($this->getHorizonJobStatus()) { + 'reserved' => true, + 'unknown' => $this->updated_at?->gt(now()->subSeconds($this->server?->settings?->dynamic_timeout ?? 3600)) ?? false, + default => false, + }; + } + public function commitMessage() { if (empty($this->commit_message) || is_null($this->commit_message)) { diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 2e9fdbb253..81f2dbe726 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2623,8 +2623,7 @@ function isAnyDeploymentInprogress(bool $showAll = false) $deploymentDetails = []; foreach ($runningJobs as $runningJob) { - $horizonJobStatus = getJobStatus($runningJob->horizon_job_id); - if ($horizonJobStatus === 'unknown' || $horizonJobStatus === 'reserved') { + if ($runningJob->isHorizonJobActive()) { $horizonJobIds[] = $runningJob->horizon_job_id; if ($showAll) { diff --git a/config/constants.php b/config/constants.php index c25c29ca9b..8c71e262dd 100644 --- a/config/constants.php +++ b/config/constants.php @@ -5,7 +5,7 @@ $sshCommandTimeout = (int) env('SSH_COMMAND_TIMEOUT', 3600); return [ 'coolify' => [ - 'version' => env('COOLIFY_VERSION') ?: '4.4.0', + 'version' => env('COOLIFY_VERSION') ?: '4.4.1', 'helper_version' => '1.0.17', 'railpack_version' => '0.23.0', 'self_hosted' => env('SELF_HOSTED', true), diff --git a/config/database.php b/config/database.php index 9238a70551..07039d6d62 100644 --- a/config/database.php +++ b/config/database.php @@ -33,6 +33,8 @@ $pgsql = [ 'prefix' => '', 'prefix_indexes' => true, 'search_path' => 'public', + // Laravel writes dates without an offset, so timestampTz columns need a UTC session. + 'timezone' => 'UTC', 'sslmode' => 'prefer', 'options' => [ (defined('Pdo\Pgsql::ATTR_DISABLE_PREPARES') ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => env('DB_DISABLE_PREPARES', false), diff --git a/other/nightly/versions.json b/other/nightly/versions.json index 81016eeeb4..bd38eada4e 100644 --- a/other/nightly/versions.json +++ b/other/nightly/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "main" + "version": "4.4.1" }, "nightly": { "version": "4.5-rc.1" diff --git a/tests/Feature/DeploymentHorizonJobActiveTest.php b/tests/Feature/DeploymentHorizonJobActiveTest.php new file mode 100644 index 0000000000..07da481f90 --- /dev/null +++ b/tests/Feature/DeploymentHorizonJobActiveTest.php @@ -0,0 +1,59 @@ +create(); + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $this->server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]); + $this->server->settings->update(['dynamic_timeout' => 3600]); + + $this->deployment = ApplicationDeploymentQueue::create([ + 'application_id' => '1', + 'deployment_uuid' => 'horizon-active-uuid', + 'server_id' => $this->server->id, + 'horizon_job_id' => 'job-1', + ]); +}); + +function fakeHorizonJobStatus(?string $status): void +{ + $jobs = $status === null ? collect() : collect([(object) ['id' => 'job-1', 'status' => $status]]); + + $repository = Mockery::mock(JobRepository::class); + $repository->shouldReceive('getJobs')->andReturn($jobs); + app()->instance(JobRepository::class, $repository); +} + +it('counts a reserved job as active', function () { + fakeHorizonJobStatus('reserved'); + + expect($this->deployment->isHorizonJobActive())->toBeTrue(); +}); + +it('does not count a finished job as active', function (string $status) { + fakeHorizonJobStatus($status); + + expect($this->deployment->isHorizonJobActive())->toBeFalse(); +})->with(['completed', 'failed']); + +it('counts a job without a Horizon record as active within the deployment timeout', function () { + fakeHorizonJobStatus(null); + $this->travel(59)->minutes(); + + expect($this->deployment->isHorizonJobActive())->toBeTrue(); +}); + +it('does not count a job without a Horizon record as active after the deployment timeout', function () { + fakeHorizonJobStatus(null); + $this->travel(61)->minutes(); + + expect($this->deployment->isHorizonJobActive())->toBeFalse(); +}); diff --git a/versions.json b/versions.json index d4a76777b3..bd38eada4e 100644 --- a/versions.json +++ b/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "4.4.0" + "version": "4.4.1" }, "nightly": { "version": "4.5-rc.1" From 56497d50cf5e9e3b58a3c6882f6e319477efda4b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:59:16 +0200 Subject: [PATCH 2/3] fix(deployments): include scheduled jobs in progress checks Count running scheduled tasks, database backups, and volume backups, excluding runs older than their job timeout. Report scheduled job counts and keep the progress check active while any scheduled jobs are running. --- bootstrap/helpers/shared.php | 43 +++++++++--- .../Feature/RunningScheduledJobCountTest.php | 66 +++++++++++++++++++ tests/Unit/RunningDeploymentsOutputTest.php | 2 +- 3 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 tests/Feature/RunningScheduledJobCountTest.php diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 81f2dbe726..90063c7c28 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2,6 +2,8 @@ use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProxyTypes; +use App\Jobs\DatabaseBackupJob; +use App\Jobs\ScheduledTaskJob; use App\Jobs\ServerFilesFromServerJob; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; @@ -14,6 +16,10 @@ use App\Models\LocalFileVolume; use App\Models\LocalPersistentVolume; use App\Models\Project; use App\Models\S3Storage; +use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledTaskExecution; +use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; @@ -35,6 +41,7 @@ use App\Models\User; use App\Support\DnsRecordHints; use Carbon\CarbonImmutable; use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException; +use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Process\Pool; @@ -2614,11 +2621,6 @@ function isAnyDeploymentInprogress(bool $showAll = false) { $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get(); - if ($runningJobs->isEmpty()) { - echo "No deployments in progress.\n"; - exit(0); - } - $horizonJobIds = []; $deploymentDetails = []; @@ -2654,20 +2656,43 @@ function isAnyDeploymentInprogress(bool $showAll = false) } } - if (count($horizonJobIds) === 0) { - echo "No active deployments in progress (all jobs completed or failed).\n"; + $scheduledJobCount = runningScheduledJobCount(); + + if (count($horizonJobIds) === 0 && $scheduledJobCount === 0) { + echo "No active deployments or scheduled jobs in progress.\n"; exit(0); } - echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll); + echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll, $scheduledJobCount); exit(1); } -function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false): string +/** + * Counts scheduled task, database backup, and volume backup runs that are still running. + * A run that started longer ago than its job timeout is stale: the worker has stopped it. + */ +function runningScheduledJobCount(): int +{ + $startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false; + + $tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get() + ->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS)); + + $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get() + ->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS)); + + $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get() + ->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT)); + + return $tasks->count() + $databaseBackups->count() + $volumeBackups->count(); +} + +function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false, int $scheduledJobCount = 0): string { $output = "\n=== Running Deployments ===\n"; $output .= 'Total active deployments: '.$activeDeploymentCount."\n"; + $output .= 'Total running scheduled jobs: '.$scheduledJobCount."\n"; if (! $showAll) { return $output; diff --git a/tests/Feature/RunningScheduledJobCountTest.php b/tests/Feature/RunningScheduledJobCountTest.php new file mode 100644 index 0000000000..37edc5d6bc --- /dev/null +++ b/tests/Feature/RunningScheduledJobCountTest.php @@ -0,0 +1,66 @@ +create(); + + $this->task = ScheduledTask::factory()->create(['team_id' => $team->id, 'timeout' => 300]); + $this->databaseBackup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'timeout' => 3600, + ]); + $this->volumeBackup = ScheduledVolumeBackup::create([ + 'uuid' => fake()->uuid(), + 'backupable_type' => 'App\Models\LocalPersistentVolume', + 'backupable_id' => 1, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + 'save_s3' => false, + 'timeout' => 7200, + ]); +}); + +it('counts running scheduled tasks, database backups, and volume backups', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']); + $this->volumeBackup->executions()->create(); + + expect(runningScheduledJobCount())->toBe(3); +}); + +it('does not count finished runs', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'success']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'failed']); + $this->volumeBackup->executions()->create(['status' => 'success']); + + expect(runningScheduledJobCount())->toBe(0); +}); + +it('does not count running rows older than their job timeout', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']); + $this->volumeBackup->executions()->create(); + + $this->travel(30)->minutes(); + expect(runningScheduledJobCount())->toBe(2); + + $this->travel(40)->minutes(); + expect(runningScheduledJobCount())->toBe(1); + + $this->travel(60)->minutes(); + expect(runningScheduledJobCount())->toBe(0); +}); diff --git a/tests/Unit/RunningDeploymentsOutputTest.php b/tests/Unit/RunningDeploymentsOutputTest.php index 62d367514c..308c67e108 100644 --- a/tests/Unit/RunningDeploymentsOutputTest.php +++ b/tests/Unit/RunningDeploymentsOutputTest.php @@ -11,7 +11,7 @@ it('shows only the active deployment count by default', function () { ]]); expect($output) - ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\n") + ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\nTotal running scheduled jobs: 0\n") ->not->toContain('secret-application') ->not->toContain('https://example.com/secret-deployment') ->not->toContain('member@example.com'); From 393dcbb6cd193e61ba14ce43fc85f685af767903 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:09:27 +0200 Subject: [PATCH 3/3] fix(logs): wrap viewer in a root element for parent re-renders --- .../livewire/project/shared/get-logs.blade.php | 2 ++ .../TaskAndLogOutputVisibilityTest.php | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/resources/views/livewire/project/shared/get-logs.blade.php b/resources/views/livewire/project/shared/get-logs.blade.php index e05ef34eaa..7b000c8a78 100644 --- a/resources/views/livewire/project/shared/get-logs.blade.php +++ b/resources/views/livewire/project/shared/get-logs.blade.php @@ -1,3 +1,4 @@ +
@if (! $canReadLogs) Container logs can contain secrets. @@ -315,3 +316,4 @@
@endif + diff --git a/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php b/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php index eb51d9c4b3..891711458b 100644 --- a/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php +++ b/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php @@ -136,3 +136,16 @@ it('shows task commands and task output to an owner in the ui', function () { Livewire::test(GetLogs::class, ['server' => $this->server, 'resource' => $this->application, 'container' => 'app-container']) ->assertDontSee('Hidden (only admins can view)'); }); + +it('renders logs inside one root element so the parent can re-render it', function (string $role) { + $this->actingAs($role === 'owner' ? $this->owner : $this->member); + session(['currentTeam' => $this->team]); + + $html = Livewire::test(GetLogs::class, ['server' => $this->server, 'resource' => $this->application, 'container' => 'app-container']) + ->html(); + + // Livewire reuses the first tag name as the placeholder for kept children; a leading comment gives an empty tag. + preg_match('/<([a-zA-Z0-9\-]*)/', $html, $matches); + + expect($matches[1])->toBe('div'); +})->with(['owner', 'member']);