diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 6ebdf255ae..5994f187de 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -177,6 +177,19 @@ class ApplicationDeploymentQueue extends Model return getJobStatus($this->horizon_job_id); } + /** + * Horizon drops its job record 'trim.pending' minutes after the push, also while the job runs, + * so a missing record counts as running only until the deployment timeout has passed. + */ + public function isHorizonJobActive(): bool + { + return match ($this->getHorizonJobStatus()) { + 'reserved' => true, + 'unknown' => $this->updated_at?->gt(now()->subSeconds($this->server?->settings?->dynamic_timeout ?? 3600)) ?? false, + default => false, + }; + } + public function commitMessage() { if (empty($this->commit_message) || is_null($this->commit_message)) { diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 2e9fdbb253..90063c7c28 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2,6 +2,8 @@ use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProxyTypes; +use App\Jobs\DatabaseBackupJob; +use App\Jobs\ScheduledTaskJob; use App\Jobs\ServerFilesFromServerJob; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; @@ -14,6 +16,10 @@ use App\Models\LocalFileVolume; use App\Models\LocalPersistentVolume; use App\Models\Project; use App\Models\S3Storage; +use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledTaskExecution; +use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; @@ -35,6 +41,7 @@ use App\Models\User; use App\Support\DnsRecordHints; use Carbon\CarbonImmutable; use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException; +use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Process\Pool; @@ -2614,17 +2621,11 @@ function isAnyDeploymentInprogress(bool $showAll = false) { $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get(); - if ($runningJobs->isEmpty()) { - echo "No deployments in progress.\n"; - exit(0); - } - $horizonJobIds = []; $deploymentDetails = []; foreach ($runningJobs as $runningJob) { - $horizonJobStatus = getJobStatus($runningJob->horizon_job_id); - if ($horizonJobStatus === 'unknown' || $horizonJobStatus === 'reserved') { + if ($runningJob->isHorizonJobActive()) { $horizonJobIds[] = $runningJob->horizon_job_id; if ($showAll) { @@ -2655,20 +2656,43 @@ function isAnyDeploymentInprogress(bool $showAll = false) } } - if (count($horizonJobIds) === 0) { - echo "No active deployments in progress (all jobs completed or failed).\n"; + $scheduledJobCount = runningScheduledJobCount(); + + if (count($horizonJobIds) === 0 && $scheduledJobCount === 0) { + echo "No active deployments or scheduled jobs in progress.\n"; exit(0); } - echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll); + echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll, $scheduledJobCount); exit(1); } -function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false): string +/** + * Counts scheduled task, database backup, and volume backup runs that are still running. + * A run that started longer ago than its job timeout is stale: the worker has stopped it. + */ +function runningScheduledJobCount(): int +{ + $startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false; + + $tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get() + ->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS)); + + $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get() + ->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS)); + + $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get() + ->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT)); + + return $tasks->count() + $databaseBackups->count() + $volumeBackups->count(); +} + +function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false, int $scheduledJobCount = 0): string { $output = "\n=== Running Deployments ===\n"; $output .= 'Total active deployments: '.$activeDeploymentCount."\n"; + $output .= 'Total running scheduled jobs: '.$scheduledJobCount."\n"; if (! $showAll) { return $output; diff --git a/config/constants.php b/config/constants.php index c25c29ca9b..8c71e262dd 100644 --- a/config/constants.php +++ b/config/constants.php @@ -5,7 +5,7 @@ $sshCommandTimeout = (int) env('SSH_COMMAND_TIMEOUT', 3600); return [ 'coolify' => [ - 'version' => env('COOLIFY_VERSION') ?: '4.4.0', + 'version' => env('COOLIFY_VERSION') ?: '4.4.1', 'helper_version' => '1.0.17', 'railpack_version' => '0.23.0', 'self_hosted' => env('SELF_HOSTED', true), diff --git a/config/database.php b/config/database.php index 9238a70551..07039d6d62 100644 --- a/config/database.php +++ b/config/database.php @@ -33,6 +33,8 @@ $pgsql = [ 'prefix' => '', 'prefix_indexes' => true, 'search_path' => 'public', + // Laravel writes dates without an offset, so timestampTz columns need a UTC session. + 'timezone' => 'UTC', 'sslmode' => 'prefer', 'options' => [ (defined('Pdo\Pgsql::ATTR_DISABLE_PREPARES') ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => env('DB_DISABLE_PREPARES', false), diff --git a/other/nightly/versions.json b/other/nightly/versions.json index 81016eeeb4..bd38eada4e 100644 --- a/other/nightly/versions.json +++ b/other/nightly/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "main" + "version": "4.4.1" }, "nightly": { "version": "4.5-rc.1" diff --git a/resources/views/livewire/project/shared/get-logs.blade.php b/resources/views/livewire/project/shared/get-logs.blade.php index e05ef34eaa..7b000c8a78 100644 --- a/resources/views/livewire/project/shared/get-logs.blade.php +++ b/resources/views/livewire/project/shared/get-logs.blade.php @@ -1,3 +1,4 @@ +
@if (! $canReadLogs) Container logs can contain secrets. @@ -315,3 +316,4 @@
@endif + diff --git a/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php b/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php index eb51d9c4b3..891711458b 100644 --- a/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php +++ b/tests/Feature/Authorization/TaskAndLogOutputVisibilityTest.php @@ -136,3 +136,16 @@ it('shows task commands and task output to an owner in the ui', function () { Livewire::test(GetLogs::class, ['server' => $this->server, 'resource' => $this->application, 'container' => 'app-container']) ->assertDontSee('Hidden (only admins can view)'); }); + +it('renders logs inside one root element so the parent can re-render it', function (string $role) { + $this->actingAs($role === 'owner' ? $this->owner : $this->member); + session(['currentTeam' => $this->team]); + + $html = Livewire::test(GetLogs::class, ['server' => $this->server, 'resource' => $this->application, 'container' => 'app-container']) + ->html(); + + // Livewire reuses the first tag name as the placeholder for kept children; a leading comment gives an empty tag. + preg_match('/<([a-zA-Z0-9\-]*)/', $html, $matches); + + expect($matches[1])->toBe('div'); +})->with(['owner', 'member']); diff --git a/tests/Feature/DeploymentHorizonJobActiveTest.php b/tests/Feature/DeploymentHorizonJobActiveTest.php new file mode 100644 index 0000000000..07da481f90 --- /dev/null +++ b/tests/Feature/DeploymentHorizonJobActiveTest.php @@ -0,0 +1,59 @@ +create(); + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $this->server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]); + $this->server->settings->update(['dynamic_timeout' => 3600]); + + $this->deployment = ApplicationDeploymentQueue::create([ + 'application_id' => '1', + 'deployment_uuid' => 'horizon-active-uuid', + 'server_id' => $this->server->id, + 'horizon_job_id' => 'job-1', + ]); +}); + +function fakeHorizonJobStatus(?string $status): void +{ + $jobs = $status === null ? collect() : collect([(object) ['id' => 'job-1', 'status' => $status]]); + + $repository = Mockery::mock(JobRepository::class); + $repository->shouldReceive('getJobs')->andReturn($jobs); + app()->instance(JobRepository::class, $repository); +} + +it('counts a reserved job as active', function () { + fakeHorizonJobStatus('reserved'); + + expect($this->deployment->isHorizonJobActive())->toBeTrue(); +}); + +it('does not count a finished job as active', function (string $status) { + fakeHorizonJobStatus($status); + + expect($this->deployment->isHorizonJobActive())->toBeFalse(); +})->with(['completed', 'failed']); + +it('counts a job without a Horizon record as active within the deployment timeout', function () { + fakeHorizonJobStatus(null); + $this->travel(59)->minutes(); + + expect($this->deployment->isHorizonJobActive())->toBeTrue(); +}); + +it('does not count a job without a Horizon record as active after the deployment timeout', function () { + fakeHorizonJobStatus(null); + $this->travel(61)->minutes(); + + expect($this->deployment->isHorizonJobActive())->toBeFalse(); +}); diff --git a/tests/Feature/RunningScheduledJobCountTest.php b/tests/Feature/RunningScheduledJobCountTest.php new file mode 100644 index 0000000000..37edc5d6bc --- /dev/null +++ b/tests/Feature/RunningScheduledJobCountTest.php @@ -0,0 +1,66 @@ +create(); + + $this->task = ScheduledTask::factory()->create(['team_id' => $team->id, 'timeout' => 300]); + $this->databaseBackup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'timeout' => 3600, + ]); + $this->volumeBackup = ScheduledVolumeBackup::create([ + 'uuid' => fake()->uuid(), + 'backupable_type' => 'App\Models\LocalPersistentVolume', + 'backupable_id' => 1, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + 'save_s3' => false, + 'timeout' => 7200, + ]); +}); + +it('counts running scheduled tasks, database backups, and volume backups', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']); + $this->volumeBackup->executions()->create(); + + expect(runningScheduledJobCount())->toBe(3); +}); + +it('does not count finished runs', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'success']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'failed']); + $this->volumeBackup->executions()->create(['status' => 'success']); + + expect(runningScheduledJobCount())->toBe(0); +}); + +it('does not count running rows older than their job timeout', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']); + $this->volumeBackup->executions()->create(); + + $this->travel(30)->minutes(); + expect(runningScheduledJobCount())->toBe(2); + + $this->travel(40)->minutes(); + expect(runningScheduledJobCount())->toBe(1); + + $this->travel(60)->minutes(); + expect(runningScheduledJobCount())->toBe(0); +}); diff --git a/tests/Unit/RunningDeploymentsOutputTest.php b/tests/Unit/RunningDeploymentsOutputTest.php index 62d367514c..308c67e108 100644 --- a/tests/Unit/RunningDeploymentsOutputTest.php +++ b/tests/Unit/RunningDeploymentsOutputTest.php @@ -11,7 +11,7 @@ it('shows only the active deployment count by default', function () { ]]); expect($output) - ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\n") + ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\nTotal running scheduled jobs: 0\n") ->not->toContain('secret-application') ->not->toContain('https://example.com/secret-deployment') ->not->toContain('member@example.com'); diff --git a/versions.json b/versions.json index d4a76777b3..bd38eada4e 100644 --- a/versions.json +++ b/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "4.4.0" + "version": "4.4.1" }, "nightly": { "version": "4.5-rc.1"