From 2a7faa12105586a0b3bec2e0d8c062cee3f8feea Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:59:22 +0200 Subject: [PATCH] fix(docker-cleanup): skip scheduled cleanups while a deployment is running Closes #12183 Co-Authored-By: Claude Opus 5.5 --- app/Actions/Server/CleanupDocker.php | 22 +++++++- app/Jobs/DockerCleanupJob.php | 11 ++++ tests/Feature/DockerCleanupJobTest.php | 74 ++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index f439e01af5..3a1415ccc6 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -2,6 +2,8 @@ namespace App\Actions\Server; +use App\Enums\ApplicationDeploymentStatus; +use App\Models\ApplicationDeploymentQueue; use App\Models\Server; use App\Support\Actions\UniqueUntilProcessingJobDecorator; use Illuminate\Contracts\Queue\ShouldBeUnique; @@ -83,9 +85,24 @@ class CleanupDocker implements ShouldBeUnique return 'docker-cleanup-last-run-'.$server->uuid; } + /** + * The deployment that is queued or running on the server (as target or build server), if any. + * The cleanup prunes the build caches, which fails a build that uses them. A deployment that + * wrote no log for longer than the deployment timeout is stuck and does not count. + */ + public static function activeDeployment(Server $server): ?ApplicationDeploymentQueue + { + return ApplicationDeploymentQueue::query() + ->where(fn ($query) => $query->where('server_id', $server->id)->orWhere('build_server_id', $server->id)) + ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value]) + ->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600)) + ->first(['id', 'deployment_uuid']); + } + /** * Cleanup after a resource stop: skipped when a cleanup ran recently, and the queued run - * only cleans when the disk usage is at or above the server cleanup threshold. + * only cleans when the disk usage is at or above the server cleanup threshold and no + * deployment is active on the server. */ public static function dispatchAfterStop(Server $server): void { @@ -125,6 +142,9 @@ class CleanupDocker implements ShouldBeUnique $this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE; if ($onlyAboveThreshold) { + if (self::activeDeployment($server)) { + return []; + } $diskUsage = $server->getDiskUsage(); if (is_numeric($diskUsage) && (int) $diskUsage < $server->settings->docker_cleanup_threshold) { return []; diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 146f751b23..9daee0c82c 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -110,6 +110,17 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue return; } + $activeDeployment = $this->manualCleanup ? null : CleanupDocker::activeDeployment($this->server); + if ($activeDeployment) { + $this->execution_log->update([ + 'status' => 'success', + 'message' => 'Docker cleanup skipped: deployment '.$activeDeployment->deployment_uuid.' is queued or running on this server. The next scheduled cleanup runs as usual.', + ]); + event(new DockerCleanupDone($this->execution_log)); + + return; + } + $this->usageBefore = $this->server->getDiskUsage(); if ($this->manualCleanup || $this->server->settings->force_docker_cleanup) { diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php index a4ec2ac720..a6443e04ef 100644 --- a/tests/Feature/DockerCleanupJobTest.php +++ b/tests/Feature/DockerCleanupJobTest.php @@ -5,6 +5,7 @@ use App\Events\DockerCleanupDone; use App\Jobs\DockerCleanupJob; use App\Jobs\ScheduledJobManager; use App\Models\Application; +use App\Models\ApplicationDeploymentQueue; use App\Models\DockerCleanupExecution; use App\Models\Environment; use App\Models\InstanceSettings; @@ -660,3 +661,76 @@ it('keeps an application image name with a single quote inside the grep pattern' expect(file_exists($marker))->toBeFalse() ->and(explode("\n", $output))->toBe(['rmi postgres:16']); })->with(['root user' => false, 'non-root user' => true]); + +function dockerCleanupDeployment(Server $server, string $status, string $column = 'server_id', ?Carbon $updatedAt = null): void +{ + $deployment = ApplicationDeploymentQueue::create([ + 'application_id' => '1', + 'deployment_uuid' => 'deployment-'.str()->random(8), + 'status' => $status, + $column => $server->id, + ]); + if ($updatedAt) { + $deployment->forceFill(['updated_at' => $updatedAt])->saveQuietly(); + } +} + +it('skips a forced scheduled cleanup while a deployment is queued or running on the server, so the build cache is not pruned', function (string $status, string $column) { + Notification::fake(); + Event::fake([DockerCleanupDone::class]); + $server = dockerCleanupReachableServer(); + $server->settings()->update(['force_docker_cleanup' => true]); + dockerCleanupDeployment($server, $status, $column); + $commands = dockerCleanupFakeProcesses(); + + (new DockerCleanupJob($server->fresh()))->handle(); + + $execution = DockerCleanupExecution::query()->sole(); + expect($commands->contains(fn (string $command) => str_contains($command, 'prune')))->toBeFalse() + ->and($execution->status)->toBe('success') + ->and($execution->message)->toContain('skipped') + ->and($execution->finished_at)->not->toBeNull(); +})->with([ + 'running on the server' => ['in_progress', 'server_id'], + 'queued on the server' => ['queued', 'server_id'], + 'building on the server' => ['in_progress', 'build_server_id'], +]); + +it('runs a forced scheduled cleanup when the only deployments are finished, on another server, or stuck', function () { + Notification::fake(); + Event::fake([DockerCleanupDone::class]); + $server = dockerCleanupReachableServer(); + $server->settings()->update(['force_docker_cleanup' => true, 'dynamic_timeout' => 3600]); + dockerCleanupDeployment($server, 'finished'); + dockerCleanupDeployment(dockerCleanupReachableServer(), 'in_progress'); + dockerCleanupDeployment($server, 'in_progress', updatedAt: now()->subSeconds(3601)); + $commands = dockerCleanupFakeProcesses(); + + (new DockerCleanupJob($server->fresh()))->handle(); + + expect($commands->contains(fn (string $command) => str_contains($command, 'docker builder prune')))->toBeTrue() + ->and(DockerCleanupExecution::query()->sole()->message)->toContain('Forced Docker cleanup'); +}); + +it('runs a manual cleanup while a deployment is running, because the user started it', function () { + Notification::fake(); + Event::fake([DockerCleanupDone::class]); + $server = dockerCleanupReachableServer(); + dockerCleanupDeployment($server, 'in_progress'); + $commands = dockerCleanupFakeProcesses(); + + (new DockerCleanupJob($server->fresh(), true))->handle(); + + expect($commands->contains(fn (string $command) => str_contains($command, 'docker builder prune')))->toBeTrue(); +}); + +it('does not clean after a stop while a deployment is running on the server', function () { + $server = dockerCleanupReachableServer(); + $server->settings()->update(['docker_cleanup_threshold' => 10]); + dockerCleanupDeployment($server, 'in_progress'); + $commands = dockerCleanupFakeProcesses(); + + expect(CleanupDocker::run($server->fresh(), false, false, true))->toBe([]) + ->and($commands)->toBeEmpty() + ->and(Cache::has(CleanupDocker::lastRunCacheKey($server)))->toBeFalse(); +});