From d542f13b9c15045367054899cf45485b02802944 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:44:43 +0200 Subject: [PATCH] fix(docker-cleanup): keep new images of deploying and Docker Compose applications Each Docker Compose service image now has its own retention, images used by any container count as in use, and applications with an active deployment keep all their images. Closes #11069 Co-Authored-By: Claude Opus 5.5 --- app/Actions/Server/CleanupDocker.php | 86 +++++++++++++----------- tests/Feature/DockerCleanupJobTest.php | 91 +++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 42 deletions(-) diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 3a1415ccc6..f132ebd17d 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -324,14 +324,11 @@ class CleanupDocker implements ShouldBeUnique $disableRetention = $server->settings->disable_application_image_retention ?? false; - // Image of each container, keyed by container name + // Images that a container uses, running or stopped $containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? '')) ->filter() - ->mapWithKeys(function (string $line) { - [$name, $image] = array_pad(explode('#', $line, 2), 2, ''); - - return [$name => $image]; - }); + ->map(fn (string $line) => array_pad(explode('#', $line, 2), 2, '')[1]) + ->unique(); $serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? '')) ->filter() @@ -347,49 +344,58 @@ class CleanupDocker implements ShouldBeUnique }) ->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== ''); + // A deployment starts the containers of its new images only after the build or pull, + // so the images of an application with a queued or running deployment all stay. + $deployingApplicationIds = ApplicationDeploymentQueue::query() + ->whereIn('application_id', $applications->map(fn ($application) => (string) $application->id)) + ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value]) + ->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600)) + ->pluck('application_id'); + $imagesToDelete = collect(); foreach ($applications as $application) { - $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2); - $imageRepository = $application->docker_registry_image_name ?? $application->uuid; - - // Tag of the image of the running container (named after the application uuid) - $currentTag = preg_match('/:([^:]+)$/', $containerImages->get($application->uuid, ''), $matches) ? $matches[1] : ''; - - // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like - // the docker images --filter reference='*' filter - $images = $serverImages->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository) - && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/')); - - if ($images->isEmpty()) { + if ($deployingApplicationIds->contains((string) $application->id)) { continue; } - // Separate images into categories - // PR images (pr-*) are always deleted - // Build images (*-build) are cleaned up to match retained regular images - $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-')); - $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build')); - $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build')); + $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2); + $imageRepository = $application->docker_registry_image_name ?? $application->uuid; - // Filter out current running image from regular images and sort by creation date - $sortedRegularImages = $regularImages - ->filter(fn ($image) => $image['tag'] !== $currentTag) - ->sortByDesc('created_at') - ->values(); + // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like + // the docker images --filter reference='*' filter. Each repository (the + // application, or one Docker Compose service) has its own retention. + $repositories = $serverImages + ->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository) + && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/')) + ->groupBy('repository'); - // Clean up build images (-build suffix) that don't correspond to retained regular images - // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers. - // If a build is in progress, docker rmi will fail silently since the image is in use. - $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag'); - if ($currentTag !== '') { - $keptTags->push($currentTag); + foreach ($repositories as $images) { + // Tags that a container uses + $currentTags = $images->filter(fn ($image) => $containerImages->contains($image['image_ref']))->pluck('tag'); + + // Separate images into categories + // PR images (pr-*) are always deleted + // Build images (*-build) are cleaned up to match retained regular images + $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-')); + $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build')); + $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build')); + + // Filter out the images in use from regular images and sort by creation date + $sortedRegularImages = $regularImages + ->reject(fn ($image) => $currentTags->contains($image['tag'])) + ->sortByDesc('created_at') + ->values(); + + // Clean up build images (-build suffix) that don't correspond to retained regular images + // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers. + $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag')->concat($currentTags); + + $imagesToDelete = $imagesToDelete + ->concat($prImages) + ->concat($sortedRegularImages->skip($imagesToKeep)) + ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag'])))); } - - $imagesToDelete = $imagesToDelete - ->concat($prImages) - ->concat($sortedRegularImages->skip($imagesToKeep)) - ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag'])))); } // Images in use fail silently; docker rmi continues with the next image diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php index a6443e04ef..d3c1b01b02 100644 --- a/tests/Feature/DockerCleanupJobTest.php +++ b/tests/Feature/DockerCleanupJobTest.php @@ -522,10 +522,10 @@ it('scans application images with a fixed number of remote commands and keeps th ->and($removals)->toHaveCount(1); $removal = $removals->first(); - foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build", "{$app->uuid}_web:v1"] as $deleted) { + foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build"] as $deleted) { expect($removal)->toContain("'{$deleted}'"); } - foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}/other"] as $kept) { + foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}_web:v1", "{$app->uuid}/other"] as $kept) { expect($removal)->not->toContain($kept); } }); @@ -734,3 +734,90 @@ it('does not clean after a stop while a deployment is running on the server', fu ->and($commands)->toBeEmpty() ->and(Cache::has(CleanupDocker::lastRunCacheKey($server)))->toBeFalse(); }); + +/** + * Runs a cleanup with the given `docker ps` and `docker images` output and returns the image + * removal commands. + * + * @param array $containers name#image lines + * @param array $images repository#tag#created lines + * @return Collection + */ +function dockerCleanupImageRemovals(Server $server, array $containers, array $images): Collection +{ + $commands = collect(); + Process::fake(function ($process) use ($commands, $containers, $images) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $commands->push($command); + + return match (true) { + str_contains($command, 'docker ps -a --format') => Process::result(output: implode("\n", $containers)), + str_contains($command, "docker images --format '{{.Repository}}#") => Process::result(output: implode("\n", $images)), + default => Process::result(output: ''), + }; + }); + + CleanupDocker::run($server); + + return $commands->filter(fn (string $command) => str_contains($command, 'docker rmi \''))->values(); +} + +function dockerCleanupApplication(Server $server): Application +{ + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $server->team_id]); + $environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]); + + return Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); +} + +it('keeps the images of each Docker Compose service separately, so a new build of one service does not remove the build of another', function () { + $server = dockerCleanupReachableServer(); + $app = dockerCleanupApplication($server); + $app->settings()->update(['docker_images_to_keep' => 1]); + + $containers = []; + $images = []; + foreach (['web', 'api', 'worker'] as $service) { + $containers[] = "{$service}-{$app->uuid}-123456789#{$app->uuid}_{$service}:old"; + $images[] = "{$app->uuid}_{$service}#oldest#2026-01-01 00:00:00 +0000 UTC"; + $images[] = "{$app->uuid}_{$service}#old#2026-01-02 00:00:00 +0000 UTC"; + $images[] = "{$app->uuid}_{$service}#new#2026-01-03 00:00:00 +0000 UTC"; + } + + $removals = dockerCleanupImageRemovals($server, $containers, $images); + + expect($removals)->toHaveCount(1); + foreach (['web', 'api', 'worker'] as $service) { + expect($removals->first())->toContain("'{$app->uuid}_{$service}:oldest'") + ->not->toContain("{$app->uuid}_{$service}:new") + ->not->toContain("{$app->uuid}_{$service}:old'"); + } +}); + +it('keeps the images of an application while it deploys, because its new images are not used by a container yet', function (string $status) { + $server = dockerCleanupReachableServer(); + $server->settings()->update(['disable_application_image_retention' => true]); + $deploying = dockerCleanupApplication($server); + $idle = dockerCleanupApplication($server); + ApplicationDeploymentQueue::create([ + 'application_id' => (string) $deploying->id, + 'deployment_uuid' => 'deployment-'.str()->random(8), + 'status' => $status, + 'server_id' => $server->id, + ]); + + $removals = dockerCleanupImageRemovals($server, [], [ + "{$deploying->uuid}_api#new#2026-01-03 00:00:00 +0000 UTC", + "{$deploying->uuid}#new#2026-01-03 00:00:00 +0000 UTC", + "{$idle->uuid}#old#2026-01-01 00:00:00 +0000 UTC", + ]); + + expect($removals)->toHaveCount(1) + ->and($removals->first())->toContain("'{$idle->uuid}:old'") + ->not->toContain($deploying->uuid); +})->with(['queued', 'in_progress']);