diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 3a1415ccc6..f132ebd17d 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -324,14 +324,11 @@ class CleanupDocker implements ShouldBeUnique $disableRetention = $server->settings->disable_application_image_retention ?? false; - // Image of each container, keyed by container name + // Images that a container uses, running or stopped $containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? '')) ->filter() - ->mapWithKeys(function (string $line) { - [$name, $image] = array_pad(explode('#', $line, 2), 2, ''); - - return [$name => $image]; - }); + ->map(fn (string $line) => array_pad(explode('#', $line, 2), 2, '')[1]) + ->unique(); $serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? '')) ->filter() @@ -347,49 +344,58 @@ class CleanupDocker implements ShouldBeUnique }) ->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== ''); + // A deployment starts the containers of its new images only after the build or pull, + // so the images of an application with a queued or running deployment all stay. + $deployingApplicationIds = ApplicationDeploymentQueue::query() + ->whereIn('application_id', $applications->map(fn ($application) => (string) $application->id)) + ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value]) + ->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600)) + ->pluck('application_id'); + $imagesToDelete = collect(); foreach ($applications as $application) { - $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2); - $imageRepository = $application->docker_registry_image_name ?? $application->uuid; - - // Tag of the image of the running container (named after the application uuid) - $currentTag = preg_match('/:([^:]+)$/', $containerImages->get($application->uuid, ''), $matches) ? $matches[1] : ''; - - // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like - // the docker images --filter reference='*' filter - $images = $serverImages->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository) - && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/')); - - if ($images->isEmpty()) { + if ($deployingApplicationIds->contains((string) $application->id)) { continue; } - // Separate images into categories - // PR images (pr-*) are always deleted - // Build images (*-build) are cleaned up to match retained regular images - $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-')); - $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build')); - $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build')); + $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2); + $imageRepository = $application->docker_registry_image_name ?? $application->uuid; - // Filter out current running image from regular images and sort by creation date - $sortedRegularImages = $regularImages - ->filter(fn ($image) => $image['tag'] !== $currentTag) - ->sortByDesc('created_at') - ->values(); + // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like + // the docker images --filter reference='*' filter. Each repository (the + // application, or one Docker Compose service) has its own retention. + $repositories = $serverImages + ->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository) + && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/')) + ->groupBy('repository'); - // Clean up build images (-build suffix) that don't correspond to retained regular images - // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers. - // If a build is in progress, docker rmi will fail silently since the image is in use. - $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag'); - if ($currentTag !== '') { - $keptTags->push($currentTag); + foreach ($repositories as $images) { + // Tags that a container uses + $currentTags = $images->filter(fn ($image) => $containerImages->contains($image['image_ref']))->pluck('tag'); + + // Separate images into categories + // PR images (pr-*) are always deleted + // Build images (*-build) are cleaned up to match retained regular images + $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-')); + $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build')); + $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build')); + + // Filter out the images in use from regular images and sort by creation date + $sortedRegularImages = $regularImages + ->reject(fn ($image) => $currentTags->contains($image['tag'])) + ->sortByDesc('created_at') + ->values(); + + // Clean up build images (-build suffix) that don't correspond to retained regular images + // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers. + $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag')->concat($currentTags); + + $imagesToDelete = $imagesToDelete + ->concat($prImages) + ->concat($sortedRegularImages->skip($imagesToKeep)) + ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag'])))); } - - $imagesToDelete = $imagesToDelete - ->concat($prImages) - ->concat($sortedRegularImages->skip($imagesToKeep)) - ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag'])))); } // Images in use fail silently; docker rmi continues with the next image diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php index a6443e04ef..d3c1b01b02 100644 --- a/tests/Feature/DockerCleanupJobTest.php +++ b/tests/Feature/DockerCleanupJobTest.php @@ -522,10 +522,10 @@ it('scans application images with a fixed number of remote commands and keeps th ->and($removals)->toHaveCount(1); $removal = $removals->first(); - foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build", "{$app->uuid}_web:v1"] as $deleted) { + foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build"] as $deleted) { expect($removal)->toContain("'{$deleted}'"); } - foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}/other"] as $kept) { + foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}_web:v1", "{$app->uuid}/other"] as $kept) { expect($removal)->not->toContain($kept); } }); @@ -734,3 +734,90 @@ it('does not clean after a stop while a deployment is running on the server', fu ->and($commands)->toBeEmpty() ->and(Cache::has(CleanupDocker::lastRunCacheKey($server)))->toBeFalse(); }); + +/** + * Runs a cleanup with the given `docker ps` and `docker images` output and returns the image + * removal commands. + * + * @param array $containers name#image lines + * @param array $images repository#tag#created lines + * @return Collection + */ +function dockerCleanupImageRemovals(Server $server, array $containers, array $images): Collection +{ + $commands = collect(); + Process::fake(function ($process) use ($commands, $containers, $images) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $commands->push($command); + + return match (true) { + str_contains($command, 'docker ps -a --format') => Process::result(output: implode("\n", $containers)), + str_contains($command, "docker images --format '{{.Repository}}#") => Process::result(output: implode("\n", $images)), + default => Process::result(output: ''), + }; + }); + + CleanupDocker::run($server); + + return $commands->filter(fn (string $command) => str_contains($command, 'docker rmi \''))->values(); +} + +function dockerCleanupApplication(Server $server): Application +{ + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $server->team_id]); + $environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]); + + return Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); +} + +it('keeps the images of each Docker Compose service separately, so a new build of one service does not remove the build of another', function () { + $server = dockerCleanupReachableServer(); + $app = dockerCleanupApplication($server); + $app->settings()->update(['docker_images_to_keep' => 1]); + + $containers = []; + $images = []; + foreach (['web', 'api', 'worker'] as $service) { + $containers[] = "{$service}-{$app->uuid}-123456789#{$app->uuid}_{$service}:old"; + $images[] = "{$app->uuid}_{$service}#oldest#2026-01-01 00:00:00 +0000 UTC"; + $images[] = "{$app->uuid}_{$service}#old#2026-01-02 00:00:00 +0000 UTC"; + $images[] = "{$app->uuid}_{$service}#new#2026-01-03 00:00:00 +0000 UTC"; + } + + $removals = dockerCleanupImageRemovals($server, $containers, $images); + + expect($removals)->toHaveCount(1); + foreach (['web', 'api', 'worker'] as $service) { + expect($removals->first())->toContain("'{$app->uuid}_{$service}:oldest'") + ->not->toContain("{$app->uuid}_{$service}:new") + ->not->toContain("{$app->uuid}_{$service}:old'"); + } +}); + +it('keeps the images of an application while it deploys, because its new images are not used by a container yet', function (string $status) { + $server = dockerCleanupReachableServer(); + $server->settings()->update(['disable_application_image_retention' => true]); + $deploying = dockerCleanupApplication($server); + $idle = dockerCleanupApplication($server); + ApplicationDeploymentQueue::create([ + 'application_id' => (string) $deploying->id, + 'deployment_uuid' => 'deployment-'.str()->random(8), + 'status' => $status, + 'server_id' => $server->id, + ]); + + $removals = dockerCleanupImageRemovals($server, [], [ + "{$deploying->uuid}_api#new#2026-01-03 00:00:00 +0000 UTC", + "{$deploying->uuid}#new#2026-01-03 00:00:00 +0000 UTC", + "{$idle->uuid}#old#2026-01-01 00:00:00 +0000 UTC", + ]); + + expect($removals)->toHaveCount(1) + ->and($removals->first())->toContain("'{$idle->uuid}:old'") + ->not->toContain($deploying->uuid); +})->with(['queued', 'in_progress']);