uuid; } /** * Lock lifetime: longer than the job timeout, so a new run cannot start while a timed-out * run is still finishing. */ public static function overlapLockExpiresAfter(): int { return self::JOB_TIMEOUT + 60; } /** * At most one queued cleanup per server. The unique lock is released when a worker starts * the job, so a stop during a running cleanup queues one follow-up cleanup. */ public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator { return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments); } public static function lastRunCacheKey(Server $server): string { return 'docker-cleanup-last-run-'.$server->uuid; } /** * The deployment that is queued or running on the server (as target or build server), if any. * The cleanup prunes the build caches, which fails a build that uses them. A deployment that * wrote no log for longer than the deployment timeout is stuck and does not count. */ public static function activeDeployment(Server $server): ?ApplicationDeploymentQueue { return ApplicationDeploymentQueue::query() ->where(fn ($query) => $query->where('server_id', $server->id)->orWhere('build_server_id', $server->id)) ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value]) ->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600)) ->first(['id', 'deployment_uuid']); } /** * Cleanup after a resource stop: skipped when a cleanup ran recently, and the queued run * only cleans when the disk usage is at or above the server cleanup threshold and no * deployment is active on the server. */ public static function dispatchAfterStop(Server $server): void { if (Cache::has(self::lastRunCacheKey($server))) { return; } static::dispatch($server, false, false, true); } public function getJobUniqueId(Server $server): string { return $server->uuid; } public function configureJob(JobDecorator $job): void { $job->onQueue(maintenance_queue())->delay(self::QUEUED_DELAY); } /** * Only applies when dispatched as a job. ::run() inside DockerCleanupJob already holds * the lock and does not go through this middleware. */ public function getJobMiddleware(Server $server): array { return [ (new WithoutOverlapping(self::overlapLockKey($server))) ->shared() ->expireAfter(self::overlapLockExpiresAfter()) ->dontRelease(), ]; } public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false, bool $onlyAboveThreshold = false) { $this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE; if ($onlyAboveThreshold) { if (self::activeDeployment($server)) { return []; } $diskUsage = $server->getDiskUsage(); if (is_numeric($diskUsage) && (int) $diskUsage < $server->settings->docker_cleanup_threshold) { return []; } } $helperImageVersion = getHelperVersion(); $helperImageWithVersion = coolifyHelperImage().":$helperImageVersion"; $cleanupLog = []; // Get all application image repositories to exclude from prune $applications = $server->applications(); $applicationImageRepos = collect($applications)->map(function ($app) { return $app->docker_registry_image_name ?? $app->uuid; })->unique()->values(); // Clean up old application images while preserving N most recent for rollback $applicationCleanupLog = $this->cleanupApplicationImages($server, $applications); $cleanupLog = array_merge($cleanupLog, $applicationCleanupLog); // Build image prune command that excludes application images and current Coolify infrastructure images // This ensures we clean up non-Coolify images while preserving rollback images and current helper image // Note: Only the current version is protected; old versions will be cleaned up by explicit commands below // We pass the version strings so all registry variants are protected (ghcr.io, docker.io, no prefix) $imagePruneCmd = $this->buildImagePruneCommand( $applicationImageRepos, $helperImageVersion ); $commands = [ $this->buildContainerPruneCommand(), $imagePruneCmd, 'docker builder prune -af', // The prune fails on a stopped builder, so start it first. railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, 'docker buildx inspect --bootstrap coolify-railpack >/dev/null 2>&1; docker buildx prune --builder coolify-railpack -af'), $this->buildCoolifyImageCleanupCommand($helperImageVersion), ]; if ($deleteUnusedVolumes) { $commands[] = 'docker volume prune -af'; } if ($deleteUnusedNetworks) { $commands[] = 'docker network prune -f'; } foreach ($commands as $command) { $commandOutput = $this->runRemoteCommand($command, $server); if ($commandOutput !== null) { $cleanupLog[] = [ 'command' => $command, 'output' => remove_iip($commandOutput), ]; } } Cache::put(self::lastRunCacheKey($server), true, self::STOP_CLEANUP_COOLDOWN); return $cleanupLog; } /** * Run one remote command with the time left before the cleanup deadline as its local * SSH timeout. Errors and timeouts of a single command are ignored, as before; once the * deadline has passed the cleanup stops and fails. */ private function runRemoteCommand(string $command, Server $server): ?string { $remaining = $this->deadline - now()->getTimestamp(); if ($remaining < 1) { throw new RuntimeException('Docker cleanup did not finish within '.self::REMOTE_COMMANDS_DEADLINE.' seconds. The remaining cleanup commands were not run.'); } return instant_remote_process([$command], $server, false, timeout: $remaining); } /** * Build a command that removes stopped Coolify containers, except the proxy and the * containers of databases, applications and services. * * docker container prune only excludes a container that matches all of its label!= filters * together, so we list the stopped containers and filter them by label in the shell. */ private function buildContainerPruneCommand(): string { return "docker ps -a --filter status=created --filter status=exited --filter status=dead --filter label=coolify.managed=true --format '{{.ID}} coolify.proxy={{.Label \"coolify.proxy\"}} coolify.type={{.Label \"coolify.type\"}}' | ". "grep -v -E 'coolify\.proxy=true |coolify\.type=(database|application|service)$' | ". "awk '{print $1}' | xargs -r docker rm"; } /** * Build a command that removes unused images of the Coolify helper, realtime and sentinel, * from any registry. They have the coolify.managed=true label, so the generic image prune * skips them. The current helper version stays, and docker rmi (without -f) keeps images * that a container uses, for example the running sentinel. */ private function buildCoolifyImageCleanupCommand(string $helperImageVersion): string { $escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion); return "docker images --format '{{.Repository}}:{{.Tag}}' | ". "grep -E '(^|/)coollabsio/(coolify-helper|coolify-realtime|sentinel):' | ". "grep -v -E ':$|(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$' | ". 'xargs -r docker rmi 2>/dev/null || true'; } /** * Build a docker image prune command that excludes application image repositories. * * Since docker image prune doesn't support excluding by repository name directly, * we use a shell script approach to delete unused images while preserving application images. */ private function buildImagePruneCommand( $applicationImageRepos, string $helperImageVersion ): string { // Step 1: Always prune dangling images (untagged) $commands = ['docker image prune -f']; // Build grep pattern to exclude application image repositories (matches repo:tag and repo_service:tag) $appExcludePatterns = $applicationImageRepos->map(function ($repo) { // Escape special characters for grep extended regex (ERE) // ERE special chars: . \ + * ? [ ^ ] $ ( ) { } | return preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $repo); })->implode('|'); // Build grep pattern to exclude Coolify infrastructure images (current version only) // This pattern matches the image name regardless of registry prefix: // - ghcr.io/coollabsio/coolify-helper:1.0.12 // - docker.io/coollabsio/coolify-helper:1.0.12 // - coollabsio/coolify-helper:1.0.12 // Pattern: (^|/)coollabsio/coolify-helper:VERSION$ $escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion); $infraExcludePattern = "(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$"; // Delete unused images that: // - Are not application images (don't match app repos) // - Are not current Coolify helper image (any registry) // - Don't have coolify.managed=true label // Images in use by containers will fail silently with docker rmi // Pattern matches both uuid:tag and uuid_servicename:tag (Docker Compose with build) // Images pulled by digest have no tag and are not dangling, so they are listed as repo@digest $grepCommands = "grep -v ''"; // Add application repo exclusion if there are applications if ($applicationImageRepos->isNotEmpty()) { // The repository names come from application data, so quote the pattern for the shell. $grepCommands .= ' | grep -v -E '.escapeshellarg("^({$appExcludePatterns})[_:@].+"); } // Add infrastructure image exclusion (matches any registry prefix) $grepCommands .= ' | grep -v -E '.escapeshellarg($infraExcludePattern); $commands[] = "docker images --digests --format '{{if eq .Tag \"\"}}{{.Repository}}@{{.Digest}}{{else}}{{.Repository}}:{{.Tag}}{{end}}' | ". $grepCommands.' | '. "xargs -r -I {} sh -c 'docker inspect --format \"{{index .Config.Labels \\\"coolify.managed\\\"}}\" \"{}\" 2>/dev/null | grep -q true || docker rmi \"{}\" 2>/dev/null' || true"; return implode(' && ', $commands); } /** * Remove old application images while the N most recent images of each application stay * for rollback. Uses a fixed number of remote commands, independent of the number of * applications on the server. */ private function cleanupApplicationImages(Server $server, $applications = null): array { $cleanupLog = []; if ($applications === null) { $applications = $server->applications(); } if ($applications->isEmpty()) { return $cleanupLog; } $applications->loadMissing('settings'); $disableRetention = $server->settings->disable_application_image_retention ?? false; // Image of each container, keyed by container name $containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? '')) ->filter() ->mapWithKeys(function (string $line) { [$name, $image] = array_pad(explode('#', $line, 2), 2, ''); return [$name => $image]; }); $serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? '')) ->filter() ->map(function (string $line) { [$repository, $tag, $createdAt] = array_pad(explode('#', $line, 3), 3, ''); return [ 'repository' => $repository, 'tag' => $tag, 'created_at' => $createdAt, 'image_ref' => "{$repository}:{$tag}", ]; }) ->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== ''); $imagesToDelete = collect(); foreach ($applications as $application) { $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2); $imageRepository = $application->docker_registry_image_name ?? $application->uuid; // Tag of the image of the running container (named after the application uuid) $currentTag = preg_match('/:([^:]+)$/', $containerImages->get($application->uuid, ''), $matches) ? $matches[1] : ''; // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like // the docker images --filter reference='*' filter $images = $serverImages->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository) && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/')); if ($images->isEmpty()) { continue; } // Separate images into categories // PR images (pr-*) are always deleted // Build images (*-build) are cleaned up to match retained regular images $prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-')); $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build')); $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build')); // Filter out current running image from regular images and sort by creation date $sortedRegularImages = $regularImages ->filter(fn ($image) => $image['tag'] !== $currentTag) ->sortByDesc('created_at') ->values(); // Clean up build images (-build suffix) that don't correspond to retained regular images // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers. // If a build is in progress, docker rmi will fail silently since the image is in use. $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag'); if ($currentTag !== '') { $keptTags->push($currentTag); } $imagesToDelete = $imagesToDelete ->concat($prImages) ->concat($sortedRegularImages->skip($imagesToKeep)) ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag'])))); } // Images in use fail silently; docker rmi continues with the next image foreach ($this->imageRemovalCommands($imagesToDelete->pluck('image_ref')->unique()) as $deleteCommand) { $deleteOutput = $this->runRemoteCommand($deleteCommand, $server); $cleanupLog[] = [ 'command' => $deleteCommand, 'output' => remove_iip($deleteOutput ?? 'Images removed or were in use'), ]; } return $cleanupLog; } /** * @param Collection $imageRefs * @return Collection */ private function imageRemovalCommands(Collection $imageRefs): Collection { return $imageRefs ->chunk(100) ->map(fn (Collection $chunk) => 'docker rmi '.$chunk->map(fn (string $ref) => escapeshellarg($ref))->implode(' ').' 2>/dev/null || true') ->values(); } }