diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 9f58fd4f88..cde75e30de 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -3,16 +3,101 @@ namespace App\Actions\Server; use App\Models\Server; +use App\Support\Actions\UniqueUntilProcessingJobDecorator; +use Illuminate\Contracts\Queue\ShouldBeUnique; +use Illuminate\Queue\Middleware\WithoutOverlapping; +use Illuminate\Support\Collection; use Lorisleiva\Actions\Concerns\AsAction; +use Lorisleiva\Actions\Decorators\JobDecorator; +use Lorisleiva\Actions\Decorators\UniqueJobDecorator; +use RuntimeException; -class CleanupDocker +class CleanupDocker implements ShouldBeUnique { use AsAction; - public string $jobQueue = 'high'; + /** + * Timeout of a queued cleanup, shared with DockerCleanupJob. + */ + public const JOB_TIMEOUT = 600; + + /** + * Time budget for all remote commands of one cleanup. It stays below JOB_TIMEOUT, so a + * slow server makes the cleanup fail with an error instead of the worker killing the job. + * A remote Docker prune that already started still finishes on the server. + */ + public const REMOTE_COMMANDS_DEADLINE = 540; + + /** + * Delay of a queued cleanup. Stop requests for one server in this window (for example a + * bulk stop through the API) become one cleanup that runs after the batch. + */ + public const QUEUED_DELAY = 60; + + public int $jobTries = 1; + + /** + * Bounds how long a lost queued run (for example a killed worker) blocks new cleanups. + */ + public int $jobUniqueFor = self::QUEUED_DELAY + 240; + + public int $jobTimeout = self::JOB_TIMEOUT; + + private int $deadline = 0; + + /** + * Overlap lock key shared by DockerCleanupJob and queued CleanupDocker runs. + */ + public static function overlapLockKey(Server $server): string + { + return 'docker-cleanup-'.$server->uuid; + } + + /** + * Lock lifetime: longer than the job timeout, so a new run cannot start while a timed-out + * run is still finishing. + */ + public static function overlapLockExpiresAfter(): int + { + return self::JOB_TIMEOUT + 60; + } + + /** + * At most one queued cleanup per server. The unique lock is released when a worker starts + * the job, so a stop during a running cleanup queues one follow-up cleanup. + */ + public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator + { + return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments); + } + + public function getJobUniqueId(Server $server): string + { + return $server->uuid; + } + + public function configureJob(JobDecorator $job): void + { + $job->onQueue(maintenance_queue())->delay(self::QUEUED_DELAY); + } + + /** + * Only applies when dispatched as a job. ::run() inside DockerCleanupJob already holds + * the lock and does not go through this middleware. + */ + public function getJobMiddleware(Server $server): array + { + return [ + (new WithoutOverlapping(self::overlapLockKey($server))) + ->shared() + ->expireAfter(self::overlapLockExpiresAfter()) + ->dontRelease(), + ]; + } public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false) { + $this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE; $helperImageVersion = getHelperVersion(); $helperImage = coolifyHelperImage(); $helperImageWithVersion = "$helperImage:$helperImageVersion"; @@ -61,7 +146,7 @@ class CleanupDocker } foreach ($commands as $command) { - $commandOutput = instant_remote_process([$command], $server, false); + $commandOutput = $this->runRemoteCommand($command, $server); if ($commandOutput !== null) { $cleanupLog[] = [ 'command' => $command, @@ -73,6 +158,21 @@ class CleanupDocker return $cleanupLog; } + /** + * Run one remote command with the time left before the cleanup deadline as its local + * SSH timeout. Errors and timeouts of a single command are ignored, as before; once the + * deadline has passed the cleanup stops and fails. + */ + private function runRemoteCommand(string $command, Server $server): ?string + { + $remaining = $this->deadline - now()->getTimestamp(); + if ($remaining < 1) { + throw new RuntimeException('Docker cleanup did not finish within '.self::REMOTE_COMMANDS_DEADLINE.' seconds. The remaining cleanup commands were not run.'); + } + + return instant_remote_process([$command], $server, false, timeout: $remaining); + } + /** * Build a docker image prune command that excludes application image repositories. * @@ -125,6 +225,11 @@ class CleanupDocker return implode(' && ', $commands); } + /** + * Remove old application images while the N most recent images of each application stay + * for rollback. Uses a fixed number of remote commands, independent of the number of + * applications on the server. + */ private function cleanupApplicationImages(Server $server, $applications = null): array { $cleanupLog = []; @@ -132,43 +237,54 @@ class CleanupDocker if ($applications === null) { $applications = $server->applications(); } + if ($applications->isEmpty()) { + return $cleanupLog; + } + $applications->loadMissing('settings'); $disableRetention = $server->settings->disable_application_image_retention ?? false; + // Image of each container, keyed by container name + $containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? '')) + ->filter() + ->mapWithKeys(function (string $line) { + [$name, $image] = array_pad(explode('#', $line, 2), 2, ''); + + return [$name => $image]; + }); + + $serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? '')) + ->filter() + ->map(function (string $line) { + [$repository, $tag, $createdAt] = array_pad(explode('#', $line, 3), 3, ''); + + return [ + 'repository' => $repository, + 'tag' => $tag, + 'created_at' => $createdAt, + 'image_ref' => "{$repository}:{$tag}", + ]; + }) + ->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== ''); + + $imagesToDelete = collect(); + foreach ($applications as $application) { $imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2); $imageRepository = $application->docker_registry_image_name ?? $application->uuid; - // Get the currently running image tag - $currentTagCommand = "docker inspect --format='{{.Config.Image}}' {$application->uuid} 2>/dev/null | grep -oP '(?<=:)[^:]+$' || true"; - $currentTag = instant_remote_process([$currentTagCommand], $server, false); - $currentTag = trim($currentTag ?? ''); + // Tag of the image of the running container (named after the application uuid) + $currentTag = preg_match('/:([^:]+)$/', $containerImages->get($application->uuid, ''), $matches) ? $matches[1] : ''; - // List all images for this application with their creation timestamps - // Use wildcard to match both uuid:tag and uuid_servicename:tag (Docker Compose with build) - $listCommand = "docker images --format '{{.Repository}}:{{.Tag}}#{{.CreatedAt}}' --filter reference='{$imageRepository}*' 2>/dev/null || true"; - $output = instant_remote_process([$listCommand], $server, false); + // Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like + // the docker images --filter reference='*' filter + $images = $serverImages->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository) + && ! str_contains(substr($image['repository'], strlen($imageRepository)), '/')); - if (empty($output)) { + if ($images->isEmpty()) { continue; } - $images = collect(explode("\n", trim($output))) - ->filter() - ->map(function ($line) { - $parts = explode('#', $line); - $imageRef = $parts[0] ?? ''; - $tagParts = explode(':', $imageRef); - - return [ - 'repository' => $tagParts[0] ?? '', - 'tag' => $tagParts[1] ?? '', - 'created_at' => $parts[1] ?? '', - 'image_ref' => $imageRef, - ]; - }) - ->filter(fn ($image) => ! empty($image['tag'])); - // Separate images into categories // PR images (pr-*) are always deleted // Build images (*-build) are cleaned up to match retained regular images @@ -176,55 +292,47 @@ class CleanupDocker $buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build')); $regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build')); - // Always delete all PR images - foreach ($prImages as $image) { - $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true"; - $deleteOutput = instant_remote_process([$deleteCommand], $server, false); - $cleanupLog[] = [ - 'command' => $deleteCommand, - 'output' => $deleteOutput ?? 'PR image removed or was in use', - ]; - } - // Filter out current running image from regular images and sort by creation date $sortedRegularImages = $regularImages ->filter(fn ($image) => $image['tag'] !== $currentTag) ->sortByDesc('created_at') ->values(); - // Keep only N images (imagesToKeep), delete the rest - $imagesToDelete = $sortedRegularImages->skip($imagesToKeep); - - foreach ($imagesToDelete as $image) { - $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true"; - $deleteOutput = instant_remote_process([$deleteCommand], $server, false); - $cleanupLog[] = [ - 'command' => $deleteCommand, - 'output' => $deleteOutput ?? 'Image removed or was in use', - ]; - } - // Clean up build images (-build suffix) that don't correspond to retained regular images // Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers. // If a build is in progress, docker rmi will fail silently since the image is in use. $keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag'); - if (! empty($currentTag)) { - $keptTags = $keptTags->push($currentTag); + if ($currentTag !== '') { + $keptTags->push($currentTag); } - foreach ($buildImages as $image) { - $baseTag = preg_replace('/-build$/', '', $image['tag']); - if (! $keptTags->contains($baseTag)) { - $deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true"; - $deleteOutput = instant_remote_process([$deleteCommand], $server, false); - $cleanupLog[] = [ - 'command' => $deleteCommand, - 'output' => $deleteOutput ?? 'Build image removed or was in use', - ]; - } - } + $imagesToDelete = $imagesToDelete + ->concat($prImages) + ->concat($sortedRegularImages->skip($imagesToKeep)) + ->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag'])))); + } + + // Images in use fail silently; docker rmi continues with the next image + foreach ($this->imageRemovalCommands($imagesToDelete->pluck('image_ref')->unique()) as $deleteCommand) { + $deleteOutput = $this->runRemoteCommand($deleteCommand, $server); + $cleanupLog[] = [ + 'command' => $deleteCommand, + 'output' => $deleteOutput ?? 'Images removed or were in use', + ]; } return $cleanupLog; } + + /** + * @param Collection $imageRefs + * @return Collection + */ + private function imageRemovalCommands(Collection $imageRefs): Collection + { + return $imageRefs + ->chunk(100) + ->map(fn (Collection $chunk) => 'docker rmi '.$chunk->map(fn (string $ref) => escapeshellarg($ref))->implode(' ').' 2>/dev/null || true') + ->values(); + } } diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 4c744a6e64..03a9207827 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -15,14 +15,13 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - public $timeout = 600; + public $timeout = CleanupDocker::JOB_TIMEOUT; public $tries = 1; @@ -30,9 +29,20 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue public ?DockerCleanupExecution $execution_log = null; + /** + * Shares the per-server lock with queued CleanupDocker runs, so two cleanups never run on + * one server at once. A scheduled run that finds the lock held marks its occurrence + * skipped (not missed); a manual run is dropped as before. The lock outlives the job + * timeout, so a new run cannot start while a timed-out run is still finishing. + */ public function middleware(): array { - return [(new WithoutOverlapping('docker-cleanup-'.$this->server->uuid))->expireAfter(600)->dontRelease()]; + return [ + ScheduledJobDeliveryService::withoutOverlapping(CleanupDocker::overlapLockKey($this->server), $this->occurrenceUuid) + ->shared() + ->expireAfter(CleanupDocker::overlapLockExpiresAfter()) + ->dontRelease(), + ]; } public function __construct( @@ -42,7 +52,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue public bool $deleteUnusedNetworks = false, public ?string $occurrenceUuid = null, ) { - $this->onQueue('high'); + $this->onQueue(maintenance_queue()); } public function handle(): void diff --git a/tests/Feature/DockerCleanupJobTest.php b/tests/Feature/DockerCleanupJobTest.php index 081b49a520..eeedb60559 100644 --- a/tests/Feature/DockerCleanupJobTest.php +++ b/tests/Feature/DockerCleanupJobTest.php @@ -1,13 +1,76 @@ InstanceSettings::firstOrCreate(['id' => 0])); + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + + return $server->fresh(); +} + +/** + * @return Collection The local commands (SSH invocations) that ran. + */ +function dockerCleanupFakeProcesses(?Closure $onRun = null): Collection +{ + $commands = collect(); + Process::fake(function ($process) use ($commands, $onRun) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $commands->push($command); + if ($onRun) { + $onRun($command); + } + + return Process::result(output: ''); + }); + + return $commands; +} + it('persists the server id when creating an execution record', function () { $user = User::factory()->create(); $team = $user->teams()->first(); @@ -86,3 +149,204 @@ it('finishes the latest running execution when the job fails after a timeout', f ->and($olderExecution->refresh()->status)->toBe('running') ->and($olderExecution->finished_at)->toBeNull(); }); + +it('skips a scheduled cleanup occurrence while another cleanup holds the server lock, so it is not reported missed later', function () { + Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')); + Notification::fake(); + $server = dockerCleanupReachableServer(); + $occurrence = ScheduledJobDelivery::create([ + 'schedule_key' => "docker-cleanup:{$server->id}", + 'scheduled_for' => now()->startOfMinute(), + 'job_type' => 'docker-cleanup', + 'resource_id' => $server->id, + 'status' => 'enqueued', + 'enqueued_at' => now(), + ]); + $job = new DockerCleanupJob($server, false, false, false, $occurrence->uuid); + $runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600); + $runningCleanupLock->get(); + $commands = dockerCleanupFakeProcesses(); + + dispatch_sync($job); + $runningCleanupLock->release(); + + expect($occurrence->fresh()->status)->toBe('skipped') + ->and(DockerCleanupExecution::query()->exists())->toBeFalse() + ->and($commands)->toBeEmpty(); + + Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1)); + Queue::fake(); + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + + Queue::assertNothingPushed(); + expect(DockerCleanupExecution::query()->exists())->toBeFalse(); + Notification::assertNotSentTo($server->team, DockerCleanupFailed::class); +}); + +it('drops a manual cleanup while another cleanup holds the server lock without touching occurrences', function () { + $server = dockerCleanupReachableServer(); + $job = new DockerCleanupJob($server, true); + $runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600); + $runningCleanupLock->get(); + $commands = dockerCleanupFakeProcesses(); + + dispatch_sync($job); + $runningCleanupLock->release(); + + expect(DockerCleanupExecution::query()->exists())->toBeFalse() + ->and(ScheduledJobDelivery::query()->exists())->toBeFalse() + ->and($commands)->toBeEmpty(); +}); + +it('shares one server lock between scheduled cleanups and queued CleanupDocker runs that outlives the job timeout', function () { + $server = dockerCleanupReachableServer(); + $cleanupJob = new DockerCleanupJob($server); + $actionJob = CleanupDocker::makeJob($server, false, false); + + $cleanupLock = $cleanupJob->middleware()[0]; + $actionLock = $actionJob->middleware()[0]; + + expect($cleanupLock->getLockKey($cleanupJob)) + ->toBe('laravel-queue-overlap:docker-cleanup-'.$server->uuid) + ->toBe($actionLock->getLockKey($actionJob)) + ->and($cleanupLock->expiresAfter)->toBeGreaterThan($cleanupJob->timeout) + ->and($actionLock->expiresAfter)->toBeGreaterThan($actionJob->timeout) + ->and($cleanupLock->releaseAfter)->toBeNull() + ->and($actionLock->releaseAfter)->toBeNull() + ->and($actionJob->timeout)->toBe(CleanupDocker::JOB_TIMEOUT) + ->and($actionJob->tries)->toBe(1); +}); + +it('drops a queued CleanupDocker run while a scheduled cleanup holds the server lock', function () { + $server = dockerCleanupReachableServer(); + $actionJob = CleanupDocker::makeJob($server, false, false); + $runningCleanupLock = Cache::lock('laravel-queue-overlap:docker-cleanup-'.$server->uuid, 3600); + $runningCleanupLock->get(); + $commands = dockerCleanupFakeProcesses(); + + dispatch_sync($actionJob); + $runningCleanupLock->release(); + + expect($commands)->toBeEmpty(); +}); + +it('routes docker cleanups to the maintenance queue on cloud and to high on self-hosted', function (bool $selfHosted, string $queue) { + config(['constants.coolify.self_hosted' => $selfHosted]); + Queue::fake(); + $server = dockerCleanupReachableServer(); + + DockerCleanupJob::dispatch($server, true); + CleanupDocker::dispatch($server, false, false); + + expect(maintenance_queue())->toBe($queue); + Queue::assertPushedOn($queue, DockerCleanupJob::class); + Queue::assertPushedOn($queue, UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->decorates(CleanupDocker::class)); +})->with([ + 'cloud' => [false, 'maintenance'], + 'self-hosted' => [true, 'high'], +]); + +it('gives remote cleanup commands a local timeout and fails once the cleanup deadline has passed', function () { + Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')); + $server = dockerCleanupReachableServer(); + $commands = dockerCleanupFakeProcesses(function (string $command) { + if (str_contains($command, 'docker ')) { + Carbon::setTestNow(now()->addSeconds(300)); + } + }); + + expect(fn () => CleanupDocker::run($server)) + ->toThrow(RuntimeException::class, 'Docker cleanup did not finish within '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' seconds'); + + $sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker container prune') || str_contains($command, 'docker image prune'))->values(); + + expect(CleanupDocker::REMOTE_COMMANDS_DEADLINE)->toBeLessThan(CleanupDocker::JOB_TIMEOUT) + ->and($sshCommands)->toHaveCount(2) + ->and($sshCommands[0])->toContain('timeout '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' ssh') + ->and($sshCommands[1])->toContain('timeout '.(CleanupDocker::REMOTE_COMMANDS_DEADLINE - 300).' ssh') + ->and($commands->contains(fn (string $command) => str_contains($command, 'docker builder prune')))->toBeFalse(); +}); + +it('coalesces queued cleanups per server and delays them until a bulk stop is over', function () { + Queue::fake(); + $first = dockerCleanupReachableServer(); + $second = dockerCleanupReachableServer(); + + foreach (range(1, 30) as $stop) { + CleanupDocker::dispatch($first, false, false); + } + CleanupDocker::dispatch($second, false, false); + + Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2); + Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->delay === CleanupDocker::QUEUED_DELAY); +}); + +it('accepts a new queued cleanup once the waiting one is lost', function () { + Queue::fake(); + $server = dockerCleanupReachableServer(); + + CleanupDocker::dispatch($server, false, false); + $this->travel(CleanupDocker::QUEUED_DELAY + 241)->seconds(); + CleanupDocker::dispatch($server, false, false); + + Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2); +}); + +it('scans application images with a fixed number of remote commands and keeps the retention rules', function () { + $server = dockerCleanupReachableServer(); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $server->team_id]); + $environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]); + $applications = collect(range(1, 5))->map(fn () => Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ])); + $app = $applications->first(); + $app->settings()->update(['docker_images_to_keep' => 1]); + + $containers = "{$app->uuid}#{$app->uuid}:v2"; + $images = implode("\n", [ + "{$app->uuid}#v1#2026-01-01 00:00:00 +0000 UTC", + "{$app->uuid}#v2#2026-01-02 00:00:00 +0000 UTC", + "{$app->uuid}#v3#2026-01-03 00:00:00 +0000 UTC", + "{$app->uuid}#v4#2026-01-04 00:00:00 +0000 UTC", + "{$app->uuid}#pr-7#2026-01-05 00:00:00 +0000 UTC", + "{$app->uuid}#v1-build#2026-01-01 00:00:00 +0000 UTC", + "{$app->uuid}#v2-build#2026-01-02 00:00:00 +0000 UTC", + "{$app->uuid}#v4-build#2026-01-04 00:00:00 +0000 UTC", + "{$app->uuid}_web#v1#2026-01-01 00:00:00 +0000 UTC", + "{$app->uuid}/other#v1#2026-01-01 00:00:00 +0000 UTC", + ]); + + $commands = collect(); + Process::fake(function ($process) use ($commands, $containers, $images) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $commands->push($command); + + return match (true) { + str_contains($command, 'docker ps -a --format') => Process::result(output: $containers), + str_contains($command, "docker images --format '{{.Repository}}#") => Process::result(output: $images), + default => Process::result(output: ''), + }; + }); + + CleanupDocker::run($server); + + $applicationScanCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker ps -a --format') + || str_contains($command, "docker images --format '{{.Repository}}#") + || str_contains($command, 'docker inspect --format=') + || str_contains($command, "--filter reference='")); + $removals = $commands->filter(fn (string $command) => str_contains($command, 'docker rmi \''))->values(); + + expect($applicationScanCommands)->toHaveCount(2) + ->and($removals)->toHaveCount(1); + + $removal = $removals->first(); + foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build", "{$app->uuid}_web:v1"] as $deleted) { + expect($removal)->toContain("'{$deleted}'"); + } + foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}/other"] as $kept) { + expect($removal)->not->toContain($kept); + } +});