mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
fix(docker-cleanup): run one bounded cleanup per server on the maintenance queue
Scheduled, manual and stop-triggered cleanups share one per-server lock. Stop-triggered cleanups wait 60 seconds and are coalesced per server, so a bulk stop queues one cleanup. Remote commands share a 540 second deadline below the 600 second job timeout, and application images are scanned with a fixed number of remote commands.
This commit is contained in:
Andras Bacsai
@@ -3,16 +3,101 @@
|
||||
namespace App\Actions\Server;
|
||||
|
||||
use App\Models\Server;
|
||||
use App\Support\Actions\UniqueUntilProcessingJobDecorator;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Support\Collection;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
use Lorisleiva\Actions\Decorators\JobDecorator;
|
||||
use Lorisleiva\Actions\Decorators\UniqueJobDecorator;
|
||||
use RuntimeException;
|
||||
|
||||
class CleanupDocker
|
||||
class CleanupDocker implements ShouldBeUnique
|
||||
{
|
||||
use AsAction;
|
||||
|
||||
public string $jobQueue = 'high';
|
||||
/**
|
||||
* Timeout of a queued cleanup, shared with DockerCleanupJob.
|
||||
*/
|
||||
public const JOB_TIMEOUT = 600;
|
||||
|
||||
/**
|
||||
* Time budget for all remote commands of one cleanup. It stays below JOB_TIMEOUT, so a
|
||||
* slow server makes the cleanup fail with an error instead of the worker killing the job.
|
||||
* A remote Docker prune that already started still finishes on the server.
|
||||
*/
|
||||
public const REMOTE_COMMANDS_DEADLINE = 540;
|
||||
|
||||
/**
|
||||
* Delay of a queued cleanup. Stop requests for one server in this window (for example a
|
||||
* bulk stop through the API) become one cleanup that runs after the batch.
|
||||
*/
|
||||
public const QUEUED_DELAY = 60;
|
||||
|
||||
public int $jobTries = 1;
|
||||
|
||||
/**
|
||||
* Bounds how long a lost queued run (for example a killed worker) blocks new cleanups.
|
||||
*/
|
||||
public int $jobUniqueFor = self::QUEUED_DELAY + 240;
|
||||
|
||||
public int $jobTimeout = self::JOB_TIMEOUT;
|
||||
|
||||
private int $deadline = 0;
|
||||
|
||||
/**
|
||||
* Overlap lock key shared by DockerCleanupJob and queued CleanupDocker runs.
|
||||
*/
|
||||
public static function overlapLockKey(Server $server): string
|
||||
{
|
||||
return 'docker-cleanup-'.$server->uuid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lock lifetime: longer than the job timeout, so a new run cannot start while a timed-out
|
||||
* run is still finishing.
|
||||
*/
|
||||
public static function overlapLockExpiresAfter(): int
|
||||
{
|
||||
return self::JOB_TIMEOUT + 60;
|
||||
}
|
||||
|
||||
/**
|
||||
* At most one queued cleanup per server. The unique lock is released when a worker starts
|
||||
* the job, so a stop during a running cleanup queues one follow-up cleanup.
|
||||
*/
|
||||
public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator
|
||||
{
|
||||
return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments);
|
||||
}
|
||||
|
||||
public function getJobUniqueId(Server $server): string
|
||||
{
|
||||
return $server->uuid;
|
||||
}
|
||||
|
||||
public function configureJob(JobDecorator $job): void
|
||||
{
|
||||
$job->onQueue(maintenance_queue())->delay(self::QUEUED_DELAY);
|
||||
}
|
||||
|
||||
/**
|
||||
* Only applies when dispatched as a job. ::run() inside DockerCleanupJob already holds
|
||||
* the lock and does not go through this middleware.
|
||||
*/
|
||||
public function getJobMiddleware(Server $server): array
|
||||
{
|
||||
return [
|
||||
(new WithoutOverlapping(self::overlapLockKey($server)))
|
||||
->shared()
|
||||
->expireAfter(self::overlapLockExpiresAfter())
|
||||
->dontRelease(),
|
||||
];
|
||||
}
|
||||
|
||||
public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false)
|
||||
{
|
||||
$this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE;
|
||||
$helperImageVersion = getHelperVersion();
|
||||
$helperImage = coolifyHelperImage();
|
||||
$helperImageWithVersion = "$helperImage:$helperImageVersion";
|
||||
@@ -61,7 +146,7 @@ class CleanupDocker
|
||||
}
|
||||
|
||||
foreach ($commands as $command) {
|
||||
$commandOutput = instant_remote_process([$command], $server, false);
|
||||
$commandOutput = $this->runRemoteCommand($command, $server);
|
||||
if ($commandOutput !== null) {
|
||||
$cleanupLog[] = [
|
||||
'command' => $command,
|
||||
@@ -73,6 +158,21 @@ class CleanupDocker
|
||||
return $cleanupLog;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one remote command with the time left before the cleanup deadline as its local
|
||||
* SSH timeout. Errors and timeouts of a single command are ignored, as before; once the
|
||||
* deadline has passed the cleanup stops and fails.
|
||||
*/
|
||||
private function runRemoteCommand(string $command, Server $server): ?string
|
||||
{
|
||||
$remaining = $this->deadline - now()->getTimestamp();
|
||||
if ($remaining < 1) {
|
||||
throw new RuntimeException('Docker cleanup did not finish within '.self::REMOTE_COMMANDS_DEADLINE.' seconds. The remaining cleanup commands were not run.');
|
||||
}
|
||||
|
||||
return instant_remote_process([$command], $server, false, timeout: $remaining);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a docker image prune command that excludes application image repositories.
|
||||
*
|
||||
@@ -125,6 +225,11 @@ class CleanupDocker
|
||||
return implode(' && ', $commands);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove old application images while the N most recent images of each application stay
|
||||
* for rollback. Uses a fixed number of remote commands, independent of the number of
|
||||
* applications on the server.
|
||||
*/
|
||||
private function cleanupApplicationImages(Server $server, $applications = null): array
|
||||
{
|
||||
$cleanupLog = [];
|
||||
@@ -132,43 +237,54 @@ class CleanupDocker
|
||||
if ($applications === null) {
|
||||
$applications = $server->applications();
|
||||
}
|
||||
if ($applications->isEmpty()) {
|
||||
return $cleanupLog;
|
||||
}
|
||||
$applications->loadMissing('settings');
|
||||
|
||||
$disableRetention = $server->settings->disable_application_image_retention ?? false;
|
||||
|
||||
// Image of each container, keyed by container name
|
||||
$containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? ''))
|
||||
->filter()
|
||||
->mapWithKeys(function (string $line) {
|
||||
[$name, $image] = array_pad(explode('#', $line, 2), 2, '');
|
||||
|
||||
return [$name => $image];
|
||||
});
|
||||
|
||||
$serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? ''))
|
||||
->filter()
|
||||
->map(function (string $line) {
|
||||
[$repository, $tag, $createdAt] = array_pad(explode('#', $line, 3), 3, '');
|
||||
|
||||
return [
|
||||
'repository' => $repository,
|
||||
'tag' => $tag,
|
||||
'created_at' => $createdAt,
|
||||
'image_ref' => "{$repository}:{$tag}",
|
||||
];
|
||||
})
|
||||
->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== '<none>');
|
||||
|
||||
$imagesToDelete = collect();
|
||||
|
||||
foreach ($applications as $application) {
|
||||
$imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2);
|
||||
$imageRepository = $application->docker_registry_image_name ?? $application->uuid;
|
||||
|
||||
// Get the currently running image tag
|
||||
$currentTagCommand = "docker inspect --format='{{.Config.Image}}' {$application->uuid} 2>/dev/null | grep -oP '(?<=:)[^:]+$' || true";
|
||||
$currentTag = instant_remote_process([$currentTagCommand], $server, false);
|
||||
$currentTag = trim($currentTag ?? '');
|
||||
// Tag of the image of the running container (named after the application uuid)
|
||||
$currentTag = preg_match('/:([^:]+)$/', $containerImages->get($application->uuid, ''), $matches) ? $matches[1] : '';
|
||||
|
||||
// List all images for this application with their creation timestamps
|
||||
// Use wildcard to match both uuid:tag and uuid_servicename:tag (Docker Compose with build)
|
||||
$listCommand = "docker images --format '{{.Repository}}:{{.Tag}}#{{.CreatedAt}}' --filter reference='{$imageRepository}*' 2>/dev/null || true";
|
||||
$output = instant_remote_process([$listCommand], $server, false);
|
||||
// Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like
|
||||
// the docker images --filter reference='<repository>*' filter
|
||||
$images = $serverImages->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository)
|
||||
&& ! str_contains(substr($image['repository'], strlen($imageRepository)), '/'));
|
||||
|
||||
if (empty($output)) {
|
||||
if ($images->isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$images = collect(explode("\n", trim($output)))
|
||||
->filter()
|
||||
->map(function ($line) {
|
||||
$parts = explode('#', $line);
|
||||
$imageRef = $parts[0] ?? '';
|
||||
$tagParts = explode(':', $imageRef);
|
||||
|
||||
return [
|
||||
'repository' => $tagParts[0] ?? '',
|
||||
'tag' => $tagParts[1] ?? '',
|
||||
'created_at' => $parts[1] ?? '',
|
||||
'image_ref' => $imageRef,
|
||||
];
|
||||
})
|
||||
->filter(fn ($image) => ! empty($image['tag']));
|
||||
|
||||
// Separate images into categories
|
||||
// PR images (pr-*) are always deleted
|
||||
// Build images (*-build) are cleaned up to match retained regular images
|
||||
@@ -176,55 +292,47 @@ class CleanupDocker
|
||||
$buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build'));
|
||||
$regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build'));
|
||||
|
||||
// Always delete all PR images
|
||||
foreach ($prImages as $image) {
|
||||
$deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
|
||||
$deleteOutput = instant_remote_process([$deleteCommand], $server, false);
|
||||
$cleanupLog[] = [
|
||||
'command' => $deleteCommand,
|
||||
'output' => $deleteOutput ?? 'PR image removed or was in use',
|
||||
];
|
||||
}
|
||||
|
||||
// Filter out current running image from regular images and sort by creation date
|
||||
$sortedRegularImages = $regularImages
|
||||
->filter(fn ($image) => $image['tag'] !== $currentTag)
|
||||
->sortByDesc('created_at')
|
||||
->values();
|
||||
|
||||
// Keep only N images (imagesToKeep), delete the rest
|
||||
$imagesToDelete = $sortedRegularImages->skip($imagesToKeep);
|
||||
|
||||
foreach ($imagesToDelete as $image) {
|
||||
$deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
|
||||
$deleteOutput = instant_remote_process([$deleteCommand], $server, false);
|
||||
$cleanupLog[] = [
|
||||
'command' => $deleteCommand,
|
||||
'output' => $deleteOutput ?? 'Image removed or was in use',
|
||||
];
|
||||
}
|
||||
|
||||
// Clean up build images (-build suffix) that don't correspond to retained regular images
|
||||
// Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers.
|
||||
// If a build is in progress, docker rmi will fail silently since the image is in use.
|
||||
$keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag');
|
||||
if (! empty($currentTag)) {
|
||||
$keptTags = $keptTags->push($currentTag);
|
||||
if ($currentTag !== '') {
|
||||
$keptTags->push($currentTag);
|
||||
}
|
||||
|
||||
foreach ($buildImages as $image) {
|
||||
$baseTag = preg_replace('/-build$/', '', $image['tag']);
|
||||
if (! $keptTags->contains($baseTag)) {
|
||||
$deleteCommand = "docker rmi {$image['image_ref']} 2>/dev/null || true";
|
||||
$deleteOutput = instant_remote_process([$deleteCommand], $server, false);
|
||||
$cleanupLog[] = [
|
||||
'command' => $deleteCommand,
|
||||
'output' => $deleteOutput ?? 'Build image removed or was in use',
|
||||
];
|
||||
}
|
||||
}
|
||||
$imagesToDelete = $imagesToDelete
|
||||
->concat($prImages)
|
||||
->concat($sortedRegularImages->skip($imagesToKeep))
|
||||
->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag']))));
|
||||
}
|
||||
|
||||
// Images in use fail silently; docker rmi continues with the next image
|
||||
foreach ($this->imageRemovalCommands($imagesToDelete->pluck('image_ref')->unique()) as $deleteCommand) {
|
||||
$deleteOutput = $this->runRemoteCommand($deleteCommand, $server);
|
||||
$cleanupLog[] = [
|
||||
'command' => $deleteCommand,
|
||||
'output' => $deleteOutput ?? 'Images removed or were in use',
|
||||
];
|
||||
}
|
||||
|
||||
return $cleanupLog;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Collection<int, string> $imageRefs
|
||||
* @return Collection<int, string>
|
||||
*/
|
||||
private function imageRemovalCommands(Collection $imageRefs): Collection
|
||||
{
|
||||
return $imageRefs
|
||||
->chunk(100)
|
||||
->map(fn (Collection $chunk) => 'docker rmi '.$chunk->map(fn (string $ref) => escapeshellarg($ref))->implode(' ').' 2>/dev/null || true')
|
||||
->values();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,14 +15,13 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public $timeout = 600;
|
||||
public $timeout = CleanupDocker::JOB_TIMEOUT;
|
||||
|
||||
public $tries = 1;
|
||||
|
||||
@@ -30,9 +29,20 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
|
||||
public ?DockerCleanupExecution $execution_log = null;
|
||||
|
||||
/**
|
||||
* Shares the per-server lock with queued CleanupDocker runs, so two cleanups never run on
|
||||
* one server at once. A scheduled run that finds the lock held marks its occurrence
|
||||
* skipped (not missed); a manual run is dropped as before. The lock outlives the job
|
||||
* timeout, so a new run cannot start while a timed-out run is still finishing.
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping('docker-cleanup-'.$this->server->uuid))->expireAfter(600)->dontRelease()];
|
||||
return [
|
||||
ScheduledJobDeliveryService::withoutOverlapping(CleanupDocker::overlapLockKey($this->server), $this->occurrenceUuid)
|
||||
->shared()
|
||||
->expireAfter(CleanupDocker::overlapLockExpiresAfter())
|
||||
->dontRelease(),
|
||||
];
|
||||
}
|
||||
|
||||
public function __construct(
|
||||
@@ -42,7 +52,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
|
||||
public bool $deleteUnusedNetworks = false,
|
||||
public ?string $occurrenceUuid = null,
|
||||
) {
|
||||
$this->onQueue('high');
|
||||
$this->onQueue(maintenance_queue());
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
|
||||
@@ -1,13 +1,76 @@
|
||||
<?php
|
||||
|
||||
use App\Actions\Server\CleanupDocker;
|
||||
use App\Jobs\DockerCleanupJob;
|
||||
use App\Models\Application;
|
||||
use App\Models\DockerCleanupExecution;
|
||||
use App\Models\Environment;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
use App\Services\ScheduledJobDeliveryService;
|
||||
use App\Support\Actions\UniqueUntilProcessingJobDecorator;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Lorisleiva\Actions\Decorators\JobDecorator;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
Storage::fake('ssh-keys');
|
||||
Storage::fake('ssh-mux');
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
Carbon::setTestNow();
|
||||
});
|
||||
|
||||
function dockerCleanupReachableServer(): Server
|
||||
{
|
||||
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
|
||||
]);
|
||||
$server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
|
||||
|
||||
return $server->fresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Collection<int, string> The local commands (SSH invocations) that ran.
|
||||
*/
|
||||
function dockerCleanupFakeProcesses(?Closure $onRun = null): Collection
|
||||
{
|
||||
$commands = collect();
|
||||
Process::fake(function ($process) use ($commands, $onRun) {
|
||||
$command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
|
||||
$commands->push($command);
|
||||
if ($onRun) {
|
||||
$onRun($command);
|
||||
}
|
||||
|
||||
return Process::result(output: '');
|
||||
});
|
||||
|
||||
return $commands;
|
||||
}
|
||||
|
||||
it('persists the server id when creating an execution record', function () {
|
||||
$user = User::factory()->create();
|
||||
$team = $user->teams()->first();
|
||||
@@ -86,3 +149,204 @@ it('finishes the latest running execution when the job fails after a timeout', f
|
||||
->and($olderExecution->refresh()->status)->toBe('running')
|
||||
->and($olderExecution->finished_at)->toBeNull();
|
||||
});
|
||||
|
||||
it('skips a scheduled cleanup occurrence while another cleanup holds the server lock, so it is not reported missed later', function () {
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
|
||||
Notification::fake();
|
||||
$server = dockerCleanupReachableServer();
|
||||
$occurrence = ScheduledJobDelivery::create([
|
||||
'schedule_key' => "docker-cleanup:{$server->id}",
|
||||
'scheduled_for' => now()->startOfMinute(),
|
||||
'job_type' => 'docker-cleanup',
|
||||
'resource_id' => $server->id,
|
||||
'status' => 'enqueued',
|
||||
'enqueued_at' => now(),
|
||||
]);
|
||||
$job = new DockerCleanupJob($server, false, false, false, $occurrence->uuid);
|
||||
$runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
|
||||
$runningCleanupLock->get();
|
||||
$commands = dockerCleanupFakeProcesses();
|
||||
|
||||
dispatch_sync($job);
|
||||
$runningCleanupLock->release();
|
||||
|
||||
expect($occurrence->fresh()->status)->toBe('skipped')
|
||||
->and(DockerCleanupExecution::query()->exists())->toBeFalse()
|
||||
->and($commands)->toBeEmpty();
|
||||
|
||||
Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1));
|
||||
Queue::fake();
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
|
||||
Queue::assertNothingPushed();
|
||||
expect(DockerCleanupExecution::query()->exists())->toBeFalse();
|
||||
Notification::assertNotSentTo($server->team, DockerCleanupFailed::class);
|
||||
});
|
||||
|
||||
it('drops a manual cleanup while another cleanup holds the server lock without touching occurrences', function () {
|
||||
$server = dockerCleanupReachableServer();
|
||||
$job = new DockerCleanupJob($server, true);
|
||||
$runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
|
||||
$runningCleanupLock->get();
|
||||
$commands = dockerCleanupFakeProcesses();
|
||||
|
||||
dispatch_sync($job);
|
||||
$runningCleanupLock->release();
|
||||
|
||||
expect(DockerCleanupExecution::query()->exists())->toBeFalse()
|
||||
->and(ScheduledJobDelivery::query()->exists())->toBeFalse()
|
||||
->and($commands)->toBeEmpty();
|
||||
});
|
||||
|
||||
it('shares one server lock between scheduled cleanups and queued CleanupDocker runs that outlives the job timeout', function () {
|
||||
$server = dockerCleanupReachableServer();
|
||||
$cleanupJob = new DockerCleanupJob($server);
|
||||
$actionJob = CleanupDocker::makeJob($server, false, false);
|
||||
|
||||
$cleanupLock = $cleanupJob->middleware()[0];
|
||||
$actionLock = $actionJob->middleware()[0];
|
||||
|
||||
expect($cleanupLock->getLockKey($cleanupJob))
|
||||
->toBe('laravel-queue-overlap:docker-cleanup-'.$server->uuid)
|
||||
->toBe($actionLock->getLockKey($actionJob))
|
||||
->and($cleanupLock->expiresAfter)->toBeGreaterThan($cleanupJob->timeout)
|
||||
->and($actionLock->expiresAfter)->toBeGreaterThan($actionJob->timeout)
|
||||
->and($cleanupLock->releaseAfter)->toBeNull()
|
||||
->and($actionLock->releaseAfter)->toBeNull()
|
||||
->and($actionJob->timeout)->toBe(CleanupDocker::JOB_TIMEOUT)
|
||||
->and($actionJob->tries)->toBe(1);
|
||||
});
|
||||
|
||||
it('drops a queued CleanupDocker run while a scheduled cleanup holds the server lock', function () {
|
||||
$server = dockerCleanupReachableServer();
|
||||
$actionJob = CleanupDocker::makeJob($server, false, false);
|
||||
$runningCleanupLock = Cache::lock('laravel-queue-overlap:docker-cleanup-'.$server->uuid, 3600);
|
||||
$runningCleanupLock->get();
|
||||
$commands = dockerCleanupFakeProcesses();
|
||||
|
||||
dispatch_sync($actionJob);
|
||||
$runningCleanupLock->release();
|
||||
|
||||
expect($commands)->toBeEmpty();
|
||||
});
|
||||
|
||||
it('routes docker cleanups to the maintenance queue on cloud and to high on self-hosted', function (bool $selfHosted, string $queue) {
|
||||
config(['constants.coolify.self_hosted' => $selfHosted]);
|
||||
Queue::fake();
|
||||
$server = dockerCleanupReachableServer();
|
||||
|
||||
DockerCleanupJob::dispatch($server, true);
|
||||
CleanupDocker::dispatch($server, false, false);
|
||||
|
||||
expect(maintenance_queue())->toBe($queue);
|
||||
Queue::assertPushedOn($queue, DockerCleanupJob::class);
|
||||
Queue::assertPushedOn($queue, UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->decorates(CleanupDocker::class));
|
||||
})->with([
|
||||
'cloud' => [false, 'maintenance'],
|
||||
'self-hosted' => [true, 'high'],
|
||||
]);
|
||||
|
||||
it('gives remote cleanup commands a local timeout and fails once the cleanup deadline has passed', function () {
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
|
||||
$server = dockerCleanupReachableServer();
|
||||
$commands = dockerCleanupFakeProcesses(function (string $command) {
|
||||
if (str_contains($command, 'docker ')) {
|
||||
Carbon::setTestNow(now()->addSeconds(300));
|
||||
}
|
||||
});
|
||||
|
||||
expect(fn () => CleanupDocker::run($server))
|
||||
->toThrow(RuntimeException::class, 'Docker cleanup did not finish within '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' seconds');
|
||||
|
||||
$sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker container prune') || str_contains($command, 'docker image prune'))->values();
|
||||
|
||||
expect(CleanupDocker::REMOTE_COMMANDS_DEADLINE)->toBeLessThan(CleanupDocker::JOB_TIMEOUT)
|
||||
->and($sshCommands)->toHaveCount(2)
|
||||
->and($sshCommands[0])->toContain('timeout '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' ssh')
|
||||
->and($sshCommands[1])->toContain('timeout '.(CleanupDocker::REMOTE_COMMANDS_DEADLINE - 300).' ssh')
|
||||
->and($commands->contains(fn (string $command) => str_contains($command, 'docker builder prune')))->toBeFalse();
|
||||
});
|
||||
|
||||
it('coalesces queued cleanups per server and delays them until a bulk stop is over', function () {
|
||||
Queue::fake();
|
||||
$first = dockerCleanupReachableServer();
|
||||
$second = dockerCleanupReachableServer();
|
||||
|
||||
foreach (range(1, 30) as $stop) {
|
||||
CleanupDocker::dispatch($first, false, false);
|
||||
}
|
||||
CleanupDocker::dispatch($second, false, false);
|
||||
|
||||
Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2);
|
||||
Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->delay === CleanupDocker::QUEUED_DELAY);
|
||||
});
|
||||
|
||||
it('accepts a new queued cleanup once the waiting one is lost', function () {
|
||||
Queue::fake();
|
||||
$server = dockerCleanupReachableServer();
|
||||
|
||||
CleanupDocker::dispatch($server, false, false);
|
||||
$this->travel(CleanupDocker::QUEUED_DELAY + 241)->seconds();
|
||||
CleanupDocker::dispatch($server, false, false);
|
||||
|
||||
Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2);
|
||||
});
|
||||
|
||||
it('scans application images with a fixed number of remote commands and keeps the retention rules', function () {
|
||||
$server = dockerCleanupReachableServer();
|
||||
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
|
||||
$project = Project::factory()->create(['team_id' => $server->team_id]);
|
||||
$environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]);
|
||||
$applications = collect(range(1, 5))->map(fn () => Application::factory()->create([
|
||||
'environment_id' => $environment->id,
|
||||
'destination_id' => $destination->id,
|
||||
'destination_type' => $destination->getMorphClass(),
|
||||
]));
|
||||
$app = $applications->first();
|
||||
$app->settings()->update(['docker_images_to_keep' => 1]);
|
||||
|
||||
$containers = "{$app->uuid}#{$app->uuid}:v2";
|
||||
$images = implode("\n", [
|
||||
"{$app->uuid}#v1#2026-01-01 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#v2#2026-01-02 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#v3#2026-01-03 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#v4#2026-01-04 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#pr-7#2026-01-05 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#v1-build#2026-01-01 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#v2-build#2026-01-02 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}#v4-build#2026-01-04 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}_web#v1#2026-01-01 00:00:00 +0000 UTC",
|
||||
"{$app->uuid}/other#v1#2026-01-01 00:00:00 +0000 UTC",
|
||||
]);
|
||||
|
||||
$commands = collect();
|
||||
Process::fake(function ($process) use ($commands, $containers, $images) {
|
||||
$command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
|
||||
$commands->push($command);
|
||||
|
||||
return match (true) {
|
||||
str_contains($command, 'docker ps -a --format') => Process::result(output: $containers),
|
||||
str_contains($command, "docker images --format '{{.Repository}}#") => Process::result(output: $images),
|
||||
default => Process::result(output: ''),
|
||||
};
|
||||
});
|
||||
|
||||
CleanupDocker::run($server);
|
||||
|
||||
$applicationScanCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker ps -a --format')
|
||||
|| str_contains($command, "docker images --format '{{.Repository}}#")
|
||||
|| str_contains($command, 'docker inspect --format=')
|
||||
|| str_contains($command, "--filter reference='"));
|
||||
$removals = $commands->filter(fn (string $command) => str_contains($command, 'docker rmi \''))->values();
|
||||
|
||||
expect($applicationScanCommands)->toHaveCount(2)
|
||||
->and($removals)->toHaveCount(1);
|
||||
|
||||
$removal = $removals->first();
|
||||
foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build", "{$app->uuid}_web:v1"] as $deleted) {
|
||||
expect($removal)->toContain("'{$deleted}'");
|
||||
}
|
||||
foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}/other"] as $kept) {
|
||||
expect($removal)->not->toContain($kept);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user