coolify/app/Jobs/DockerCleanupJob.php

<?php

namespace App\Jobs;

use App\Actions\Server\CleanupDocker;
use App\Events\DockerCleanupDone;
use App\Models\DockerCleanupExecution;
use App\Models\Server;
use App\Notifications\Server\DockerCleanupFailed;
use App\Notifications\Server\DockerCleanupSuccess;
use App\Services\ScheduledJobDeliveryService;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;

class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    public $timeout = CleanupDocker::JOB_TIMEOUT;

    public $tries = 1;

    public ?string $usageBefore = null;

    public ?DockerCleanupExecution $execution_log = null;

    /**
     * Shares the per-server lock with queued CleanupDocker runs, so two cleanups never run on
     * one server at once. A scheduled run that finds the lock held marks its occurrence
     * skipped (not missed); a manual run is dropped as before. The lock outlives the job
     * timeout, so a new run cannot start while a timed-out run is still finishing.
     */
    public function middleware(): array
    {
        return [
            ScheduledJobDeliveryService::withoutOverlapping(CleanupDocker::overlapLockKey($this->server), $this->occurrenceUuid)
                ->shared()
                ->expireAfter(CleanupDocker::overlapLockExpiresAfter())
                ->dontRelease(),
        ];
    }

    public function __construct(
        public Server $server,
        public bool $manualCleanup = false,
        public bool $deleteUnusedVolumes = false,
        public bool $deleteUnusedNetworks = false,
        public ?string $occurrenceUuid = null,
    ) {
        $this->onQueue(maintenance_queue());
    }

    public function handle(): void
    {
        if ($this->occurrenceUuid && ! app(ScheduledJobDeliveryService::class)->claim($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid)) {
            return;
        }

        $failed = false;

        try {
            $this->execution_log = DockerCleanupExecution::create([
                'server_id' => $this->server->id,
            ]);

            if (! $this->server->isFunctional()) {
                $this->execution_log->update([
                    'status' => 'failed',
                    'message' => 'Server is not functional (unreachable, unusable, or disabled)',
                    'finished_at' => Carbon::now()->toImmutable(),
                ]);

                return;
            }

            $this->usageBefore = $this->server->getDiskUsage();

            if ($this->manualCleanup || $this->server->settings->force_docker_cleanup) {
                $cleanup_log = CleanupDocker::run(
                    server: $this->server,
                    deleteUnusedVolumes: $this->deleteUnusedVolumes,
                    deleteUnusedNetworks: $this->deleteUnusedNetworks
                );
                $usageAfter = $this->server->getDiskUsage();
                $message = ($this->manualCleanup ? 'Manual' : 'Forced').' Docker cleanup job executed successfully. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.';

                $this->execution_log->update([
                    'status' => 'success',
                    'message' => $message,
                    'cleanup_log' => $cleanup_log,
                ]);

                $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
                event(new DockerCleanupDone($this->execution_log));

                return;
            }

            if (str($this->usageBefore)->isEmpty() || $this->usageBefore === null || $this->usageBefore === 0) {
                $cleanup_log = CleanupDocker::run(
                    server: $this->server,
                    deleteUnusedVolumes: $this->deleteUnusedVolumes,
                    deleteUnusedNetworks: $this->deleteUnusedNetworks
                );
                $message = 'Docker cleanup job executed successfully, but no disk usage could be determined.';

                $this->execution_log->update([
                    'status' => 'success',
                    'message' => $message,
                    'cleanup_log' => $cleanup_log,
                ]);

                $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
                event(new DockerCleanupDone($this->execution_log));

                return;
            }

            if ($this->usageBefore >= $this->server->settings->docker_cleanup_threshold) {
                $cleanup_log = CleanupDocker::run(
                    server: $this->server,
                    deleteUnusedVolumes: $this->deleteUnusedVolumes,
                    deleteUnusedNetworks: $this->deleteUnusedNetworks
                );
                $usageAfter = $this->server->getDiskUsage();
                $diskSaved = $this->usageBefore - $usageAfter;

                if ($diskSaved > 0) {
                    $message = 'Saved '.$diskSaved.'% disk space. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.';
                } else {
                    $message = 'Docker cleanup job executed successfully, but no disk space was saved. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.';
                }

                $this->execution_log->update([
                    'status' => 'success',
                    'message' => $message,
                    'cleanup_log' => $cleanup_log,
                ]);

                $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
                event(new DockerCleanupDone($this->execution_log));
            } else {
                $message = 'No cleanup needed for '.$this->server->name;

                $this->execution_log->update([
                    'status' => 'success',
                    'message' => $message,
                ]);

                $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
                event(new DockerCleanupDone($this->execution_log));
            }
        } catch (\Throwable $e) {
            $failed = true;
            if ($this->execution_log) {
                $this->execution_log->update([
                    'status' => 'failed',
                    'message' => $e->getMessage(),
                ]);
                event(new DockerCleanupDone($this->execution_log));
            }
            $this->server->team?->notify(new DockerCleanupFailed($this->server, 'Docker cleanup job failed with the following error: '.$e->getMessage()));
            throw $e;
        } finally {
            if (! $failed && $this->occurrenceUuid) {
                app(ScheduledJobDeliveryService::class)->complete($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid);
            }

            if ($this->execution_log) {
                $this->execution_log->update([
                    'finished_at' => Carbon::now()->toImmutable(),
                ]);
            }
        }
    }

    public function failed(?\Throwable $exception): void
    {
        if ($this->occurrenceUuid) {
            app(ScheduledJobDeliveryService::class)->fail($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid);
        }

        $execution = DockerCleanupExecution::query()
            ->where('server_id', $this->server->id)
            ->where('status', 'running')
            ->whereNull('finished_at')
            ->latest('id')
            ->first();

        if (! $execution) {
            return;
        }

        $message = $exception?->getMessage() ?? 'Docker cleanup job failed without an exception.';

        $updated = DockerCleanupExecution::query()
            ->whereKey($execution->id)
            ->where('status', 'running')
            ->whereNull('finished_at')
            ->update([
                'status' => 'failed',
                'message' => $message,
                'finished_at' => Carbon::now()->toImmutable(),
            ]);

        if ($updated === 0) {
            return;
        }

        $execution->refresh();
        event(new DockerCleanupDone($execution));
        $this->server->team?->notify(new DockerCleanupFailed($this->server, 'Docker cleanup job failed with the following error: '.$message));
    }
}