server), $this->occurrenceUuid) ->shared() ->expireAfter(CleanupDocker::overlapLockExpiresAfter()); return [ $this->manualCleanup ? $withoutOverlapping->releaseAfter(self::MANUAL_RELEASE_DELAY) : $withoutOverlapping->dontRelease(), ]; } /** * A manual run may wait for one full cleanup that holds the lock. Fixed at dispatch, so a manual run of a * killed worker, which comes back after retry_after, fails instead of running late. Other runs try once. */ public function retryUntil(): ?\DateTimeInterface { if (! $this->manualCleanup) { return null; } return now()->addSeconds(CleanupDocker::overlapLockExpiresAfter() + self::MANUAL_RELEASE_DELAY * 2); } public function __construct( public Server $server, public bool $manualCleanup = false, public bool $deleteUnusedVolumes = false, public bool $deleteUnusedNetworks = false, public ?string $occurrenceUuid = null, ) { $this->onQueue(maintenance_queue()); } public function handle(): void { if ($this->occurrenceUuid && ! app(ScheduledJobDeliveryService::class)->claim($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid)) { return; } $failed = false; try { $this->execution_log = DockerCleanupExecution::create([ 'server_id' => $this->server->id, ]); $this->rememberExecution(); if (! $this->server->isFunctional()) { $this->execution_log->update([ 'status' => 'failed', 'message' => 'Server is not functional (unreachable, unusable, or disabled)', 'finished_at' => Carbon::now()->toImmutable(), ]); return; } $this->usageBefore = $this->server->getDiskUsage(); if ($this->manualCleanup || $this->server->settings->force_docker_cleanup) { $cleanup_log = CleanupDocker::run( server: $this->server, deleteUnusedVolumes: $this->deleteUnusedVolumes, deleteUnusedNetworks: $this->deleteUnusedNetworks, pruneBuildCacheDuringDeployment: $this->manualCleanup ); $usageAfter = $this->server->getDiskUsage(); $message = ($this->manualCleanup ? 'Manual' : 'Forced').' Docker cleanup job executed successfully. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.'; $message = $this->withBuildCacheKeptReason($message, $cleanup_log); $this->execution_log->update([ 'status' => 'success', 'message' => $message, 'cleanup_log' => $cleanup_log, ]); $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); event(new DockerCleanupDone($this->execution_log)); return; } if (str($this->usageBefore)->isEmpty() || $this->usageBefore === null || $this->usageBefore === 0) { $cleanup_log = CleanupDocker::run( server: $this->server, deleteUnusedVolumes: $this->deleteUnusedVolumes, deleteUnusedNetworks: $this->deleteUnusedNetworks, pruneBuildCacheDuringDeployment: $this->manualCleanup ); $message = 'Docker cleanup job executed successfully, but no disk usage could be determined.'; $message = $this->withBuildCacheKeptReason($message, $cleanup_log); $this->execution_log->update([ 'status' => 'success', 'message' => $message, 'cleanup_log' => $cleanup_log, ]); $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); event(new DockerCleanupDone($this->execution_log)); return; } if ($this->usageBefore >= $this->server->settings->docker_cleanup_threshold) { $cleanup_log = CleanupDocker::run( server: $this->server, deleteUnusedVolumes: $this->deleteUnusedVolumes, deleteUnusedNetworks: $this->deleteUnusedNetworks, pruneBuildCacheDuringDeployment: $this->manualCleanup ); $usageAfter = $this->server->getDiskUsage(); $diskSaved = $this->usageBefore - $usageAfter; if ($diskSaved > 0) { $message = 'Saved '.$diskSaved.'% disk space. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.'; } else { $message = 'Docker cleanup job executed successfully, but no disk space was saved. Disk usage before: '.$this->usageBefore.'%, Disk usage after: '.$usageAfter.'%.'; } $message = $this->withBuildCacheKeptReason($message, $cleanup_log); $this->execution_log->update([ 'status' => 'success', 'message' => $message, 'cleanup_log' => $cleanup_log, ]); $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); event(new DockerCleanupDone($this->execution_log)); } else { $message = 'No cleanup needed for '.$this->server->name; $this->execution_log->update([ 'status' => 'success', 'message' => $message, ]); $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); event(new DockerCleanupDone($this->execution_log)); } } catch (\Throwable $e) { $failed = true; $errorMessage = remove_iip($e->getMessage()); if ($this->execution_log) { $this->execution_log->update([ 'status' => 'failed', 'message' => $errorMessage, ]); event(new DockerCleanupDone($this->execution_log)); } $this->server->team?->notify(new DockerCleanupFailed($this->server, 'Docker cleanup job failed with the following error: '.$errorMessage)); throw $e; } finally { if (! $failed && $this->occurrenceUuid) { app(ScheduledJobDeliveryService::class)->complete($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid); } if ($this->execution_log) { $this->execution_log->update([ 'finished_at' => Carbon::now()->toImmutable(), ]); } if (! $failed && $this->executionCacheKey()) { Cache::forget($this->executionCacheKey()); } } } private function withBuildCacheKeptReason(string $message, array $cleanupLog): string { $reason = CleanupDocker::buildCacheKeptReason($cleanupLog); return $reason ? $message.' '.$reason : $message; } /** * failed() runs on a fresh job copy built from the queue payload (for example after a worker timeout), * so the execution this run created is kept under the queue job uuid instead of on the job. */ private function rememberExecution(): void { if ($this->executionCacheKey()) { Cache::put($this->executionCacheKey(), $this->execution_log->id, now()->addSeconds((int) config('queue.connections.redis.retry_after', 86400) + 3600)); } } private function executionCacheKey(): ?string { $jobUuid = $this->job?->uuid(); return $jobUuid ? 'docker-cleanup-execution:'.$jobUuid : null; } /** * Fails running executions that are older than the server lock. The worker kills a cleanup at the * job timeout, which is shorter than the lock, so such a run lost its worker before failed() ran * (for example a killed container) and would otherwise stay running. */ public static function failInterruptedExecutions(): void { $executions = DockerCleanupExecution::query() ->with('server.team') ->where('status', 'running') ->whereNull('finished_at') ->where('created_at', '<', now()->subSeconds(CleanupDocker::overlapLockExpiresAfter())) ->get(); foreach ($executions as $execution) { $updated = DockerCleanupExecution::query() ->whereKey($execution->id) ->where('status', 'running') ->whereNull('finished_at') ->update([ 'status' => 'failed', 'message' => self::INTERRUPTED_MESSAGE, 'finished_at' => Carbon::now()->toImmutable(), ]); if ($updated === 0) { continue; } event(new DockerCleanupDone($execution->refresh())); $execution->server?->team?->notify(new DockerCleanupFailed($execution->server, 'Docker cleanup job failed with the following error: '.self::INTERRUPTED_MESSAGE)); } } public function failed(?\Throwable $exception): void { if ($this->occurrenceUuid) { app(ScheduledJobDeliveryService::class)->fail($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid); } // Only this run's own execution. A run that never started (e.g. a manual run whose wait for the // server lock ran out) created none and must not fail another running cleanup. $executionId = $this->executionCacheKey() ? Cache::pull($this->executionCacheKey()) : null; $execution = $executionId ? DockerCleanupExecution::query()->find($executionId) : null; if (! $execution) { return; } $message = remove_iip($exception?->getMessage() ?? 'Docker cleanup job failed without an exception.'); $updated = DockerCleanupExecution::query() ->whereKey($execution->id) ->where('status', 'running') ->whereNull('finished_at') ->update([ 'status' => 'failed', 'message' => $message, 'finished_at' => Carbon::now()->toImmutable(), ]); if ($updated === 0) { return; } $execution->refresh(); event(new DockerCleanupDone($execution)); $this->server->team?->notify(new DockerCleanupFailed($this->server, 'Docker cleanup job failed with the following error: '.$message)); } }