timeout = $timeout; } $this->onQueue($queue); } /** * Execute the job. */ public function handle(): void { // A database import that Coolify stopped after a restart must not run again when // the queue retries this job. Its cleanup is already queued. if (DatabaseImportCleanup::stopRequested($this->activity)) { RemoteProcessCommand::forget($this->activity); return; } $remote_process = resolve(RunRemoteProcess::class, [ 'activity' => $this->activity, 'ignore_errors' => $this->ignore_errors, 'call_event_on_finish' => $this->call_event_on_finish, 'call_event_data' => $this->call_event_data, ]); $remote_process(); // The task is finished. A failed run throws before this line and is removed in failed(), // because a retry of the job must still be able to read the command. RemoteProcessCommand::forget($this->activity); } /** * Calculate the number of seconds to wait before retrying the job. */ public function backoff(): array { return [30, 90, 180]; // 30s, 90s, 180s between retries } /** * Handle a job failure. */ public function failed(?\Throwable $exception): void { Log::channel('scheduled-errors')->error('CoolifyTask permanently failed', [ 'job' => 'CoolifyTask', 'activity_id' => $this->activity->id, 'server_uuid' => $this->activity->getExtraProperty('server_uuid'), 'error' => $exception ? remove_iip($exception->getMessage()) : null, 'total_attempts' => $this->attempts(), 'trace' => $exception?->getTraceAsString(), ]); // A database import whose restore can still run in the database container stays in progress // until its stop has run, so it keeps blocking other operations on the database. // The message can contain command output or the SSH command, so store only a redacted copy. $message = filled($exception?->getMessage()) ? remove_iip($exception->getMessage()) : null; if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $message ?: 'Job permanently failed')) { RemoteProcessCommand::forget($this->activity); return; } // Update activity status to reflect permanent failure // A stopped database import already has the message that explains the stop; the process // error ("Terminated") would hide it. $stopRequested = DatabaseImportCleanup::stopRequested($this->activity); $this->activity->properties = $this->activity->properties->merge(array_filter([ 'status' => ProcessStatus::ERROR->value, 'error' => $stopRequested ? null : ($message ?? 'Job permanently failed'), 'failed_at' => now()->toIso8601String(), ], fn ($value) => $value !== null)); $this->activity->save(); // No attempt is left, so the command (which can contain secrets) is no longer needed. RemoteProcessCommand::forget($this->activity); // Dispatch cleanup event on failure (same as on success) if ($this->call_event_on_finish) { try { RunRemoteProcess::dispatchFinishEvent($this->activity, $this->call_event_on_finish, $this->call_event_data); Log::info('Cleanup event dispatched after job failure', [ 'event' => $this->call_event_on_finish, ]); } catch (\Throwable $e) { Log::error('Error dispatching cleanup event on failure: '.$e->getMessage()); } } } }