From 4a1228c611a7ed66e55460070797bc8ba041b282 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:17:49 +0100 Subject: [PATCH] feat(scheduled-tasks): disable tasks that keep failing Disable a scheduled task after at least 5 failed runs in a row with no success for 3 days, notify the team, and show the reason on the task page. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + app/Jobs/ScheduledTaskJob.php | 12 ++ app/Models/ScheduledTask.php | 63 +++++++++ .../ScheduledTask/TaskAutoDisabled.php | 126 ++++++++++++++++++ .../ServerTransfer/ServerTransferBundle.php | 3 + .../ServerTransfer/ServerTransferExporter.php | 3 + .../ServerTransfer/ServerTransferImporter.php | 5 +- ...uto_disable_columns_to_scheduled_tasks.php | 34 +++++ docs/server-transfer-full-coverage.md | 2 +- .../scheduled-task-auto-disabled.blade.php | 5 + .../shared/scheduled-task/show.blade.php | 7 + 11 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 app/Notifications/ScheduledTask/TaskAutoDisabled.php create mode 100644 database/migrations/2026_11_05_120000_add_auto_disable_columns_to_scheduled_tasks.php create mode 100644 resources/views/emails/scheduled-task-auto-disabled.blade.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 8deeaef144..5fdfc33186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Add SMTP templates for common email providers, such as PostShiba, Amazon SES, Mailgun, SendGrid, Postmark, and Brevo, which fill in the host, port, and encryption - Save more than one named SMTP server on the transactional email and email notification pages, and switch between them - Give each SMTP server and Resend its own sender name and address, because each provider sends from its own verified domain +- Disable a scheduled task automatically when it fails at least 5 times in a row and has had no successful run for 3 days. Coolify sends a notification on the scheduled task failure channels and shows the reason on the task page until you enable the task again ### Changed diff --git a/app/Jobs/ScheduledTaskJob.php b/app/Jobs/ScheduledTaskJob.php index 7f59fcfc3f..d682a586b8 100644 --- a/app/Jobs/ScheduledTaskJob.php +++ b/app/Jobs/ScheduledTaskJob.php @@ -11,6 +11,7 @@ use App\Models\ScheduledTaskExecution; use App\Models\Server; use App\Models\Service; use App\Models\Team; +use App\Notifications\ScheduledTask\TaskAutoDisabled; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\ScheduledTask\TaskSuccess; use App\Services\ScheduledJobDeliveryService; @@ -199,6 +200,7 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue 'status' => 'success', 'message' => $this->task_output, ]); + $this->task->recordRunSuccess(); $this->team?->notify(new TaskSuccess($this->task, $this->task_output)); @@ -346,5 +348,15 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue // Notify team about permanent failure $this->team?->notify(new TaskFailed($this->task, $safeErrorMessage)); + + if ($this->task->recordRunFailure()) { + auditLog('scheduled_task.auto_disabled', [ + 'team_id' => $this->task->team_id, + 'scheduled_task_uuid' => $this->task->uuid, + 'scheduled_task_name' => $this->task->name, + 'failure_count' => $this->task->failure_count, + ], 'warning'); + $this->team?->notify(new TaskAutoDisabled($this->task)); + } } } diff --git a/app/Models/ScheduledTask.php b/app/Models/ScheduledTask.php index 0a53395d31..59b34375b8 100644 --- a/app/Models/ScheduledTask.php +++ b/app/Models/ScheduledTask.php @@ -20,6 +20,9 @@ use OpenApi\Attributes as OA; 'frequency' => ['type' => 'string', 'description' => 'The frequency of the scheduled task.'], 'container' => ['type' => 'string', 'nullable' => true, 'description' => 'The container where the command should be executed.'], 'timeout' => ['type' => 'integer', 'description' => 'The timeout of the scheduled task in seconds.'], + 'failure_count' => ['type' => 'integer', 'description' => 'The number of failed runs in a row.'], + 'failing_since' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The date and time of the first failed run in the current series of failures.'], + 'auto_disabled_at' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The date and time when Coolify disabled the task because it kept failing.'], 'created_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was created.'], 'updated_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was last updated.'], ], @@ -42,6 +45,31 @@ class ScheduledTask extends BaseModel 'service_id', ]; + /** + * A task is disabled automatically when it has failed at least this many times in a row... + */ + public const AUTO_DISABLE_MIN_FAILURES = 5; + + /** + * ...and has not had a successful run for at least this many days. Both limits must be + * reached, so a short outage does not disable a task that runs every minute. + */ + public const AUTO_DISABLE_MIN_DAYS = 3; + + /** + * A task that is enabled again starts with a clean failure series. + */ + protected static function booted(): void + { + static::saving(function (self $task) { + if ($task->isDirty('enabled') && $task->enabled) { + $task->failure_count = 0; + $task->failing_since = null; + $task->auto_disabled_at = null; + } + }); + } + public static function ownedByCurrentTeamAPI(int $teamId) { return static::where('team_id', $teamId)->orderBy('created_at', 'desc'); @@ -52,9 +80,44 @@ class ScheduledTask extends BaseModel return [ 'enabled' => 'boolean', 'timeout' => 'integer', + 'failure_count' => 'integer', + 'failing_since' => 'datetime', + 'auto_disabled_at' => 'datetime', ]; } + public function recordRunSuccess(): void + { + if ($this->failure_count > 0 || $this->failing_since !== null) { + $this->failure_count = 0; + $this->failing_since = null; + $this->save(); + } + } + + /** + * Count a failed run and disable the task when it keeps failing. + * + * @return bool True when this failure disabled the task. + */ + public function recordRunFailure(): bool + { + $this->refresh(); + $this->increment('failure_count', 1, ['failing_since' => $this->failing_since ?? now()]); + + if (! $this->enabled + || $this->failure_count < self::AUTO_DISABLE_MIN_FAILURES + || $this->failing_since->gt(now()->subDays(self::AUTO_DISABLE_MIN_DAYS))) { + return false; + } + + $this->enabled = false; + $this->auto_disabled_at = now(); + $this->save(); + + return true; + } + public function service() { return $this->belongsTo(Service::class); diff --git a/app/Notifications/ScheduledTask/TaskAutoDisabled.php b/app/Notifications/ScheduledTask/TaskAutoDisabled.php new file mode 100644 index 0000000000..6ab926d308 --- /dev/null +++ b/app/Notifications/ScheduledTask/TaskAutoDisabled.php @@ -0,0 +1,126 @@ +onQueue('high'); + $resource = $task->application ?? $task->service; + if ($resource) { + $type = $resource instanceof Application ? 'application' : 'service'; + $this->url = base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$task->uuid}"; + } + + $this->summary = "Scheduled task ({$task->name}) was disabled after {$task->failure_count} failed runs in a row since {$task->failing_since?->toDayDateTimeString()} UTC. Fix the task and enable it again."; + } + + public function via(object $notifiable): array + { + return $notifiable->getEnabledChannels('scheduled_task_failure'); + } + + public function toMail(): MailMessage + { + $mail = new MailMessage; + $mail->subject("Coolify: [ACTION REQUIRED] Scheduled task ({$this->task->name}) was disabled."); + $mail->view('emails.scheduled-task-auto-disabled', [ + 'summary' => $this->summary, + 'url' => $this->url, + ]); + + return $mail; + } + + public function toDiscord(): DiscordMessage + { + $message = new DiscordMessage( + title: ':no_entry: Scheduled task disabled', + description: $this->summary, + color: DiscordMessage::errorColor(), + ); + + if ($this->url) { + $message->addField('Scheduled task', '[Link]('.$this->url.')'); + } + + return $message; + } + + public function toTelegram(): array + { + return [ + 'message' => "Coolify: {$this->summary}", + ]; + } + + public function toPushover(): PushoverMessage + { + $buttons = []; + if ($this->url) { + $buttons[] = [ + 'text' => 'Open task in Coolify', + 'url' => (string) $this->url, + ]; + } + + return new PushoverMessage( + title: 'Scheduled task disabled', + level: 'error', + message: $this->summary, + buttons: $buttons, + ); + } + + public function toSlack(): SlackMessage + { + $description = $this->summary; + if ($this->url) { + $description .= "\n\n*Task URL:* {$this->url}"; + } + + return new SlackMessage( + title: 'Scheduled task disabled', + description: $description, + color: SlackMessage::errorColor() + ); + } + + public function toWebhook(): array + { + $data = [ + 'success' => false, + 'message' => 'Scheduled task disabled', + 'event' => 'task_auto_disabled', + 'task_name' => $this->task->name, + 'task_uuid' => $this->task->uuid, + 'failure_count' => $this->task->failure_count, + 'failing_since' => $this->task->failing_since?->toIso8601String(), + ]; + + if ($this->task->application) { + $data['application_uuid'] = $this->task->application->uuid; + } elseif ($this->task->service) { + $data['service_uuid'] = $this->task->service->uuid; + } + + if ($this->url) { + $data['url'] = $this->url; + } + + return $data; + } +} diff --git a/app/Services/ServerTransfer/ServerTransferBundle.php b/app/Services/ServerTransfer/ServerTransferBundle.php index 09cee9d951..8a3d023b88 100644 --- a/app/Services/ServerTransfer/ServerTransferBundle.php +++ b/app/Services/ServerTransfer/ServerTransferBundle.php @@ -382,6 +382,9 @@ class ServerTransferBundle self::check($errors, "{$at}.scheduled_tasks.{$i}.command", $task['command'] ?? null, ['required', 'string']); self::check($errors, "{$at}.scheduled_tasks.{$i}.container", $task['container'] ?? null, ['nullable', ...ValidationPatterns::containerNameRules()]); self::checkCron($errors, "{$at}.scheduled_tasks.{$i}.frequency", $task['frequency'] ?? ''); + self::check($errors, "{$at}.scheduled_tasks.{$i}.failure_count", $task['failure_count'] ?? null, ['nullable', 'integer', 'min:0']); + self::check($errors, "{$at}.scheduled_tasks.{$i}.failing_since", $task['failing_since'] ?? null, ['nullable', 'date']); + self::check($errors, "{$at}.scheduled_tasks.{$i}.auto_disabled_at", $task['auto_disabled_at'] ?? null, ['nullable', 'date']); } } diff --git a/app/Services/ServerTransfer/ServerTransferExporter.php b/app/Services/ServerTransfer/ServerTransferExporter.php index f3a54bce3d..659deebeef 100644 --- a/app/Services/ServerTransfer/ServerTransferExporter.php +++ b/app/Services/ServerTransfer/ServerTransferExporter.php @@ -1336,6 +1336,9 @@ class ServerTransferExporter 'container' => $task->container, 'timeout' => $task->timeout, 'enabled' => (bool) ($task->enabled ?? true), + 'failure_count' => (int) $task->failure_count, + 'failing_since' => $task->failing_since?->toIso8601String(), + 'auto_disabled_at' => $task->auto_disabled_at?->toIso8601String(), ]; } diff --git a/app/Services/ServerTransfer/ServerTransferImporter.php b/app/Services/ServerTransfer/ServerTransferImporter.php index bb270f1d04..fcabc4a63a 100644 --- a/app/Services/ServerTransfer/ServerTransferImporter.php +++ b/app/Services/ServerTransfer/ServerTransferImporter.php @@ -1688,6 +1688,9 @@ class ServerTransferImporter 'container' => data_get($task, 'container'), 'timeout' => data_get($task, 'timeout'), 'enabled' => (bool) data_get($task, 'enabled', true), + 'failure_count' => (int) data_get($task, 'failure_count', 0), + 'failing_since' => data_get($task, 'failing_since'), + 'auto_disabled_at' => data_get($task, 'auto_disabled_at'), 'team_id' => $teamId, ]; @@ -1697,7 +1700,7 @@ class ServerTransferImporter $payload['service_id'] = $resource->id; } - ScheduledTask::create($payload); + ScheduledTask::forceCreate($payload); } } diff --git a/database/migrations/2026_11_05_120000_add_auto_disable_columns_to_scheduled_tasks.php b/database/migrations/2026_11_05_120000_add_auto_disable_columns_to_scheduled_tasks.php new file mode 100644 index 0000000000..d8918a5413 --- /dev/null +++ b/database/migrations/2026_11_05_120000_add_auto_disable_columns_to_scheduled_tasks.php @@ -0,0 +1,34 @@ +unsignedInteger('failure_count')->default(0); + $table->timestampTz('failing_since')->nullable(); + $table->timestampTz('auto_disabled_at')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('scheduled_tasks', function (Blueprint $table) { + $table->dropColumn(['failure_count', 'failing_since', 'auto_disabled_at']); + }); + } +}; diff --git a/docs/server-transfer-full-coverage.md b/docs/server-transfer-full-coverage.md index 3391120322..665f524af7 100644 --- a/docs/server-transfer-full-coverage.md +++ b/docs/server-transfer-full-coverage.md @@ -101,7 +101,7 @@ When you add a model or a setting to the transfer, add it to this file too. | ✅ | Persistent volume | the seed has one; add a host-path bind mount and preview suffix on | | ✅ | File mount + directory mount | content, chown, chmod | | ✅ | Volume backup | on the persistent volume: S3 on, stop during backup, all retention values, timeout, missing-backup days | -| ✅ | Scheduled tasks | the seed has one; add a disabled one with container + timeout | +| ✅ | Scheduled tasks | the seed has one; add a disabled one with container + timeout, and one that Coolify disabled automatically (the "Disabled automatically" warning shows on the target) | | ✅ | Tag | | | ✅ | Preview deployment | PR preview with domain port overrides + max restart count + keeps production values (previews from before v4.5.1) (needs the GitHub App, or create it in the database) | | ✅ | Secret manager link | the app loads variables from the secret manager token | diff --git a/resources/views/emails/scheduled-task-auto-disabled.blade.php b/resources/views/emails/scheduled-task-auto-disabled.blade.php new file mode 100644 index 0000000000..b1b1c0dc9c --- /dev/null +++ b/resources/views/emails/scheduled-task-auto-disabled.blade.php @@ -0,0 +1,5 @@ + +{{ $summary }} + +Click [here]({{ $url }}) to view the task. + diff --git a/resources/views/livewire/project/shared/scheduled-task/show.blade.php b/resources/views/livewire/project/shared/scheduled-task/show.blade.php index 66d6bb8afe..0febba56be 100644 --- a/resources/views/livewire/project/shared/scheduled-task/show.blade.php +++ b/resources/views/livewire/project/shared/scheduled-task/show.blade.php @@ -14,6 +14,13 @@
+ @if (! $isEnabled && $task->auto_disabled_at) + + Coolify disabled this task on {{ $task->auto_disabled_at->toDayDateTimeString() }} UTC after + {{ $task->failure_count }} failed runs in a row since {{ $task->failing_since?->toDayDateTimeString() }} UTC. + Check the executions, fix the task, and enable it again. + + @endif