diff --git a/CHANGELOG.md b/CHANGELOG.md index 4288272018..c3aa74002b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Changed +- Stop the notification storm from scheduled jobs that keep failing. A failed scheduled task, database backup, storage backup, or Docker cleanup now notifies on the first failure, then at most once a day while it keeps failing. The first successful run after a failure sends one "works again" notification to the failure channels. A database backup whose S3 upload keeps failing is also notified at most once a day +- Disable a scheduled task automatically when it failed 10 times in a row and its first failure is more than 24 hours old, and send one notification about it. The task page shows why the task was disabled. Enable the task again to clear the reason. The scheduled tasks API returns `consecutive_failures`, `failing_since`, and `disabled_reason`. Backups and Docker cleanup are never disabled automatically, so they start again by themselves after an outage +- Skip a scheduled task when its application or service has no running container, instead of a failed run with "No containers running". A skipped run sends no notification and does not count toward the automatic disable - Move **Import Backup** from the resource menu to an **Import backup** button on the **Backups** page of databases and services. The button opens the import in a dialog. Old **Import Backup** links open the **Backups** page ([#11818](https://github.com/coollabsio/coolify/discussions/11818)) ### Fixed diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 51950a001f..d1c7e80ee6 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -5,6 +5,7 @@ namespace App\Jobs; use App\Events\BackupCreated; use App\Helpers\SshMultiplexingHelper; use App\Helpers\SshRetryHandler; +use App\Models\NotificationThrottle; use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledDatabaseBackupExecution; @@ -20,6 +21,7 @@ use App\Models\Team; use App\Notifications\Database\BackupFailed; use App\Notifications\Database\BackupSuccess; use App\Notifications\Database\BackupSuccessWithS3Warning; +use App\Notifications\ScheduledJobRecovered; use App\Services\ScheduledJobDeliveryService; use App\Services\Security\SensitiveDataRedactor; use App\Support\BackupCompression; @@ -316,6 +318,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $ip = Str::slug($this->server->ip); $this->backup_dir = backup_dir().'/coolify'."/coolify-db-$ip"; } + $runHadProblem = false; foreach ($databasesToBackup as $database) { // Reset per-database state, so a failure before this database's execution exists // does not mark the previous database's execution failed or delete its backup file. @@ -450,6 +453,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue throw new \Exception('Local backup file is empty or was not created'); } } catch (Throwable $e) { + $runHadProblem = true; // Local backup failed: the partial file is not tracked, so retention would never remove it. deleteBackupsLocally($this->backup_location, $this->server); $failureMessage = $this->redact($this->error_output ?? $this->backup_output ?? $e->getMessage()); @@ -498,6 +502,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } // Step 3: Update status and send notifications based on results + if ($s3UploadError) { + $runHadProblem = true; + } if ($localBackupSucceeded) { $message = $this->backup_output === null ? null : $this->redact($this->backup_output); @@ -536,6 +543,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue if ($this->backup_log && $this->backup_log->status === 'success') { $this->removeExpiredBackups(); } + if (! $runHadProblem) { + $this->notifyIfRecovered(); + } } catch (Throwable $e) { $failed = true; throw $e; @@ -1097,6 +1107,33 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue * * @return string|array|null */ + /** + * After a run without problems, tell the team once that a backup that failed before works again. + */ + private function notifyIfRecovered(): void + { + $wasFailing = NotificationThrottle::release($this->backup, BackupFailed::class); + $hadS3Warning = NotificationThrottle::release($this->backup, BackupSuccessWithS3Warning::class); + if (! $wasFailing && ! $hadS3Warning) { + return; + } + + try { + $this->team?->notify(new ScheduledJobRecovered( + failureEvent: 'backup_failure', + title: "Database backup for {$this->database->name} works again.", + webhookEvent: 'backup_recovered', + url: base_url().'/project/'.data_get($this->database, 'environment.project.uuid').'/environment/'.data_get($this->database, 'environment.uuid').'/database/'.$this->database->uuid, + webhookData: ['database_name' => $this->database->name, 'database_uuid' => $this->database->uuid, 'backup_uuid' => $this->backup->uuid], + )); + } catch (Throwable $e) { + Log::channel('scheduled-errors')->warning('Failed to send backup recovery notification', [ + 'backup_id' => $this->backup->uuid, + 'error' => $e->getMessage(), + ]); + } + } + private function configuredDatabasesToBackup(): string|array|null { if ($this->backup->dump_all) { diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 3f70534301..3e5caa3037 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -5,7 +5,9 @@ namespace App\Jobs; use App\Actions\Server\CleanupDocker; use App\Events\DockerCleanupDone; use App\Models\DockerCleanupExecution; +use App\Models\NotificationThrottle; use App\Models\Server; +use App\Notifications\ScheduledJobRecovered; use App\Notifications\Server\DockerCleanupFailed; use App\Notifications\Server\DockerCleanupSuccess; use App\Services\ScheduledJobDeliveryService; @@ -132,7 +134,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue 'cleanup_log' => $cleanup_log, ]); - $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); + $this->notifySuccess($message); event(new DockerCleanupDone($this->execution_log)); return; @@ -154,7 +156,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue 'cleanup_log' => $cleanup_log, ]); - $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); + $this->notifySuccess($message); event(new DockerCleanupDone($this->execution_log)); return; @@ -183,7 +185,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue 'cleanup_log' => $cleanup_log, ]); - $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); + $this->notifySuccess($message); event(new DockerCleanupDone($this->execution_log)); } else { $message = 'No cleanup needed for '.$this->server->name; @@ -193,7 +195,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue 'message' => $message, ]); - $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); + $this->notifySuccess($message); event(new DockerCleanupDone($this->execution_log)); } } catch (\Throwable $e) { @@ -285,6 +287,23 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue }); } + /** + * Also tells the team once that a cleanup that failed before works again. + */ + private function notifySuccess(string $message): void + { + $this->server->team?->notify(new DockerCleanupSuccess($this->server, $message)); + if (NotificationThrottle::release($this->server, DockerCleanupFailed::class)) { + $this->server->team?->notify(new ScheduledJobRecovered( + failureEvent: 'docker_cleanup_failure', + title: "Docker cleanup job works again on {$this->server->name}.", + webhookEvent: 'docker_cleanup_recovered', + url: base_url().'/server/'.$this->server->uuid, + webhookData: ['server_name' => $this->server->name, 'server_uuid' => $this->server->uuid], + )); + } + } + public function failed(?\Throwable $exception): void { if ($this->occurrenceUuid) { diff --git a/app/Jobs/ScheduledTaskJob.php b/app/Jobs/ScheduledTaskJob.php index 7f59fcfc3f..28c54a3457 100644 --- a/app/Jobs/ScheduledTaskJob.php +++ b/app/Jobs/ScheduledTaskJob.php @@ -11,6 +11,7 @@ use App\Models\ScheduledTaskExecution; use App\Models\Server; use App\Models\Service; use App\Models\Team; +use App\Notifications\ScheduledJobRecovered; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\ScheduledTask\TaskSuccess; use App\Services\ScheduledJobDeliveryService; @@ -178,8 +179,14 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue } }); } + // A stopped resource is not a task failure: it sends no notification and does not count toward auto-disable. if (count($this->containers) == 0) { - throw new \Exception('ScheduledTaskJob failed: No containers running.'); + $this->task_log->update([ + 'status' => 'skipped', + 'message' => 'Skipped: the resource has no running container.', + ]); + + return; } if (count($this->containers) > 1 && empty($this->task->container)) { @@ -201,6 +208,15 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue ]); $this->team?->notify(new TaskSuccess($this->task, $this->task_output)); + if ($this->task->recordSuccess()) { + $this->team?->notify(new ScheduledJobRecovered( + failureEvent: 'scheduled_task_failure', + title: "Scheduled task ({$this->task->name}) works again.", + webhookEvent: 'task_recovered', + url: $this->task->url(), + webhookData: ['task_name' => $this->task->name, 'task_uuid' => $this->task->uuid], + )); + } return; } @@ -344,7 +360,16 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue ]); } - // Notify team about permanent failure - $this->team?->notify(new TaskFailed($this->task, $safeErrorMessage)); + try { + $disabled = $this->task->recordFailure(); + } catch (\Throwable $e) { + Log::channel('scheduled-errors')->warning('Could not record the scheduled task failure', [ + 'task_id' => $this->task->uuid, + 'error' => $e->getMessage(), + ]); + $disabled = false; + } + + $this->team?->notify(new TaskFailed($this->task, $safeErrorMessage, disabled: $disabled)); } } diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index 50ab3a48ed..916b8479a2 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -6,11 +6,14 @@ use App\Events\BackupCreated; use App\Helpers\SshMultiplexingHelper; use App\Helpers\SshRetryHandler; use App\Models\LocalPersistentVolume; +use App\Models\NotificationThrottle; use App\Models\S3Storage; use App\Models\ScheduledVolumeBackup; use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; +use App\Models\Team; use App\Models\VolumeBackupS3Replica; +use App\Notifications\ScheduledJobRecovered; use App\Notifications\VolumeBackup\BackupFailed; use App\Notifications\VolumeBackup\BackupSuccess; use App\Services\ScheduledJobDeliveryService; @@ -269,6 +272,29 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } $team->notify(new BackupSuccess($this->backup, $warning)); + if ($warning === null) { + $this->notifyIfRecovered($team); + } + } + + /** + * After a run without warning, tell the team once that a backup that failed before works again. + */ + private function notifyIfRecovered(Team $team): void + { + $wasFailing = NotificationThrottle::release($this->backup, BackupFailed::class); + $hadWarning = NotificationThrottle::release($this->backup, BackupSuccess::class); + if (! $wasFailing && ! $hadWarning) { + return; + } + + $team->notify(new ScheduledJobRecovered( + failureEvent: 'backup_failure', + title: 'Storage backup of '.$this->backup->targetType().' '.$this->backup->targetName().' works again.', + webhookEvent: 'volume_backup_recovered', + url: $this->backup->resourceUrl(), + webhookData: ['backup_uuid' => $this->backup->uuid], + )); } public function failed(?Throwable $exception): void diff --git a/app/Models/NotificationThrottle.php b/app/Models/NotificationThrottle.php index a43e24c433..4121160f2e 100644 --- a/app/Models/NotificationThrottle.php +++ b/app/Models/NotificationThrottle.php @@ -16,6 +16,13 @@ use Throwable; */ class NotificationThrottle extends Model { + /** + * While a scheduled job keeps failing, its failure notification is sent again at most this often. + */ + public const FAILURE_REMINDER_INTERVAL_MINUTES = 24 * 60; + + public const FAILURE_REMINDER_TEXT = 'While it keeps failing, you get at most one notification each day.'; + public $timestamps = false; protected $fillable = [ diff --git a/app/Models/ScheduledTask.php b/app/Models/ScheduledTask.php index 0a53395d31..c6224f2eda 100644 --- a/app/Models/ScheduledTask.php +++ b/app/Models/ScheduledTask.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Notifications\ScheduledTask\TaskFailed; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; @@ -20,6 +21,9 @@ use OpenApi\Attributes as OA; 'frequency' => ['type' => 'string', 'description' => 'The frequency of the scheduled task.'], 'container' => ['type' => 'string', 'nullable' => true, 'description' => 'The container where the command should be executed.'], 'timeout' => ['type' => 'integer', 'description' => 'The timeout of the scheduled task in seconds.'], + 'consecutive_failures' => ['type' => 'integer', 'description' => 'The number of failed runs since the last successful run.'], + 'failing_since' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The date and time of the first failed run since the last successful run.'], + 'disabled_reason' => ['type' => 'string', 'nullable' => true, 'description' => 'Why Coolify disabled the task automatically. It is cleared when the task is enabled again.'], 'created_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was created.'], 'updated_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was last updated.'], ], @@ -42,6 +46,33 @@ class ScheduledTask extends BaseModel 'service_id', ]; + /** + * A task is disabled automatically only when it failed this many times in a row... + */ + public const AUTO_DISABLE_AFTER_FAILURES = 10; + + /** + * ...and its first failure is this old, so a short outage does not disable a task that runs often. + */ + public const AUTO_DISABLE_AFTER_HOURS = 24; + + protected static function booted(): void + { + // A task that is enabled again starts a new failure streak, and its next failure notifies at once. + static::saving(function (ScheduledTask $task) { + if ($task->isDirty('enabled') && $task->enabled) { + $task->consecutive_failures = 0; + $task->failing_since = null; + $task->disabled_reason = null; + } + }); + static::saved(function (ScheduledTask $task) { + if ($task->wasChanged('enabled') && $task->enabled) { + NotificationThrottle::release($task, TaskFailed::class); + } + }); + } + public static function ownedByCurrentTeamAPI(int $teamId) { return static::where('team_id', $teamId)->orderBy('created_at', 'desc'); @@ -52,9 +83,59 @@ class ScheduledTask extends BaseModel return [ 'enabled' => 'boolean', 'timeout' => 'integer', + 'consecutive_failures' => 'integer', + 'failing_since' => 'datetime', ]; } + /** + * Count a failed run. Disables the task when it failed too often for too long. + * + * @return bool True when this failure disabled the task. + */ + public function recordFailure(): bool + { + $this->refresh(); + $this->consecutive_failures++; + $this->failing_since ??= now(); + + $disable = $this->enabled + && $this->consecutive_failures >= self::AUTO_DISABLE_AFTER_FAILURES + && $this->failing_since->lte(now()->subHours(self::AUTO_DISABLE_AFTER_HOURS)); + if ($disable) { + $this->enabled = false; + $this->disabled_reason = "Disabled automatically after {$this->consecutive_failures} failed runs in a row since {$this->failing_since->toDateTimeString()} UTC."; + } + $this->save(); + + return $disable; + } + + /** + * End the failure streak after a successful run. + * + * @return bool True when the team was notified about the failure, so it must be told that the task works again. + */ + public function recordSuccess(): bool + { + if ($this->consecutive_failures > 0 || $this->failing_since !== null) { + $this->forceFill(['consecutive_failures' => 0, 'failing_since' => null])->save(); + } + + return NotificationThrottle::release($this, TaskFailed::class); + } + + public function url(): ?string + { + $resource = $this->application ?? $this->service; + if (! $resource) { + return null; + } + $type = $resource instanceof Application ? 'application' : 'service'; + + return base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$this->uuid}"; + } + public function service() { return $this->belongsTo(Service::class); diff --git a/app/Notifications/Database/BackupFailed.php b/app/Notifications/Database/BackupFailed.php index c2b21b1d51..29d4f51333 100644 --- a/app/Notifications/Database/BackupFailed.php +++ b/app/Notifications/Database/BackupFailed.php @@ -2,26 +2,42 @@ namespace App\Notifications\Database; +use App\Contracts\ThrottledNotification; +use App\Models\NotificationThrottle; use App\Models\ScheduledDatabaseBackup; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class BackupFailed extends CustomEmailNotification +/** + * Sent on the first failure, then at most once a day until a run of the backup succeeds again. + */ +class BackupFailed extends CustomEmailNotification implements ThrottledNotification { public string $name; public string $frequency; - public function __construct(ScheduledDatabaseBackup $backup, public $database, public $output, public $database_name) + public function __construct(public ScheduledDatabaseBackup $backup, public $database, public $output, public $database_name) { $this->onQueue('high'); $this->name = $database->name; $this->frequency = $backup->frequency; } + public function throttleSubject(): ?Model + { + return $this->backup; + } + + public function throttleIntervalMinutes(): int + { + return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES; + } + public function via(object $notifiable): array { return $notifiable->getEnabledChannels('backup_failure'); @@ -36,6 +52,7 @@ class BackupFailed extends CustomEmailNotification 'database_name' => $this->database_name, 'frequency' => $this->frequency, 'output' => $this->output, + 'reminder' => NotificationThrottle::FAILURE_REMINDER_TEXT, ]); return $mail; diff --git a/app/Notifications/Database/BackupSuccessWithS3Warning.php b/app/Notifications/Database/BackupSuccessWithS3Warning.php index ee24ef17de..cbed5addf5 100644 --- a/app/Notifications/Database/BackupSuccessWithS3Warning.php +++ b/app/Notifications/Database/BackupSuccessWithS3Warning.php @@ -2,14 +2,20 @@ namespace App\Notifications\Database; +use App\Contracts\ThrottledNotification; +use App\Models\NotificationThrottle; use App\Models\ScheduledDatabaseBackup; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class BackupSuccessWithS3Warning extends CustomEmailNotification +/** + * Sent on the first failed S3 upload, then at most once a day until a run of the backup has no problem again. + */ +class BackupSuccessWithS3Warning extends CustomEmailNotification implements ThrottledNotification { public string $name; @@ -17,7 +23,7 @@ class BackupSuccessWithS3Warning extends CustomEmailNotification public ?string $s3_storage_url = null; - public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error) + public function __construct(public ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error) { $this->onQueue('high'); @@ -29,6 +35,16 @@ class BackupSuccessWithS3Warning extends CustomEmailNotification } } + public function throttleSubject(): ?Model + { + return $this->backup; + } + + public function throttleIntervalMinutes(): int + { + return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES; + } + public function via(object $notifiable): array { return $notifiable->getEnabledChannels('backup_failure'); diff --git a/app/Notifications/ScheduledJobRecovered.php b/app/Notifications/ScheduledJobRecovered.php new file mode 100644 index 0000000000..2d3a20667a --- /dev/null +++ b/app/Notifications/ScheduledJobRecovered.php @@ -0,0 +1,104 @@ + $webhookData Extra fields of the webhook payload. + */ + public function __construct( + public string $failureEvent, + public string $title, + public string $webhookEvent, + public ?string $url = null, + public array $webhookData = [], + ) { + $this->onQueue('high'); + } + + public function via(object $notifiable): array + { + return $notifiable->getEnabledChannels($this->failureEvent); + } + + public function toMail(): MailMessage + { + $mail = new MailMessage; + $mail->subject("Coolify: {$this->title}"); + $mail->view('emails.scheduled-job-recovered', [ + 'title' => $this->title, + 'url' => $this->url, + ]); + + return $mail; + } + + public function toDiscord(): DiscordMessage + { + $message = new DiscordMessage( + title: ':white_check_mark: '.$this->title, + description: 'The last run succeeded after failed runs.', + color: DiscordMessage::successColor(), + ); + + if ($this->url) { + $message->addField('Link', '[Open in Coolify]('.$this->url.')'); + } + + return $message; + } + + public function toTelegram(): array + { + return [ + 'message' => "Coolify: {$this->title}\n\nThe last run succeeded after failed runs.", + ]; + } + + public function toPushover(): PushoverMessage + { + return new PushoverMessage( + title: $this->title, + level: 'success', + message: 'The last run succeeded after failed runs.', + buttons: $this->url ? [['text' => 'Open in Coolify', 'url' => $this->url]] : [], + ); + } + + public function toSlack(): SlackMessage + { + $description = 'The last run succeeded after failed runs.'; + if ($this->url) { + $description .= "\n\n*Link:* {$this->url}"; + } + + return new SlackMessage( + title: $this->title, + description: $description, + color: SlackMessage::successColor(), + ); + } + + public function toWebhook(): array + { + return array_filter([ + 'success' => true, + 'message' => $this->title, + 'event' => $this->webhookEvent, + ...$this->webhookData, + 'url' => $this->url, + ], fn ($value) => $value !== null); + } +} diff --git a/app/Notifications/ScheduledTask/TaskFailed.php b/app/Notifications/ScheduledTask/TaskFailed.php index 2ca3874ba0..83d14a6535 100644 --- a/app/Notifications/ScheduledTask/TaskFailed.php +++ b/app/Notifications/ScheduledTask/TaskFailed.php @@ -2,26 +2,28 @@ namespace App\Notifications\ScheduledTask; -use App\Models\Application; +use App\Contracts\ThrottledNotification; +use App\Models\NotificationThrottle; use App\Models\ScheduledTask; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class TaskFailed extends CustomEmailNotification +/** + * Sent on the first failure of a streak, then at most once a day until the task succeeds again. + * With $disabled, it tells that Coolify disabled the task, and it is always sent. + */ +class TaskFailed extends CustomEmailNotification implements ThrottledNotification { public ?string $url = null; - public function __construct(public ScheduledTask $task, public string $output) + public function __construct(public ScheduledTask $task, public string $output, public bool $disabled = false) { $this->onQueue('high'); - $resource = $task->application ?? $task->service; - if ($resource) { - $type = $resource instanceof Application ? 'application' : 'service'; - $this->url = base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$task->uuid}"; - } + $this->url = $task->url(); } public function via(object $notifiable): array @@ -29,14 +31,52 @@ class TaskFailed extends CustomEmailNotification return $notifiable->getEnabledChannels('scheduled_task_failure'); } + public function throttleSubject(): ?Model + { + return $this->disabled ? null : $this->task; + } + + public function throttleIntervalMinutes(): int + { + return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES; + } + + public function title(): string + { + return $this->disabled + ? "Scheduled task ({$this->task->name}) was disabled because it keeps failing." + : "Scheduled task ({$this->task->name}) failed."; + } + + /** + * Tells how long the task fails, and what happens next. + */ + public function streakSummary(): string + { + if ($this->disabled) { + return ($this->task->disabled_reason ?? 'Disabled automatically.').' Fix the task, then enable it again.'; + } + + $failures = (int) $this->task->consecutive_failures; + $summary = $failures > 1 && $this->task->failing_since + ? "It failed {$failures} times in a row since {$this->task->failing_since->toDateTimeString()} UTC. " + : ''; + + return $summary.'While it keeps failing, you get at most one notification each day. After ' + .ScheduledTask::AUTO_DISABLE_AFTER_FAILURES.' failures in a row over more than ' + .ScheduledTask::AUTO_DISABLE_AFTER_HOURS.' hours, Coolify disables the task.'; + } + public function toMail(): MailMessage { $mail = new MailMessage; - $mail->subject("Coolify: [ACTION REQUIRED] Scheduled task ({$this->task->name}) failed."); + $mail->subject("Coolify: [ACTION REQUIRED] {$this->title()}"); $mail->view('emails.scheduled-task-failed', [ 'task' => $this->task, 'url' => $this->url, 'output' => $this->output, + 'title' => $this->title(), + 'summary' => $this->streakSummary(), ]); return $mail; @@ -45,8 +85,8 @@ class TaskFailed extends CustomEmailNotification public function toDiscord(): DiscordMessage { $message = new DiscordMessage( - title: ':cross_mark: Scheduled task failed', - description: "Scheduled task ({$this->task->name}) failed.", + title: $this->disabled ? ':cross_mark: Scheduled task disabled' : ':cross_mark: Scheduled task failed', + description: $this->title()."\n\n".$this->streakSummary(), color: DiscordMessage::errorColor(), ); @@ -59,7 +99,7 @@ class TaskFailed extends CustomEmailNotification public function toTelegram(): array { - $message = "Coolify: Scheduled task ({$this->task->name}) failed with output: {$this->output}"; + $message = "Coolify: {$this->title()}\n\n{$this->streakSummary()}\n\nOutput: {$this->output}"; if ($this->url) { $buttons[] = [ 'text' => 'Open task in Coolify', @@ -74,7 +114,7 @@ class TaskFailed extends CustomEmailNotification public function toPushover(): PushoverMessage { - $message = "Scheduled task ({$this->task->name}) failed
"; + $message = "{$this->title()}
{$this->streakSummary()}
"; if ($this->output) { $message .= "
Error Output:{$this->output}"; @@ -89,7 +129,7 @@ class TaskFailed extends CustomEmailNotification } return new PushoverMessage( - title: 'Scheduled task failed', + title: $this->disabled ? 'Scheduled task disabled' : 'Scheduled task failed', level: 'error', message: $message, buttons: $buttons, @@ -98,8 +138,8 @@ class TaskFailed extends CustomEmailNotification public function toSlack(): SlackMessage { - $title = 'Scheduled task failed'; - $description = "Scheduled task ({$this->task->name}) failed."; + $title = $this->disabled ? 'Scheduled task disabled' : 'Scheduled task failed'; + $description = $this->title()."\n\n".$this->streakSummary(); if ($this->output) { $description .= "\n\n*Error Output:* {$this->output}"; @@ -120,11 +160,12 @@ class TaskFailed extends CustomEmailNotification { $data = [ 'success' => false, - 'message' => 'Scheduled task failed', - 'event' => 'task_failed', + 'message' => $this->disabled ? 'Scheduled task disabled' : 'Scheduled task failed', + 'event' => $this->disabled ? 'task_disabled' : 'task_failed', 'task_name' => $this->task->name, 'task_uuid' => $this->task->uuid, 'output' => $this->output, + 'consecutive_failures' => (int) $this->task->consecutive_failures, ]; if ($this->task->application) { diff --git a/app/Notifications/ScheduledTask/TaskSuccess.php b/app/Notifications/ScheduledTask/TaskSuccess.php index 2978eaed32..17d3b5f320 100644 --- a/app/Notifications/ScheduledTask/TaskSuccess.php +++ b/app/Notifications/ScheduledTask/TaskSuccess.php @@ -2,7 +2,6 @@ namespace App\Notifications\ScheduledTask; -use App\Models\Application; use App\Models\ScheduledTask; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; @@ -17,11 +16,7 @@ class TaskSuccess extends CustomEmailNotification public function __construct(public ScheduledTask $task, public string $output) { $this->onQueue('high'); - $resource = $task->application ?? $task->service; - if ($resource) { - $type = $resource instanceof Application ? 'application' : 'service'; - $this->url = base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$task->uuid}"; - } + $this->url = $task->url(); } public function via(object $notifiable): array diff --git a/app/Notifications/Server/DockerCleanupFailed.php b/app/Notifications/Server/DockerCleanupFailed.php index 9cbdeb4888..43daefc5b5 100644 --- a/app/Notifications/Server/DockerCleanupFailed.php +++ b/app/Notifications/Server/DockerCleanupFailed.php @@ -2,20 +2,36 @@ namespace App\Notifications\Server; +use App\Contracts\ThrottledNotification; +use App\Models\NotificationThrottle; use App\Models\Server; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class DockerCleanupFailed extends CustomEmailNotification +/** + * Sent on the first failure, then at most once a day until a cleanup on the server succeeds again. + */ +class DockerCleanupFailed extends CustomEmailNotification implements ThrottledNotification { public function __construct(public Server $server, public string $message) { $this->onQueue('high'); } + public function throttleSubject(): ?Model + { + return $this->server; + } + + public function throttleIntervalMinutes(): int + { + return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES; + } + public function via(object $notifiable): array { return $notifiable->getEnabledChannels('docker_cleanup_failure'); @@ -28,6 +44,7 @@ class DockerCleanupFailed extends CustomEmailNotification $mail->view('emails.docker-cleanup-failed', [ 'name' => $this->server->name, 'text' => $this->message, + 'reminder' => NotificationThrottle::FAILURE_REMINDER_TEXT, ]); return $mail; diff --git a/app/Notifications/VolumeBackup/BackupFailed.php b/app/Notifications/VolumeBackup/BackupFailed.php index c1dc753cd7..59942a93c2 100644 --- a/app/Notifications/VolumeBackup/BackupFailed.php +++ b/app/Notifications/VolumeBackup/BackupFailed.php @@ -2,14 +2,20 @@ namespace App\Notifications\VolumeBackup; +use App\Contracts\ThrottledNotification; +use App\Models\NotificationThrottle; use App\Models\ScheduledVolumeBackup; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class BackupFailed extends CustomEmailNotification +/** + * Sent on the first failure, then at most once a day until a run of the backup succeeds again. + */ +class BackupFailed extends CustomEmailNotification implements ThrottledNotification { public string $resourceName; @@ -19,7 +25,7 @@ class BackupFailed extends CustomEmailNotification public ?string $url = null; - public function __construct(ScheduledVolumeBackup $backup, public string $output) + public function __construct(public ScheduledVolumeBackup $backup, public string $output) { $this->onQueue('high'); $resource = $backup->targetResource(); @@ -30,6 +36,16 @@ class BackupFailed extends CustomEmailNotification $this->url = $backup->resourceUrl(); } + public function throttleSubject(): ?Model + { + return $this->backup; + } + + public function throttleIntervalMinutes(): int + { + return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES; + } + public function via(object $notifiable): array { return $notifiable->getEnabledChannels('backup_failure'); @@ -49,6 +65,7 @@ class BackupFailed extends CustomEmailNotification 'frequency' => $this->frequency, 'output' => $this->output, 'url' => $this->url, + 'reminder' => NotificationThrottle::FAILURE_REMINDER_TEXT, ]); return $mail; diff --git a/app/Notifications/VolumeBackup/BackupSuccess.php b/app/Notifications/VolumeBackup/BackupSuccess.php index eaa4c823fb..5d37e6b7fa 100644 --- a/app/Notifications/VolumeBackup/BackupSuccess.php +++ b/app/Notifications/VolumeBackup/BackupSuccess.php @@ -2,14 +2,20 @@ namespace App\Notifications\VolumeBackup; +use App\Contracts\ThrottledNotification; +use App\Models\NotificationThrottle; use App\Models\ScheduledVolumeBackup; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class BackupSuccess extends CustomEmailNotification +/** + * With a warning, it goes to the failure channels: then it is sent at most once a day until a run has no warning again. + */ +class BackupSuccess extends CustomEmailNotification implements ThrottledNotification { public string $resourceName; @@ -19,7 +25,7 @@ class BackupSuccess extends CustomEmailNotification public ?string $url = null; - public function __construct(ScheduledVolumeBackup $backup, public ?string $warning = null) + public function __construct(public ScheduledVolumeBackup $backup, public ?string $warning = null) { $this->onQueue('high'); $resource = $backup->targetResource(); @@ -30,6 +36,16 @@ class BackupSuccess extends CustomEmailNotification $this->url = $backup->resourceUrl(); } + public function throttleSubject(): ?Model + { + return $this->warning ? $this->backup : null; + } + + public function throttleIntervalMinutes(): int + { + return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES; + } + public function via(object $notifiable): array { return $notifiable->getEnabledChannels($this->warning ? 'backup_failure' : 'backup_success'); diff --git a/app/Services/ServerTransfer/ServerTransferBundle.php b/app/Services/ServerTransfer/ServerTransferBundle.php index 09cee9d951..7092dda1fa 100644 --- a/app/Services/ServerTransfer/ServerTransferBundle.php +++ b/app/Services/ServerTransfer/ServerTransferBundle.php @@ -382,6 +382,7 @@ class ServerTransferBundle self::check($errors, "{$at}.scheduled_tasks.{$i}.command", $task['command'] ?? null, ['required', 'string']); self::check($errors, "{$at}.scheduled_tasks.{$i}.container", $task['container'] ?? null, ['nullable', ...ValidationPatterns::containerNameRules()]); self::checkCron($errors, "{$at}.scheduled_tasks.{$i}.frequency", $task['frequency'] ?? ''); + self::check($errors, "{$at}.scheduled_tasks.{$i}.disabled_reason", $task['disabled_reason'] ?? null, ['nullable', 'string', 'max:1000']); } } diff --git a/app/Services/ServerTransfer/ServerTransferCoverage.php b/app/Services/ServerTransfer/ServerTransferCoverage.php index 6558fc2edf..559208cd9d 100644 --- a/app/Services/ServerTransfer/ServerTransferCoverage.php +++ b/app/Services/ServerTransfer/ServerTransferCoverage.php @@ -256,7 +256,10 @@ final class ServerTransferCoverage 'container_id' => 'Not used: only listed in $fillable.', ], LocalFileVolume::class => ['resource_type' => self::LINKED, 'resource_id' => self::LINKED], - ScheduledTask::class => ['team_id' => self::TEAM, 'application_id' => self::LINKED, 'service_id' => self::LINKED], + ScheduledTask::class => [ + 'team_id' => self::TEAM, 'application_id' => self::LINKED, 'service_id' => self::LINKED, + 'consecutive_failures' => self::RUNTIME, 'failing_since' => self::RUNTIME, + ], Tag::class => ['team_id' => self::TEAM], ScheduledDatabaseBackup::class => [ 'team_id' => self::TEAM, 'database_type' => self::LINKED, 'database_id' => self::LINKED, 's3_storage_id' => self::LINKED, diff --git a/app/Services/ServerTransfer/ServerTransferExporter.php b/app/Services/ServerTransfer/ServerTransferExporter.php index d99110437f..64040ed02f 100644 --- a/app/Services/ServerTransfer/ServerTransferExporter.php +++ b/app/Services/ServerTransfer/ServerTransferExporter.php @@ -1325,6 +1325,7 @@ class ServerTransferExporter 'container' => $task->container, 'timeout' => $task->timeout, 'enabled' => (bool) ($task->enabled ?? true), + 'disabled_reason' => $task->disabled_reason, ]; } diff --git a/app/Services/ServerTransfer/ServerTransferImporter.php b/app/Services/ServerTransfer/ServerTransferImporter.php index 5091dcdf91..0746c3f39e 100644 --- a/app/Services/ServerTransfer/ServerTransferImporter.php +++ b/app/Services/ServerTransfer/ServerTransferImporter.php @@ -1688,6 +1688,7 @@ class ServerTransferImporter 'container' => data_get($task, 'container'), 'timeout' => data_get($task, 'timeout'), 'enabled' => (bool) data_get($task, 'enabled', true), + 'disabled_reason' => data_get($task, 'disabled_reason'), 'team_id' => $teamId, ]; @@ -1697,7 +1698,7 @@ class ServerTransferImporter $payload['service_id'] = $resource->id; } - ScheduledTask::create($payload); + ScheduledTask::forceCreate($payload); } } diff --git a/database/migrations/2026_11_03_192735_add_failure_tracking_to_scheduled_tasks_table.php b/database/migrations/2026_11_03_192735_add_failure_tracking_to_scheduled_tasks_table.php new file mode 100644 index 0000000000..14cc190166 --- /dev/null +++ b/database/migrations/2026_11_03_192735_add_failure_tracking_to_scheduled_tasks_table.php @@ -0,0 +1,48 @@ +unsignedInteger('consecutive_failures')->default(0); + $table->timestamp('failing_since')->nullable(); + $table->text('disabled_reason')->nullable(); + }); + + $this->setExecutionStatuses(['success', 'failed', 'running', 'skipped']); + } + + public function down(): void + { + DB::table('scheduled_task_executions')->where('status', 'skipped')->update(['status' => 'success']); + $this->setExecutionStatuses(['success', 'failed', 'running']); + + Schema::table('scheduled_tasks', function (Blueprint $table) { + $table->dropColumn(['consecutive_failures', 'failing_since', 'disabled_reason']); + }); + } + + /** + * @param list $statuses + */ + private function setExecutionStatuses(array $statuses): void + { + if (DB::getDriverName() === 'pgsql') { + $allowed = implode(', ', array_map(fn (string $status) => "'{$status}'", $statuses)); + DB::statement('ALTER TABLE scheduled_task_executions DROP CONSTRAINT IF EXISTS scheduled_task_executions_status_check'); + DB::statement("ALTER TABLE scheduled_task_executions ADD CONSTRAINT scheduled_task_executions_status_check CHECK (status IN ({$allowed}))"); + + return; + } + + Schema::table('scheduled_task_executions', function (Blueprint $table) use ($statuses) { + $table->enum('status', $statuses)->default('running')->change(); + }); + } +}; diff --git a/docs/server-transfer-full-coverage.md b/docs/server-transfer-full-coverage.md index 1e0e50ecae..acc7c4f8ed 100644 --- a/docs/server-transfer-full-coverage.md +++ b/docs/server-transfer-full-coverage.md @@ -101,7 +101,7 @@ When you add a model or a setting to the transfer, add it to this file too. | ✅ | Persistent volume | the seed has one; add a host-path bind mount and preview suffix on | | ✅ | File mount + directory mount | content, chown, chmod | | ✅ | Volume backup | on the persistent volume: S3 on, stop during backup, all retention values, timeout, missing-backup days | -| ✅ | Scheduled tasks | the seed has one; add a disabled one with container + timeout | +| ✅ | Scheduled tasks | the seed has one; add a disabled one with container + timeout; an auto-disabled task keeps its disabled reason | | ✅ | Tag | | | ✅ | Preview deployment | PR preview with domain port overrides + max restart count + keeps production values (previews from before v4.5.1) (needs the GitHub App, or create it in the database) | | ✅ | Secret manager link | the app loads variables from the secret manager token | diff --git a/openapi.json b/openapi.json index 601e8d1b5d..fa3d83c6c5 100644 --- a/openapi.json +++ b/openapi.json @@ -27638,6 +27638,21 @@ "type": "integer", "description": "The timeout of the scheduled task in seconds." }, + "consecutive_failures": { + "type": "integer", + "description": "The number of failed runs since the last successful run." + }, + "failing_since": { + "type": "string", + "format": "date-time", + "nullable": true, + "description": "The date and time of the first failed run since the last successful run." + }, + "disabled_reason": { + "type": "string", + "nullable": true, + "description": "Why Coolify disabled the task automatically. It is cleared when the task is enabled again." + }, "created_at": { "type": "string", "format": "date-time", diff --git a/openapi.yaml b/openapi.yaml index 30b79a471f..baed9a27ea 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -17673,6 +17673,18 @@ components: timeout: type: integer description: 'The timeout of the scheduled task in seconds.' + consecutive_failures: + type: integer + description: 'The number of failed runs since the last successful run.' + failing_since: + type: string + format: date-time + nullable: true + description: 'The date and time of the first failed run since the last successful run.' + disabled_reason: + type: string + nullable: true + description: 'Why Coolify disabled the task automatically. It is cleared when the task is enabled again.' created_at: type: string format: date-time diff --git a/resources/views/emails/backup-failed.blade.php b/resources/views/emails/backup-failed.blade.php index 013c8bc98f..f955d1d818 100644 --- a/resources/views/emails/backup-failed.blade.php +++ b/resources/views/emails/backup-failed.blade.php @@ -4,4 +4,8 @@ Database backup for {{ $name }} @if($database_name)(db:{{ $database_name }})@end ### Reason {{ $output }} + +@isset($reminder) +{{ $reminder }} +@endisset diff --git a/resources/views/emails/docker-cleanup-failed.blade.php b/resources/views/emails/docker-cleanup-failed.blade.php index a12f20fa8e..e3051f545e 100644 --- a/resources/views/emails/docker-cleanup-failed.blade.php +++ b/resources/views/emails/docker-cleanup-failed.blade.php @@ -5,4 +5,8 @@ Docker Cleanup on {{ $name }} FAILED with the following error: {{ $text }} +@isset($reminder) +{{ $reminder }} +@endisset + diff --git a/resources/views/emails/scheduled-job-recovered.blade.php b/resources/views/emails/scheduled-job-recovered.blade.php new file mode 100644 index 0000000000..72d43b5bf7 --- /dev/null +++ b/resources/views/emails/scheduled-job-recovered.blade.php @@ -0,0 +1,9 @@ + +{{ $title }} + +The last run succeeded after failed runs. + +@if ($url) +Click [here]({{ $url }}) to open it in Coolify. +@endif + diff --git a/resources/views/emails/scheduled-task-failed.blade.php b/resources/views/emails/scheduled-task-failed.blade.php index 60e451823f..4022205cec 100644 --- a/resources/views/emails/scheduled-task-failed.blade.php +++ b/resources/views/emails/scheduled-task-failed.blade.php @@ -1,5 +1,9 @@ -Scheduled task ({{ $task->name }}) was FAILED with the following error: +{{ $title ?? "Scheduled task ({$task->name}) failed." }} + +{{ $summary ?? '' }} + +Last error:
 {{ $output }}
diff --git a/resources/views/emails/volume-backup-result.blade.php b/resources/views/emails/volume-backup-result.blade.php
index b271e11097..a56922f704 100644
--- a/resources/views/emails/volume-backup-result.blade.php
+++ b/resources/views/emails/volume-backup-result.blade.php
@@ -9,6 +9,10 @@ Frequency: {{ $frequency }}
 {{ $output }}
 @endisset
 
+@isset($reminder)
+{{ $reminder }}
+@endisset
+
 @if ($url)
 Click [here]({{ $url }}) to open the resource.
 @endif
diff --git a/resources/views/livewire/project/shared/scheduled-task/show.blade.php b/resources/views/livewire/project/shared/scheduled-task/show.blade.php
index 66d6bb8afe..8730e620c7 100644
--- a/resources/views/livewire/project/shared/scheduled-task/show.blade.php
+++ b/resources/views/livewire/project/shared/scheduled-task/show.blade.php
@@ -12,6 +12,12 @@
         @endforeach
     
 
+    @if (! $isEnabled && $task->disabled_reason)
+        
+            {{ $task->disabled_reason }} Fix the task, then click Enable.
+        
+    @endif
+