starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-06 13:10:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(notifications): throttle repeated scheduled job failures and disable failing tasks

Browse Source
This commit is contained in:
Andras Bacsai
2026-11-03 20:42:03 +01:00
parent 1ff99b0dc8
commit 06c41f57d6
29 changed files with 577 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3
View File
@@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
### Changed
- Stop the notification storm from scheduled jobs that keep failing. A failed scheduled task, database backup, storage backup, or Docker cleanup now notifies on the first failure, then at most once a day while it keeps failing. The first successful run after a failure sends one "works again" notification to the failure channels. A database backup whose S3 upload keeps failing is also notified at most once a day
- Disable a scheduled task automatically when it failed 10 times in a row and its first failure is more than 24 hours old, and send one notification about it. The task page shows why the task was disabled. Enable the task again to clear the reason. The scheduled tasks API returns `consecutive_failures`, `failing_since`, and `disabled_reason`. Backups and Docker cleanup are never disabled automatically, so they start again by themselves after an outage
- Skip a scheduled task when its application or service has no running container, instead of a failed run with "No containers running". A skipped run sends no notification and does not count toward the automatic disable
- Move **Import Backup** from the resource menu to an **Import backup** button on the **Backups** page of databases and services. The button opens the import in a dialog. Old **Import Backup** links open the **Backups** page ([#11818](https://github.com/coollabsio/coolify/discussions/11818))
### Fixed
app/Jobs/DatabaseBackupJob.php
+37
View File
@@ -5,6 +5,7 @@ namespace App\Jobs;
use App\Events\BackupCreated;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SshRetryHandler;
use App\Models\NotificationThrottle;
use App\Models\S3Storage;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
@@ -20,6 +21,7 @@ use App\Models\Team;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupSuccess;
use App\Notifications\Database\BackupSuccessWithS3Warning;
use App\Notifications\ScheduledJobRecovered;
use App\Services\ScheduledJobDeliveryService;
use App\Services\Security\SensitiveDataRedactor;
use App\Support\BackupCompression;
@@ -316,6 +318,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$ip = Str::slug($this->server->ip);
$this->backup_dir = backup_dir().'/coolify'."/coolify-db-$ip";
}
$runHadProblem = false;
foreach ($databasesToBackup as $database) {
// Reset per-database state, so a failure before this database's execution exists
// does not mark the previous database's execution failed or delete its backup file.
@@ -450,6 +453,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
throw new \Exception('Local backup file is empty or was not created');
}
} catch (Throwable $e) {
$runHadProblem = true;
// Local backup failed: the partial file is not tracked, so retention would never remove it.
deleteBackupsLocally($this->backup_location, $this->server);
$failureMessage = $this->redact($this->error_output ?? $this->backup_output ?? $e->getMessage());
@@ -498,6 +502,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
// Step 3: Update status and send notifications based on results
if ($s3UploadError) {
$runHadProblem = true;
}
if ($localBackupSucceeded) {
$message = $this->backup_output === null ? null : $this->redact($this->backup_output);
@@ -536,6 +543,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->backup_log && $this->backup_log->status === 'success') {
$this->removeExpiredBackups();
}
if (! $runHadProblem) {
$this->notifyIfRecovered();
}
} catch (Throwable $e) {
$failed = true;
throw $e;
@@ -1097,6 +1107,33 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
*
* @return string|array<int, string>|null
*/
/**
* After a run without problems, tell the team once that a backup that failed before works again.
*/
private function notifyIfRecovered(): void
{
$wasFailing = NotificationThrottle::release($this->backup, BackupFailed::class);
$hadS3Warning = NotificationThrottle::release($this->backup, BackupSuccessWithS3Warning::class);
if (! $wasFailing && ! $hadS3Warning) {
return;
}
try {
$this->team?->notify(new ScheduledJobRecovered(
failureEvent: 'backup_failure',
title: "Database backup for {$this->database->name} works again.",
webhookEvent: 'backup_recovered',
url: base_url().'/project/'.data_get($this->database, 'environment.project.uuid').'/environment/'.data_get($this->database, 'environment.uuid').'/database/'.$this->database->uuid,
webhookData: ['database_name' => $this->database->name, 'database_uuid' => $this->database->uuid, 'backup_uuid' => $this->backup->uuid],
));
} catch (Throwable $e) {
Log::channel('scheduled-errors')->warning('Failed to send backup recovery notification', [
'backup_id' => $this->backup->uuid,
'error' => $e->getMessage(),
]);
}
}
private function configuredDatabasesToBackup(): string|array|null
{
if ($this->backup->dump_all) {
app/Jobs/DockerCleanupJob.php
+23 -4
View File
@@ -5,7 +5,9 @@ namespace App\Jobs;
use App\Actions\Server\CleanupDocker;
use App\Events\DockerCleanupDone;
use App\Models\DockerCleanupExecution;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Notifications\ScheduledJobRecovered;
use App\Notifications\Server\DockerCleanupFailed;
use App\Notifications\Server\DockerCleanupSuccess;
use App\Services\ScheduledJobDeliveryService;
@@ -132,7 +134,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
'cleanup_log' => $cleanup_log,
]);
$this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
$this->notifySuccess($message);
event(new DockerCleanupDone($this->execution_log));
return;
@@ -154,7 +156,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
'cleanup_log' => $cleanup_log,
]);
$this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
$this->notifySuccess($message);
event(new DockerCleanupDone($this->execution_log));
return;
@@ -183,7 +185,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
'cleanup_log' => $cleanup_log,
]);
$this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
$this->notifySuccess($message);
event(new DockerCleanupDone($this->execution_log));
} else {
$message = 'No cleanup needed for '.$this->server->name;
@@ -193,7 +195,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
'message' => $message,
]);
$this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
$this->notifySuccess($message);
event(new DockerCleanupDone($this->execution_log));
}
} catch (\Throwable $e) {
@@ -285,6 +287,23 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
});
}
/**
* Also tells the team once that a cleanup that failed before works again.
*/
private function notifySuccess(string $message): void
{
$this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
if (NotificationThrottle::release($this->server, DockerCleanupFailed::class)) {
$this->server->team?->notify(new ScheduledJobRecovered(
failureEvent: 'docker_cleanup_failure',
title: "Docker cleanup job works again on {$this->server->name}.",
webhookEvent: 'docker_cleanup_recovered',
url: base_url().'/server/'.$this->server->uuid,
webhookData: ['server_name' => $this->server->name, 'server_uuid' => $this->server->uuid],
));
}
}
public function failed(?\Throwable $exception): void
{
if ($this->occurrenceUuid) {
app/Jobs/ScheduledTaskJob.php
+28 -3
View File
@@ -11,6 +11,7 @@ use App\Models\ScheduledTaskExecution;
use App\Models\Server;
use App\Models\Service;
use App\Models\Team;
use App\Notifications\ScheduledJobRecovered;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\ScheduledTask\TaskSuccess;
use App\Services\ScheduledJobDeliveryService;
@@ -178,8 +179,14 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
}
});
}
// A stopped resource is not a task failure: it sends no notification and does not count toward auto-disable.
if (count($this->containers) == 0) {
throw new \Exception('ScheduledTaskJob failed: No containers running.');
$this->task_log->update([
'status' => 'skipped',
'message' => 'Skipped: the resource has no running container.',
]);
return;
}
if (count($this->containers) > 1 && empty($this->task->container)) {
@@ -201,6 +208,15 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
]);
$this->team?->notify(new TaskSuccess($this->task, $this->task_output));
if ($this->task->recordSuccess()) {
$this->team?->notify(new ScheduledJobRecovered(
failureEvent: 'scheduled_task_failure',
title: "Scheduled task ({$this->task->name}) works again.",
webhookEvent: 'task_recovered',
url: $this->task->url(),
webhookData: ['task_name' => $this->task->name, 'task_uuid' => $this->task->uuid],
));
}
return;
}
@@ -344,7 +360,16 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
]);
}
// Notify team about permanent failure
$this->team?->notify(new TaskFailed($this->task, $safeErrorMessage));
try {
$disabled = $this->task->recordFailure();
} catch (\Throwable $e) {
Log::channel('scheduled-errors')->warning('Could not record the scheduled task failure', [
'task_id' => $this->task->uuid,
'error' => $e->getMessage(),
]);
$disabled = false;
}
$this->team?->notify(new TaskFailed($this->task, $safeErrorMessage, disabled: $disabled));
}
}
app/Jobs/VolumeBackupJob.php
+26
View File
@@ -6,11 +6,14 @@ use App\Events\BackupCreated;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SshRetryHandler;
use App\Models\LocalPersistentVolume;
use App\Models\NotificationThrottle;
use App\Models\S3Storage;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Models\Team;
use App\Models\VolumeBackupS3Replica;
use App\Notifications\ScheduledJobRecovered;
use App\Notifications\VolumeBackup\BackupFailed;
use App\Notifications\VolumeBackup\BackupSuccess;
use App\Services\ScheduledJobDeliveryService;
@@ -269,6 +272,29 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
$team->notify(new BackupSuccess($this->backup, $warning));
if ($warning === null) {
$this->notifyIfRecovered($team);
}
}
/**
* After a run without warning, tell the team once that a backup that failed before works again.
*/
private function notifyIfRecovered(Team $team): void
{
$wasFailing = NotificationThrottle::release($this->backup, BackupFailed::class);
$hadWarning = NotificationThrottle::release($this->backup, BackupSuccess::class);
if (! $wasFailing && ! $hadWarning) {
return;
}
$team->notify(new ScheduledJobRecovered(
failureEvent: 'backup_failure',
title: 'Storage backup of '.$this->backup->targetType().' '.$this->backup->targetName().' works again.',
webhookEvent: 'volume_backup_recovered',
url: $this->backup->resourceUrl(),
webhookData: ['backup_uuid' => $this->backup->uuid],
));
}
public function failed(?Throwable $exception): void
app/Models/NotificationThrottle.php
+7
View File
@@ -16,6 +16,13 @@ use Throwable;
*/
class NotificationThrottle extends Model
{
/**
* While a scheduled job keeps failing, its failure notification is sent again at most this often.
*/
public const FAILURE_REMINDER_INTERVAL_MINUTES = 24 * 60;
public const FAILURE_REMINDER_TEXT = 'While it keeps failing, you get at most one notification each day.';
public $timestamps = false;
protected $fillable = [
app/Models/ScheduledTask.php
+81
View File
@@ -2,6 +2,7 @@
namespace App\Models;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
@@ -20,6 +21,9 @@ use OpenApi\Attributes as OA;
'frequency' => ['type' => 'string', 'description' => 'The frequency of the scheduled task.'],
'container' => ['type' => 'string', 'nullable' => true, 'description' => 'The container where the command should be executed.'],
'timeout' => ['type' => 'integer', 'description' => 'The timeout of the scheduled task in seconds.'],
'consecutive_failures' => ['type' => 'integer', 'description' => 'The number of failed runs since the last successful run.'],
'failing_since' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The date and time of the first failed run since the last successful run.'],
'disabled_reason' => ['type' => 'string', 'nullable' => true, 'description' => 'Why Coolify disabled the task automatically. It is cleared when the task is enabled again.'],
'created_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was created.'],
'updated_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was last updated.'],
],
@@ -42,6 +46,33 @@ class ScheduledTask extends BaseModel
'service_id',
];
/**
* A task is disabled automatically only when it failed this many times in a row...
*/
public const AUTO_DISABLE_AFTER_FAILURES = 10;
/**
* ...and its first failure is this old, so a short outage does not disable a task that runs often.
*/
public const AUTO_DISABLE_AFTER_HOURS = 24;
protected static function booted(): void
{
// A task that is enabled again starts a new failure streak, and its next failure notifies at once.
static::saving(function (ScheduledTask $task) {
if ($task->isDirty('enabled') && $task->enabled) {
$task->consecutive_failures = 0;
$task->failing_since = null;
$task->disabled_reason = null;
}
});
static::saved(function (ScheduledTask $task) {
if ($task->wasChanged('enabled') && $task->enabled) {
NotificationThrottle::release($task, TaskFailed::class);
}
});
}
public static function ownedByCurrentTeamAPI(int $teamId)
{
return static::where('team_id', $teamId)->orderBy('created_at', 'desc');
@@ -52,9 +83,59 @@ class ScheduledTask extends BaseModel
return [
'enabled' => 'boolean',
'timeout' => 'integer',
'consecutive_failures' => 'integer',
'failing_since' => 'datetime',
];
}
/**
* Count a failed run. Disables the task when it failed too often for too long.
*
* @return bool True when this failure disabled the task.
*/
public function recordFailure(): bool
{
$this->refresh();
$this->consecutive_failures++;
$this->failing_since ??= now();
$disable = $this->enabled
&& $this->consecutive_failures >= self::AUTO_DISABLE_AFTER_FAILURES
&& $this->failing_since->lte(now()->subHours(self::AUTO_DISABLE_AFTER_HOURS));
if ($disable) {
$this->enabled = false;
$this->disabled_reason = "Disabled automatically after {$this->consecutive_failures} failed runs in a row since {$this->failing_since->toDateTimeString()} UTC.";
}
$this->save();
return $disable;
}
/**
* End the failure streak after a successful run.
*
* @return bool True when the team was notified about the failure, so it must be told that the task works again.
*/
public function recordSuccess(): bool
{
if ($this->consecutive_failures > 0 || $this->failing_since !== null) {
$this->forceFill(['consecutive_failures' => 0, 'failing_since' => null])->save();
}
return NotificationThrottle::release($this, TaskFailed::class);
}
public function url(): ?string
{
$resource = $this->application ?? $this->service;
if (! $resource) {
return null;
}
$type = $resource instanceof Application ? 'application' : 'service';
return base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$this->uuid}";
}
public function service()
{
return $this->belongsTo(Service::class);
app/Notifications/Database/BackupFailed.php
+19 -2
View File
@@ -2,26 +2,42 @@
namespace App\Notifications\Database;
use App\Contracts\ThrottledNotification;
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class BackupFailed extends CustomEmailNotification
/**
* Sent on the first failure, then at most once a day until a run of the backup succeeds again.
*/
class BackupFailed extends CustomEmailNotification implements ThrottledNotification
{
public string $name;
public string $frequency;
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $output, public $database_name)
public function __construct(public ScheduledDatabaseBackup $backup, public $database, public $output, public $database_name)
{
$this->onQueue('high');
$this->name = $database->name;
$this->frequency = $backup->frequency;
}
public function throttleSubject(): ?Model
{
return $this->backup;
}
public function throttleIntervalMinutes(): int
{
return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES;
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('backup_failure');
@@ -36,6 +52,7 @@ class BackupFailed extends CustomEmailNotification
'database_name' => $this->database_name,
'frequency' => $this->frequency,
'output' => $this->output,
'reminder' => NotificationThrottle::FAILURE_REMINDER_TEXT,
]);
return $mail;
app/Notifications/Database/BackupSuccessWithS3Warning.php
+18 -2
View File
@@ -2,14 +2,20 @@
namespace App\Notifications\Database;
use App\Contracts\ThrottledNotification;
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class BackupSuccessWithS3Warning extends CustomEmailNotification
/**
* Sent on the first failed S3 upload, then at most once a day until a run of the backup has no problem again.
*/
class BackupSuccessWithS3Warning extends CustomEmailNotification implements ThrottledNotification
{
public string $name;
@@ -17,7 +23,7 @@ class BackupSuccessWithS3Warning extends CustomEmailNotification
public ?string $s3_storage_url = null;
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error)
public function __construct(public ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error)
{
$this->onQueue('high');
@@ -29,6 +35,16 @@ class BackupSuccessWithS3Warning extends CustomEmailNotification
}
}
public function throttleSubject(): ?Model
{
return $this->backup;
}
public function throttleIntervalMinutes(): int
{
return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES;
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('backup_failure');
app/Notifications/ScheduledJobRecovered.php
+104
View File
@@ -0,0 +1,104 @@
<?php
namespace App\Notifications;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Notifications\Messages\MailMessage;
/**
* Sent once on the first success after the team was notified that a scheduled job failed.
* It goes to the channels of the failure notification, so everyone who got the failure learns that it works again.
*/
class ScheduledJobRecovered extends CustomEmailNotification
{
/**
* @param string $failureEvent The notification setting of the failure, for example `backup_failure`.
* @param string $webhookEvent The `event` value of the webhook payload.
* @param array<string, mixed> $webhookData Extra fields of the webhook payload.
*/
public function __construct(
public string $failureEvent,
public string $title,
public string $webhookEvent,
public ?string $url = null,
public array $webhookData = [],
) {
$this->onQueue('high');
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels($this->failureEvent);
}
public function toMail(): MailMessage
{
$mail = new MailMessage;
$mail->subject("Coolify: {$this->title}");
$mail->view('emails.scheduled-job-recovered', [
'title' => $this->title,
'url' => $this->url,
]);
return $mail;
}
public function toDiscord(): DiscordMessage
{
$message = new DiscordMessage(
title: ':white_check_mark: '.$this->title,
description: 'The last run succeeded after failed runs.',
color: DiscordMessage::successColor(),
);
if ($this->url) {
$message->addField('Link', '[Open in Coolify]('.$this->url.')');
}
return $message;
}
public function toTelegram(): array
{
return [
'message' => "Coolify: {$this->title}\n\nThe last run succeeded after failed runs.",
];
}
public function toPushover(): PushoverMessage
{
return new PushoverMessage(
title: $this->title,
level: 'success',
message: 'The last run succeeded after failed runs.',
buttons: $this->url ? [['text' => 'Open in Coolify', 'url' => $this->url]] : [],
);
}
public function toSlack(): SlackMessage
{
$description = 'The last run succeeded after failed runs.';
if ($this->url) {
$description .= "\n\n*Link:* {$this->url}";
}
return new SlackMessage(
title: $this->title,
description: $description,
color: SlackMessage::successColor(),
);
}
public function toWebhook(): array
{
return array_filter([
'success' => true,
'message' => $this->title,
'event' => $this->webhookEvent,
...$this->webhookData,
'url' => $this->url,
], fn ($value) => $value !== null);
}
}
app/Notifications/ScheduledTask/TaskFailed.php
+59 -18
View File
@@ -2,26 +2,28 @@
namespace App\Notifications\ScheduledTask;
use App\Models\Application;
use App\Contracts\ThrottledNotification;
use App\Models\NotificationThrottle;
use App\Models\ScheduledTask;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class TaskFailed extends CustomEmailNotification
/**
* Sent on the first failure of a streak, then at most once a day until the task succeeds again.
* With $disabled, it tells that Coolify disabled the task, and it is always sent.
*/
class TaskFailed extends CustomEmailNotification implements ThrottledNotification
{
public ?string $url = null;
public function __construct(public ScheduledTask $task, public string $output)
public function __construct(public ScheduledTask $task, public string $output, public bool $disabled = false)
{
$this->onQueue('high');
$resource = $task->application ?? $task->service;
if ($resource) {
$type = $resource instanceof Application ? 'application' : 'service';
$this->url = base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$task->uuid}";
}
$this->url = $task->url();
}
public function via(object $notifiable): array
@@ -29,14 +31,52 @@ class TaskFailed extends CustomEmailNotification
return $notifiable->getEnabledChannels('scheduled_task_failure');
}
public function throttleSubject(): ?Model
{
return $this->disabled ? null : $this->task;
}
public function throttleIntervalMinutes(): int
{
return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES;
}
public function title(): string
{
return $this->disabled
? "Scheduled task ({$this->task->name}) was disabled because it keeps failing."
: "Scheduled task ({$this->task->name}) failed.";
}
/**
* Tells how long the task fails, and what happens next.
*/
public function streakSummary(): string
{
if ($this->disabled) {
return ($this->task->disabled_reason ?? 'Disabled automatically.').' Fix the task, then enable it again.';
}
$failures = (int) $this->task->consecutive_failures;
$summary = $failures > 1 && $this->task->failing_since
? "It failed {$failures} times in a row since {$this->task->failing_since->toDateTimeString()} UTC. "
: '';
return $summary.'While it keeps failing, you get at most one notification each day. After '
.ScheduledTask::AUTO_DISABLE_AFTER_FAILURES.' failures in a row over more than '
.ScheduledTask::AUTO_DISABLE_AFTER_HOURS.' hours, Coolify disables the task.';
}
public function toMail(): MailMessage
{
$mail = new MailMessage;
$mail->subject("Coolify: [ACTION REQUIRED] Scheduled task ({$this->task->name}) failed.");
$mail->subject("Coolify: [ACTION REQUIRED] {$this->title()}");
$mail->view('emails.scheduled-task-failed', [
'task' => $this->task,
'url' => $this->url,
'output' => $this->output,
'title' => $this->title(),
'summary' => $this->streakSummary(),
]);
return $mail;
@@ -45,8 +85,8 @@ class TaskFailed extends CustomEmailNotification
public function toDiscord(): DiscordMessage
{
$message = new DiscordMessage(
title: ':cross_mark: Scheduled task failed',
description: "Scheduled task ({$this->task->name}) failed.",
title: $this->disabled ? ':cross_mark: Scheduled task disabled' : ':cross_mark: Scheduled task failed',
description: $this->title()."\n\n".$this->streakSummary(),
color: DiscordMessage::errorColor(),
);
@@ -59,7 +99,7 @@ class TaskFailed extends CustomEmailNotification
public function toTelegram(): array
{
$message = "Coolify: Scheduled task ({$this->task->name}) failed with output: {$this->output}";
$message = "Coolify: {$this->title()}\n\n{$this->streakSummary()}\n\nOutput: {$this->output}";
if ($this->url) {
$buttons[] = [
'text' => 'Open task in Coolify',
@@ -74,7 +114,7 @@ class TaskFailed extends CustomEmailNotification
public function toPushover(): PushoverMessage
{
$message = "Scheduled task ({$this->task->name}) failed<br/>";
$message = "{$this->title()}<br/>{$this->streakSummary()}<br/>";
if ($this->output) {
$message .= "<br/><b>Error Output:</b>{$this->output}";
@@ -89,7 +129,7 @@ class TaskFailed extends CustomEmailNotification
}
return new PushoverMessage(
title: 'Scheduled task failed',
title: $this->disabled ? 'Scheduled task disabled' : 'Scheduled task failed',
level: 'error',
message: $message,
buttons: $buttons,
@@ -98,8 +138,8 @@ class TaskFailed extends CustomEmailNotification
public function toSlack(): SlackMessage
{
$title = 'Scheduled task failed';
$description = "Scheduled task ({$this->task->name}) failed.";
$title = $this->disabled ? 'Scheduled task disabled' : 'Scheduled task failed';
$description = $this->title()."\n\n".$this->streakSummary();
if ($this->output) {
$description .= "\n\n*Error Output:* {$this->output}";
@@ -120,11 +160,12 @@ class TaskFailed extends CustomEmailNotification
{
$data = [
'success' => false,
'message' => 'Scheduled task failed',
'event' => 'task_failed',
'message' => $this->disabled ? 'Scheduled task disabled' : 'Scheduled task failed',
'event' => $this->disabled ? 'task_disabled' : 'task_failed',
'task_name' => $this->task->name,
'task_uuid' => $this->task->uuid,
'output' => $this->output,
'consecutive_failures' => (int) $this->task->consecutive_failures,
];
if ($this->task->application) {
app/Notifications/ScheduledTask/TaskSuccess.php
+1 -6
View File
@@ -2,7 +2,6 @@
namespace App\Notifications\ScheduledTask;
use App\Models\Application;
use App\Models\ScheduledTask;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
@@ -17,11 +16,7 @@ class TaskSuccess extends CustomEmailNotification
public function __construct(public ScheduledTask $task, public string $output)
{
$this->onQueue('high');
$resource = $task->application ?? $task->service;
if ($resource) {
$type = $resource instanceof Application ? 'application' : 'service';
$this->url = base_url().'/project/'.data_get($resource, 'environment.project.uuid').'/environment/'.data_get($resource, 'environment.uuid')."/{$type}/{$resource->uuid}/tasks/{$task->uuid}";
}
$this->url = $task->url();
}
public function via(object $notifiable): array
app/Notifications/Server/DockerCleanupFailed.php
+18 -1
View File
@@ -2,20 +2,36 @@
namespace App\Notifications\Server;
use App\Contracts\ThrottledNotification;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class DockerCleanupFailed extends CustomEmailNotification
/**
* Sent on the first failure, then at most once a day until a cleanup on the server succeeds again.
*/
class DockerCleanupFailed extends CustomEmailNotification implements ThrottledNotification
{
public function __construct(public Server $server, public string $message)
{
$this->onQueue('high');
}
public function throttleSubject(): ?Model
{
return $this->server;
}
public function throttleIntervalMinutes(): int
{
return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES;
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('docker_cleanup_failure');
@@ -28,6 +44,7 @@ class DockerCleanupFailed extends CustomEmailNotification
$mail->view('emails.docker-cleanup-failed', [
'name' => $this->server->name,
'text' => $this->message,
'reminder' => NotificationThrottle::FAILURE_REMINDER_TEXT,
]);
return $mail;
app/Notifications/VolumeBackup/BackupFailed.php
+19 -2
View File
@@ -2,14 +2,20 @@
namespace App\Notifications\VolumeBackup;
use App\Contracts\ThrottledNotification;
use App\Models\NotificationThrottle;
use App\Models\ScheduledVolumeBackup;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class BackupFailed extends CustomEmailNotification
/**
* Sent on the first failure, then at most once a day until a run of the backup succeeds again.
*/
class BackupFailed extends CustomEmailNotification implements ThrottledNotification
{
public string $resourceName;
@@ -19,7 +25,7 @@ class BackupFailed extends CustomEmailNotification
public ?string $url = null;
public function __construct(ScheduledVolumeBackup $backup, public string $output)
public function __construct(public ScheduledVolumeBackup $backup, public string $output)
{
$this->onQueue('high');
$resource = $backup->targetResource();
@@ -30,6 +36,16 @@ class BackupFailed extends CustomEmailNotification
$this->url = $backup->resourceUrl();
}
public function throttleSubject(): ?Model
{
return $this->backup;
}
public function throttleIntervalMinutes(): int
{
return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES;
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('backup_failure');
@@ -49,6 +65,7 @@ class BackupFailed extends CustomEmailNotification
'frequency' => $this->frequency,
'output' => $this->output,
'url' => $this->url,
'reminder' => NotificationThrottle::FAILURE_REMINDER_TEXT,
]);
return $mail;
app/Notifications/VolumeBackup/BackupSuccess.php
+18 -2
View File
@@ -2,14 +2,20 @@
namespace App\Notifications\VolumeBackup;
use App\Contracts\ThrottledNotification;
use App\Models\NotificationThrottle;
use App\Models\ScheduledVolumeBackup;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class BackupSuccess extends CustomEmailNotification
/**
* With a warning, it goes to the failure channels: then it is sent at most once a day until a run has no warning again.
*/
class BackupSuccess extends CustomEmailNotification implements ThrottledNotification
{
public string $resourceName;
@@ -19,7 +25,7 @@ class BackupSuccess extends CustomEmailNotification
public ?string $url = null;
public function __construct(ScheduledVolumeBackup $backup, public ?string $warning = null)
public function __construct(public ScheduledVolumeBackup $backup, public ?string $warning = null)
{
$this->onQueue('high');
$resource = $backup->targetResource();
@@ -30,6 +36,16 @@ class BackupSuccess extends CustomEmailNotification
$this->url = $backup->resourceUrl();
}
public function throttleSubject(): ?Model
{
return $this->warning ? $this->backup : null;
}
public function throttleIntervalMinutes(): int
{
return NotificationThrottle::FAILURE_REMINDER_INTERVAL_MINUTES;
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels($this->warning ? 'backup_failure' : 'backup_success');
app/Services/ServerTransfer/ServerTransferBundle.php
+1
View File
@@ -382,6 +382,7 @@ class ServerTransferBundle
self::check($errors, "{$at}.scheduled_tasks.{$i}.command", $task['command'] ?? null, ['required', 'string']);
self::check($errors, "{$at}.scheduled_tasks.{$i}.container", $task['container'] ?? null, ['nullable', ...ValidationPatterns::containerNameRules()]);
self::checkCron($errors, "{$at}.scheduled_tasks.{$i}.frequency", $task['frequency'] ?? '');
self::check($errors, "{$at}.scheduled_tasks.{$i}.disabled_reason", $task['disabled_reason'] ?? null, ['nullable', 'string', 'max:1000']);
}
}
app/Services/ServerTransfer/ServerTransferCoverage.php
+4 -1
View File
@@ -256,7 +256,10 @@ final class ServerTransferCoverage
'container_id' => 'Not used: only listed in $fillable.',
],
LocalFileVolume::class => ['resource_type' => self::LINKED, 'resource_id' => self::LINKED],
ScheduledTask::class => ['team_id' => self::TEAM, 'application_id' => self::LINKED, 'service_id' => self::LINKED],
ScheduledTask::class => [
'team_id' => self::TEAM, 'application_id' => self::LINKED, 'service_id' => self::LINKED,
'consecutive_failures' => self::RUNTIME, 'failing_since' => self::RUNTIME,
],
Tag::class => ['team_id' => self::TEAM],
ScheduledDatabaseBackup::class => [
'team_id' => self::TEAM, 'database_type' => self::LINKED, 'database_id' => self::LINKED, 's3_storage_id' => self::LINKED,
app/Services/ServerTransfer/ServerTransferExporter.php
+1
View File
@@ -1325,6 +1325,7 @@ class ServerTransferExporter
'container' => $task->container,
'timeout' => $task->timeout,
'enabled' => (bool) ($task->enabled ?? true),
'disabled_reason' => $task->disabled_reason,
];
}
app/Services/ServerTransfer/ServerTransferImporter.php
+2 -1
View File
@@ -1688,6 +1688,7 @@ class ServerTransferImporter
'container' => data_get($task, 'container'),
'timeout' => data_get($task, 'timeout'),
'enabled' => (bool) data_get($task, 'enabled', true),
'disabled_reason' => data_get($task, 'disabled_reason'),
'team_id' => $teamId,
];
@@ -1697,7 +1698,7 @@ class ServerTransferImporter
$payload['service_id'] = $resource->id;
}
ScheduledTask::create($payload);
ScheduledTask::forceCreate($payload);
}
}
database/migrations/2026_11_03_192735_add_failure_tracking_to_scheduled_tasks_table.php
+48
View File
@@ -0,0 +1,48 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('scheduled_tasks', function (Blueprint $table) {
$table->unsignedInteger('consecutive_failures')->default(0);
$table->timestamp('failing_since')->nullable();
$table->text('disabled_reason')->nullable();
});
$this->setExecutionStatuses(['success', 'failed', 'running', 'skipped']);
}
public function down(): void
{
DB::table('scheduled_task_executions')->where('status', 'skipped')->update(['status' => 'success']);
$this->setExecutionStatuses(['success', 'failed', 'running']);
Schema::table('scheduled_tasks', function (Blueprint $table) {
$table->dropColumn(['consecutive_failures', 'failing_since', 'disabled_reason']);
});
}
/**
* @param list<string> $statuses
*/
private function setExecutionStatuses(array $statuses): void
{
if (DB::getDriverName() === 'pgsql') {
$allowed = implode(', ', array_map(fn (string $status) => "'{$status}'", $statuses));
DB::statement('ALTER TABLE scheduled_task_executions DROP CONSTRAINT IF EXISTS scheduled_task_executions_status_check');
DB::statement("ALTER TABLE scheduled_task_executions ADD CONSTRAINT scheduled_task_executions_status_check CHECK (status IN ({$allowed}))");
return;
}
Schema::table('scheduled_task_executions', function (Blueprint $table) use ($statuses) {
$table->enum('status', $statuses)->default('running')->change();
});
}
};
docs/server-transfer-full-coverage.md
+1 -1
View File
@@ -101,7 +101,7 @@ When you add a model or a setting to the transfer, add it to this file too.
| ✅ | Persistent volume | the seed has one; add a host-path bind mount and preview suffix on |
| ✅ | File mount + directory mount | content, chown, chmod |
| ✅ | Volume backup | on the persistent volume: S3 on, stop during backup, all retention values, timeout, missing-backup days |
| ✅ | Scheduled tasks | the seed has one; add a disabled one with container + timeout |
| ✅ | Scheduled tasks | the seed has one; add a disabled one with container + timeout; an auto-disabled task keeps its disabled reason |
| ✅ | Tag | |
| ✅ | Preview deployment | PR preview with domain port overrides + max restart count + keeps production values (previews from before v4.5.1) (needs the GitHub App, or create it in the database) |
| ✅ | Secret manager link | the app loads variables from the secret manager token |
openapi.json
+15
View File
@@ -27638,6 +27638,21 @@
"type": "integer",
"description": "The timeout of the scheduled task in seconds."
},
"consecutive_failures": {
"type": "integer",
"description": "The number of failed runs since the last successful run."
},
"failing_since": {
"type": "string",
"format": "date-time",
"nullable": true,
"description": "The date and time of the first failed run since the last successful run."
},
"disabled_reason": {
"type": "string",
"nullable": true,
"description": "Why Coolify disabled the task automatically. It is cleared when the task is enabled again."
},
"created_at": {
"type": "string",
"format": "date-time",
openapi.yaml
+12
View File
@@ -17673,6 +17673,18 @@ components:
timeout:
type: integer
description: 'The timeout of the scheduled task in seconds.'
consecutive_failures:
type: integer
description: 'The number of failed runs since the last successful run.'
failing_since:
type: string
format: date-time
nullable: true
description: 'The date and time of the first failed run since the last successful run.'
disabled_reason:
type: string
nullable: true
description: 'Why Coolify disabled the task automatically. It is cleared when the task is enabled again.'
created_at:
type: string
format: date-time
resources/views/emails/backup-failed.blade.php
+4
View File
@@ -4,4 +4,8 @@ Database backup for {{ $name }} @if($database_name)(db:{{ $database_name }})@end
### Reason
{{ $output }}
@isset($reminder)
{{ $reminder }}
@endisset
</x-emails.layout>
resources/views/emails/docker-cleanup-failed.blade.php
+4
View File
@@ -5,4 +5,8 @@ Docker Cleanup on {{ $name }} FAILED with the following error:
{{ $text }}
</pre>
@isset($reminder)
{{ $reminder }}
@endisset
</x-emails.layout>
resources/views/emails/scheduled-job-recovered.blade.php
+9
View File
@@ -0,0 +1,9 @@
<x-emails.layout>
{{ $title }}
The last run succeeded after failed runs.
@if ($url)
Click [here]({{ $url }}) to open it in Coolify.
@endif
</x-emails.layout>
resources/views/emails/scheduled-task-failed.blade.php
+5 -1
View File
@@ -1,5 +1,9 @@
<x-emails.layout>
Scheduled task ({{ $task->name }}) was FAILED with the following error:
{{ $title ?? "Scheduled task ({$task->name}) failed." }}
{{ $summary ?? '' }}
Last error:
<pre>
{{ $output }}
resources/views/emails/volume-backup-result.blade.php
+4
View File
@@ -9,6 +9,10 @@ Frequency: {{ $frequency }}
{{ $output }}
@endisset
@isset($reminder)
{{ $reminder }}
@endisset
@if ($url)
Click [here]({{ $url }}) to open the resource.
@endif
resources/views/livewire/project/shared/scheduled-task/show.blade.php
+6
View File
@@ -12,6 +12,12 @@
@endforeach
</nav>
@if (! $isEnabled && $task->disabled_reason)
<x-callout type="danger" title="Disabled because it keeps failing">
{{ $task->disabled_reason }} Fix the task, then click Enable.
</x-callout>
@endif
<form x-show="activeSection === 'general'" wire:submit="submit" class="application-settings-form flex flex-col">
<x-unsaved-bar action="submit" />
<x-application.settings-section id="task-configuration-section" title="Task configuration"