From 407c384b09ae0ec044966967e4c13a70066c4c89 Mon Sep 17 00:00:00 2001
From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com>
Date: Wed, 7 Oct 2026 09:37:35 +0200
Subject: [PATCH] feat(backups): add configurable missing volume backup alerts
Check enabled volume backup schedules hourly and notify teams when no
execution occurs within the configured number of days. Expose the alert
threshold in the UI and API, with alerts disabled by default.
Route volume backup results and warnings to the appropriate Telegram
backup threads.
---
app/Console/Kernel.php | 2 +
.../Api/VolumeBackupsController.php | 10 +-
app/Jobs/CheckMissingVolumeBackupsJob.php | 69 +++++++++++
.../Project/Shared/Storages/VolumeBackups.php | 5 +
app/Models/ScheduledVolumeBackup.php | 2 +
.../Channels/TelegramChannel.php | 14 ++-
.../VolumeBackup/BackupMissing.php | 117 ++++++++++++++++++
app/Services/ScheduledJobDeliveryService.php | 39 +++++-
...days_to_scheduled_volume_backups_table.php | 22 ++++
.../emails/volume-backup-missing.blade.php | 13 ++
.../storages/volume-backups/general.blade.php | 3 +
.../Api/VolumeBackupScheduleApiTest.php | 20 ++-
.../MissingVolumeBackupNotificationTest.php | 109 ++++++++++++++++
.../ScheduledInstanceJobsUniqueLockTest.php | 2 +
.../ScheduledJobManagerDispatchTest.php | 27 ++++
.../TelegramNotificationThreadRoutingTest.php | 22 ++++
tests/Feature/VolumeBackupTest.php | 51 ++++++++
17 files changed, 522 insertions(+), 5 deletions(-)
create mode 100644 app/Jobs/CheckMissingVolumeBackupsJob.php
create mode 100644 app/Notifications/VolumeBackup/BackupMissing.php
create mode 100644 database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php
create mode 100644 resources/views/emails/volume-backup-missing.blade.php
create mode 100644 tests/Feature/MissingVolumeBackupNotificationTest.php
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index b13d8c99c4..a88ba64a81 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -6,6 +6,7 @@ use App\Jobs\ApiTokenExpirationWarningJob;
use App\Jobs\CheckForUpdatesJob;
use App\Jobs\CheckHelperImageJob;
use App\Jobs\CheckMissingDatabaseBackupsJob;
+use App\Jobs\CheckMissingVolumeBackupsJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\CleanupOrphanedPreviewContainersJob;
use App\Jobs\CleanupStaleMultiplexedConnections;
@@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel
->runInBackground();
$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();
+ $this->scheduleInstance->job(new CheckMissingVolumeBackupsJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer();
diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.php
index 26ff938a10..e3b48a367a 100644
--- a/app/Http/Controllers/Api/VolumeBackupsController.php
+++ b/app/Http/Controllers/Api/VolumeBackupsController.php
@@ -35,13 +35,14 @@ use RuntimeException;
new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000),
+ new OA\Property(property: 'missing_backup_notification_days', type: 'integer', description: 'Alert after this many days without an execution; 0 disables alerts', default: 0, minimum: 0, maximum: 365),
],
type: 'object',
additionalProperties: false,
)]
#[OA\Schema(
schema: 'VolumeBackupScheduleResponse',
- required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout'],
+ required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout', 'missing_backup_notification_days'],
properties: [
new OA\Property(property: 'uuid', type: 'string'),
new OA\Property(property: 'message', type: 'string'),
@@ -60,6 +61,7 @@ use RuntimeException;
new OA\Property(property: 'retention_days_s3', type: 'integer'),
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float'),
new OA\Property(property: 'timeout', type: 'integer'),
+ new OA\Property(property: 'missing_backup_notification_days', type: 'integer'),
],
type: 'object',
)]
@@ -197,6 +199,7 @@ class VolumeBackupsController extends Controller
'retention_days_s3' => 'integer|min:0|max:2147483647',
'retention_max_storage_s3' => 'numeric|min:0|max:9999999999',
'timeout' => 'integer|min:60|max:36000',
+ 'missing_backup_notification_days' => 'integer|min:0|max:365',
]);
$errors = $validator->errors();
$allowedFields = [
@@ -213,6 +216,7 @@ class VolumeBackupsController extends Controller
'retention_days_s3',
'retention_max_storage_s3',
'timeout',
+ 'missing_backup_notification_days',
];
foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) {
@@ -279,6 +283,9 @@ class VolumeBackupsController extends Controller
if ($request->has('timeout')) {
$attributes['timeout'] = $request->integer('timeout');
}
+ if ($request->has('missing_backup_notification_days')) {
+ $attributes['missing_backup_notification_days'] = $request->integer('missing_backup_notification_days');
+ }
$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);
$created = $backup->wasRecentlyCreated;
@@ -445,6 +452,7 @@ class VolumeBackupsController extends Controller
'retention_days_s3' => $backup->retention_days_s3,
'retention_max_storage_s3' => $backup->retention_max_storage_s3,
'timeout' => $backup->timeout,
+ 'missing_backup_notification_days' => $backup->missing_backup_notification_days ?? 0,
];
}
diff --git a/app/Jobs/CheckMissingVolumeBackupsJob.php b/app/Jobs/CheckMissingVolumeBackupsJob.php
new file mode 100644
index 0000000000..1f8ab30b4f
--- /dev/null
+++ b/app/Jobs/CheckMissingVolumeBackupsJob.php
@@ -0,0 +1,69 @@
+with(['team', 'latestExecution'])
+ ->where('enabled', true)
+ ->where('missing_backup_notification_days', '>', 0)
+ ->chunkById(100, function ($backups): void {
+ foreach ($backups as $backup) {
+ $this->notifyIfMissing($backup);
+ }
+ });
+ }
+
+ private function notifyIfMissing(ScheduledVolumeBackup $backup): void
+ {
+ $lastExecutionAt = $backup->latestExecution?->created_at;
+ $lastActivityAt = $lastExecutionAt ?? $backup->created_at;
+
+ if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) {
+ return;
+ }
+
+ if (! $backup->team) {
+ Log::warning("Cannot send missing volume backup notification for backup {$backup->id}: team not found");
+
+ return;
+ }
+
+ if ($backup->team->getEnabledChannels('backup_failure') === []) {
+ return;
+ }
+
+ // Send once for each period without backup activity.
+ NotificationThrottle::sendOnce(
+ $backup,
+ BackupMissing::class,
+ $lastActivityAt,
+ fn () => $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)),
+ );
+ }
+}
diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php
index 215c8a68f9..d741bcc0e4 100644
--- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php
+++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php
@@ -61,6 +61,8 @@ class VolumeBackups extends Component
public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;
+ public int $missingBackupNotificationDays = 0;
+
public int $perPage = 10;
public function updatedPerPage(): void
@@ -94,6 +96,7 @@ class VolumeBackups extends Component
'retentionDaysS3' => ['required', 'integer', 'min:0'],
'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
+ 'missingBackupNotificationDays' => ['required', 'integer', 'min:0', 'max:365'],
];
}
@@ -123,6 +126,7 @@ class VolumeBackups extends Component
$this->retentionDaysS3 = $this->backup->retention_days_s3;
$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
$this->timeout = $this->backup->timeout;
+ $this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;
} else {
$this->s3StorageId = $this->availableS3Storages->first()?->id;
}
@@ -435,6 +439,7 @@ class VolumeBackups extends Component
'retention_days_s3' => $this->retentionDaysS3,
'retention_max_storage_s3' => $this->retentionMaxStorageS3,
'timeout' => $this->timeout,
+ 'missing_backup_notification_days' => $this->missingBackupNotificationDays,
]);
$changedFields = auditChangedFields($backup);
$backup->save();
diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.php
index 7f33fd92b9..9ad676d953 100644
--- a/app/Models/ScheduledVolumeBackup.php
+++ b/app/Models/ScheduledVolumeBackup.php
@@ -31,6 +31,7 @@ class ScheduledVolumeBackup extends BaseModel
'retention_days_s3',
'retention_max_storage_s3',
'timeout',
+ 'missing_backup_notification_days',
];
protected function casts(): array
@@ -47,6 +48,7 @@ class ScheduledVolumeBackup extends BaseModel
'retention_days_s3' => 'integer',
'retention_max_storage_s3' => 'float',
'timeout' => 'integer',
+ 'missing_backup_notification_days' => 'integer',
];
}
diff --git a/app/Notifications/Channels/TelegramChannel.php b/app/Notifications/Channels/TelegramChannel.php
index 118ad4269c..d0e1c95ed9 100644
--- a/app/Notifications/Channels/TelegramChannel.php
+++ b/app/Notifications/Channels/TelegramChannel.php
@@ -11,6 +11,7 @@ use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupMissing;
use App\Notifications\Database\BackupSuccess;
+use App\Notifications\Database\BackupSuccessWithS3Warning;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\ScheduledTask\TaskSuccess;
use App\Notifications\Server\DockerCleanupFailed;
@@ -20,6 +21,10 @@ use App\Notifications\Server\Reachable;
use App\Notifications\Server\ServerPatchCheck;
use App\Notifications\Server\TraefikVersionOutdated;
use App\Notifications\Server\Unreachable;
+use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
+use App\Notifications\VolumeBackup\BackupMissing as VolumeBackupMissing;
+use App\Notifications\VolumeBackup\BackupSuccess as VolumeBackupSuccess;
+use App\Notifications\VolumeBackup\RecoveryFailed as VolumeBackupRecoveryFailed;
class TelegramChannel
{
@@ -41,8 +46,15 @@ class TelegramChannel
RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id,
BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,
+ VolumeBackupSuccess::class => $notification->warning
+ ? $settings->telegram_notifications_backup_failure_thread_id
+ : $settings->telegram_notifications_backup_success_thread_id,
BackupFailed::class,
- BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id,
+ BackupMissing::class,
+ BackupSuccessWithS3Warning::class,
+ VolumeBackupFailed::class,
+ VolumeBackupMissing::class,
+ VolumeBackupRecoveryFailed::class => $settings->telegram_notifications_backup_failure_thread_id,
TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,
TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,
diff --git a/app/Notifications/VolumeBackup/BackupMissing.php b/app/Notifications/VolumeBackup/BackupMissing.php
new file mode 100644
index 0000000000..63282e3c13
--- /dev/null
+++ b/app/Notifications/VolumeBackup/BackupMissing.php
@@ -0,0 +1,117 @@
+onQueue('high');
+ $resource = $backup->targetResource();
+ $this->resourceName = $resource?->name ?? 'Unknown resource';
+ $this->target = $backup->targetType().' '.$backup->targetName();
+
+ $linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;
+ if ($linkable && method_exists($linkable, 'link')) {
+ $this->url = $linkable->link();
+ }
+ }
+
+ public function via(object $notifiable): array
+ {
+ return $notifiable->getEnabledChannels('backup_failure');
+ }
+
+ public function summary(): string
+ {
+ return "The enabled storage backup schedule for {$this->target} ({$this->resourceName}) has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s).";
+ }
+
+ public function toMail(): MailMessage
+ {
+ return (new MailMessage)
+ ->subject("Coolify: [ACTION REQUIRED] No recent storage backup for {$this->resourceName}")
+ ->view('emails.volume-backup-missing', [
+ 'summary' => $this->summary(),
+ 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
+ 'url' => $this->url,
+ ]);
+ }
+
+ public function toDiscord(): DiscordMessage
+ {
+ $message = new DiscordMessage(
+ title: ':warning: Scheduled storage backup missing',
+ description: $this->summary(),
+ color: DiscordMessage::errorColor(),
+ isCritical: true,
+ );
+
+ if ($this->url) {
+ $message->addField('Resource', '[Link]('.$this->url.')');
+ }
+
+ return $message;
+ }
+
+ public function toTelegram(): array
+ {
+ $data = ['message' => "Coolify: {$this->summary()}"];
+
+ if ($this->url) {
+ $data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];
+ }
+
+ return $data;
+ }
+
+ public function toPushover(): PushoverMessage
+ {
+ return new PushoverMessage(
+ title: 'Scheduled storage backup missing',
+ level: 'error',
+ message: $this->summary(),
+ buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],
+ );
+ }
+
+ public function toSlack(): SlackMessage
+ {
+ return new SlackMessage(
+ title: 'Scheduled storage backup missing',
+ description: $this->summary().($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),
+ color: SlackMessage::errorColor(),
+ );
+ }
+
+ public function toWebhook(): array
+ {
+ return [
+ 'success' => false,
+ 'message' => 'Scheduled storage backup missing',
+ 'event' => 'volume_backup_missing',
+ 'resource_name' => $this->resourceName,
+ 'storage' => $this->target,
+ 'backup_uuid' => $this->backup->uuid,
+ 'days' => $this->backup->missing_backup_notification_days,
+ 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
+ 'url' => $this->url,
+ ];
+ }
+}
diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.php
index 8609b78b80..0d5ddd633a 100644
--- a/app/Services/ScheduledJobDeliveryService.php
+++ b/app/Services/ScheduledJobDeliveryService.php
@@ -2,6 +2,7 @@
namespace App\Services;
+use App\Events\BackupCreated;
use App\Events\DockerCleanupDone;
use App\Events\ScheduledTaskDone;
use App\Jobs\DatabaseBackupJob;
@@ -11,15 +12,19 @@ use App\Jobs\VolumeBackupJob;
use App\Models\DockerCleanupExecution;
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\ScheduledTaskExecution;
use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Models\Team;
+use App\Notifications\Database\BackupFailed;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\Server\DockerCleanupFailed;
+use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
use Closure;
use Cron\CronExpression;
use Illuminate\Database\Eloquent\Model;
@@ -353,7 +358,7 @@ class ScheduledJobDeliveryService
}
/**
- * Show a missed task or Docker cleanup as a failed execution and send the failure notification.
+ * Show a missed task, Docker cleanup, or backup as a failed execution and send the failure notification.
* The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,
* and claim() rejects the late job if it starts afterwards.
*/
@@ -391,6 +396,38 @@ class ScheduledJobDeliveryService
]);
event(new DockerCleanupDone($execution));
$this->notifyMissedOccurrence($server, fn () => $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}")));
+ } elseif ($occurrence->job_type === 'database-backup') {
+ $backup = ScheduledDatabaseBackup::with(['team', 'database'])->find($occurrence->resource_id);
+ if (! $backup?->database) {
+ return;
+ }
+
+ $databaseName = $backup->dump_all ? 'all' : $backup->databases_to_backup;
+ ScheduledDatabaseBackupExecution::create([
+ 'uuid' => new_public_id(),
+ 'database_name' => filled($databaseName) ? str($databaseName)->limit(250)->toString() : null,
+ 'scheduled_database_backup_id' => $backup->id,
+ 'status' => 'failed',
+ 'message' => $message,
+ 'local_storage_deleted' => false,
+ 'finished_at' => now(),
+ ]);
+ BackupCreated::dispatch($backup->team_id);
+ $this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new BackupFailed($backup, $backup->database, $message, $databaseName ?? 'unknown')));
+ } elseif ($occurrence->job_type === 'volume-backup') {
+ $backup = ScheduledVolumeBackup::with(['team', 'backupable.resource'])->find($occurrence->resource_id);
+ if (! $backup) {
+ return;
+ }
+
+ ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'failed',
+ 'message' => $message,
+ 'finished_at' => now(),
+ ]);
+ BackupCreated::dispatch($backup->team_id);
+ $this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new VolumeBackupFailed($backup, $message)));
}
} catch (\Throwable $e) {
Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [
diff --git a/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php b/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php
new file mode 100644
index 0000000000..b3aae9916b
--- /dev/null
+++ b/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php
@@ -0,0 +1,22 @@
+unsignedInteger('missing_backup_notification_days')->default(0);
+ });
+ }
+
+ public function down(): void
+ {
+ Schema::table('scheduled_volume_backups', function (Blueprint $table) {
+ $table->dropColumn('missing_backup_notification_days');
+ });
+ }
+};
diff --git a/resources/views/emails/volume-backup-missing.blade.php b/resources/views/emails/volume-backup-missing.blade.php
new file mode 100644
index 0000000000..e9952ff24a
--- /dev/null
+++ b/resources/views/emails/volume-backup-missing.blade.php
@@ -0,0 +1,13 @@
+
+{{ $summary }}
+
+@if ($last_execution_at)
+The last execution was at {{ $last_execution_at }}.
+@else
+This schedule has never produced an execution.
+@endif
+
+@if ($url)
+Click [here]({{ $url }}) to open the resource.
+@endif
+
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php
index 5b4ece52f1..a48b6cb303 100644
--- a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php
+++ b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php
@@ -44,6 +44,9 @@
helper="Uses the backup server timezone, or the instance timezone when none is configured." required />
+
diff --git a/tests/Feature/Api/VolumeBackupScheduleApiTest.php b/tests/Feature/Api/VolumeBackupScheduleApiTest.php
index e60f2919da..e494261d51 100644
--- a/tests/Feature/Api/VolumeBackupScheduleApiTest.php
+++ b/tests/Feature/Api/VolumeBackupScheduleApiTest.php
@@ -101,9 +101,12 @@ it('sets an application volume backup schedule through the API', function () {
'retention_days_s3' => 7,
'retention_max_storage_s3' => 8.5,
'timeout' => 600,
+ 'missing_backup_notification_days' => 5,
]);
- $response->assertCreated()->assertJsonStructure(['uuid', 'message']);
+ $response->assertCreated()
+ ->assertJsonStructure(['uuid', 'message'])
+ ->assertJsonPath('missing_backup_notification_days', 5);
$backup = ScheduledVolumeBackup::query()->sole();
expect($backup->backupable->is($this->volume))->toBeTrue()
@@ -120,7 +123,20 @@ it('sets an application volume backup schedule through the API', function () {
->and($backup->retention_amount_s3)->toBe(6)
->and($backup->retention_days_s3)->toBe(7)
->and($backup->retention_max_storage_s3)->toBe(8.5)
- ->and($backup->timeout)->toBe(600);
+ ->and($backup->timeout)->toBe(600)
+ ->and($backup->missing_backup_notification_days)->toBe(5);
+});
+
+it('rejects a missing backup alert period above one year through the API', function () {
+ $this->withHeaders($this->headers)
+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
+ 'frequency' => 'daily',
+ 'missing_backup_notification_days' => 366,
+ ])
+ ->assertUnprocessable()
+ ->assertJsonValidationErrors('missing_backup_notification_days');
+
+ expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
});
it('updates the existing backup schedule instead of creating another one', function () {
diff --git a/tests/Feature/MissingVolumeBackupNotificationTest.php b/tests/Feature/MissingVolumeBackupNotificationTest.php
new file mode 100644
index 0000000000..18f5c05c5f
--- /dev/null
+++ b/tests/Feature/MissingVolumeBackupNotificationTest.php
@@ -0,0 +1,109 @@
+ Carbon::setTestNow());
+
+function missingVolumeBackupSchedule(Team $team, array $attributes = []): ScheduledVolumeBackup
+{
+ $backup = ScheduledVolumeBackup::create(array_merge([
+ 'backupable_type' => (new LocalPersistentVolume)->getMorphClass(),
+ 'backupable_id' => ScheduledVolumeBackup::query()->max('backupable_id') + 1000,
+ 'team_id' => $team->id,
+ 'frequency' => '0 0 * * *',
+ 'enabled' => true,
+ 'missing_backup_notification_days' => 2,
+ ], $attributes));
+
+ ScheduledVolumeBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]);
+
+ return $backup->refresh();
+}
+
+function teamWithVolumeBackupFailureNotifications(): Team
+{
+ $team = Team::create(['name' => 'Test team']);
+ $team->emailNotificationSettings->update([
+ 'smtp_enabled' => true,
+ 'backup_failure_email_notifications' => true,
+ ]);
+
+ return $team;
+}
+
+it('notifies the team when an enabled volume backup has no executions for the configured days', function () {
+ Notification::fake();
+ $team = teamWithVolumeBackupFailureNotifications();
+ $backup = missingVolumeBackupSchedule($team);
+
+ (new CheckMissingVolumeBackupsJob)->handle();
+
+ Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)
+ && $notification->lastExecutionAt === null
+ && $notification->toWebhook()['event'] === 'volume_backup_missing');
+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue()
+ ->and((string) (new BackupMissing($backup, null))->toMail()->render())->toContain('This schedule has never produced an execution.');
+});
+
+it('does not notify for recent, disabled or unconfigured volume backup schedules', function () {
+ Notification::fake();
+ $team = teamWithVolumeBackupFailureNotifications();
+ missingVolumeBackupSchedule($team, ['enabled' => false]);
+ missingVolumeBackupSchedule($team, ['missing_backup_notification_days' => 0]);
+ $recentlyCreated = missingVolumeBackupSchedule($team);
+ ScheduledVolumeBackup::whereKey($recentlyCreated->id)->update(['created_at' => now()->subDay()]);
+ $recentlyExecuted = missingVolumeBackupSchedule($team);
+ ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $recentlyExecuted->id,
+ 'status' => 'success',
+ ]);
+
+ (new CheckMissingVolumeBackupsJob)->handle();
+
+ Notification::assertNothingSent();
+});
+
+it('does not notify when the team has no backup failure channels', function () {
+ Notification::fake();
+ $team = Team::create(['name' => 'Test team']);
+ $backup = missingVolumeBackupSchedule($team);
+
+ (new CheckMissingVolumeBackupsJob)->handle();
+
+ Notification::assertNothingSent();
+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
+});
+
+it('notifies once per period without volume backup executions and rearms after another execution', function () {
+ Notification::fake();
+ $team = teamWithVolumeBackupFailureNotifications();
+ $backup = missingVolumeBackupSchedule($team);
+
+ (new CheckMissingVolumeBackupsJob)->handle();
+ (new CheckMissingVolumeBackupsJob)->handle();
+
+ Notification::assertSentToTimes($team, BackupMissing::class, 1);
+
+ Carbon::setTestNow(now()->addMinute());
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'success',
+ ]);
+ Carbon::setTestNow(now()->addDays(3));
+
+ (new CheckMissingVolumeBackupsJob)->handle();
+
+ Notification::assertSentToTimes($team, BackupMissing::class, 2);
+ Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->lastExecutionAt?->equalTo($execution->created_at) === true);
+});
diff --git a/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php b/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php
index 4ee8d66780..4da77c49d4 100644
--- a/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php
+++ b/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php
@@ -1,6 +1,7 @@
[CleanupOrphanedPreviewContainersJob::class, 600],
'instance cleanup' => [CleanupInstanceStuffsJob::class, 60],
'missing database backups check' => [CheckMissingDatabaseBackupsJob::class, 1800],
+ 'missing volume backups check' => [CheckMissingVolumeBackupsJob::class, 1800],
]);
it('does not queue the job again while its unique lock is held', function (string $jobClass, int $lockSeconds) {
diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.php
index 82f024c848..d09bb865a1 100644
--- a/tests/Feature/ScheduledJobManagerDispatchTest.php
+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php
@@ -13,6 +13,7 @@ use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
@@ -24,6 +25,7 @@ use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
+use App\Notifications\Database\BackupFailed;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\Server\DockerCleanupFailed;
use App\Services\ScheduledJobDeliveryService;
@@ -752,6 +754,31 @@ it('records a missed Docker cleanup as a failed execution and notifies the team
Event::assertDispatched(DockerCleanupDone::class, 1);
});
+it('records a missed database backup that is too old to run late as a failed execution and notifies the team once', function () {
+ config(['constants.coolify.self_hosted' => true]);
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
+ Queue::fake();
+ Notification::fake();
+ Event::fake([BackupCreated::class]);
+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
+ $backup->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);
+ $occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'));
+
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+
+ $executions = ScheduledDatabaseBackupExecution::query()->where('scheduled_database_backup_id', $backup->id)->get();
+ expect($executions)->toHaveCount(1)
+ ->and($executions->first()->status)->toBe('failed')
+ ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
+ ->and($executions->first()->finished_at)->not->toBeNull()
+ ->and($occurrence->fresh()->status)->toBe('failed');
+ Queue::assertNotPushed(DatabaseBackupJob::class);
+ Notification::assertSentToTimes($backup->team, BackupFailed::class, 1);
+ Notification::assertSentTo($backup->team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
+ Event::assertDispatched(BackupCreated::class);
+});
+
it('sends at most one missed-run notification per hour for the same schedule', function (string $jobType) {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
diff --git a/tests/Feature/TelegramNotificationThreadRoutingTest.php b/tests/Feature/TelegramNotificationThreadRoutingTest.php
index 474d2b157d..00a8bc327a 100644
--- a/tests/Feature/TelegramNotificationThreadRoutingTest.php
+++ b/tests/Feature/TelegramNotificationThreadRoutingTest.php
@@ -2,11 +2,14 @@
use App\Jobs\SendMessageToTelegramJob;
use App\Models\InstanceSettings;
+use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Channels\TelegramChannel;
use App\Notifications\Server\ServerPatchCheck;
use App\Notifications\Server\TraefikVersionOutdated;
+use App\Notifications\VolumeBackup\BackupFailed;
+use App\Notifications\VolumeBackup\BackupSuccess;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
@@ -87,3 +90,22 @@ it('falls back to the main chat when no topic is configured', function () {
fn (SendMessageToTelegramJob $job) => $job->threadId === null
);
});
+
+it('sends volume backup notifications to the backup topics', function (Closure $makeNotification, string $threadId) {
+ $this->team->telegramNotificationSettings->update([
+ 'telegram_notifications_backup_success_thread_id' => '11',
+ 'telegram_notifications_backup_failure_thread_id' => '12',
+ ]);
+ $backup = new ScheduledVolumeBackup(['frequency' => 'daily']);
+
+ (new TelegramChannel)->send($this->team->fresh(), $makeNotification($backup));
+
+ Queue::assertPushed(
+ SendMessageToTelegramJob::class,
+ fn (SendMessageToTelegramJob $job) => $job->threadId === $threadId
+ );
+})->with([
+ 'success' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup), '11'],
+ 'success with warning' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup, 'S3 upload failed'), '12'],
+ 'failed' => [fn (ScheduledVolumeBackup $backup) => new BackupFailed($backup, 'Archive command failed'), '12'],
+]);
diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php
index 75fbe6b9c6..d247a7a4e6 100644
--- a/tests/Feature/VolumeBackupTest.php
+++ b/tests/Feature/VolumeBackupTest.php
@@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\S3Storage;
+use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
@@ -29,6 +30,7 @@ use App\Models\Team;
use App\Models\User;
use App\Notifications\VolumeBackup\BackupFailed;
use App\Notifications\VolumeBackup\BackupSuccess;
+use App\Services\ScheduledJobDeliveryService;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Filesystem\FilesystemAdapter;
use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -1142,6 +1144,19 @@ it('does not change S3 storage when another volume backup setting is invalid', f
->and($backup->frequency)->toBe('daily');
});
+it('rejects a missing volume backup alert period above one year', function () {
+ $team = Team::factory()->create();
+ signInForVolumeBackups($this, $team);
+ [$application, $volume] = createVolumeBackupApplication($team);
+
+ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
+ ->set('missingBackupNotificationDays', 366)
+ ->call('save')
+ ->assertHasErrors(['missingBackupNotificationDays' => 'max']);
+
+ expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
+});
+
function createVolumeBackupApplication(Team $team): array
{
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
@@ -1208,12 +1223,14 @@ it('creates a local scheduled backup for a persistent volume', function () {
->set('retentionDaysLocally', 14)
->set('retentionMaxStorageLocally', 1.5)
->set('stopDuringBackup', true)
+ ->set('missingBackupNotificationDays', 3)
->call('save')
->assertDispatched('success');
$backup = ScheduledVolumeBackup::query()->sole();
expect($backup->backupable->is($volume))->toBeTrue()
+ ->and($backup->missing_backup_notification_days)->toBe(3)
->and($backup->team_id)->toBe($team->id)
->and($backup->frequency)->toBe('daily')
->and($backup->retention_amount_locally)->toBe(5)
@@ -2850,3 +2867,37 @@ it('sends volume backup success with a warning through the backup failure channe
'without warning' => [null, 'backup_success'],
'with warning' => ['S3 upload failed: denied', 'backup_failure'],
]);
+
+it('records a missed volume backup that is too old to run late as a failed execution and notifies the team once', function () {
+ config(['broadcasting.default' => 'null']);
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
+ Queue::fake();
+ Notification::fake();
+ $team = Team::factory()->create();
+ $team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]);
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ ]);
+ $occurrence = ScheduledJobDelivery::create([
+ 'schedule_key' => "scheduled-volume-backup:{$backup->id}",
+ 'scheduled_for' => Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'),
+ 'job_type' => 'volume-backup',
+ 'resource_id' => $backup->id,
+ 'status' => 'enqueued',
+ 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),
+ ]);
+
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+
+ $execution = ScheduledVolumeBackupExecution::query()->sole();
+ expect($execution->status)->toBe('failed')
+ ->and($execution->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
+ ->and($execution->finished_at)->not->toBeNull()
+ ->and($occurrence->fresh()->status)->toBe('failed');
+ Queue::assertNotPushed(VolumeBackupJob::class);
+ Notification::assertSentToTimes($team, BackupFailed::class, 1);
+ Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
+});