diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index b13d8c99c4..a88ba64a81 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -6,6 +6,7 @@ use App\Jobs\ApiTokenExpirationWarningJob; use App\Jobs\CheckForUpdatesJob; use App\Jobs\CheckHelperImageJob; use App\Jobs\CheckMissingDatabaseBackupsJob; +use App\Jobs\CheckMissingVolumeBackupsJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\CleanupOrphanedPreviewContainersJob; use App\Jobs\CleanupStaleMultiplexedConnections; @@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel ->runInBackground(); $this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer(); $this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer(); + $this->scheduleInstance->job(new CheckMissingVolumeBackupsJob)->hourly()->onOneServer(); $this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer(); $this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer(); diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.php index 26ff938a10..e3b48a367a 100644 --- a/app/Http/Controllers/Api/VolumeBackupsController.php +++ b/app/Http/Controllers/Api/VolumeBackupsController.php @@ -35,13 +35,14 @@ use RuntimeException; new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0), new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0), new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000), + new OA\Property(property: 'missing_backup_notification_days', type: 'integer', description: 'Alert after this many days without an execution; 0 disables alerts', default: 0, minimum: 0, maximum: 365), ], type: 'object', additionalProperties: false, )] #[OA\Schema( schema: 'VolumeBackupScheduleResponse', - required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout'], + required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout', 'missing_backup_notification_days'], properties: [ new OA\Property(property: 'uuid', type: 'string'), new OA\Property(property: 'message', type: 'string'), @@ -60,6 +61,7 @@ use RuntimeException; new OA\Property(property: 'retention_days_s3', type: 'integer'), new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float'), new OA\Property(property: 'timeout', type: 'integer'), + new OA\Property(property: 'missing_backup_notification_days', type: 'integer'), ], type: 'object', )] @@ -197,6 +199,7 @@ class VolumeBackupsController extends Controller 'retention_days_s3' => 'integer|min:0|max:2147483647', 'retention_max_storage_s3' => 'numeric|min:0|max:9999999999', 'timeout' => 'integer|min:60|max:36000', + 'missing_backup_notification_days' => 'integer|min:0|max:365', ]); $errors = $validator->errors(); $allowedFields = [ @@ -213,6 +216,7 @@ class VolumeBackupsController extends Controller 'retention_days_s3', 'retention_max_storage_s3', 'timeout', + 'missing_backup_notification_days', ]; foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) { @@ -279,6 +283,9 @@ class VolumeBackupsController extends Controller if ($request->has('timeout')) { $attributes['timeout'] = $request->integer('timeout'); } + if ($request->has('missing_backup_notification_days')) { + $attributes['missing_backup_notification_days'] = $request->integer('missing_backup_notification_days'); + } $backup = $storage->scheduledBackups()->updateOrCreate([], $attributes); $created = $backup->wasRecentlyCreated; @@ -445,6 +452,7 @@ class VolumeBackupsController extends Controller 'retention_days_s3' => $backup->retention_days_s3, 'retention_max_storage_s3' => $backup->retention_max_storage_s3, 'timeout' => $backup->timeout, + 'missing_backup_notification_days' => $backup->missing_backup_notification_days ?? 0, ]; } diff --git a/app/Jobs/CheckMissingVolumeBackupsJob.php b/app/Jobs/CheckMissingVolumeBackupsJob.php new file mode 100644 index 0000000000..1f8ab30b4f --- /dev/null +++ b/app/Jobs/CheckMissingVolumeBackupsJob.php @@ -0,0 +1,69 @@ +with(['team', 'latestExecution']) + ->where('enabled', true) + ->where('missing_backup_notification_days', '>', 0) + ->chunkById(100, function ($backups): void { + foreach ($backups as $backup) { + $this->notifyIfMissing($backup); + } + }); + } + + private function notifyIfMissing(ScheduledVolumeBackup $backup): void + { + $lastExecutionAt = $backup->latestExecution?->created_at; + $lastActivityAt = $lastExecutionAt ?? $backup->created_at; + + if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) { + return; + } + + if (! $backup->team) { + Log::warning("Cannot send missing volume backup notification for backup {$backup->id}: team not found"); + + return; + } + + if ($backup->team->getEnabledChannels('backup_failure') === []) { + return; + } + + // Send once for each period without backup activity. + NotificationThrottle::sendOnce( + $backup, + BackupMissing::class, + $lastActivityAt, + fn () => $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)), + ); + } +} diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php index 215c8a68f9..d741bcc0e4 100644 --- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -61,6 +61,8 @@ class VolumeBackups extends Component public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT; + public int $missingBackupNotificationDays = 0; + public int $perPage = 10; public function updatedPerPage(): void @@ -94,6 +96,7 @@ class VolumeBackups extends Component 'retentionDaysS3' => ['required', 'integer', 'min:0'], 'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'], 'timeout' => ['required', 'integer', 'min:60', 'max:36000'], + 'missingBackupNotificationDays' => ['required', 'integer', 'min:0', 'max:365'], ]; } @@ -123,6 +126,7 @@ class VolumeBackups extends Component $this->retentionDaysS3 = $this->backup->retention_days_s3; $this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3; $this->timeout = $this->backup->timeout; + $this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days; } else { $this->s3StorageId = $this->availableS3Storages->first()?->id; } @@ -435,6 +439,7 @@ class VolumeBackups extends Component 'retention_days_s3' => $this->retentionDaysS3, 'retention_max_storage_s3' => $this->retentionMaxStorageS3, 'timeout' => $this->timeout, + 'missing_backup_notification_days' => $this->missingBackupNotificationDays, ]); $changedFields = auditChangedFields($backup); $backup->save(); diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.php index 7f33fd92b9..9ad676d953 100644 --- a/app/Models/ScheduledVolumeBackup.php +++ b/app/Models/ScheduledVolumeBackup.php @@ -31,6 +31,7 @@ class ScheduledVolumeBackup extends BaseModel 'retention_days_s3', 'retention_max_storage_s3', 'timeout', + 'missing_backup_notification_days', ]; protected function casts(): array @@ -47,6 +48,7 @@ class ScheduledVolumeBackup extends BaseModel 'retention_days_s3' => 'integer', 'retention_max_storage_s3' => 'float', 'timeout' => 'integer', + 'missing_backup_notification_days' => 'integer', ]; } diff --git a/app/Notifications/Channels/TelegramChannel.php b/app/Notifications/Channels/TelegramChannel.php index 118ad4269c..d0e1c95ed9 100644 --- a/app/Notifications/Channels/TelegramChannel.php +++ b/app/Notifications/Channels/TelegramChannel.php @@ -11,6 +11,7 @@ use App\Notifications\Container\ContainerRestarted; use App\Notifications\Database\BackupFailed; use App\Notifications\Database\BackupMissing; use App\Notifications\Database\BackupSuccess; +use App\Notifications\Database\BackupSuccessWithS3Warning; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\ScheduledTask\TaskSuccess; use App\Notifications\Server\DockerCleanupFailed; @@ -20,6 +21,10 @@ use App\Notifications\Server\Reachable; use App\Notifications\Server\ServerPatchCheck; use App\Notifications\Server\TraefikVersionOutdated; use App\Notifications\Server\Unreachable; +use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed; +use App\Notifications\VolumeBackup\BackupMissing as VolumeBackupMissing; +use App\Notifications\VolumeBackup\BackupSuccess as VolumeBackupSuccess; +use App\Notifications\VolumeBackup\RecoveryFailed as VolumeBackupRecoveryFailed; class TelegramChannel { @@ -41,8 +46,15 @@ class TelegramChannel RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id, BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id, + VolumeBackupSuccess::class => $notification->warning + ? $settings->telegram_notifications_backup_failure_thread_id + : $settings->telegram_notifications_backup_success_thread_id, BackupFailed::class, - BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id, + BackupMissing::class, + BackupSuccessWithS3Warning::class, + VolumeBackupFailed::class, + VolumeBackupMissing::class, + VolumeBackupRecoveryFailed::class => $settings->telegram_notifications_backup_failure_thread_id, TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id, TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id, diff --git a/app/Notifications/VolumeBackup/BackupMissing.php b/app/Notifications/VolumeBackup/BackupMissing.php new file mode 100644 index 0000000000..63282e3c13 --- /dev/null +++ b/app/Notifications/VolumeBackup/BackupMissing.php @@ -0,0 +1,117 @@ +onQueue('high'); + $resource = $backup->targetResource(); + $this->resourceName = $resource?->name ?? 'Unknown resource'; + $this->target = $backup->targetType().' '.$backup->targetName(); + + $linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource; + if ($linkable && method_exists($linkable, 'link')) { + $this->url = $linkable->link(); + } + } + + public function via(object $notifiable): array + { + return $notifiable->getEnabledChannels('backup_failure'); + } + + public function summary(): string + { + return "The enabled storage backup schedule for {$this->target} ({$this->resourceName}) has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s)."; + } + + public function toMail(): MailMessage + { + return (new MailMessage) + ->subject("Coolify: [ACTION REQUIRED] No recent storage backup for {$this->resourceName}") + ->view('emails.volume-backup-missing', [ + 'summary' => $this->summary(), + 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(), + 'url' => $this->url, + ]); + } + + public function toDiscord(): DiscordMessage + { + $message = new DiscordMessage( + title: ':warning: Scheduled storage backup missing', + description: $this->summary(), + color: DiscordMessage::errorColor(), + isCritical: true, + ); + + if ($this->url) { + $message->addField('Resource', '[Link]('.$this->url.')'); + } + + return $message; + } + + public function toTelegram(): array + { + $data = ['message' => "Coolify: {$this->summary()}"]; + + if ($this->url) { + $data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]]; + } + + return $data; + } + + public function toPushover(): PushoverMessage + { + return new PushoverMessage( + title: 'Scheduled storage backup missing', + level: 'error', + message: $this->summary(), + buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [], + ); + } + + public function toSlack(): SlackMessage + { + return new SlackMessage( + title: 'Scheduled storage backup missing', + description: $this->summary().($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''), + color: SlackMessage::errorColor(), + ); + } + + public function toWebhook(): array + { + return [ + 'success' => false, + 'message' => 'Scheduled storage backup missing', + 'event' => 'volume_backup_missing', + 'resource_name' => $this->resourceName, + 'storage' => $this->target, + 'backup_uuid' => $this->backup->uuid, + 'days' => $this->backup->missing_backup_notification_days, + 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(), + 'url' => $this->url, + ]; + } +} diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.php index 8609b78b80..0d5ddd633a 100644 --- a/app/Services/ScheduledJobDeliveryService.php +++ b/app/Services/ScheduledJobDeliveryService.php @@ -2,6 +2,7 @@ namespace App\Services; +use App\Events\BackupCreated; use App\Events\DockerCleanupDone; use App\Events\ScheduledTaskDone; use App\Jobs\DatabaseBackupJob; @@ -11,15 +12,19 @@ use App\Jobs\VolumeBackupJob; use App\Models\DockerCleanupExecution; use App\Models\NotificationThrottle; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledDatabaseBackupExecution; use App\Models\ScheduledJobDelivery; use App\Models\ScheduledJobState; use App\Models\ScheduledTask; use App\Models\ScheduledTaskExecution; use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\Team; +use App\Notifications\Database\BackupFailed; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\Server\DockerCleanupFailed; +use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed; use Closure; use Cron\CronExpression; use Illuminate\Database\Eloquent\Model; @@ -353,7 +358,7 @@ class ScheduledJobDeliveryService } /** - * Show a missed task or Docker cleanup as a failed execution and send the failure notification. + * Show a missed task, Docker cleanup, or backup as a failed execution and send the failure notification. * The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence, * and claim() rejects the late job if it starts afterwards. */ @@ -391,6 +396,38 @@ class ScheduledJobDeliveryService ]); event(new DockerCleanupDone($execution)); $this->notifyMissedOccurrence($server, fn () => $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}"))); + } elseif ($occurrence->job_type === 'database-backup') { + $backup = ScheduledDatabaseBackup::with(['team', 'database'])->find($occurrence->resource_id); + if (! $backup?->database) { + return; + } + + $databaseName = $backup->dump_all ? 'all' : $backup->databases_to_backup; + ScheduledDatabaseBackupExecution::create([ + 'uuid' => new_public_id(), + 'database_name' => filled($databaseName) ? str($databaseName)->limit(250)->toString() : null, + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'failed', + 'message' => $message, + 'local_storage_deleted' => false, + 'finished_at' => now(), + ]); + BackupCreated::dispatch($backup->team_id); + $this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new BackupFailed($backup, $backup->database, $message, $databaseName ?? 'unknown'))); + } elseif ($occurrence->job_type === 'volume-backup') { + $backup = ScheduledVolumeBackup::with(['team', 'backupable.resource'])->find($occurrence->resource_id); + if (! $backup) { + return; + } + + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + 'message' => $message, + 'finished_at' => now(), + ]); + BackupCreated::dispatch($backup->team_id); + $this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new VolumeBackupFailed($backup, $message))); } } catch (\Throwable $e) { Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [ diff --git a/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php b/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php new file mode 100644 index 0000000000..b3aae9916b --- /dev/null +++ b/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php @@ -0,0 +1,22 @@ +unsignedInteger('missing_backup_notification_days')->default(0); + }); + } + + public function down(): void + { + Schema::table('scheduled_volume_backups', function (Blueprint $table) { + $table->dropColumn('missing_backup_notification_days'); + }); + } +}; diff --git a/resources/views/emails/volume-backup-missing.blade.php b/resources/views/emails/volume-backup-missing.blade.php new file mode 100644 index 0000000000..e9952ff24a --- /dev/null +++ b/resources/views/emails/volume-backup-missing.blade.php @@ -0,0 +1,13 @@ + +{{ $summary }} + +@if ($last_execution_at) +The last execution was at {{ $last_execution_at }}. +@else +This schedule has never produced an execution. +@endif + +@if ($url) +Click [here]({{ $url }}) to open the resource. +@endif + diff --git a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php index 5b4ece52f1..a48b6cb303 100644 --- a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php +++ b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php @@ -44,6 +44,9 @@ helper="Uses the backup server timezone, or the instance timezone when none is configured." required /> + diff --git a/tests/Feature/Api/VolumeBackupScheduleApiTest.php b/tests/Feature/Api/VolumeBackupScheduleApiTest.php index e60f2919da..e494261d51 100644 --- a/tests/Feature/Api/VolumeBackupScheduleApiTest.php +++ b/tests/Feature/Api/VolumeBackupScheduleApiTest.php @@ -101,9 +101,12 @@ it('sets an application volume backup schedule through the API', function () { 'retention_days_s3' => 7, 'retention_max_storage_s3' => 8.5, 'timeout' => 600, + 'missing_backup_notification_days' => 5, ]); - $response->assertCreated()->assertJsonStructure(['uuid', 'message']); + $response->assertCreated() + ->assertJsonStructure(['uuid', 'message']) + ->assertJsonPath('missing_backup_notification_days', 5); $backup = ScheduledVolumeBackup::query()->sole(); expect($backup->backupable->is($this->volume))->toBeTrue() @@ -120,7 +123,20 @@ it('sets an application volume backup schedule through the API', function () { ->and($backup->retention_amount_s3)->toBe(6) ->and($backup->retention_days_s3)->toBe(7) ->and($backup->retention_max_storage_s3)->toBe(8.5) - ->and($backup->timeout)->toBe(600); + ->and($backup->timeout)->toBe(600) + ->and($backup->missing_backup_notification_days)->toBe(5); +}); + +it('rejects a missing backup alert period above one year through the API', function () { + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'daily', + 'missing_backup_notification_days' => 366, + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors('missing_backup_notification_days'); + + expect(ScheduledVolumeBackup::query()->exists())->toBeFalse(); }); it('updates the existing backup schedule instead of creating another one', function () { diff --git a/tests/Feature/MissingVolumeBackupNotificationTest.php b/tests/Feature/MissingVolumeBackupNotificationTest.php new file mode 100644 index 0000000000..18f5c05c5f --- /dev/null +++ b/tests/Feature/MissingVolumeBackupNotificationTest.php @@ -0,0 +1,109 @@ + Carbon::setTestNow()); + +function missingVolumeBackupSchedule(Team $team, array $attributes = []): ScheduledVolumeBackup +{ + $backup = ScheduledVolumeBackup::create(array_merge([ + 'backupable_type' => (new LocalPersistentVolume)->getMorphClass(), + 'backupable_id' => ScheduledVolumeBackup::query()->max('backupable_id') + 1000, + 'team_id' => $team->id, + 'frequency' => '0 0 * * *', + 'enabled' => true, + 'missing_backup_notification_days' => 2, + ], $attributes)); + + ScheduledVolumeBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]); + + return $backup->refresh(); +} + +function teamWithVolumeBackupFailureNotifications(): Team +{ + $team = Team::create(['name' => 'Test team']); + $team->emailNotificationSettings->update([ + 'smtp_enabled' => true, + 'backup_failure_email_notifications' => true, + ]); + + return $team; +} + +it('notifies the team when an enabled volume backup has no executions for the configured days', function () { + Notification::fake(); + $team = teamWithVolumeBackupFailureNotifications(); + $backup = missingVolumeBackupSchedule($team); + + (new CheckMissingVolumeBackupsJob)->handle(); + + Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup) + && $notification->lastExecutionAt === null + && $notification->toWebhook()['event'] === 'volume_backup_missing'); + expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue() + ->and((string) (new BackupMissing($backup, null))->toMail()->render())->toContain('This schedule has never produced an execution.'); +}); + +it('does not notify for recent, disabled or unconfigured volume backup schedules', function () { + Notification::fake(); + $team = teamWithVolumeBackupFailureNotifications(); + missingVolumeBackupSchedule($team, ['enabled' => false]); + missingVolumeBackupSchedule($team, ['missing_backup_notification_days' => 0]); + $recentlyCreated = missingVolumeBackupSchedule($team); + ScheduledVolumeBackup::whereKey($recentlyCreated->id)->update(['created_at' => now()->subDay()]); + $recentlyExecuted = missingVolumeBackupSchedule($team); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $recentlyExecuted->id, + 'status' => 'success', + ]); + + (new CheckMissingVolumeBackupsJob)->handle(); + + Notification::assertNothingSent(); +}); + +it('does not notify when the team has no backup failure channels', function () { + Notification::fake(); + $team = Team::create(['name' => 'Test team']); + $backup = missingVolumeBackupSchedule($team); + + (new CheckMissingVolumeBackupsJob)->handle(); + + Notification::assertNothingSent(); + expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse(); +}); + +it('notifies once per period without volume backup executions and rearms after another execution', function () { + Notification::fake(); + $team = teamWithVolumeBackupFailureNotifications(); + $backup = missingVolumeBackupSchedule($team); + + (new CheckMissingVolumeBackupsJob)->handle(); + (new CheckMissingVolumeBackupsJob)->handle(); + + Notification::assertSentToTimes($team, BackupMissing::class, 1); + + Carbon::setTestNow(now()->addMinute()); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + ]); + Carbon::setTestNow(now()->addDays(3)); + + (new CheckMissingVolumeBackupsJob)->handle(); + + Notification::assertSentToTimes($team, BackupMissing::class, 2); + Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->lastExecutionAt?->equalTo($execution->created_at) === true); +}); diff --git a/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php b/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php index 4ee8d66780..4da77c49d4 100644 --- a/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php +++ b/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php @@ -1,6 +1,7 @@ [CleanupOrphanedPreviewContainersJob::class, 600], 'instance cleanup' => [CleanupInstanceStuffsJob::class, 60], 'missing database backups check' => [CheckMissingDatabaseBackupsJob::class, 1800], + 'missing volume backups check' => [CheckMissingVolumeBackupsJob::class, 1800], ]); it('does not queue the job again while its unique lock is held', function (string $jobClass, int $lockSeconds) { diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.php index 82f024c848..d09bb865a1 100644 --- a/tests/Feature/ScheduledJobManagerDispatchTest.php +++ b/tests/Feature/ScheduledJobManagerDispatchTest.php @@ -13,6 +13,7 @@ use App\Models\InstanceSettings; use App\Models\PrivateKey; use App\Models\Project; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledDatabaseBackupExecution; use App\Models\ScheduledJobDelivery; use App\Models\ScheduledJobState; use App\Models\ScheduledTask; @@ -24,6 +25,7 @@ use App\Models\ServiceDatabase; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\Team; +use App\Notifications\Database\BackupFailed; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\Server\DockerCleanupFailed; use App\Services\ScheduledJobDeliveryService; @@ -752,6 +754,31 @@ it('records a missed Docker cleanup as a failed execution and notifies the team Event::assertDispatched(DockerCleanupDone::class, 1); }); +it('records a missed database backup that is too old to run late as a failed execution and notifies the team once', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC')); + Queue::fake(); + Notification::fake(); + Event::fake([BackupCreated::class]); + $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']); + $backup->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]); + $occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC')); + + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + + $executions = ScheduledDatabaseBackupExecution::query()->where('scheduled_database_backup_id', $backup->id)->get(); + expect($executions)->toHaveCount(1) + ->and($executions->first()->status)->toBe('failed') + ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.') + ->and($executions->first()->finished_at)->not->toBeNull() + ->and($occurrence->fresh()->status)->toBe('failed'); + Queue::assertNotPushed(DatabaseBackupJob::class); + Notification::assertSentToTimes($backup->team, BackupFailed::class, 1); + Notification::assertSentTo($backup->team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.'); + Event::assertDispatched(BackupCreated::class); +}); + it('sends at most one missed-run notification per hour for the same schedule', function (string $jobType) { config(['constants.coolify.self_hosted' => true]); Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC')); diff --git a/tests/Feature/TelegramNotificationThreadRoutingTest.php b/tests/Feature/TelegramNotificationThreadRoutingTest.php index 474d2b157d..00a8bc327a 100644 --- a/tests/Feature/TelegramNotificationThreadRoutingTest.php +++ b/tests/Feature/TelegramNotificationThreadRoutingTest.php @@ -2,11 +2,14 @@ use App\Jobs\SendMessageToTelegramJob; use App\Models\InstanceSettings; +use App\Models\ScheduledVolumeBackup; use App\Models\Server; use App\Models\Team; use App\Notifications\Channels\TelegramChannel; use App\Notifications\Server\ServerPatchCheck; use App\Notifications\Server\TraefikVersionOutdated; +use App\Notifications\VolumeBackup\BackupFailed; +use App\Notifications\VolumeBackup\BackupSuccess; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; @@ -87,3 +90,22 @@ it('falls back to the main chat when no topic is configured', function () { fn (SendMessageToTelegramJob $job) => $job->threadId === null ); }); + +it('sends volume backup notifications to the backup topics', function (Closure $makeNotification, string $threadId) { + $this->team->telegramNotificationSettings->update([ + 'telegram_notifications_backup_success_thread_id' => '11', + 'telegram_notifications_backup_failure_thread_id' => '12', + ]); + $backup = new ScheduledVolumeBackup(['frequency' => 'daily']); + + (new TelegramChannel)->send($this->team->fresh(), $makeNotification($backup)); + + Queue::assertPushed( + SendMessageToTelegramJob::class, + fn (SendMessageToTelegramJob $job) => $job->threadId === $threadId + ); +})->with([ + 'success' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup), '11'], + 'success with warning' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup, 'S3 upload failed'), '12'], + 'failed' => [fn (ScheduledVolumeBackup $backup) => new BackupFailed($backup, 'Archive command failed'), '12'], +]); diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php index 75fbe6b9c6..d247a7a4e6 100644 --- a/tests/Feature/VolumeBackupTest.php +++ b/tests/Feature/VolumeBackupTest.php @@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume; use App\Models\PrivateKey; use App\Models\Project; use App\Models\S3Storage; +use App\Models\ScheduledJobDelivery; use App\Models\ScheduledVolumeBackup; use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; @@ -29,6 +30,7 @@ use App\Models\Team; use App\Models\User; use App\Notifications\VolumeBackup\BackupFailed; use App\Notifications\VolumeBackup\BackupSuccess; +use App\Services\ScheduledJobDeliveryService; use Illuminate\Database\Eloquent\Collection; use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -1142,6 +1144,19 @@ it('does not change S3 storage when another volume backup setting is invalid', f ->and($backup->frequency)->toBe('daily'); }); +it('rejects a missing volume backup alert period above one year', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->set('missingBackupNotificationDays', 366) + ->call('save') + ->assertHasErrors(['missingBackupNotificationDays' => 'max']); + + expect(ScheduledVolumeBackup::query()->exists())->toBeFalse(); +}); + function createVolumeBackupApplication(Team $team): array { InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); @@ -1208,12 +1223,14 @@ it('creates a local scheduled backup for a persistent volume', function () { ->set('retentionDaysLocally', 14) ->set('retentionMaxStorageLocally', 1.5) ->set('stopDuringBackup', true) + ->set('missingBackupNotificationDays', 3) ->call('save') ->assertDispatched('success'); $backup = ScheduledVolumeBackup::query()->sole(); expect($backup->backupable->is($volume))->toBeTrue() + ->and($backup->missing_backup_notification_days)->toBe(3) ->and($backup->team_id)->toBe($team->id) ->and($backup->frequency)->toBe('daily') ->and($backup->retention_amount_locally)->toBe(5) @@ -2850,3 +2867,37 @@ it('sends volume backup success with a warning through the backup failure channe 'without warning' => [null, 'backup_success'], 'with warning' => ['S3 upload failed: denied', 'backup_failure'], ]); + +it('records a missed volume backup that is too old to run late as a failed execution and notifies the team once', function () { + config(['broadcasting.default' => 'null']); + Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC')); + Queue::fake(); + Notification::fake(); + $team = Team::factory()->create(); + $team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $occurrence = ScheduledJobDelivery::create([ + 'schedule_key' => "scheduled-volume-backup:{$backup->id}", + 'scheduled_for' => Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'), + 'job_type' => 'volume-backup', + 'resource_id' => $backup->id, + 'status' => 'enqueued', + 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1), + ]); + + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + expect($execution->status)->toBe('failed') + ->and($execution->message)->toBe('Skipped: the queued job did not start within 60 minutes.') + ->and($execution->finished_at)->not->toBeNull() + ->and($occurrence->fresh()->status)->toBe('failed'); + Queue::assertNotPushed(VolumeBackupJob::class); + Notification::assertSentToTimes($team, BackupFailed::class, 1); + Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.'); +});