mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
feat(backups): add configurable missing volume backup alerts
Check enabled volume backup schedules hourly and notify teams when no execution occurs within the configured number of days. Expose the alert threshold in the UI and API, with alerts disabled by default. Route volume backup results and warnings to the appropriate Telegram backup threads.
This commit is contained in:
@@ -6,6 +6,7 @@ use App\Jobs\ApiTokenExpirationWarningJob;
|
||||
use App\Jobs\CheckForUpdatesJob;
|
||||
use App\Jobs\CheckHelperImageJob;
|
||||
use App\Jobs\CheckMissingDatabaseBackupsJob;
|
||||
use App\Jobs\CheckMissingVolumeBackupsJob;
|
||||
use App\Jobs\CleanupInstanceStuffsJob;
|
||||
use App\Jobs\CleanupOrphanedPreviewContainersJob;
|
||||
use App\Jobs\CleanupStaleMultiplexedConnections;
|
||||
@@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel
|
||||
->runInBackground();
|
||||
$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new CheckMissingVolumeBackupsJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer();
|
||||
|
||||
|
||||
@@ -35,13 +35,14 @@ use RuntimeException;
|
||||
new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
|
||||
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
|
||||
new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000),
|
||||
new OA\Property(property: 'missing_backup_notification_days', type: 'integer', description: 'Alert after this many days without an execution; 0 disables alerts', default: 0, minimum: 0, maximum: 365),
|
||||
],
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
)]
|
||||
#[OA\Schema(
|
||||
schema: 'VolumeBackupScheduleResponse',
|
||||
required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout'],
|
||||
required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout', 'missing_backup_notification_days'],
|
||||
properties: [
|
||||
new OA\Property(property: 'uuid', type: 'string'),
|
||||
new OA\Property(property: 'message', type: 'string'),
|
||||
@@ -60,6 +61,7 @@ use RuntimeException;
|
||||
new OA\Property(property: 'retention_days_s3', type: 'integer'),
|
||||
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float'),
|
||||
new OA\Property(property: 'timeout', type: 'integer'),
|
||||
new OA\Property(property: 'missing_backup_notification_days', type: 'integer'),
|
||||
],
|
||||
type: 'object',
|
||||
)]
|
||||
@@ -197,6 +199,7 @@ class VolumeBackupsController extends Controller
|
||||
'retention_days_s3' => 'integer|min:0|max:2147483647',
|
||||
'retention_max_storage_s3' => 'numeric|min:0|max:9999999999',
|
||||
'timeout' => 'integer|min:60|max:36000',
|
||||
'missing_backup_notification_days' => 'integer|min:0|max:365',
|
||||
]);
|
||||
$errors = $validator->errors();
|
||||
$allowedFields = [
|
||||
@@ -213,6 +216,7 @@ class VolumeBackupsController extends Controller
|
||||
'retention_days_s3',
|
||||
'retention_max_storage_s3',
|
||||
'timeout',
|
||||
'missing_backup_notification_days',
|
||||
];
|
||||
|
||||
foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) {
|
||||
@@ -279,6 +283,9 @@ class VolumeBackupsController extends Controller
|
||||
if ($request->has('timeout')) {
|
||||
$attributes['timeout'] = $request->integer('timeout');
|
||||
}
|
||||
if ($request->has('missing_backup_notification_days')) {
|
||||
$attributes['missing_backup_notification_days'] = $request->integer('missing_backup_notification_days');
|
||||
}
|
||||
|
||||
$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);
|
||||
$created = $backup->wasRecentlyCreated;
|
||||
@@ -445,6 +452,7 @@ class VolumeBackupsController extends Controller
|
||||
'retention_days_s3' => $backup->retention_days_s3,
|
||||
'retention_max_storage_s3' => $backup->retention_max_storage_s3,
|
||||
'timeout' => $backup->timeout,
|
||||
'missing_backup_notification_days' => $backup->missing_backup_notification_days ?? 0,
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Notifications\VolumeBackup\BackupMissing;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class CheckMissingVolumeBackupsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public $timeout = 1800;
|
||||
|
||||
/**
|
||||
* Releases the unique lock of a killed worker after one run (the timeout), so the next hourly run is not
|
||||
* blocked until the queue retry_after.
|
||||
*/
|
||||
public int $uniqueFor = 1800;
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
ScheduledVolumeBackup::query()
|
||||
->with(['team', 'latestExecution'])
|
||||
->where('enabled', true)
|
||||
->where('missing_backup_notification_days', '>', 0)
|
||||
->chunkById(100, function ($backups): void {
|
||||
foreach ($backups as $backup) {
|
||||
$this->notifyIfMissing($backup);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function notifyIfMissing(ScheduledVolumeBackup $backup): void
|
||||
{
|
||||
$lastExecutionAt = $backup->latestExecution?->created_at;
|
||||
$lastActivityAt = $lastExecutionAt ?? $backup->created_at;
|
||||
|
||||
if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $backup->team) {
|
||||
Log::warning("Cannot send missing volume backup notification for backup {$backup->id}: team not found");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if ($backup->team->getEnabledChannels('backup_failure') === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Send once for each period without backup activity.
|
||||
NotificationThrottle::sendOnce(
|
||||
$backup,
|
||||
BackupMissing::class,
|
||||
$lastActivityAt,
|
||||
fn () => $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,8 @@ class VolumeBackups extends Component
|
||||
|
||||
public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;
|
||||
|
||||
public int $missingBackupNotificationDays = 0;
|
||||
|
||||
public int $perPage = 10;
|
||||
|
||||
public function updatedPerPage(): void
|
||||
@@ -94,6 +96,7 @@ class VolumeBackups extends Component
|
||||
'retentionDaysS3' => ['required', 'integer', 'min:0'],
|
||||
'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
|
||||
'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
|
||||
'missingBackupNotificationDays' => ['required', 'integer', 'min:0', 'max:365'],
|
||||
];
|
||||
}
|
||||
|
||||
@@ -123,6 +126,7 @@ class VolumeBackups extends Component
|
||||
$this->retentionDaysS3 = $this->backup->retention_days_s3;
|
||||
$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
|
||||
$this->timeout = $this->backup->timeout;
|
||||
$this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;
|
||||
} else {
|
||||
$this->s3StorageId = $this->availableS3Storages->first()?->id;
|
||||
}
|
||||
@@ -435,6 +439,7 @@ class VolumeBackups extends Component
|
||||
'retention_days_s3' => $this->retentionDaysS3,
|
||||
'retention_max_storage_s3' => $this->retentionMaxStorageS3,
|
||||
'timeout' => $this->timeout,
|
||||
'missing_backup_notification_days' => $this->missingBackupNotificationDays,
|
||||
]);
|
||||
$changedFields = auditChangedFields($backup);
|
||||
$backup->save();
|
||||
|
||||
@@ -31,6 +31,7 @@ class ScheduledVolumeBackup extends BaseModel
|
||||
'retention_days_s3',
|
||||
'retention_max_storage_s3',
|
||||
'timeout',
|
||||
'missing_backup_notification_days',
|
||||
];
|
||||
|
||||
protected function casts(): array
|
||||
@@ -47,6 +48,7 @@ class ScheduledVolumeBackup extends BaseModel
|
||||
'retention_days_s3' => 'integer',
|
||||
'retention_max_storage_s3' => 'float',
|
||||
'timeout' => 'integer',
|
||||
'missing_backup_notification_days' => 'integer',
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ use App\Notifications\Container\ContainerRestarted;
|
||||
use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\Database\BackupMissing;
|
||||
use App\Notifications\Database\BackupSuccess;
|
||||
use App\Notifications\Database\BackupSuccessWithS3Warning;
|
||||
use App\Notifications\ScheduledTask\TaskFailed;
|
||||
use App\Notifications\ScheduledTask\TaskSuccess;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
@@ -20,6 +21,10 @@ use App\Notifications\Server\Reachable;
|
||||
use App\Notifications\Server\ServerPatchCheck;
|
||||
use App\Notifications\Server\TraefikVersionOutdated;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupMissing as VolumeBackupMissing;
|
||||
use App\Notifications\VolumeBackup\BackupSuccess as VolumeBackupSuccess;
|
||||
use App\Notifications\VolumeBackup\RecoveryFailed as VolumeBackupRecoveryFailed;
|
||||
|
||||
class TelegramChannel
|
||||
{
|
||||
@@ -41,8 +46,15 @@ class TelegramChannel
|
||||
RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id,
|
||||
|
||||
BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,
|
||||
VolumeBackupSuccess::class => $notification->warning
|
||||
? $settings->telegram_notifications_backup_failure_thread_id
|
||||
: $settings->telegram_notifications_backup_success_thread_id,
|
||||
BackupFailed::class,
|
||||
BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id,
|
||||
BackupMissing::class,
|
||||
BackupSuccessWithS3Warning::class,
|
||||
VolumeBackupFailed::class,
|
||||
VolumeBackupMissing::class,
|
||||
VolumeBackupRecoveryFailed::class => $settings->telegram_notifications_backup_failure_thread_id,
|
||||
|
||||
TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,
|
||||
TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
<?php
|
||||
|
||||
namespace App\Notifications\VolumeBackup;
|
||||
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ServiceApplication;
|
||||
use App\Models\ServiceDatabase;
|
||||
use App\Notifications\CustomEmailNotification;
|
||||
use App\Notifications\Dto\DiscordMessage;
|
||||
use App\Notifications\Dto\PushoverMessage;
|
||||
use App\Notifications\Dto\SlackMessage;
|
||||
use Illuminate\Notifications\Messages\MailMessage;
|
||||
use Illuminate\Support\Carbon;
|
||||
|
||||
class BackupMissing extends CustomEmailNotification
|
||||
{
|
||||
public string $resourceName;
|
||||
|
||||
public string $target;
|
||||
|
||||
public ?string $url = null;
|
||||
|
||||
public function __construct(public ScheduledVolumeBackup $backup, public ?Carbon $lastExecutionAt)
|
||||
{
|
||||
$this->onQueue('high');
|
||||
$resource = $backup->targetResource();
|
||||
$this->resourceName = $resource?->name ?? 'Unknown resource';
|
||||
$this->target = $backup->targetType().' '.$backup->targetName();
|
||||
|
||||
$linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;
|
||||
if ($linkable && method_exists($linkable, 'link')) {
|
||||
$this->url = $linkable->link();
|
||||
}
|
||||
}
|
||||
|
||||
public function via(object $notifiable): array
|
||||
{
|
||||
return $notifiable->getEnabledChannels('backup_failure');
|
||||
}
|
||||
|
||||
public function summary(): string
|
||||
{
|
||||
return "The enabled storage backup schedule for {$this->target} ({$this->resourceName}) has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s).";
|
||||
}
|
||||
|
||||
public function toMail(): MailMessage
|
||||
{
|
||||
return (new MailMessage)
|
||||
->subject("Coolify: [ACTION REQUIRED] No recent storage backup for {$this->resourceName}")
|
||||
->view('emails.volume-backup-missing', [
|
||||
'summary' => $this->summary(),
|
||||
'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
|
||||
'url' => $this->url,
|
||||
]);
|
||||
}
|
||||
|
||||
public function toDiscord(): DiscordMessage
|
||||
{
|
||||
$message = new DiscordMessage(
|
||||
title: ':warning: Scheduled storage backup missing',
|
||||
description: $this->summary(),
|
||||
color: DiscordMessage::errorColor(),
|
||||
isCritical: true,
|
||||
);
|
||||
|
||||
if ($this->url) {
|
||||
$message->addField('Resource', '[Link]('.$this->url.')');
|
||||
}
|
||||
|
||||
return $message;
|
||||
}
|
||||
|
||||
public function toTelegram(): array
|
||||
{
|
||||
$data = ['message' => "Coolify: {$this->summary()}"];
|
||||
|
||||
if ($this->url) {
|
||||
$data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function toPushover(): PushoverMessage
|
||||
{
|
||||
return new PushoverMessage(
|
||||
title: 'Scheduled storage backup missing',
|
||||
level: 'error',
|
||||
message: $this->summary(),
|
||||
buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],
|
||||
);
|
||||
}
|
||||
|
||||
public function toSlack(): SlackMessage
|
||||
{
|
||||
return new SlackMessage(
|
||||
title: 'Scheduled storage backup missing',
|
||||
description: $this->summary().($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),
|
||||
color: SlackMessage::errorColor(),
|
||||
);
|
||||
}
|
||||
|
||||
public function toWebhook(): array
|
||||
{
|
||||
return [
|
||||
'success' => false,
|
||||
'message' => 'Scheduled storage backup missing',
|
||||
'event' => 'volume_backup_missing',
|
||||
'resource_name' => $this->resourceName,
|
||||
'storage' => $this->target,
|
||||
'backup_uuid' => $this->backup->uuid,
|
||||
'days' => $this->backup->missing_backup_notification_days,
|
||||
'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
|
||||
'url' => $this->url,
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Events\BackupCreated;
|
||||
use App\Events\DockerCleanupDone;
|
||||
use App\Events\ScheduledTaskDone;
|
||||
use App\Jobs\DatabaseBackupJob;
|
||||
@@ -11,15 +12,19 @@ use App\Jobs\VolumeBackupJob;
|
||||
use App\Models\DockerCleanupExecution;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledDatabaseBackupExecution;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\ScheduledJobState;
|
||||
use App\Models\ScheduledTask;
|
||||
use App\Models\ScheduledTaskExecution;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\ScheduledTask\TaskFailed;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
|
||||
use Closure;
|
||||
use Cron\CronExpression;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
@@ -353,7 +358,7 @@ class ScheduledJobDeliveryService
|
||||
}
|
||||
|
||||
/**
|
||||
* Show a missed task or Docker cleanup as a failed execution and send the failure notification.
|
||||
* Show a missed task, Docker cleanup, or backup as a failed execution and send the failure notification.
|
||||
* The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,
|
||||
* and claim() rejects the late job if it starts afterwards.
|
||||
*/
|
||||
@@ -391,6 +396,38 @@ class ScheduledJobDeliveryService
|
||||
]);
|
||||
event(new DockerCleanupDone($execution));
|
||||
$this->notifyMissedOccurrence($server, fn () => $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}")));
|
||||
} elseif ($occurrence->job_type === 'database-backup') {
|
||||
$backup = ScheduledDatabaseBackup::with(['team', 'database'])->find($occurrence->resource_id);
|
||||
if (! $backup?->database) {
|
||||
return;
|
||||
}
|
||||
|
||||
$databaseName = $backup->dump_all ? 'all' : $backup->databases_to_backup;
|
||||
ScheduledDatabaseBackupExecution::create([
|
||||
'uuid' => new_public_id(),
|
||||
'database_name' => filled($databaseName) ? str($databaseName)->limit(250)->toString() : null,
|
||||
'scheduled_database_backup_id' => $backup->id,
|
||||
'status' => 'failed',
|
||||
'message' => $message,
|
||||
'local_storage_deleted' => false,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
BackupCreated::dispatch($backup->team_id);
|
||||
$this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new BackupFailed($backup, $backup->database, $message, $databaseName ?? 'unknown')));
|
||||
} elseif ($occurrence->job_type === 'volume-backup') {
|
||||
$backup = ScheduledVolumeBackup::with(['team', 'backupable.resource'])->find($occurrence->resource_id);
|
||||
if (! $backup) {
|
||||
return;
|
||||
}
|
||||
|
||||
ScheduledVolumeBackupExecution::create([
|
||||
'scheduled_volume_backup_id' => $backup->id,
|
||||
'status' => 'failed',
|
||||
'message' => $message,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
BackupCreated::dispatch($backup->team_id);
|
||||
$this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new VolumeBackupFailed($backup, $message)));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
|
||||
$table->unsignedInteger('missing_backup_notification_days')->default(0);
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
|
||||
$table->dropColumn('missing_backup_notification_days');
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
<x-emails.layout>
|
||||
{{ $summary }}
|
||||
|
||||
@if ($last_execution_at)
|
||||
The last execution was at {{ $last_execution_at }}.
|
||||
@else
|
||||
This schedule has never produced an execution.
|
||||
@endif
|
||||
|
||||
@if ($url)
|
||||
Click [here]({{ $url }}) to open the resource.
|
||||
@endif
|
||||
</x-emails.layout>
|
||||
@@ -44,6 +44,9 @@
|
||||
helper="Uses the backup server timezone, or the instance timezone when none is configured." required />
|
||||
<x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"
|
||||
helper="Maximum backup runtime in seconds." required />
|
||||
<x-forms.input label="Missing backup alert after" id="missingBackupNotificationDays" type="number"
|
||||
min="0" max="365" suffix="days"
|
||||
helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required />
|
||||
</div>
|
||||
</x-application.settings-section>
|
||||
</form>
|
||||
|
||||
@@ -101,9 +101,12 @@ it('sets an application volume backup schedule through the API', function () {
|
||||
'retention_days_s3' => 7,
|
||||
'retention_max_storage_s3' => 8.5,
|
||||
'timeout' => 600,
|
||||
'missing_backup_notification_days' => 5,
|
||||
]);
|
||||
|
||||
$response->assertCreated()->assertJsonStructure(['uuid', 'message']);
|
||||
$response->assertCreated()
|
||||
->assertJsonStructure(['uuid', 'message'])
|
||||
->assertJsonPath('missing_backup_notification_days', 5);
|
||||
|
||||
$backup = ScheduledVolumeBackup::query()->sole();
|
||||
expect($backup->backupable->is($this->volume))->toBeTrue()
|
||||
@@ -120,7 +123,20 @@ it('sets an application volume backup schedule through the API', function () {
|
||||
->and($backup->retention_amount_s3)->toBe(6)
|
||||
->and($backup->retention_days_s3)->toBe(7)
|
||||
->and($backup->retention_max_storage_s3)->toBe(8.5)
|
||||
->and($backup->timeout)->toBe(600);
|
||||
->and($backup->timeout)->toBe(600)
|
||||
->and($backup->missing_backup_notification_days)->toBe(5);
|
||||
});
|
||||
|
||||
it('rejects a missing backup alert period above one year through the API', function () {
|
||||
$this->withHeaders($this->headers)
|
||||
->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
|
||||
'frequency' => 'daily',
|
||||
'missing_backup_notification_days' => 366,
|
||||
])
|
||||
->assertUnprocessable()
|
||||
->assertJsonValidationErrors('missing_backup_notification_days');
|
||||
|
||||
expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
|
||||
});
|
||||
|
||||
it('updates the existing backup schedule instead of creating another one', function () {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CheckMissingVolumeBackupsJob;
|
||||
use App\Models\LocalPersistentVolume;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\VolumeBackup\BackupMissing;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
afterEach(fn () => Carbon::setTestNow());
|
||||
|
||||
function missingVolumeBackupSchedule(Team $team, array $attributes = []): ScheduledVolumeBackup
|
||||
{
|
||||
$backup = ScheduledVolumeBackup::create(array_merge([
|
||||
'backupable_type' => (new LocalPersistentVolume)->getMorphClass(),
|
||||
'backupable_id' => ScheduledVolumeBackup::query()->max('backupable_id') + 1000,
|
||||
'team_id' => $team->id,
|
||||
'frequency' => '0 0 * * *',
|
||||
'enabled' => true,
|
||||
'missing_backup_notification_days' => 2,
|
||||
], $attributes));
|
||||
|
||||
ScheduledVolumeBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]);
|
||||
|
||||
return $backup->refresh();
|
||||
}
|
||||
|
||||
function teamWithVolumeBackupFailureNotifications(): Team
|
||||
{
|
||||
$team = Team::create(['name' => 'Test team']);
|
||||
$team->emailNotificationSettings->update([
|
||||
'smtp_enabled' => true,
|
||||
'backup_failure_email_notifications' => true,
|
||||
]);
|
||||
|
||||
return $team;
|
||||
}
|
||||
|
||||
it('notifies the team when an enabled volume backup has no executions for the configured days', function () {
|
||||
Notification::fake();
|
||||
$team = teamWithVolumeBackupFailureNotifications();
|
||||
$backup = missingVolumeBackupSchedule($team);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)
|
||||
&& $notification->lastExecutionAt === null
|
||||
&& $notification->toWebhook()['event'] === 'volume_backup_missing');
|
||||
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue()
|
||||
->and((string) (new BackupMissing($backup, null))->toMail()->render())->toContain('This schedule has never produced an execution.');
|
||||
});
|
||||
|
||||
it('does not notify for recent, disabled or unconfigured volume backup schedules', function () {
|
||||
Notification::fake();
|
||||
$team = teamWithVolumeBackupFailureNotifications();
|
||||
missingVolumeBackupSchedule($team, ['enabled' => false]);
|
||||
missingVolumeBackupSchedule($team, ['missing_backup_notification_days' => 0]);
|
||||
$recentlyCreated = missingVolumeBackupSchedule($team);
|
||||
ScheduledVolumeBackup::whereKey($recentlyCreated->id)->update(['created_at' => now()->subDay()]);
|
||||
$recentlyExecuted = missingVolumeBackupSchedule($team);
|
||||
ScheduledVolumeBackupExecution::create([
|
||||
'scheduled_volume_backup_id' => $recentlyExecuted->id,
|
||||
'status' => 'success',
|
||||
]);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertNothingSent();
|
||||
});
|
||||
|
||||
it('does not notify when the team has no backup failure channels', function () {
|
||||
Notification::fake();
|
||||
$team = Team::create(['name' => 'Test team']);
|
||||
$backup = missingVolumeBackupSchedule($team);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertNothingSent();
|
||||
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
|
||||
});
|
||||
|
||||
it('notifies once per period without volume backup executions and rearms after another execution', function () {
|
||||
Notification::fake();
|
||||
$team = teamWithVolumeBackupFailureNotifications();
|
||||
$backup = missingVolumeBackupSchedule($team);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentToTimes($team, BackupMissing::class, 1);
|
||||
|
||||
Carbon::setTestNow(now()->addMinute());
|
||||
$execution = ScheduledVolumeBackupExecution::create([
|
||||
'scheduled_volume_backup_id' => $backup->id,
|
||||
'status' => 'success',
|
||||
]);
|
||||
Carbon::setTestNow(now()->addDays(3));
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentToTimes($team, BackupMissing::class, 2);
|
||||
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->lastExecutionAt?->equalTo($execution->created_at) === true);
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CheckMissingDatabaseBackupsJob;
|
||||
use App\Jobs\CheckMissingVolumeBackupsJob;
|
||||
use App\Jobs\CleanupInstanceStuffsJob;
|
||||
use App\Jobs\CleanupOrphanedPreviewContainersJob;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
@@ -14,6 +15,7 @@ dataset('scheduled instance jobs', [
|
||||
'orphaned preview containers cleanup' => [CleanupOrphanedPreviewContainersJob::class, 600],
|
||||
'instance cleanup' => [CleanupInstanceStuffsJob::class, 60],
|
||||
'missing database backups check' => [CheckMissingDatabaseBackupsJob::class, 1800],
|
||||
'missing volume backups check' => [CheckMissingVolumeBackupsJob::class, 1800],
|
||||
]);
|
||||
|
||||
it('does not queue the job again while its unique lock is held', function (string $jobClass, int $lockSeconds) {
|
||||
|
||||
@@ -13,6 +13,7 @@ use App\Models\InstanceSettings;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledDatabaseBackupExecution;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\ScheduledJobState;
|
||||
use App\Models\ScheduledTask;
|
||||
@@ -24,6 +25,7 @@ use App\Models\ServiceDatabase;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\ScheduledTask\TaskFailed;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
use App\Services\ScheduledJobDeliveryService;
|
||||
@@ -752,6 +754,31 @@ it('records a missed Docker cleanup as a failed execution and notifies the team
|
||||
Event::assertDispatched(DockerCleanupDone::class, 1);
|
||||
});
|
||||
|
||||
it('records a missed database backup that is too old to run late as a failed execution and notifies the team once', function () {
|
||||
config(['constants.coolify.self_hosted' => true]);
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
|
||||
Queue::fake();
|
||||
Notification::fake();
|
||||
Event::fake([BackupCreated::class]);
|
||||
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
|
||||
$backup->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);
|
||||
$occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'));
|
||||
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
|
||||
$executions = ScheduledDatabaseBackupExecution::query()->where('scheduled_database_backup_id', $backup->id)->get();
|
||||
expect($executions)->toHaveCount(1)
|
||||
->and($executions->first()->status)->toBe('failed')
|
||||
->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
|
||||
->and($executions->first()->finished_at)->not->toBeNull()
|
||||
->and($occurrence->fresh()->status)->toBe('failed');
|
||||
Queue::assertNotPushed(DatabaseBackupJob::class);
|
||||
Notification::assertSentToTimes($backup->team, BackupFailed::class, 1);
|
||||
Notification::assertSentTo($backup->team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
|
||||
Event::assertDispatched(BackupCreated::class);
|
||||
});
|
||||
|
||||
it('sends at most one missed-run notification per hour for the same schedule', function (string $jobType) {
|
||||
config(['constants.coolify.self_hosted' => true]);
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
|
||||
|
||||
@@ -2,11 +2,14 @@
|
||||
|
||||
use App\Jobs\SendMessageToTelegramJob;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Channels\TelegramChannel;
|
||||
use App\Notifications\Server\ServerPatchCheck;
|
||||
use App\Notifications\Server\TraefikVersionOutdated;
|
||||
use App\Notifications\VolumeBackup\BackupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupSuccess;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
@@ -87,3 +90,22 @@ it('falls back to the main chat when no topic is configured', function () {
|
||||
fn (SendMessageToTelegramJob $job) => $job->threadId === null
|
||||
);
|
||||
});
|
||||
|
||||
it('sends volume backup notifications to the backup topics', function (Closure $makeNotification, string $threadId) {
|
||||
$this->team->telegramNotificationSettings->update([
|
||||
'telegram_notifications_backup_success_thread_id' => '11',
|
||||
'telegram_notifications_backup_failure_thread_id' => '12',
|
||||
]);
|
||||
$backup = new ScheduledVolumeBackup(['frequency' => 'daily']);
|
||||
|
||||
(new TelegramChannel)->send($this->team->fresh(), $makeNotification($backup));
|
||||
|
||||
Queue::assertPushed(
|
||||
SendMessageToTelegramJob::class,
|
||||
fn (SendMessageToTelegramJob $job) => $job->threadId === $threadId
|
||||
);
|
||||
})->with([
|
||||
'success' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup), '11'],
|
||||
'success with warning' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup, 'S3 upload failed'), '12'],
|
||||
'failed' => [fn (ScheduledVolumeBackup $backup) => new BackupFailed($backup, 'Archive command failed'), '12'],
|
||||
]);
|
||||
|
||||
@@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
@@ -29,6 +30,7 @@ use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\VolumeBackup\BackupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupSuccess;
|
||||
use App\Services\ScheduledJobDeliveryService;
|
||||
use Illuminate\Database\Eloquent\Collection;
|
||||
use Illuminate\Filesystem\FilesystemAdapter;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
@@ -1142,6 +1144,19 @@ it('does not change S3 storage when another volume backup setting is invalid', f
|
||||
->and($backup->frequency)->toBe('daily');
|
||||
});
|
||||
|
||||
it('rejects a missing volume backup alert period above one year', function () {
|
||||
$team = Team::factory()->create();
|
||||
signInForVolumeBackups($this, $team);
|
||||
[$application, $volume] = createVolumeBackupApplication($team);
|
||||
|
||||
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
|
||||
->set('missingBackupNotificationDays', 366)
|
||||
->call('save')
|
||||
->assertHasErrors(['missingBackupNotificationDays' => 'max']);
|
||||
|
||||
expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
|
||||
});
|
||||
|
||||
function createVolumeBackupApplication(Team $team): array
|
||||
{
|
||||
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
|
||||
@@ -1208,12 +1223,14 @@ it('creates a local scheduled backup for a persistent volume', function () {
|
||||
->set('retentionDaysLocally', 14)
|
||||
->set('retentionMaxStorageLocally', 1.5)
|
||||
->set('stopDuringBackup', true)
|
||||
->set('missingBackupNotificationDays', 3)
|
||||
->call('save')
|
||||
->assertDispatched('success');
|
||||
|
||||
$backup = ScheduledVolumeBackup::query()->sole();
|
||||
|
||||
expect($backup->backupable->is($volume))->toBeTrue()
|
||||
->and($backup->missing_backup_notification_days)->toBe(3)
|
||||
->and($backup->team_id)->toBe($team->id)
|
||||
->and($backup->frequency)->toBe('daily')
|
||||
->and($backup->retention_amount_locally)->toBe(5)
|
||||
@@ -2850,3 +2867,37 @@ it('sends volume backup success with a warning through the backup failure channe
|
||||
'without warning' => [null, 'backup_success'],
|
||||
'with warning' => ['S3 upload failed: denied', 'backup_failure'],
|
||||
]);
|
||||
|
||||
it('records a missed volume backup that is too old to run late as a failed execution and notifies the team once', function () {
|
||||
config(['broadcasting.default' => 'null']);
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
|
||||
Queue::fake();
|
||||
Notification::fake();
|
||||
$team = Team::factory()->create();
|
||||
$team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]);
|
||||
[$application, $volume] = createVolumeBackupApplication($team);
|
||||
$backup = $volume->scheduledBackups()->create([
|
||||
'team_id' => $team->id,
|
||||
'frequency' => 'daily',
|
||||
]);
|
||||
$occurrence = ScheduledJobDelivery::create([
|
||||
'schedule_key' => "scheduled-volume-backup:{$backup->id}",
|
||||
'scheduled_for' => Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'),
|
||||
'job_type' => 'volume-backup',
|
||||
'resource_id' => $backup->id,
|
||||
'status' => 'enqueued',
|
||||
'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),
|
||||
]);
|
||||
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
|
||||
$execution = ScheduledVolumeBackupExecution::query()->sole();
|
||||
expect($execution->status)->toBe('failed')
|
||||
->and($execution->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
|
||||
->and($execution->finished_at)->not->toBeNull()
|
||||
->and($occurrence->fresh()->status)->toBe('failed');
|
||||
Queue::assertNotPushed(VolumeBackupJob::class);
|
||||
Notification::assertSentToTimes($team, BackupFailed::class, 1);
|
||||
Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user