mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 13:10:37 -04:00
Merge remote-tracking branch 'origin/main' into next
This commit is contained in:
@@ -6,6 +6,7 @@ use App\Jobs\ApiTokenExpirationWarningJob;
|
||||
use App\Jobs\CheckForUpdatesJob;
|
||||
use App\Jobs\CheckHelperImageJob;
|
||||
use App\Jobs\CheckMissingDatabaseBackupsJob;
|
||||
use App\Jobs\CheckMissingVolumeBackupsJob;
|
||||
use App\Jobs\CleanupInstanceStuffsJob;
|
||||
use App\Jobs\CleanupOrphanedPreviewContainersJob;
|
||||
use App\Jobs\CleanupStaleMultiplexedConnections;
|
||||
@@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel
|
||||
->runInBackground();
|
||||
$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new CheckMissingVolumeBackupsJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();
|
||||
$this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer();
|
||||
|
||||
|
||||
@@ -35,13 +35,14 @@ use RuntimeException;
|
||||
new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
|
||||
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
|
||||
new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000),
|
||||
new OA\Property(property: 'missing_backup_notification_days', type: 'integer', description: 'Alert after this many days without an execution; 0 disables alerts', default: 0, minimum: 0, maximum: 365),
|
||||
],
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
)]
|
||||
#[OA\Schema(
|
||||
schema: 'VolumeBackupScheduleResponse',
|
||||
required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout'],
|
||||
required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout', 'missing_backup_notification_days'],
|
||||
properties: [
|
||||
new OA\Property(property: 'uuid', type: 'string'),
|
||||
new OA\Property(property: 'message', type: 'string'),
|
||||
@@ -60,6 +61,7 @@ use RuntimeException;
|
||||
new OA\Property(property: 'retention_days_s3', type: 'integer'),
|
||||
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float'),
|
||||
new OA\Property(property: 'timeout', type: 'integer'),
|
||||
new OA\Property(property: 'missing_backup_notification_days', type: 'integer'),
|
||||
],
|
||||
type: 'object',
|
||||
)]
|
||||
@@ -197,6 +199,7 @@ class VolumeBackupsController extends Controller
|
||||
'retention_days_s3' => 'integer|min:0|max:2147483647',
|
||||
'retention_max_storage_s3' => 'numeric|min:0|max:9999999999',
|
||||
'timeout' => 'integer|min:60|max:36000',
|
||||
'missing_backup_notification_days' => 'integer|min:0|max:365',
|
||||
]);
|
||||
$errors = $validator->errors();
|
||||
$allowedFields = [
|
||||
@@ -213,6 +216,7 @@ class VolumeBackupsController extends Controller
|
||||
'retention_days_s3',
|
||||
'retention_max_storage_s3',
|
||||
'timeout',
|
||||
'missing_backup_notification_days',
|
||||
];
|
||||
|
||||
foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) {
|
||||
@@ -279,6 +283,9 @@ class VolumeBackupsController extends Controller
|
||||
if ($request->has('timeout')) {
|
||||
$attributes['timeout'] = $request->integer('timeout');
|
||||
}
|
||||
if ($request->has('missing_backup_notification_days')) {
|
||||
$attributes['missing_backup_notification_days'] = $request->integer('missing_backup_notification_days');
|
||||
}
|
||||
|
||||
$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);
|
||||
$created = $backup->wasRecentlyCreated;
|
||||
@@ -445,6 +452,7 @@ class VolumeBackupsController extends Controller
|
||||
'retention_days_s3' => $backup->retention_days_s3,
|
||||
'retention_max_storage_s3' => $backup->retention_max_storage_s3,
|
||||
'timeout' => $backup->timeout,
|
||||
'missing_backup_notification_days' => $backup->missing_backup_notification_days ?? 0,
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Notifications\VolumeBackup\BackupMissing;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class CheckMissingVolumeBackupsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public $timeout = 1800;
|
||||
|
||||
/**
|
||||
* Releases the unique lock of a killed worker after one run (the timeout), so the next hourly run is not
|
||||
* blocked until the queue retry_after.
|
||||
*/
|
||||
public int $uniqueFor = 1800;
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
ScheduledVolumeBackup::query()
|
||||
->with(['team', 'latestExecution'])
|
||||
->where('enabled', true)
|
||||
->where('missing_backup_notification_days', '>', 0)
|
||||
->chunkById(100, function ($backups): void {
|
||||
foreach ($backups as $backup) {
|
||||
$this->notifyIfMissing($backup);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function notifyIfMissing(ScheduledVolumeBackup $backup): void
|
||||
{
|
||||
$lastExecutionAt = $backup->latestExecution?->created_at;
|
||||
$lastActivityAt = $lastExecutionAt ?? $backup->created_at;
|
||||
|
||||
if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $backup->team) {
|
||||
Log::warning("Cannot send missing volume backup notification for backup {$backup->id}: team not found");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if ($backup->team->getEnabledChannels('backup_failure') === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Send once for each period without backup activity.
|
||||
NotificationThrottle::sendOnce(
|
||||
$backup,
|
||||
BackupMissing::class,
|
||||
$lastActivityAt,
|
||||
fn () => $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,8 @@ class VolumeBackups extends Component
|
||||
|
||||
public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;
|
||||
|
||||
public int $missingBackupNotificationDays = 0;
|
||||
|
||||
public int $perPage = 10;
|
||||
|
||||
public function updatedPerPage(): void
|
||||
@@ -94,6 +96,7 @@ class VolumeBackups extends Component
|
||||
'retentionDaysS3' => ['required', 'integer', 'min:0'],
|
||||
'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
|
||||
'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
|
||||
'missingBackupNotificationDays' => ['required', 'integer', 'min:0', 'max:365'],
|
||||
];
|
||||
}
|
||||
|
||||
@@ -123,6 +126,7 @@ class VolumeBackups extends Component
|
||||
$this->retentionDaysS3 = $this->backup->retention_days_s3;
|
||||
$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
|
||||
$this->timeout = $this->backup->timeout;
|
||||
$this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;
|
||||
} else {
|
||||
$this->s3StorageId = $this->availableS3Storages->first()?->id;
|
||||
}
|
||||
@@ -435,6 +439,7 @@ class VolumeBackups extends Component
|
||||
'retention_days_s3' => $this->retentionDaysS3,
|
||||
'retention_max_storage_s3' => $this->retentionMaxStorageS3,
|
||||
'timeout' => $this->timeout,
|
||||
'missing_backup_notification_days' => $this->missingBackupNotificationDays,
|
||||
]);
|
||||
$changedFields = auditChangedFields($backup);
|
||||
$backup->save();
|
||||
|
||||
@@ -31,6 +31,7 @@ class ScheduledVolumeBackup extends BaseModel
|
||||
'retention_days_s3',
|
||||
'retention_max_storage_s3',
|
||||
'timeout',
|
||||
'missing_backup_notification_days',
|
||||
];
|
||||
|
||||
protected function casts(): array
|
||||
@@ -47,6 +48,7 @@ class ScheduledVolumeBackup extends BaseModel
|
||||
'retention_days_s3' => 'integer',
|
||||
'retention_max_storage_s3' => 'float',
|
||||
'timeout' => 'integer',
|
||||
'missing_backup_notification_days' => 'integer',
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ use App\Notifications\Container\ContainerRestarted;
|
||||
use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\Database\BackupMissing;
|
||||
use App\Notifications\Database\BackupSuccess;
|
||||
use App\Notifications\Database\BackupSuccessWithS3Warning;
|
||||
use App\Notifications\ScheduledTask\TaskFailed;
|
||||
use App\Notifications\ScheduledTask\TaskSuccess;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
@@ -20,6 +21,10 @@ use App\Notifications\Server\Reachable;
|
||||
use App\Notifications\Server\ServerPatchCheck;
|
||||
use App\Notifications\Server\TraefikVersionOutdated;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupMissing as VolumeBackupMissing;
|
||||
use App\Notifications\VolumeBackup\BackupSuccess as VolumeBackupSuccess;
|
||||
use App\Notifications\VolumeBackup\RecoveryFailed as VolumeBackupRecoveryFailed;
|
||||
|
||||
class TelegramChannel
|
||||
{
|
||||
@@ -41,8 +46,15 @@ class TelegramChannel
|
||||
RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id,
|
||||
|
||||
BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,
|
||||
VolumeBackupSuccess::class => $notification->warning
|
||||
? $settings->telegram_notifications_backup_failure_thread_id
|
||||
: $settings->telegram_notifications_backup_success_thread_id,
|
||||
BackupFailed::class,
|
||||
BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id,
|
||||
BackupMissing::class,
|
||||
BackupSuccessWithS3Warning::class,
|
||||
VolumeBackupFailed::class,
|
||||
VolumeBackupMissing::class,
|
||||
VolumeBackupRecoveryFailed::class => $settings->telegram_notifications_backup_failure_thread_id,
|
||||
|
||||
TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,
|
||||
TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
<?php
|
||||
|
||||
namespace App\Notifications\VolumeBackup;
|
||||
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ServiceApplication;
|
||||
use App\Models\ServiceDatabase;
|
||||
use App\Notifications\CustomEmailNotification;
|
||||
use App\Notifications\Dto\DiscordMessage;
|
||||
use App\Notifications\Dto\PushoverMessage;
|
||||
use App\Notifications\Dto\SlackMessage;
|
||||
use Illuminate\Notifications\Messages\MailMessage;
|
||||
use Illuminate\Support\Carbon;
|
||||
|
||||
class BackupMissing extends CustomEmailNotification
|
||||
{
|
||||
public string $resourceName;
|
||||
|
||||
public string $target;
|
||||
|
||||
public ?string $url = null;
|
||||
|
||||
public function __construct(public ScheduledVolumeBackup $backup, public ?Carbon $lastExecutionAt)
|
||||
{
|
||||
$this->onQueue('high');
|
||||
$resource = $backup->targetResource();
|
||||
$this->resourceName = $resource?->name ?? 'Unknown resource';
|
||||
$this->target = $backup->targetType().' '.$backup->targetName();
|
||||
|
||||
$linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;
|
||||
if ($linkable && method_exists($linkable, 'link')) {
|
||||
$this->url = $linkable->link();
|
||||
}
|
||||
}
|
||||
|
||||
public function via(object $notifiable): array
|
||||
{
|
||||
return $notifiable->getEnabledChannels('backup_failure');
|
||||
}
|
||||
|
||||
public function summary(): string
|
||||
{
|
||||
return "The enabled storage backup schedule for {$this->target} ({$this->resourceName}) has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s).";
|
||||
}
|
||||
|
||||
public function toMail(): MailMessage
|
||||
{
|
||||
return (new MailMessage)
|
||||
->subject("Coolify: [ACTION REQUIRED] No recent storage backup for {$this->resourceName}")
|
||||
->view('emails.volume-backup-missing', [
|
||||
'summary' => $this->summary(),
|
||||
'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
|
||||
'url' => $this->url,
|
||||
]);
|
||||
}
|
||||
|
||||
public function toDiscord(): DiscordMessage
|
||||
{
|
||||
$message = new DiscordMessage(
|
||||
title: ':warning: Scheduled storage backup missing',
|
||||
description: $this->summary(),
|
||||
color: DiscordMessage::errorColor(),
|
||||
isCritical: true,
|
||||
);
|
||||
|
||||
if ($this->url) {
|
||||
$message->addField('Resource', '[Link]('.$this->url.')');
|
||||
}
|
||||
|
||||
return $message;
|
||||
}
|
||||
|
||||
public function toTelegram(): array
|
||||
{
|
||||
$data = ['message' => "Coolify: {$this->summary()}"];
|
||||
|
||||
if ($this->url) {
|
||||
$data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function toPushover(): PushoverMessage
|
||||
{
|
||||
return new PushoverMessage(
|
||||
title: 'Scheduled storage backup missing',
|
||||
level: 'error',
|
||||
message: $this->summary(),
|
||||
buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],
|
||||
);
|
||||
}
|
||||
|
||||
public function toSlack(): SlackMessage
|
||||
{
|
||||
return new SlackMessage(
|
||||
title: 'Scheduled storage backup missing',
|
||||
description: $this->summary().($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),
|
||||
color: SlackMessage::errorColor(),
|
||||
);
|
||||
}
|
||||
|
||||
public function toWebhook(): array
|
||||
{
|
||||
return [
|
||||
'success' => false,
|
||||
'message' => 'Scheduled storage backup missing',
|
||||
'event' => 'volume_backup_missing',
|
||||
'resource_name' => $this->resourceName,
|
||||
'storage' => $this->target,
|
||||
'backup_uuid' => $this->backup->uuid,
|
||||
'days' => $this->backup->missing_backup_notification_days,
|
||||
'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
|
||||
'url' => $this->url,
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Events\BackupCreated;
|
||||
use App\Events\DockerCleanupDone;
|
||||
use App\Events\ScheduledTaskDone;
|
||||
use App\Jobs\DatabaseBackupJob;
|
||||
@@ -11,15 +12,19 @@ use App\Jobs\VolumeBackupJob;
|
||||
use App\Models\DockerCleanupExecution;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledDatabaseBackupExecution;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\ScheduledJobState;
|
||||
use App\Models\ScheduledTask;
|
||||
use App\Models\ScheduledTaskExecution;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\ScheduledTask\TaskFailed;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
|
||||
use Closure;
|
||||
use Cron\CronExpression;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
@@ -353,7 +358,7 @@ class ScheduledJobDeliveryService
|
||||
}
|
||||
|
||||
/**
|
||||
* Show a missed task or Docker cleanup as a failed execution and send the failure notification.
|
||||
* Show a missed task, Docker cleanup, or backup as a failed execution and send the failure notification.
|
||||
* The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,
|
||||
* and claim() rejects the late job if it starts afterwards.
|
||||
*/
|
||||
@@ -391,6 +396,38 @@ class ScheduledJobDeliveryService
|
||||
]);
|
||||
event(new DockerCleanupDone($execution));
|
||||
$this->notifyMissedOccurrence($server, fn () => $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}")));
|
||||
} elseif ($occurrence->job_type === 'database-backup') {
|
||||
$backup = ScheduledDatabaseBackup::with(['team', 'database'])->find($occurrence->resource_id);
|
||||
if (! $backup?->database) {
|
||||
return;
|
||||
}
|
||||
|
||||
$databaseName = $backup->dump_all ? 'all' : $backup->databases_to_backup;
|
||||
ScheduledDatabaseBackupExecution::create([
|
||||
'uuid' => new_public_id(),
|
||||
'database_name' => filled($databaseName) ? str($databaseName)->limit(250)->toString() : null,
|
||||
'scheduled_database_backup_id' => $backup->id,
|
||||
'status' => 'failed',
|
||||
'message' => $message,
|
||||
'local_storage_deleted' => false,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
BackupCreated::dispatch($backup->team_id);
|
||||
$this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new BackupFailed($backup, $backup->database, $message, $databaseName ?? 'unknown')));
|
||||
} elseif ($occurrence->job_type === 'volume-backup') {
|
||||
$backup = ScheduledVolumeBackup::with(['team', 'backupable.resource'])->find($occurrence->resource_id);
|
||||
if (! $backup) {
|
||||
return;
|
||||
}
|
||||
|
||||
ScheduledVolumeBackupExecution::create([
|
||||
'scheduled_volume_backup_id' => $backup->id,
|
||||
'status' => 'failed',
|
||||
'message' => $message,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
BackupCreated::dispatch($backup->team_id);
|
||||
$this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new VolumeBackupFailed($backup, $message)));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
|
||||
$table->unsignedInteger('missing_backup_notification_days')->default(0);
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
|
||||
$table->dropColumn('missing_backup_notification_days');
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
<x-emails.layout>
|
||||
{{ $summary }}
|
||||
|
||||
@if ($last_execution_at)
|
||||
The last execution was at {{ $last_execution_at }}.
|
||||
@else
|
||||
This schedule has never produced an execution.
|
||||
@endif
|
||||
|
||||
@if ($url)
|
||||
Click [here]({{ $url }}) to open the resource.
|
||||
@endif
|
||||
</x-emails.layout>
|
||||
@@ -44,6 +44,9 @@
|
||||
helper="Uses the backup server timezone, or the instance timezone when none is configured." required />
|
||||
<x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"
|
||||
helper="Maximum backup runtime in seconds." required />
|
||||
<x-forms.input label="Missing backup alert after" id="missingBackupNotificationDays" type="number"
|
||||
min="0" max="365" suffix="days"
|
||||
helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required />
|
||||
</div>
|
||||
</x-application.settings-section>
|
||||
</form>
|
||||
|
||||
@@ -101,9 +101,12 @@ it('sets an application volume backup schedule through the API', function () {
|
||||
'retention_days_s3' => 7,
|
||||
'retention_max_storage_s3' => 8.5,
|
||||
'timeout' => 600,
|
||||
'missing_backup_notification_days' => 5,
|
||||
]);
|
||||
|
||||
$response->assertCreated()->assertJsonStructure(['uuid', 'message']);
|
||||
$response->assertCreated()
|
||||
->assertJsonStructure(['uuid', 'message'])
|
||||
->assertJsonPath('missing_backup_notification_days', 5);
|
||||
|
||||
$backup = ScheduledVolumeBackup::query()->sole();
|
||||
expect($backup->backupable->is($this->volume))->toBeTrue()
|
||||
@@ -120,7 +123,20 @@ it('sets an application volume backup schedule through the API', function () {
|
||||
->and($backup->retention_amount_s3)->toBe(6)
|
||||
->and($backup->retention_days_s3)->toBe(7)
|
||||
->and($backup->retention_max_storage_s3)->toBe(8.5)
|
||||
->and($backup->timeout)->toBe(600);
|
||||
->and($backup->timeout)->toBe(600)
|
||||
->and($backup->missing_backup_notification_days)->toBe(5);
|
||||
});
|
||||
|
||||
it('rejects a missing backup alert period above one year through the API', function () {
|
||||
$this->withHeaders($this->headers)
|
||||
->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
|
||||
'frequency' => 'daily',
|
||||
'missing_backup_notification_days' => 366,
|
||||
])
|
||||
->assertUnprocessable()
|
||||
->assertJsonValidationErrors('missing_backup_notification_days');
|
||||
|
||||
expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
|
||||
});
|
||||
|
||||
it('updates the existing backup schedule instead of creating another one', function () {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CheckMissingVolumeBackupsJob;
|
||||
use App\Models\LocalPersistentVolume;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\VolumeBackup\BackupMissing;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
afterEach(fn () => Carbon::setTestNow());
|
||||
|
||||
function missingVolumeBackupSchedule(Team $team, array $attributes = []): ScheduledVolumeBackup
|
||||
{
|
||||
$backup = ScheduledVolumeBackup::create(array_merge([
|
||||
'backupable_type' => (new LocalPersistentVolume)->getMorphClass(),
|
||||
'backupable_id' => ScheduledVolumeBackup::query()->max('backupable_id') + 1000,
|
||||
'team_id' => $team->id,
|
||||
'frequency' => '0 0 * * *',
|
||||
'enabled' => true,
|
||||
'missing_backup_notification_days' => 2,
|
||||
], $attributes));
|
||||
|
||||
ScheduledVolumeBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]);
|
||||
|
||||
return $backup->refresh();
|
||||
}
|
||||
|
||||
function teamWithVolumeBackupFailureNotifications(): Team
|
||||
{
|
||||
$team = Team::create(['name' => 'Test team']);
|
||||
$team->emailNotificationSettings->update([
|
||||
'smtp_enabled' => true,
|
||||
'backup_failure_email_notifications' => true,
|
||||
]);
|
||||
|
||||
return $team;
|
||||
}
|
||||
|
||||
it('notifies the team when an enabled volume backup has no executions for the configured days', function () {
|
||||
Notification::fake();
|
||||
$team = teamWithVolumeBackupFailureNotifications();
|
||||
$backup = missingVolumeBackupSchedule($team);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)
|
||||
&& $notification->lastExecutionAt === null
|
||||
&& $notification->toWebhook()['event'] === 'volume_backup_missing');
|
||||
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue()
|
||||
->and((string) (new BackupMissing($backup, null))->toMail()->render())->toContain('This schedule has never produced an execution.');
|
||||
});
|
||||
|
||||
it('does not notify for recent, disabled or unconfigured volume backup schedules', function () {
|
||||
Notification::fake();
|
||||
$team = teamWithVolumeBackupFailureNotifications();
|
||||
missingVolumeBackupSchedule($team, ['enabled' => false]);
|
||||
missingVolumeBackupSchedule($team, ['missing_backup_notification_days' => 0]);
|
||||
$recentlyCreated = missingVolumeBackupSchedule($team);
|
||||
ScheduledVolumeBackup::whereKey($recentlyCreated->id)->update(['created_at' => now()->subDay()]);
|
||||
$recentlyExecuted = missingVolumeBackupSchedule($team);
|
||||
ScheduledVolumeBackupExecution::create([
|
||||
'scheduled_volume_backup_id' => $recentlyExecuted->id,
|
||||
'status' => 'success',
|
||||
]);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertNothingSent();
|
||||
});
|
||||
|
||||
it('does not notify when the team has no backup failure channels', function () {
|
||||
Notification::fake();
|
||||
$team = Team::create(['name' => 'Test team']);
|
||||
$backup = missingVolumeBackupSchedule($team);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertNothingSent();
|
||||
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
|
||||
});
|
||||
|
||||
it('notifies once per period without volume backup executions and rearms after another execution', function () {
|
||||
Notification::fake();
|
||||
$team = teamWithVolumeBackupFailureNotifications();
|
||||
$backup = missingVolumeBackupSchedule($team);
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentToTimes($team, BackupMissing::class, 1);
|
||||
|
||||
Carbon::setTestNow(now()->addMinute());
|
||||
$execution = ScheduledVolumeBackupExecution::create([
|
||||
'scheduled_volume_backup_id' => $backup->id,
|
||||
'status' => 'success',
|
||||
]);
|
||||
Carbon::setTestNow(now()->addDays(3));
|
||||
|
||||
(new CheckMissingVolumeBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentToTimes($team, BackupMissing::class, 2);
|
||||
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->lastExecutionAt?->equalTo($execution->created_at) === true);
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CheckMissingDatabaseBackupsJob;
|
||||
use App\Jobs\CheckMissingVolumeBackupsJob;
|
||||
use App\Jobs\CleanupInstanceStuffsJob;
|
||||
use App\Jobs\CleanupOrphanedPreviewContainersJob;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
@@ -14,6 +15,7 @@ dataset('scheduled instance jobs', [
|
||||
'orphaned preview containers cleanup' => [CleanupOrphanedPreviewContainersJob::class, 600],
|
||||
'instance cleanup' => [CleanupInstanceStuffsJob::class, 60],
|
||||
'missing database backups check' => [CheckMissingDatabaseBackupsJob::class, 1800],
|
||||
'missing volume backups check' => [CheckMissingVolumeBackupsJob::class, 1800],
|
||||
]);
|
||||
|
||||
it('does not queue the job again while its unique lock is held', function (string $jobClass, int $lockSeconds) {
|
||||
|
||||
@@ -13,6 +13,7 @@ use App\Models\InstanceSettings;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledDatabaseBackupExecution;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\ScheduledJobState;
|
||||
use App\Models\ScheduledTask;
|
||||
@@ -24,6 +25,7 @@ use App\Models\ServiceDatabase;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Database\BackupFailed;
|
||||
use App\Notifications\ScheduledTask\TaskFailed;
|
||||
use App\Notifications\Server\DockerCleanupFailed;
|
||||
use App\Services\ScheduledJobDeliveryService;
|
||||
@@ -752,6 +754,31 @@ it('records a missed Docker cleanup as a failed execution and notifies the team
|
||||
Event::assertDispatched(DockerCleanupDone::class, 1);
|
||||
});
|
||||
|
||||
it('records a missed database backup that is too old to run late as a failed execution and notifies the team once', function () {
|
||||
config(['constants.coolify.self_hosted' => true]);
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
|
||||
Queue::fake();
|
||||
Notification::fake();
|
||||
Event::fake([BackupCreated::class]);
|
||||
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
|
||||
$backup->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);
|
||||
$occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'));
|
||||
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
|
||||
$executions = ScheduledDatabaseBackupExecution::query()->where('scheduled_database_backup_id', $backup->id)->get();
|
||||
expect($executions)->toHaveCount(1)
|
||||
->and($executions->first()->status)->toBe('failed')
|
||||
->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
|
||||
->and($executions->first()->finished_at)->not->toBeNull()
|
||||
->and($occurrence->fresh()->status)->toBe('failed');
|
||||
Queue::assertNotPushed(DatabaseBackupJob::class);
|
||||
Notification::assertSentToTimes($backup->team, BackupFailed::class, 1);
|
||||
Notification::assertSentTo($backup->team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
|
||||
Event::assertDispatched(BackupCreated::class);
|
||||
});
|
||||
|
||||
it('sends at most one missed-run notification per hour for the same schedule', function (string $jobType) {
|
||||
config(['constants.coolify.self_hosted' => true]);
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
|
||||
|
||||
@@ -2,11 +2,14 @@
|
||||
|
||||
use App\Jobs\SendMessageToTelegramJob;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Channels\TelegramChannel;
|
||||
use App\Notifications\Server\ServerPatchCheck;
|
||||
use App\Notifications\Server\TraefikVersionOutdated;
|
||||
use App\Notifications\VolumeBackup\BackupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupSuccess;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
@@ -87,3 +90,22 @@ it('falls back to the main chat when no topic is configured', function () {
|
||||
fn (SendMessageToTelegramJob $job) => $job->threadId === null
|
||||
);
|
||||
});
|
||||
|
||||
it('sends volume backup notifications to the backup topics', function (Closure $makeNotification, string $threadId) {
|
||||
$this->team->telegramNotificationSettings->update([
|
||||
'telegram_notifications_backup_success_thread_id' => '11',
|
||||
'telegram_notifications_backup_failure_thread_id' => '12',
|
||||
]);
|
||||
$backup = new ScheduledVolumeBackup(['frequency' => 'daily']);
|
||||
|
||||
(new TelegramChannel)->send($this->team->fresh(), $makeNotification($backup));
|
||||
|
||||
Queue::assertPushed(
|
||||
SendMessageToTelegramJob::class,
|
||||
fn (SendMessageToTelegramJob $job) => $job->threadId === $threadId
|
||||
);
|
||||
})->with([
|
||||
'success' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup), '11'],
|
||||
'success with warning' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup, 'S3 upload failed'), '12'],
|
||||
'failed' => [fn (ScheduledVolumeBackup $backup) => new BackupFailed($backup, 'Archive command failed'), '12'],
|
||||
]);
|
||||
|
||||
@@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\ScheduledJobDelivery;
|
||||
use App\Models\ScheduledVolumeBackup;
|
||||
use App\Models\ScheduledVolumeBackupExecution;
|
||||
use App\Models\Server;
|
||||
@@ -29,6 +30,7 @@ use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\VolumeBackup\BackupFailed;
|
||||
use App\Notifications\VolumeBackup\BackupSuccess;
|
||||
use App\Services\ScheduledJobDeliveryService;
|
||||
use Illuminate\Database\Eloquent\Collection;
|
||||
use Illuminate\Filesystem\FilesystemAdapter;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
@@ -1142,6 +1144,19 @@ it('does not change S3 storage when another volume backup setting is invalid', f
|
||||
->and($backup->frequency)->toBe('daily');
|
||||
});
|
||||
|
||||
it('rejects a missing volume backup alert period above one year', function () {
|
||||
$team = Team::factory()->create();
|
||||
signInForVolumeBackups($this, $team);
|
||||
[$application, $volume] = createVolumeBackupApplication($team);
|
||||
|
||||
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
|
||||
->set('missingBackupNotificationDays', 366)
|
||||
->call('save')
|
||||
->assertHasErrors(['missingBackupNotificationDays' => 'max']);
|
||||
|
||||
expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
|
||||
});
|
||||
|
||||
function createVolumeBackupApplication(Team $team): array
|
||||
{
|
||||
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
|
||||
@@ -1208,12 +1223,14 @@ it('creates a local scheduled backup for a persistent volume', function () {
|
||||
->set('retentionDaysLocally', 14)
|
||||
->set('retentionMaxStorageLocally', 1.5)
|
||||
->set('stopDuringBackup', true)
|
||||
->set('missingBackupNotificationDays', 3)
|
||||
->call('save')
|
||||
->assertDispatched('success');
|
||||
|
||||
$backup = ScheduledVolumeBackup::query()->sole();
|
||||
|
||||
expect($backup->backupable->is($volume))->toBeTrue()
|
||||
->and($backup->missing_backup_notification_days)->toBe(3)
|
||||
->and($backup->team_id)->toBe($team->id)
|
||||
->and($backup->frequency)->toBe('daily')
|
||||
->and($backup->retention_amount_locally)->toBe(5)
|
||||
@@ -2850,3 +2867,37 @@ it('sends volume backup success with a warning through the backup failure channe
|
||||
'without warning' => [null, 'backup_success'],
|
||||
'with warning' => ['S3 upload failed: denied', 'backup_failure'],
|
||||
]);
|
||||
|
||||
it('records a missed volume backup that is too old to run late as a failed execution and notifies the team once', function () {
|
||||
config(['broadcasting.default' => 'null']);
|
||||
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
|
||||
Queue::fake();
|
||||
Notification::fake();
|
||||
$team = Team::factory()->create();
|
||||
$team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]);
|
||||
[$application, $volume] = createVolumeBackupApplication($team);
|
||||
$backup = $volume->scheduledBackups()->create([
|
||||
'team_id' => $team->id,
|
||||
'frequency' => 'daily',
|
||||
]);
|
||||
$occurrence = ScheduledJobDelivery::create([
|
||||
'schedule_key' => "scheduled-volume-backup:{$backup->id}",
|
||||
'scheduled_for' => Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'),
|
||||
'job_type' => 'volume-backup',
|
||||
'resource_id' => $backup->id,
|
||||
'status' => 'enqueued',
|
||||
'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),
|
||||
]);
|
||||
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
||||
|
||||
$execution = ScheduledVolumeBackupExecution::query()->sole();
|
||||
expect($execution->status)->toBe('failed')
|
||||
->and($execution->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
|
||||
->and($execution->finished_at)->not->toBeNull()
|
||||
->and($occurrence->fresh()->status)->toBe('failed');
|
||||
Queue::assertNotPushed(VolumeBackupJob::class);
|
||||
Notification::assertSentToTimes($team, BackupFailed::class, 1);
|
||||
Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user