Merge remote-tracking branch 'origin/main' into next

This commit is contained in:
github-actions[bot]
2026-10-07 07:54:43 +00:00
17 changed files with 522 additions and 5 deletions
+2
View File
@@ -6,6 +6,7 @@ use App\Jobs\ApiTokenExpirationWarningJob;
use App\Jobs\CheckForUpdatesJob;
use App\Jobs\CheckHelperImageJob;
use App\Jobs\CheckMissingDatabaseBackupsJob;
use App\Jobs\CheckMissingVolumeBackupsJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\CleanupOrphanedPreviewContainersJob;
use App\Jobs\CleanupStaleMultiplexedConnections;
@@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel
->runInBackground();
$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new CheckMissingVolumeBackupsJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer();
@@ -35,13 +35,14 @@ use RuntimeException;
new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000),
new OA\Property(property: 'missing_backup_notification_days', type: 'integer', description: 'Alert after this many days without an execution; 0 disables alerts', default: 0, minimum: 0, maximum: 365),
],
type: 'object',
additionalProperties: false,
)]
#[OA\Schema(
schema: 'VolumeBackupScheduleResponse',
required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout'],
required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout', 'missing_backup_notification_days'],
properties: [
new OA\Property(property: 'uuid', type: 'string'),
new OA\Property(property: 'message', type: 'string'),
@@ -60,6 +61,7 @@ use RuntimeException;
new OA\Property(property: 'retention_days_s3', type: 'integer'),
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float'),
new OA\Property(property: 'timeout', type: 'integer'),
new OA\Property(property: 'missing_backup_notification_days', type: 'integer'),
],
type: 'object',
)]
@@ -197,6 +199,7 @@ class VolumeBackupsController extends Controller
'retention_days_s3' => 'integer|min:0|max:2147483647',
'retention_max_storage_s3' => 'numeric|min:0|max:9999999999',
'timeout' => 'integer|min:60|max:36000',
'missing_backup_notification_days' => 'integer|min:0|max:365',
]);
$errors = $validator->errors();
$allowedFields = [
@@ -213,6 +216,7 @@ class VolumeBackupsController extends Controller
'retention_days_s3',
'retention_max_storage_s3',
'timeout',
'missing_backup_notification_days',
];
foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) {
@@ -279,6 +283,9 @@ class VolumeBackupsController extends Controller
if ($request->has('timeout')) {
$attributes['timeout'] = $request->integer('timeout');
}
if ($request->has('missing_backup_notification_days')) {
$attributes['missing_backup_notification_days'] = $request->integer('missing_backup_notification_days');
}
$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);
$created = $backup->wasRecentlyCreated;
@@ -445,6 +452,7 @@ class VolumeBackupsController extends Controller
'retention_days_s3' => $backup->retention_days_s3,
'retention_max_storage_s3' => $backup->retention_max_storage_s3,
'timeout' => $backup->timeout,
'missing_backup_notification_days' => $backup->missing_backup_notification_days ?? 0,
];
}
+69
View File
@@ -0,0 +1,69 @@
<?php
namespace App\Jobs;
use App\Models\NotificationThrottle;
use App\Models\ScheduledVolumeBackup;
use App\Notifications\VolumeBackup\BackupMissing;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
class CheckMissingVolumeBackupsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 1800;
/**
* Releases the unique lock of a killed worker after one run (the timeout), so the next hourly run is not
* blocked until the queue retry_after.
*/
public int $uniqueFor = 1800;
public function handle(): void
{
ScheduledVolumeBackup::query()
->with(['team', 'latestExecution'])
->where('enabled', true)
->where('missing_backup_notification_days', '>', 0)
->chunkById(100, function ($backups): void {
foreach ($backups as $backup) {
$this->notifyIfMissing($backup);
}
});
}
private function notifyIfMissing(ScheduledVolumeBackup $backup): void
{
$lastExecutionAt = $backup->latestExecution?->created_at;
$lastActivityAt = $lastExecutionAt ?? $backup->created_at;
if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) {
return;
}
if (! $backup->team) {
Log::warning("Cannot send missing volume backup notification for backup {$backup->id}: team not found");
return;
}
if ($backup->team->getEnabledChannels('backup_failure') === []) {
return;
}
// Send once for each period without backup activity.
NotificationThrottle::sendOnce(
$backup,
BackupMissing::class,
$lastActivityAt,
fn () => $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)),
);
}
}
@@ -61,6 +61,8 @@ class VolumeBackups extends Component
public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;
public int $missingBackupNotificationDays = 0;
public int $perPage = 10;
public function updatedPerPage(): void
@@ -94,6 +96,7 @@ class VolumeBackups extends Component
'retentionDaysS3' => ['required', 'integer', 'min:0'],
'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],
'timeout' => ['required', 'integer', 'min:60', 'max:36000'],
'missingBackupNotificationDays' => ['required', 'integer', 'min:0', 'max:365'],
];
}
@@ -123,6 +126,7 @@ class VolumeBackups extends Component
$this->retentionDaysS3 = $this->backup->retention_days_s3;
$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;
$this->timeout = $this->backup->timeout;
$this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;
} else {
$this->s3StorageId = $this->availableS3Storages->first()?->id;
}
@@ -435,6 +439,7 @@ class VolumeBackups extends Component
'retention_days_s3' => $this->retentionDaysS3,
'retention_max_storage_s3' => $this->retentionMaxStorageS3,
'timeout' => $this->timeout,
'missing_backup_notification_days' => $this->missingBackupNotificationDays,
]);
$changedFields = auditChangedFields($backup);
$backup->save();
+2
View File
@@ -31,6 +31,7 @@ class ScheduledVolumeBackup extends BaseModel
'retention_days_s3',
'retention_max_storage_s3',
'timeout',
'missing_backup_notification_days',
];
protected function casts(): array
@@ -47,6 +48,7 @@ class ScheduledVolumeBackup extends BaseModel
'retention_days_s3' => 'integer',
'retention_max_storage_s3' => 'float',
'timeout' => 'integer',
'missing_backup_notification_days' => 'integer',
];
}
+13 -1
View File
@@ -11,6 +11,7 @@ use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupMissing;
use App\Notifications\Database\BackupSuccess;
use App\Notifications\Database\BackupSuccessWithS3Warning;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\ScheduledTask\TaskSuccess;
use App\Notifications\Server\DockerCleanupFailed;
@@ -20,6 +21,10 @@ use App\Notifications\Server\Reachable;
use App\Notifications\Server\ServerPatchCheck;
use App\Notifications\Server\TraefikVersionOutdated;
use App\Notifications\Server\Unreachable;
use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
use App\Notifications\VolumeBackup\BackupMissing as VolumeBackupMissing;
use App\Notifications\VolumeBackup\BackupSuccess as VolumeBackupSuccess;
use App\Notifications\VolumeBackup\RecoveryFailed as VolumeBackupRecoveryFailed;
class TelegramChannel
{
@@ -41,8 +46,15 @@ class TelegramChannel
RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id,
BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,
VolumeBackupSuccess::class => $notification->warning
? $settings->telegram_notifications_backup_failure_thread_id
: $settings->telegram_notifications_backup_success_thread_id,
BackupFailed::class,
BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id,
BackupMissing::class,
BackupSuccessWithS3Warning::class,
VolumeBackupFailed::class,
VolumeBackupMissing::class,
VolumeBackupRecoveryFailed::class => $settings->telegram_notifications_backup_failure_thread_id,
TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,
TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,
@@ -0,0 +1,117 @@
<?php
namespace App\Notifications\VolumeBackup;
use App\Models\ScheduledVolumeBackup;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Support\Carbon;
class BackupMissing extends CustomEmailNotification
{
public string $resourceName;
public string $target;
public ?string $url = null;
public function __construct(public ScheduledVolumeBackup $backup, public ?Carbon $lastExecutionAt)
{
$this->onQueue('high');
$resource = $backup->targetResource();
$this->resourceName = $resource?->name ?? 'Unknown resource';
$this->target = $backup->targetType().' '.$backup->targetName();
$linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;
if ($linkable && method_exists($linkable, 'link')) {
$this->url = $linkable->link();
}
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('backup_failure');
}
public function summary(): string
{
return "The enabled storage backup schedule for {$this->target} ({$this->resourceName}) has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s).";
}
public function toMail(): MailMessage
{
return (new MailMessage)
->subject("Coolify: [ACTION REQUIRED] No recent storage backup for {$this->resourceName}")
->view('emails.volume-backup-missing', [
'summary' => $this->summary(),
'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
'url' => $this->url,
]);
}
public function toDiscord(): DiscordMessage
{
$message = new DiscordMessage(
title: ':warning: Scheduled storage backup missing',
description: $this->summary(),
color: DiscordMessage::errorColor(),
isCritical: true,
);
if ($this->url) {
$message->addField('Resource', '[Link]('.$this->url.')');
}
return $message;
}
public function toTelegram(): array
{
$data = ['message' => "Coolify: {$this->summary()}"];
if ($this->url) {
$data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];
}
return $data;
}
public function toPushover(): PushoverMessage
{
return new PushoverMessage(
title: 'Scheduled storage backup missing',
level: 'error',
message: $this->summary(),
buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],
);
}
public function toSlack(): SlackMessage
{
return new SlackMessage(
title: 'Scheduled storage backup missing',
description: $this->summary().($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),
color: SlackMessage::errorColor(),
);
}
public function toWebhook(): array
{
return [
'success' => false,
'message' => 'Scheduled storage backup missing',
'event' => 'volume_backup_missing',
'resource_name' => $this->resourceName,
'storage' => $this->target,
'backup_uuid' => $this->backup->uuid,
'days' => $this->backup->missing_backup_notification_days,
'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),
'url' => $this->url,
];
}
}
+38 -1
View File
@@ -2,6 +2,7 @@
namespace App\Services;
use App\Events\BackupCreated;
use App\Events\DockerCleanupDone;
use App\Events\ScheduledTaskDone;
use App\Jobs\DatabaseBackupJob;
@@ -11,15 +12,19 @@ use App\Jobs\VolumeBackupJob;
use App\Models\DockerCleanupExecution;
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\ScheduledTaskExecution;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Database\BackupFailed;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\Server\DockerCleanupFailed;
use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;
use Closure;
use Cron\CronExpression;
use Illuminate\Database\Eloquent\Model;
@@ -353,7 +358,7 @@ class ScheduledJobDeliveryService
}
/**
* Show a missed task or Docker cleanup as a failed execution and send the failure notification.
* Show a missed task, Docker cleanup, or backup as a failed execution and send the failure notification.
* The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,
* and claim() rejects the late job if it starts afterwards.
*/
@@ -391,6 +396,38 @@ class ScheduledJobDeliveryService
]);
event(new DockerCleanupDone($execution));
$this->notifyMissedOccurrence($server, fn () => $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}")));
} elseif ($occurrence->job_type === 'database-backup') {
$backup = ScheduledDatabaseBackup::with(['team', 'database'])->find($occurrence->resource_id);
if (! $backup?->database) {
return;
}
$databaseName = $backup->dump_all ? 'all' : $backup->databases_to_backup;
ScheduledDatabaseBackupExecution::create([
'uuid' => new_public_id(),
'database_name' => filled($databaseName) ? str($databaseName)->limit(250)->toString() : null,
'scheduled_database_backup_id' => $backup->id,
'status' => 'failed',
'message' => $message,
'local_storage_deleted' => false,
'finished_at' => now(),
]);
BackupCreated::dispatch($backup->team_id);
$this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new BackupFailed($backup, $backup->database, $message, $databaseName ?? 'unknown')));
} elseif ($occurrence->job_type === 'volume-backup') {
$backup = ScheduledVolumeBackup::with(['team', 'backupable.resource'])->find($occurrence->resource_id);
if (! $backup) {
return;
}
ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
'status' => 'failed',
'message' => $message,
'finished_at' => now(),
]);
BackupCreated::dispatch($backup->team_id);
$this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new VolumeBackupFailed($backup, $message)));
}
} catch (\Throwable $e) {
Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [
@@ -0,0 +1,22 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
$table->unsignedInteger('missing_backup_notification_days')->default(0);
});
}
public function down(): void
{
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
$table->dropColumn('missing_backup_notification_days');
});
}
};
@@ -0,0 +1,13 @@
<x-emails.layout>
{{ $summary }}
@if ($last_execution_at)
The last execution was at {{ $last_execution_at }}.
@else
This schedule has never produced an execution.
@endif
@if ($url)
Click [here]({{ $url }}) to open the resource.
@endif
</x-emails.layout>
@@ -44,6 +44,9 @@
helper="Uses the backup server timezone, or the instance timezone when none is configured." required />
<x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"
helper="Maximum backup runtime in seconds." required />
<x-forms.input label="Missing backup alert after" id="missingBackupNotificationDays" type="number"
min="0" max="365" suffix="days"
helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required />
</div>
</x-application.settings-section>
</form>
@@ -101,9 +101,12 @@ it('sets an application volume backup schedule through the API', function () {
'retention_days_s3' => 7,
'retention_max_storage_s3' => 8.5,
'timeout' => 600,
'missing_backup_notification_days' => 5,
]);
$response->assertCreated()->assertJsonStructure(['uuid', 'message']);
$response->assertCreated()
->assertJsonStructure(['uuid', 'message'])
->assertJsonPath('missing_backup_notification_days', 5);
$backup = ScheduledVolumeBackup::query()->sole();
expect($backup->backupable->is($this->volume))->toBeTrue()
@@ -120,7 +123,20 @@ it('sets an application volume backup schedule through the API', function () {
->and($backup->retention_amount_s3)->toBe(6)
->and($backup->retention_days_s3)->toBe(7)
->and($backup->retention_max_storage_s3)->toBe(8.5)
->and($backup->timeout)->toBe(600);
->and($backup->timeout)->toBe(600)
->and($backup->missing_backup_notification_days)->toBe(5);
});
it('rejects a missing backup alert period above one year through the API', function () {
$this->withHeaders($this->headers)
->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
'frequency' => 'daily',
'missing_backup_notification_days' => 366,
])
->assertUnprocessable()
->assertJsonValidationErrors('missing_backup_notification_days');
expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
});
it('updates the existing backup schedule instead of creating another one', function () {
@@ -0,0 +1,109 @@
<?php
use App\Jobs\CheckMissingVolumeBackupsJob;
use App\Models\LocalPersistentVolume;
use App\Models\NotificationThrottle;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Team;
use App\Notifications\VolumeBackup\BackupMissing;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Notification;
uses(RefreshDatabase::class);
afterEach(fn () => Carbon::setTestNow());
function missingVolumeBackupSchedule(Team $team, array $attributes = []): ScheduledVolumeBackup
{
$backup = ScheduledVolumeBackup::create(array_merge([
'backupable_type' => (new LocalPersistentVolume)->getMorphClass(),
'backupable_id' => ScheduledVolumeBackup::query()->max('backupable_id') + 1000,
'team_id' => $team->id,
'frequency' => '0 0 * * *',
'enabled' => true,
'missing_backup_notification_days' => 2,
], $attributes));
ScheduledVolumeBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]);
return $backup->refresh();
}
function teamWithVolumeBackupFailureNotifications(): Team
{
$team = Team::create(['name' => 'Test team']);
$team->emailNotificationSettings->update([
'smtp_enabled' => true,
'backup_failure_email_notifications' => true,
]);
return $team;
}
it('notifies the team when an enabled volume backup has no executions for the configured days', function () {
Notification::fake();
$team = teamWithVolumeBackupFailureNotifications();
$backup = missingVolumeBackupSchedule($team);
(new CheckMissingVolumeBackupsJob)->handle();
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)
&& $notification->lastExecutionAt === null
&& $notification->toWebhook()['event'] === 'volume_backup_missing');
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue()
->and((string) (new BackupMissing($backup, null))->toMail()->render())->toContain('This schedule has never produced an execution.');
});
it('does not notify for recent, disabled or unconfigured volume backup schedules', function () {
Notification::fake();
$team = teamWithVolumeBackupFailureNotifications();
missingVolumeBackupSchedule($team, ['enabled' => false]);
missingVolumeBackupSchedule($team, ['missing_backup_notification_days' => 0]);
$recentlyCreated = missingVolumeBackupSchedule($team);
ScheduledVolumeBackup::whereKey($recentlyCreated->id)->update(['created_at' => now()->subDay()]);
$recentlyExecuted = missingVolumeBackupSchedule($team);
ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $recentlyExecuted->id,
'status' => 'success',
]);
(new CheckMissingVolumeBackupsJob)->handle();
Notification::assertNothingSent();
});
it('does not notify when the team has no backup failure channels', function () {
Notification::fake();
$team = Team::create(['name' => 'Test team']);
$backup = missingVolumeBackupSchedule($team);
(new CheckMissingVolumeBackupsJob)->handle();
Notification::assertNothingSent();
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
});
it('notifies once per period without volume backup executions and rearms after another execution', function () {
Notification::fake();
$team = teamWithVolumeBackupFailureNotifications();
$backup = missingVolumeBackupSchedule($team);
(new CheckMissingVolumeBackupsJob)->handle();
(new CheckMissingVolumeBackupsJob)->handle();
Notification::assertSentToTimes($team, BackupMissing::class, 1);
Carbon::setTestNow(now()->addMinute());
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
'status' => 'success',
]);
Carbon::setTestNow(now()->addDays(3));
(new CheckMissingVolumeBackupsJob)->handle();
Notification::assertSentToTimes($team, BackupMissing::class, 2);
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->lastExecutionAt?->equalTo($execution->created_at) === true);
});
@@ -1,6 +1,7 @@
<?php
use App\Jobs\CheckMissingDatabaseBackupsJob;
use App\Jobs\CheckMissingVolumeBackupsJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\CleanupOrphanedPreviewContainersJob;
use Illuminate\Support\Facades\Queue;
@@ -14,6 +15,7 @@ dataset('scheduled instance jobs', [
'orphaned preview containers cleanup' => [CleanupOrphanedPreviewContainersJob::class, 600],
'instance cleanup' => [CleanupInstanceStuffsJob::class, 60],
'missing database backups check' => [CheckMissingDatabaseBackupsJob::class, 1800],
'missing volume backups check' => [CheckMissingVolumeBackupsJob::class, 1800],
]);
it('does not queue the job again while its unique lock is held', function (string $jobClass, int $lockSeconds) {
@@ -13,6 +13,7 @@ use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
@@ -24,6 +25,7 @@ use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Notifications\Database\BackupFailed;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\Server\DockerCleanupFailed;
use App\Services\ScheduledJobDeliveryService;
@@ -752,6 +754,31 @@ it('records a missed Docker cleanup as a failed execution and notifies the team
Event::assertDispatched(DockerCleanupDone::class, 1);
});
it('records a missed database backup that is too old to run late as a failed execution and notifies the team once', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
Queue::fake();
Notification::fake();
Event::fake([BackupCreated::class]);
$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);
$backup->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);
$occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'));
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
$executions = ScheduledDatabaseBackupExecution::query()->where('scheduled_database_backup_id', $backup->id)->get();
expect($executions)->toHaveCount(1)
->and($executions->first()->status)->toBe('failed')
->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
->and($executions->first()->finished_at)->not->toBeNull()
->and($occurrence->fresh()->status)->toBe('failed');
Queue::assertNotPushed(DatabaseBackupJob::class);
Notification::assertSentToTimes($backup->team, BackupFailed::class, 1);
Notification::assertSentTo($backup->team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
Event::assertDispatched(BackupCreated::class);
});
it('sends at most one missed-run notification per hour for the same schedule', function (string $jobType) {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
@@ -2,11 +2,14 @@
use App\Jobs\SendMessageToTelegramJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Channels\TelegramChannel;
use App\Notifications\Server\ServerPatchCheck;
use App\Notifications\Server\TraefikVersionOutdated;
use App\Notifications\VolumeBackup\BackupFailed;
use App\Notifications\VolumeBackup\BackupSuccess;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
@@ -87,3 +90,22 @@ it('falls back to the main chat when no topic is configured', function () {
fn (SendMessageToTelegramJob $job) => $job->threadId === null
);
});
it('sends volume backup notifications to the backup topics', function (Closure $makeNotification, string $threadId) {
$this->team->telegramNotificationSettings->update([
'telegram_notifications_backup_success_thread_id' => '11',
'telegram_notifications_backup_failure_thread_id' => '12',
]);
$backup = new ScheduledVolumeBackup(['frequency' => 'daily']);
(new TelegramChannel)->send($this->team->fresh(), $makeNotification($backup));
Queue::assertPushed(
SendMessageToTelegramJob::class,
fn (SendMessageToTelegramJob $job) => $job->threadId === $threadId
);
})->with([
'success' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup), '11'],
'success with warning' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup, 'S3 upload failed'), '12'],
'failed' => [fn (ScheduledVolumeBackup $backup) => new BackupFailed($backup, 'Archive command failed'), '12'],
]);
+51
View File
@@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\S3Storage;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
@@ -29,6 +30,7 @@ use App\Models\Team;
use App\Models\User;
use App\Notifications\VolumeBackup\BackupFailed;
use App\Notifications\VolumeBackup\BackupSuccess;
use App\Services\ScheduledJobDeliveryService;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Filesystem\FilesystemAdapter;
use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -1142,6 +1144,19 @@ it('does not change S3 storage when another volume backup setting is invalid', f
->and($backup->frequency)->toBe('daily');
});
it('rejects a missing volume backup alert period above one year', function () {
$team = Team::factory()->create();
signInForVolumeBackups($this, $team);
[$application, $volume] = createVolumeBackupApplication($team);
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
->set('missingBackupNotificationDays', 366)
->call('save')
->assertHasErrors(['missingBackupNotificationDays' => 'max']);
expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();
});
function createVolumeBackupApplication(Team $team): array
{
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
@@ -1208,12 +1223,14 @@ it('creates a local scheduled backup for a persistent volume', function () {
->set('retentionDaysLocally', 14)
->set('retentionMaxStorageLocally', 1.5)
->set('stopDuringBackup', true)
->set('missingBackupNotificationDays', 3)
->call('save')
->assertDispatched('success');
$backup = ScheduledVolumeBackup::query()->sole();
expect($backup->backupable->is($volume))->toBeTrue()
->and($backup->missing_backup_notification_days)->toBe(3)
->and($backup->team_id)->toBe($team->id)
->and($backup->frequency)->toBe('daily')
->and($backup->retention_amount_locally)->toBe(5)
@@ -2850,3 +2867,37 @@ it('sends volume backup success with a warning through the backup failure channe
'without warning' => [null, 'backup_success'],
'with warning' => ['S3 upload failed: denied', 'backup_failure'],
]);
it('records a missed volume backup that is too old to run late as a failed execution and notifies the team once', function () {
config(['broadcasting.default' => 'null']);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
Queue::fake();
Notification::fake();
$team = Team::factory()->create();
$team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]);
[$application, $volume] = createVolumeBackupApplication($team);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
'frequency' => 'daily',
]);
$occurrence = ScheduledJobDelivery::create([
'schedule_key' => "scheduled-volume-backup:{$backup->id}",
'scheduled_for' => Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'),
'job_type' => 'volume-backup',
'resource_id' => $backup->id,
'status' => 'enqueued',
'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),
]);
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
$execution = ScheduledVolumeBackupExecution::query()->sole();
expect($execution->status)->toBe('failed')
->and($execution->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
->and($execution->finished_at)->not->toBeNull()
->and($occurrence->fresh()->status)->toBe('failed');
Queue::assertNotPushed(VolumeBackupJob::class);
Notification::assertSentToTimes($team, BackupFailed::class, 1);
Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
});