feat(backups): notify teams of volume backup results

Send success and failure notifications through enabled backup channels. Route successful backups with warnings through failure channels.
2026-11-06 13:10:37 -04:00 · 2026-10-07 09:24:10 +02:00
parent d54c03730d
commit 58db0eba6f
5 changed files with 335 additions and 0 deletions
@@ -7,6 +7,8 @@ use App\Models\LocalPersistentVolume;
 use App\Models\ScheduledVolumeBackup;
 use App\Models\ScheduledVolumeBackupExecution;
 use App\Models\Server;
+use App\Notifications\VolumeBackup\BackupFailed;
+use App\Notifications\VolumeBackup\BackupSuccess;
 use App\Rules\SafeWebhookUrl;
 use App\Services\ScheduledJobDeliveryService;
 use App\Support\BackupCompression;
@@ -238,6 +240,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
            $this->execution->update(['finished_at' => now()]);
            BackupCreated::dispatch($team->id);
        }
+
+        $team->notify(new BackupSuccess($this->backup, $warning));
    }

    public function failed(?Throwable $exception): void
@@ -277,6 +281,11 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                'local_storage_deleted' => $localStorageDeleted,
            ]);
        }
+
+        $this->backup->team?->notify(new BackupFailed(
+            $this->backup,
+            $exception?->getMessage() ?? 'Volume backup timed out or was terminated.',
+        ));
    }

    /**
@@ -0,0 +1,123 @@
+<?php
+
+namespace App\Notifications\VolumeBackup;
+
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ServiceApplication;
+use App\Models\ServiceDatabase;
+use App\Notifications\CustomEmailNotification;
+use App\Notifications\Dto\DiscordMessage;
+use App\Notifications\Dto\PushoverMessage;
+use App\Notifications\Dto\SlackMessage;
+use Illuminate\Notifications\Messages\MailMessage;
+
+class BackupFailed extends CustomEmailNotification
+{
+    public string $resourceName;
+
+    public string $target;
+
+    public string $frequency;
+
+    public ?string $url = null;
+
+    public function __construct(ScheduledVolumeBackup $backup, public string $output)
+    {
+        $this->onQueue('high');
+        $resource = $backup->targetResource();
+        $this->resourceName = $resource?->name ?? 'Unknown resource';
+        $this->target = $backup->targetType().' '.$backup->targetName();
+        $this->frequency = $backup->frequency;
+
+        $linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;
+        if ($linkable && method_exists($linkable, 'link')) {
+            $this->url = $linkable->link();
+        }
+    }
+
+    public function via(object $notifiable): array
+    {
+        return $notifiable->getEnabledChannels('backup_failure');
+    }
+
+    public function summary(): string
+    {
+        return "Storage backup of {$this->target} ({$this->resourceName}) has FAILED.";
+    }
+
+    public function toMail(): MailMessage
+    {
+        $mail = new MailMessage;
+        $mail->subject("Coolify: [ACTION REQUIRED] Storage backup FAILED for {$this->resourceName}");
+        $mail->view('emails.volume-backup-result', [
+            'summary' => $this->summary(),
+            'frequency' => $this->frequency,
+            'output' => $this->output,
+            'url' => $this->url,
+        ]);
+
+        return $mail;
+    }
+
+    public function toDiscord(): DiscordMessage
+    {
+        $message = new DiscordMessage(
+            title: ':cross_mark: Storage backup failed',
+            description: $this->summary(),
+            color: DiscordMessage::errorColor(),
+            isCritical: true,
+        );
+
+        $message->addField('Frequency', $this->frequency, true);
+        $message->addField('Output', $this->output);
+        if ($this->url) {
+            $message->addField('Resource', '[Link]('.$this->url.')');
+        }
+
+        return $message;
+    }
+
+    public function toTelegram(): array
+    {
+        $data = ['message' => "Coolify: {$this->summary()}\n\nFrequency: {$this->frequency}\n\nReason:\n{$this->output}"];
+
+        if ($this->url) {
+            $data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];
+        }
+
+        return $data;
+    }
+
+    public function toPushover(): PushoverMessage
+    {
+        return new PushoverMessage(
+            title: 'Storage backup failed',
+            level: 'error',
+            message: "{$this->summary()}<br/><br/><b>Frequency:</b> {$this->frequency}.<br/><b>Reason:</b> {$this->output}",
+            buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],
+        );
+    }
+
+    public function toSlack(): SlackMessage
+    {
+        return new SlackMessage(
+            title: 'Storage backup failed',
+            description: $this->summary()."\n\n*Frequency:* {$this->frequency}\n\n*Error Output:* {$this->output}".($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),
+            color: SlackMessage::errorColor(),
+        );
+    }
+
+    public function toWebhook(): array
+    {
+        return [
+            'success' => false,
+            'message' => 'Storage backup failed',
+            'event' => 'volume_backup_failed',
+            'resource_name' => $this->resourceName,
+            'storage' => $this->target,
+            'frequency' => $this->frequency,
+            'error_output' => $this->output,
+            'url' => $this->url,
+        ];
+    }
+}
@@ -0,0 +1,124 @@
+<?php
+
+namespace App\Notifications\VolumeBackup;
+
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ServiceApplication;
+use App\Models\ServiceDatabase;
+use App\Notifications\CustomEmailNotification;
+use App\Notifications\Dto\DiscordMessage;
+use App\Notifications\Dto\PushoverMessage;
+use App\Notifications\Dto\SlackMessage;
+use Illuminate\Notifications\Messages\MailMessage;
+
+class BackupSuccess extends CustomEmailNotification
+{
+    public string $resourceName;
+
+    public string $target;
+
+    public string $frequency;
+
+    public ?string $url = null;
+
+    public function __construct(ScheduledVolumeBackup $backup, public ?string $warning = null)
+    {
+        $this->onQueue('high');
+        $resource = $backup->targetResource();
+        $this->resourceName = $resource?->name ?? 'Unknown resource';
+        $this->target = $backup->targetType().' '.$backup->targetName();
+        $this->frequency = $backup->frequency;
+
+        $linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;
+        if ($linkable && method_exists($linkable, 'link')) {
+            $this->url = $linkable->link();
+        }
+    }
+
+    public function via(object $notifiable): array
+    {
+        return $notifiable->getEnabledChannels($this->warning ? 'backup_failure' : 'backup_success');
+    }
+
+    public function summary(): string
+    {
+        $summary = "Storage backup of {$this->target} ({$this->resourceName}) was successful.";
+
+        return $this->warning ? $summary." Warning: {$this->warning}" : $summary;
+    }
+
+    public function toMail(): MailMessage
+    {
+        $mail = new MailMessage;
+        $mail->subject($this->warning
+            ? "Coolify: Storage backup succeeded with a warning for {$this->resourceName}"
+            : "Coolify: Storage backup successfully done for {$this->resourceName}");
+        $mail->view('emails.volume-backup-result', [
+            'summary' => $this->summary(),
+            'frequency' => $this->frequency,
+            'url' => $this->url,
+        ]);
+
+        return $mail;
+    }
+
+    public function toDiscord(): DiscordMessage
+    {
+        $message = new DiscordMessage(
+            title: $this->warning ? ':warning: Storage backup succeeded with a warning' : ':white_check_mark: Storage backup successful',
+            description: $this->summary(),
+            color: $this->warning ? DiscordMessage::warningColor() : DiscordMessage::successColor(),
+        );
+
+        $message->addField('Frequency', $this->frequency, true);
+        if ($this->url) {
+            $message->addField('Resource', '[Link]('.$this->url.')');
+        }
+
+        return $message;
+    }
+
+    public function toTelegram(): array
+    {
+        $data = ['message' => "Coolify: {$this->summary()}\n\nFrequency: {$this->frequency}"];
+
+        if ($this->url) {
+            $data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];
+        }
+
+        return $data;
+    }
+
+    public function toPushover(): PushoverMessage
+    {
+        return new PushoverMessage(
+            title: $this->warning ? 'Storage backup succeeded with a warning' : 'Storage backup successful',
+            level: $this->warning ? 'warning' : 'success',
+            message: "{$this->summary()}<br/><br/><b>Frequency:</b> {$this->frequency}.",
+            buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],
+        );
+    }
+
+    public function toSlack(): SlackMessage
+    {
+        return new SlackMessage(
+            title: $this->warning ? 'Storage backup succeeded with a warning' : 'Storage backup successful',
+            description: $this->summary()."\n\n*Frequency:* {$this->frequency}".($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),
+            color: $this->warning ? SlackMessage::warningColor() : SlackMessage::successColor(),
+        );
+    }
+
+    public function toWebhook(): array
+    {
+        return [
+            'success' => true,
+            'message' => $this->warning ? 'Storage backup succeeded with a warning' : 'Storage backup successful',
+            'event' => 'volume_backup_success',
+            'resource_name' => $this->resourceName,
+            'storage' => $this->target,
+            'frequency' => $this->frequency,
+            'warning' => $this->warning,
+            'url' => $this->url,
+        ];
+    }
+}
@@ -0,0 +1,15 @@
+<x-emails.layout>
+{{ $summary }}
+
+Frequency: {{ $frequency }}
+
+@isset($output)
+### Reason
+
+{{ $output }}
+@endisset
+
+@if ($url)
+Click [here]({{ $url }}) to open the resource.
+@endif
+</x-emails.layout>
@@ -27,6 +27,8 @@ use App\Models\ServiceDatabase;
 use App\Models\StandaloneDocker;
 use App\Models\Team;
 use App\Models\User;
+use App\Notifications\VolumeBackup\BackupFailed;
+use App\Notifications\VolumeBackup\BackupSuccess;
 use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Filesystem\FilesystemAdapter;
 use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -37,6 +39,7 @@ use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Notification;
 use Illuminate\Support\Facades\Process;
 use Illuminate\Support\Facades\Queue;
 use Illuminate\Support\Facades\Route;
@@ -2786,3 +2789,64 @@ it('stops S3 upload cleanup when the S3 storage no longer exists', function () {
        ->and($execution->fresh()->s3_storage_deleted)->toBeFalsy()
        ->and($execution->fresh()->message)->toContain('S3 upload cleanup was skipped');
 });
+
+it('notifies the team when a volume backup succeeds', function () {
+    config(['broadcasting.default' => 'null']);
+    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+    Notification::fake();
+    $team = Team::factory()->create();
+    $team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_success_discord_notifications' => true, 'backup_failure_discord_notifications' => true]);
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+    ]);
+    Process::fake([
+        '*du -b*' => '128',
+        '*' => '',
+    ]);
+
+    (new VolumeBackupJob($backup))->handle();
+
+    Notification::assertSentToTimes($team, BackupSuccess::class, 1);
+    Notification::assertSentTo($team, BackupSuccess::class, fn (BackupSuccess $notification) => $notification->warning === null
+        && $notification->resourceName === $application->name
+        && $notification->target === 'Volume '.$volume->name
+        && str_contains((string) $notification->toMail()->render(), 'was successful'));
+    Notification::assertNotSentTo($team, BackupFailed::class);
+});
+
+it('notifies the team once when a volume backup fails', function () {
+    Process::fake();
+    Notification::fake();
+    $team = Team::factory()->create();
+    $team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_success_discord_notifications' => true, 'backup_failure_discord_notifications' => true]);
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+    ]);
+
+    (new VolumeBackupJob($backup))->failed(new RuntimeException('Archive command failed'));
+
+    Notification::assertSentToTimes($team, BackupFailed::class, 1);
+    Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Archive command failed'
+        && str_contains((string) $notification->toMail()->render(), 'Archive command failed'));
+    Notification::assertNotSentTo($team, BackupSuccess::class);
+});
+
+it('sends volume backup success with a warning through the backup failure channels', function (?string $warning, string $event) {
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+    ]);
+    $notifiable = Mockery::mock();
+    $notifiable->shouldReceive('getEnabledChannels')->once()->with($event)->andReturn([]);
+
+    (new BackupSuccess($backup, $warning))->via($notifiable);
+})->with([
+    'without warning' => [null, 'backup_success'],
+    'with warning' => ['S3 upload failed: denied', 'backup_failure'],
+]);