fix(backups): pin S3 per volume execution and harden deletes

Store s3_storage_id on scheduled volume backup executions so retention and recovery use the S3 that received the upload. Extract DeleteScheduledVolumeBackup for UI and resource deletion, and isolate database backup retention failures so cleanup errors do not fail the job.
2026-08-24 10:05:47 -05:00 · 2026-07-16 14:30:58 +02:00
parent 8cfa41a7d9
commit ac36a59088
14 changed files with 556 additions and 77 deletions
@@ -0,0 +1,71 @@
+<?php
+
+namespace App\Actions\Shared;
+
+use App\Jobs\VolumeBackupJob;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\Server;
+use Illuminate\Support\Facades\Cache;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+class DeleteScheduledVolumeBackup
+{
+    use AsAction;
+
+    public function handle(ScheduledVolumeBackup $backup, ?Server $server = null): void
+    {
+        $lock = Cache::lock(VolumeBackupJob::lockKey($backup->id), $backup->timeout + 300);
+
+        if (! $lock->get()) {
+            throw new \RuntimeException('Wait for the queued or running storage backup to finish before deleting this schedule.');
+        }
+
+        try {
+            if ($backup->executions()
+                ->where(fn ($query) => $query
+                    ->where('status', 'running')
+                    ->orWhere('stop_recovery_pending', true)
+                    ->orWhere('s3_cleanup_pending', true))
+                ->exists()) {
+                throw new \RuntimeException('Wait for the running storage backup and recovery operations to finish before deleting this schedule.');
+            }
+
+            $localFilenames = $backup->executions()
+                ->where('local_storage_deleted', false)
+                ->pluck('filename')
+                ->filter()
+                ->all();
+
+            if ($localFilenames !== []) {
+                $server ??= $backup->server();
+                if (! $server) {
+                    throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
+                }
+
+                deleteBackupsLocally($localFilenames, $server, throwError: true);
+            }
+
+            $s3Executions = $backup->executions()
+                ->with('s3')
+                ->where('s3_uploaded', true)
+                ->where('s3_storage_deleted', false)
+                ->get();
+
+            foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {
+                $s3 = $executions->first()->s3;
+                if (! $s3) {
+                    throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
+                }
+
+                $filenames = $executions->pluck('filename')->filter()->all();
+                if ($filenames !== []) {
+                    deleteBackupsS3($filenames, $s3);
+                }
+            }
+
+            $backup->delete();
+        } finally {
+            $lock->release();
+        }
+    }
+}
@@ -497,7 +497,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
                }
            }
            if ($this->backup_log && $this->backup_log->status === 'success') {
-                removeOldBackups($this->backup);
+                $this->removeExpiredBackups();
            }
        } catch (Throwable $e) {
            throw $e;
@@ -513,6 +513,19 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
        }
    }

+    private function removeExpiredBackups(): void
+    {
+        try {
+            removeOldBackups($this->backup);
+        } catch (Throwable $exception) {
+            Log::channel('scheduled-errors')->warning('Database backup retention cleanup failed', [
+                'backup_id' => $this->backup->id,
+                'execution_id' => $this->backup_log?->id,
+                'error' => $exception->getMessage(),
+            ]);
+        }
+    }
+
    private function backup_standalone_mongodb(string $databaseWithCollections): void
    {
        try {
@@ -7,7 +7,10 @@ use App\Actions\Database\StopDatabase;
 use App\Actions\Server\CleanupDocker;
 use App\Actions\Service\DeleteService;
 use App\Actions\Service\StopService;
+use App\Actions\Shared\DeleteScheduledVolumeBackup;
+use App\Enums\ApplicationDeploymentStatus;
 use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
 use App\Models\ApplicationPreview;
 use App\Models\Service;
 use App\Models\StandaloneClickhouse;
@@ -42,6 +45,10 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue

    public function handle()
    {
+        if (! $this->resource instanceof ApplicationPreview) {
+            $this->deleteScheduledVolumeBackups();
+        }
+
        try {
            // Handle ApplicationPreview instances separately
            if ($this->resource instanceof ApplicationPreview) {
@@ -113,6 +120,24 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
        }
    }

+    private function deleteScheduledVolumeBackups(): void
+    {
+        $server = data_get($this->resource, 'server') ?? data_get($this->resource, 'destination.server');
+        $resources = $this->resource instanceof Service
+            ? $this->resource->applications()->get()->concat($this->resource->databases()->get())
+            : collect([$this->resource]);
+
+        foreach ($resources as $resource) {
+            $storages = $resource->persistentStorages()->get()->concat($resource->fileStorages()->get());
+
+            foreach ($storages as $storage) {
+                foreach ($storage->scheduledBackups()->get() as $backup) {
+                    DeleteScheduledVolumeBackup::run($backup, $server);
+                }
+            }
+        }
+    }
+
    private function deleteApplicationPreview()
    {
        $application = $this->resource->application;
@@ -125,11 +150,11 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
        }

        // Cancel any active deployments for this PR (same logic as API cancel_deployment)
-        $activeDeployments = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
+        $activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id)
            ->where('pull_request_id', $pull_request_id)
            ->whereIn('status', [
-                \App\Enums\ApplicationDeploymentStatus::QUEUED->value,
-                \App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
+                ApplicationDeploymentStatus::QUEUED->value,
+                ApplicationDeploymentStatus::IN_PROGRESS->value,
            ])
            ->get();

@@ -137,7 +162,7 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
            try {
                // Mark deployment as cancelled
                $activeDeployment->update([
-                    'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
+                    'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
                ]);

                // Add cancellation log entry
@@ -63,7 +63,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
            throw new \RuntimeException('The storage backup resource, team, or server no longer exists.');
        }

-        $this->execution = $this->backup->executions()->create();
+        $this->execution = $this->backup->executions()->create([
+            's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null,
+        ]);
        BackupCreated::dispatch($team->id);

        $backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid;
@@ -352,6 +354,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue

        if ($this->backup->save_s3 && $this->backup->s3) {
            $s3Executions = $this->backup->executions()
+                ->with('s3')
                ->where('status', 'success')
                ->where('s3_uploaded', true)
                ->where('s3_storage_deleted', false)
@@ -363,11 +366,18 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
                $this->backup->retention_max_storage_s3,
            );

-            $filenames = $s3Executions->pluck('filename')->filter()->all();
-            if ($filenames !== []) {
-                deleteBackupsS3($filenames, $this->backup->s3);
-                $this->backup->executions()->whereKey($s3Executions->pluck('id')->all())
-                    ->update(['s3_storage_deleted' => true]);
+            foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {
+                $s3 = $executions->first()->s3;
+                if (! $s3) {
+                    throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
+                }
+
+                $filenames = $executions->pluck('filename')->filter()->all();
+                if ($filenames !== []) {
+                    deleteBackupsS3($filenames, $s3);
+                    $this->backup->executions()->whereKey($executions->pluck('id')->all())
+                        ->update(['s3_storage_deleted' => true]);
+                }
            }
        }

@@ -96,8 +96,8 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue

    public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void
    {
-        $execution->loadMissing('scheduledVolumeBackup.s3');
-        $s3 = $execution->scheduledVolumeBackup?->s3;
+        $execution->loadMissing('s3');
+        $s3 = $execution->s3;

        if (! $s3 || blank($execution->filename)) {
            throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.');
@@ -3,6 +3,7 @@
 namespace App\Livewire\Project\Database;

 use App\Models\ServiceDatabase;
+use Illuminate\Contracts\View\View;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Livewire\Component;

@@ -63,4 +64,11 @@ class ScheduledBackups extends Component
        $this->database->refresh();
        $this->dispatch('refreshScheduledBackups');
    }
+
+    public function render(): View
+    {
+        $this->database->loadMissing('scheduledBackups.s3');
+
+        return view('livewire.project.database.scheduled-backups');
+    }
 }
@@ -2,6 +2,7 @@

 namespace App\Livewire\Project\Shared\Storages;

+use App\Actions\Shared\DeleteScheduledVolumeBackup;
 use App\Jobs\VolumeBackupJob;
 use App\Models\LocalFileVolume;
 use App\Models\LocalPersistentVolume;
@@ -9,7 +10,6 @@ use App\Models\S3Storage;
 use App\Models\ScheduledVolumeBackup;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Facades\Cache;
 use Livewire\Component;
 use Livewire\WithPagination;
 use Throwable;
@@ -214,57 +214,8 @@ class VolumeBackups extends Component
            return false;
        }

-        $lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300);
-
-        if (! $lock->get()) {
-            $this->dispatch('error', 'Wait for the queued or running storage backup to finish before deleting this schedule.');
-
-            return false;
-        }
-
        try {
-            if ($this->backup->executions()
-                ->where(fn ($query) => $query
-                    ->where('status', 'running')
-                    ->orWhere('stop_recovery_pending', true)
-                    ->orWhere('s3_cleanup_pending', true))
-                ->exists()) {
-                $this->dispatch('error', 'Wait for the running storage backup and container recovery to finish before deleting this schedule.');
-
-                return false;
-            }
-
-            $localFilenames = $this->backup->executions()
-                ->where('local_storage_deleted', false)
-                ->pluck('filename')
-                ->filter()
-                ->all();
-            $server = $this->backup->server();
-
-            if ($localFilenames !== []) {
-                if (! $server) {
-                    throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
-                }
-
-                deleteBackupsLocally($localFilenames, $server, throwError: true);
-            }
-
-            $s3Filenames = $this->backup->executions()
-                ->where('s3_uploaded', true)
-                ->where('s3_storage_deleted', false)
-                ->pluck('filename')
-                ->filter()
-                ->all();
-
-            if ($s3Filenames !== []) {
-                if (! $this->backup->s3) {
-                    throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.');
-                }
-
-                deleteBackupsS3($s3Filenames, $this->backup->s3);
-            }
-
-            $this->backup->delete();
+            DeleteScheduledVolumeBackup::run($this->backup);
            $this->backup = null;
            $this->dispatch('success', 'Storage backup schedule and archives deleted.');
            $this->redirectRoute('project.application.backup.index', [
@@ -278,8 +229,6 @@ class VolumeBackups extends Component
            $this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage());

            return false;
-        } finally {
-            $lock->release();
        }
    }

@@ -69,13 +69,12 @@ class S3Storage extends BaseModel
                'save_s3' => false,
                's3_storage_id' => null,
            ]);
-            $volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id');
-            ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds)
+            ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id)
                ->update([
                    's3_storage_deleted' => true,
                    's3_cleanup_pending' => false,
                ]);
-            ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([
+            ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([
                'save_s3' => false,
                's3_storage_id' => null,
            ]);
@@ -9,6 +9,7 @@ class ScheduledVolumeBackupExecution extends BaseModel
    protected $fillable = [
        'uuid',
        'scheduled_volume_backup_id',
+        's3_storage_id',
        'status',
        'message',
        'size',
@@ -40,4 +41,9 @@ class ScheduledVolumeBackupExecution extends BaseModel
    {
        return $this->belongsTo(ScheduledVolumeBackup::class);
    }
+
+    public function s3(): BelongsTo
+    {
+        return $this->belongsTo(S3Storage::class, 's3_storage_id');
+    }
 }
@@ -15,6 +15,7 @@ return new class extends Migration
            $table->id();
            $table->string('uuid')->unique();
            $table->foreignId('scheduled_volume_backup_id')->constrained()->cascadeOnDelete();
+            $table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete();
            $table->enum('status', ['success', 'failed', 'running'])->default('running');
            $table->longText('message')->nullable();
            $table->unsignedBigInteger('size')->default(0);
@@ -3,11 +3,12 @@
    backups: @js($database->scheduledBackups->map(fn ($backup) => [
        'name' => strtolower($database->name),
        'frequency' => strtolower($backup->frequency),
+        's3_storage' => strtolower($backup->s3?->name ?? ''),
    ])->values()),
    hasMatches() {
        const query = this.search.toLowerCase();

-        return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query));
+        return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query) || backup.s3_storage.includes(query));
    },
 }">
    <div class="flex flex-col gap-2">
@@ -31,14 +32,14 @@
        @else
            <div class="max-w-md pb-4">
                <x-forms.input id="null" type="search" x-model="search"
-                    placeholder="Search by database name or frequency..." />
+                    placeholder="Search by database name, frequency, or S3 storage..." />
            </div>
            <div x-cloak x-show="search !== '' && backups.length > 0 && !hasMatches()">
                No scheduled backups match your search.
            </div>
            @forelse($database->scheduledBackups as $backup)
                @if ($type == 'database')
-                    <a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase())" @class([
+                    <a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase()) || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())" @class([
                        'flex flex-col border-l-2 transition-colors p-4 cursor-pointer bg-white hover:bg-gray-100 dark:bg-coolgray-100 dark:hover:bg-coolgray-200 text-black dark:text-white',
                        'border-blue-500/50 border-dashed' =>
                            $backup->latest_log &&
@@ -112,18 +113,18 @@
                                    @endif
                                @endif
                                @if ($backup->save_s3)
-                                    • S3: Enabled
+                                    • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
                                @endif
                            @else
                                Last Run: Never • Total Executions: 0
                                @if ($backup->save_s3)
-                                    • S3: Enabled
+                                    • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
                                @endif
                            @endif
                        </div>
                    </a>
                @else
-                    <a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase())" @class([
+                    <a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase()) || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())" @class([
                        'flex flex-col border-l-2 transition-colors p-4 cursor-pointer bg-white hover:bg-gray-100 dark:bg-coolgray-100 dark:hover:bg-coolgray-200 text-black dark:text-white',
                        'border-blue-500/50 border-dashed' =>
                            $backup->latest_log &&
@@ -197,7 +198,7 @@
                                    @endif
                                @endif
                                @if ($backup->save_s3)
-                                    • S3: Enabled
+                                    • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
                                @endif
                                <br>Total Executions: {{ $backup->executions()->count() }}
                                @php
@@ -217,7 +218,7 @@
                            @else
                                Last Run: Never • Total Executions: 0
                                @if ($backup->save_s3)
-                                    • S3: Enabled
+                                    • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
                                @endif
                            @endif
                        </div>
@@ -5,6 +5,7 @@ use App\Models\Environment;
 use App\Models\InstanceSettings;
 use App\Models\LocalPersistentVolume;
 use App\Models\Project;
+use App\Models\S3Storage;
 use App\Models\Server;
 use App\Models\StandaloneDocker;
 use App\Models\StandalonePostgresql;
@@ -70,9 +71,21 @@ it('renders frontend-only database backup search data for database names and fre
        'destination_id' => $destination->id,
        'destination_type' => $destination->getMorphClass(),
    ]);
+    $storage = S3Storage::create([
+        'name' => 'Archive-Bucket',
+        'region' => 'us-east-1',
+        'key' => 'test-key',
+        'secret' => 'test-secret',
+        'bucket' => 'test-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'is_usable' => true,
+        'team_id' => $this->team->id,
+    ]);
    $database->scheduledBackups()->create([
        'team_id' => $this->team->id,
        'frequency' => 'daily',
+        'save_s3' => true,
+        's3_storage_id' => $storage->id,
    ]);
    $database->scheduledBackups()->create([
        'team_id' => $this->team->id,
@@ -89,11 +102,56 @@ it('renders frontend-only database backup search data for database names and fre
        ->assertOk()
        ->assertSee('<h3 class="font-semibold">0 3 * * 1</h3>', false)
        ->assertSee('<h3 class="font-semibold">daily</h3>', false)
+        ->assertSee('S3: Archive-Bucket')
+        ->assertSee('archive-bucket', false)
+        ->assertSee('backup.s3_storage.includes(query)', false)
+        ->assertSee('Search by database name, frequency, or S3 storage...')
        ->assertSee('x-model="search"', false)
        ->assertSee('x-show=', false)
        ->assertSee('No scheduled backups match your search.');
 });

+it('renders unavailable S3 storage only for S3-enabled database backups', function () {
+    $server = Server::factory()->create(['team_id' => $this->team->id]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $this->team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+    $database = StandalonePostgresql::create([
+        'name' => 'Orders-Primary',
+        'image' => 'postgres:16-alpine',
+        'postgres_user' => 'postgres',
+        'postgres_password' => 'password',
+        'postgres_db' => 'postgres',
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ]);
+    $database->scheduledBackups()->create([
+        'team_id' => $this->team->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        's3_storage_id' => null,
+    ]);
+    $database->scheduledBackups()->create([
+        'team_id' => $this->team->id,
+        'frequency' => 'weekly',
+        'save_s3' => false,
+        's3_storage_id' => null,
+    ]);
+
+    $parameters = [
+        'project_uuid' => $project->uuid,
+        'environment_uuid' => $environment->uuid,
+        'database_uuid' => $database->uuid,
+    ];
+
+    $response = $this->get(route('project.database.backup.index', $parameters))
+        ->assertOk()
+        ->assertSee('S3: Storage unavailable');
+
+    expect(substr_count($response->getContent(), 'S3:'))->toBe(1);
+});
+
 function createBackupSearchApplication(Team $team): Application
 {
    $server = Server::factory()->create(['team_id' => $team->id]);
@@ -7,6 +7,7 @@ use App\Models\ScheduledDatabaseBackupExecution;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\Storage;

 uses(RefreshDatabase::class);

@@ -238,6 +239,57 @@ test('failed method updates status when backup was not successful', function ()
    expect($log->message)->toContain('Some real failure');
 });

+test('retention cleanup failure does not fail a successful database backup', function () {
+    $team = Team::factory()->create();
+    $s3 = S3Storage::create([
+        'name' => 'Test S3',
+        'region' => 'us-east-1',
+        'key' => 'test-key',
+        'secret' => 'test-secret',
+        'bucket' => 'test-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+    ]);
+    $backup = ScheduledDatabaseBackup::create([
+        'frequency' => '0 0 * * *',
+        'save_s3' => true,
+        'disable_local_backup' => true,
+        's3_storage_id' => $s3->id,
+        'database_type' => 'App\Models\StandalonePostgresql',
+        'database_id' => 1,
+        'team_id' => $team->id,
+        'database_backup_retention_amount_s3' => 1,
+    ]);
+    $oldExecution = ScheduledDatabaseBackupExecution::create([
+        'uuid' => 'old-backup',
+        'database_name' => 'database',
+        'filename' => '/backup/old.dmp',
+        'scheduled_database_backup_id' => $backup->id,
+        'status' => 'success',
+        's3_uploaded' => true,
+        'local_storage_deleted' => true,
+        'created_at' => now()->subDay(),
+    ]);
+    ScheduledDatabaseBackupExecution::create([
+        'uuid' => 'new-backup',
+        'database_name' => 'database',
+        'filename' => '/backup/new.dmp',
+        'scheduled_database_backup_id' => $backup->id,
+        'status' => 'success',
+        's3_uploaded' => true,
+        'local_storage_deleted' => true,
+    ]);
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse();
+    Storage::shouldReceive('build')->once()->andReturn($disk);
+    $job = new DatabaseBackupJob($backup);
+
+    expect(fn () => (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job))
+        ->not->toThrow(Throwable::class);
+
+    expect($oldExecution->fresh()->s3_storage_deleted)->toBeFalse();
+});
+
 test('s3 storage has scheduled backups relationship', function () {
    $team = Team::factory()->create();

@@ -1,5 +1,6 @@
 <?php

+use App\Jobs\DeleteResourceJob;
 use App\Jobs\ScheduledJobManager;
 use App\Jobs\VolumeBackupJob;
 use App\Jobs\VolumeBackupRecoveryJob;
@@ -467,6 +468,11 @@ it('stores volume backup schedules and executions', function () {
        ]))->toBeTrue();
 });

+it('records the S3 storage used by each volume backup execution', function () {
+    expect(Schema::hasColumn('scheduled_volume_backup_executions', 's3_storage_id'))->toBeTrue()
+        ->and((new ScheduledVolumeBackupExecution)->s3())->not->toBeNull();
+});
+
 it('exposes actions to manage and run volume backups', function () {
    expect(method_exists(VolumeBackups::class, 'save'))->toBeTrue()
        ->and(method_exists(VolumeBackups::class, 'backupNow'))->toBeTrue()
@@ -998,6 +1004,7 @@ it('disables S3 volume backups when the storage is deleted', function () {
    ]);
    $execution = ScheduledVolumeBackupExecution::create([
        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $s3Storage->id,
        'status' => 'success',
        'filename' => '/data/coolify/backups/volumes/test/s3.tar.gz',
        's3_uploaded' => true,
@@ -1012,6 +1019,50 @@ it('disables S3 volume backups when the storage is deleted', function () {
        ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse();
 });

+it('marks historical executions deleted when their recorded S3 storage is removed', function () {
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $originalStorage = S3Storage::create([
+        'name' => 'Original storage',
+        'region' => 'us-east-1',
+        'key' => 'original-key',
+        'secret' => 'secret',
+        'bucket' => 'original-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+    ]);
+    $newStorage = S3Storage::create([
+        'name' => 'New storage',
+        'region' => 'us-east-1',
+        'key' => 'new-key',
+        'secret' => 'secret',
+        'bucket' => 'new-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        's3_storage_id' => $newStorage->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+    ]);
+    $execution = ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $originalStorage->id,
+        'status' => 'success',
+        'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz',
+        's3_uploaded' => true,
+        's3_cleanup_pending' => true,
+    ]);
+
+    $originalStorage->delete();
+
+    expect($backup->fresh()->s3_storage_id)->toBe($newStorage->id)
+        ->and($execution->fresh()->s3_storage_id)->toBeNull()
+        ->and($execution->fresh()->s3_storage_deleted)->toBeTrue()
+        ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse();
+});
+
 it('queues a manual backup before a schedule has been saved', function () {
    Queue::fake();
    $team = Team::factory()->create();
@@ -1100,6 +1151,61 @@ it('deletes local archives before deleting a volume backup schedule', function (
        && str_contains($process->command, 'archive.tar.gz'));
 });

+it('deletes S3 archives from the storage recorded on each execution', function () {
+    $team = Team::factory()->create();
+    signInForVolumeBackups($this, $team);
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $originalStorage = S3Storage::create([
+        'name' => 'Original storage',
+        'region' => 'us-east-1',
+        'key' => 'original-key',
+        'secret' => 'secret',
+        'bucket' => 'original-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $newStorage = S3Storage::create([
+        'name' => 'New storage',
+        'region' => 'us-east-1',
+        'key' => 'new-key',
+        'secret' => 'secret',
+        'bucket' => 'new-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        's3_storage_id' => $newStorage->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+    ]);
+    ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $originalStorage->id,
+        'status' => 'success',
+        'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz',
+        'local_storage_deleted' => true,
+        's3_uploaded' => true,
+    ]);
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')
+        ->once()
+        ->with(['/data/coolify/backups/volumes/test/historical.tar.gz'])
+        ->andReturnTrue();
+    Storage::shouldReceive('build')
+        ->once()
+        ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key'))
+        ->andReturn($disk);
+
+    Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
+        ->call('delete', 'password')
+        ->assertDispatched('success');
+
+    expect($backup->fresh())->toBeNull();
+});
+
 it('refuses to delete a schedule while its backup is running', function () {
    Process::fake();
    $team = Team::factory()->create();
@@ -1156,6 +1262,31 @@ it('prevents a volume with tracked backup archives from being deleted', function
    expect($volume->fresh())->not->toBeNull();
 });

+it('deletes disabled volume backup archives before deleting their application', function () {
+    Process::fake();
+    Queue::fake();
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'enabled' => false,
+    ]);
+    $execution = ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        'status' => 'success',
+        'filename' => '/data/coolify/backups/volumes/test/application-delete.tar.gz',
+    ]);
+    $application->delete();
+
+    (new DeleteResourceJob($application))->handle();
+
+    expect($backup->fresh())->toBeNull()
+        ->and($execution->fresh())->toBeNull();
+    Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f')
+        && str_contains($process->command, 'application-delete.tar.gz'));
+});
+
 it('marks a running execution failed even when the job instance lost its execution state', function () {
    Process::fake();
    $team = Team::factory()->create();
@@ -1222,6 +1353,46 @@ it('archives a named volume on its server', function () {
        && ! str_contains($process->command, ':/backup'));
 });

+it('keeps the upload destination on the volume backup execution', function () {
+    config(['broadcasting.default' => 'null']);
+    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $s3Storage = S3Storage::create([
+        'name' => 'Execution destination',
+        'region' => 'us-east-1',
+        'key' => 'key',
+        'secret' => 'secret',
+        'bucket' => 'bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        's3_storage_id' => $s3Storage->id,
+    ]);
+    $sshDisk = Storage::fake('ssh-keys');
+    $disk = Mockery::mock();
+    $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
+    $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+    Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+    Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
+    Process::fake([
+        '*du -b*' => '128',
+        '*' => '',
+    ]);
+
+    (new VolumeBackupJob($backup))->handle();
+
+    $execution = ScheduledVolumeBackupExecution::query()->sole();
+
+    expect($execution->s3_storage_id)->toBe($s3Storage->id)
+        ->and($execution->s3->is($s3Storage))->toBeTrue();
+});
+
 it('removes local volume backups older than the configured retention days', function () {
    config(['broadcasting.default' => 'null']);
    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
@@ -1295,6 +1466,70 @@ it('removes oldest local volume backups over the configured storage limit', func
        && str_contains($process->command, 'over-limit.tar.gz'));
 });

+it('removes retained S3 archives from the storage recorded on each execution', function () {
+    $team = Team::factory()->create();
+    [$application, $volume, $server] = createVolumeBackupApplication($team);
+    $originalStorage = S3Storage::create([
+        'name' => 'Original storage',
+        'region' => 'us-east-1',
+        'key' => 'original-key',
+        'secret' => 'secret',
+        'bucket' => 'original-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $newStorage = S3Storage::create([
+        'name' => 'New storage',
+        'region' => 'us-east-1',
+        'key' => 'new-key',
+        'secret' => 'secret',
+        'bucket' => 'new-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        's3_storage_id' => $newStorage->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        'retention_amount_locally' => 0,
+        'retention_days_locally' => 0,
+        'retention_max_storage_locally' => 0,
+        'retention_amount_s3' => 1,
+        'retention_days_s3' => 0,
+        'retention_max_storage_s3' => 0,
+    ]);
+    $oldExecution = ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $originalStorage->id,
+        'status' => 'success',
+        'filename' => '/data/coolify/backups/volumes/test/old.tar.gz',
+        's3_uploaded' => true,
+        'created_at' => now()->subDay(),
+    ]);
+    ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $newStorage->id,
+        'status' => 'success',
+        'filename' => '/data/coolify/backups/volumes/test/new.tar.gz',
+        's3_uploaded' => true,
+    ]);
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue();
+    Storage::shouldReceive('build')
+        ->once()
+        ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key'))
+        ->andReturn($disk);
+    $job = new VolumeBackupJob($backup);
+    $method = (new ReflectionClass($job))->getMethod('removeExpiredBackups');
+
+    $method->invoke($job, $server);
+
+    expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue();
+});
+
 it('keeps a successful backup successful when retention cleanup fails', function () {
    config(['broadcasting.default' => 'null']);
    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
@@ -1511,6 +1746,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc
    ]);
    $execution = ScheduledVolumeBackupExecution::create([
        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $s3Storage->id,
        'status' => 'failed',
        'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
        's3_cleanup_pending' => true,
@@ -1527,6 +1763,55 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc
        ->and($execution->fresh()->s3_storage_deleted)->toBeTrue();
 });

+it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () {
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $originalStorage = S3Storage::create([
+        'name' => 'Original storage',
+        'region' => 'us-east-1',
+        'key' => 'original-key',
+        'secret' => 'secret',
+        'bucket' => 'original-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $newStorage = S3Storage::create([
+        'name' => 'New storage',
+        'region' => 'us-east-1',
+        'key' => 'new-key',
+        'secret' => 'secret',
+        'bucket' => 'new-bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        's3_storage_id' => $newStorage->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+    ]);
+    $execution = ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $originalStorage->id,
+        'status' => 'failed',
+        'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
+        's3_cleanup_pending' => true,
+    ]);
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->once()->andReturnTrue();
+    Storage::shouldReceive('build')
+        ->once()
+        ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key'))
+        ->andReturn($disk);
+
+    VolumeBackupRecoveryJob::cleanupS3Upload($execution);
+
+    expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()
+        ->and($execution->fresh()->s3_storage_deleted)->toBeTrue();
+});
+
 it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () {
    Queue::fake();
    Process::fake();
@@ -1550,6 +1835,7 @@ it('keeps the S3 key tracked when interrupted upload cleanup must be retried', f
    ]);
    $execution = ScheduledVolumeBackupExecution::create([
        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $s3Storage->id,
        'status' => 'running',
        'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
        's3_cleanup_pending' => true,