fix(volume-backups): back off failed container and S3 recovery

Recovery attempts now back off exponentially, stop after 8 attempts, and
mark the execution as needing attention with a failure category. The
scheduler dispatches at most 100 recovery jobs per run, and a pending S3
cleanup no longer blocks new backups. S3 delete errors now raise with
their cause.

app/Jobs/ScheduledJobManager.php

@@ -38,6 +38,11 @@ class ScheduledJobManager implements ShouldQueue
      */
     private const VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES = 5;
 
+    /**
+     * Bounds the volume backup recovery jobs dispatched in one scheduler run.
+     */
+    private const VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES = 100;
+
     /**
      * The schedule types that run as separate dispatchers, with the delivery job types they own.
      * Each type has its own overlap lock, so a slow type cannot make another type skip a run.
@@ -485,16 +490,29 @@ class ScheduledJobManager implements ShouldQueue
 
     private function recoverStoppedVolumeBackupContainers(): void
     {
+        $dispatched = 0;
+
         ScheduledVolumeBackupExecution::query()
             ->where(fn (Builder $query) => $query
                 ->where('stop_recovery_pending', true)
                 ->orWhere('s3_cleanup_pending', true))
-            ->chunkById(self::CHUNK_SIZE, function ($executions): void {
+            ->where('recovery_needs_attention', false)
+            ->where(fn (Builder $query) => $query
+                ->whereNull('recovery_next_retry_at')
+                ->orWhere('recovery_next_retry_at', '<=', now()))
+            ->chunkById(self::CHUNK_SIZE, function ($executions) use (&$dispatched): bool {
                 foreach ($executions as $execution) {
-                    if (Cache::add("volume-backup-recovery-dispatched:{$execution->id}", true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {
+                    if ($dispatched >= self::VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES) {
+                        return false;
+                    }
+
+                    if (Cache::add(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {
                         VolumeBackupRecoveryJob::dispatch($execution);
+                        $dispatched++;
                     }
                 }
+
+                return true;
             });
     }
 
@@ -503,13 +521,9 @@ class ScheduledJobManager implements ShouldQueue
         try {
             $server = $backup->server();
 
-            if ($backup->executions()
-                ->where(fn (Builder $query) => $query
-                    ->where('stop_recovery_pending', true)
-                    ->orWhere('s3_cleanup_pending', true))
-                ->exists()) {
+            if ($backup->executions()->where('stop_recovery_pending', true)->exists()) {
                 $this->skippedCount++;
-                $this->logSkip('volume_backup', 'container_recovery_pending', [
+                $this->logSkip('volume_backup', 'stopped_container_recovery_pending', [
                     'backup_id' => $backup->id,
                     'team_id' => $backup->team_id,
                 ]);

app/Jobs/VolumeBackupJob.php

@@ -488,6 +488,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
 
         $this->backup->executions()
             ->where('local_storage_deleted', true)
+            ->where('stop_recovery_pending', false)
+            ->where('s3_cleanup_pending', false)
             ->where(function (Builder $query): void {
                 $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');
             })

app/Jobs/VolumeBackupRecoveryJob.php

@@ -3,6 +3,7 @@
 namespace App\Jobs;
 
 use App\Models\ScheduledVolumeBackupExecution;
+use Aws\S3\Exception\S3Exception;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
@@ -12,13 +13,15 @@ use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
+use RuntimeException;
+use Throwable;
 
 class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     /**
-     * ScheduledJobManager dispatches pending recovery again, so this job does not retry by itself.
+     * ScheduledJobManager dispatches pending recovery again after a backoff, so this job does not retry by itself.
      */
     public int $tries = 1;
 
@@ -26,6 +29,34 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
 
     public int $uniqueFor = 300;
 
+    public const MAX_ATTEMPTS = 8;
+
+    public const BACKOFF_BASE_MINUTES = 5;
+
+    public const BACKOFF_CAP_MINUTES = 360;
+
+    public const CATEGORY_S3_AUTH = 's3_auth';
+
+    public const CATEGORY_S3_BUCKET = 's3_bucket';
+
+    public const CATEGORY_SERVER_UNREACHABLE = 'server_unreachable';
+
+    public const CATEGORY_REMOTE_COMMAND = 'remote_command';
+
+    public const CATEGORY_UNKNOWN = 'unknown';
+
+    private const UNREACHABLE_PATTERNS = [
+        'ssh connection failed',
+        'connection refused',
+        'connection timed out',
+        'operation timed out',
+        'no route to host',
+        'network is unreachable',
+        'could not resolve hostname',
+        'permission denied (publickey',
+        'host key verification failed',
+    ];
+
     public function __construct(public ScheduledVolumeBackupExecution $execution)
     {
         $this->onQueue(crons_queue());
@@ -51,20 +82,173 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
 
     public function handle(): void
     {
-        self::recover($this->execution);
+        self::recoverWithBounds($this->execution);
     }
 
+    /**
+     * Runs one recovery attempt and records its outcome instead of throwing, so a persistent failure backs off
+     * and eventually needs attention rather than creating a failed job every few minutes.
+     */
+    public static function recoverWithBounds(ScheduledVolumeBackupExecution $execution): void
+    {
+        $current = $execution->fresh();
+        if (! $current) {
+            return;
+        }
+
+        $execution->setRawAttributes($current->getAttributes(), true);
+
+        if (! $execution->hasPendingRecovery() || $execution->recovery_needs_attention) {
+            return;
+        }
+
+        try {
+            $failure = self::runRecoverySteps($execution);
+        } catch (Throwable $exception) {
+            $failure = ['category' => self::CATEGORY_UNKNOWN, 'exception' => $exception];
+        }
+
+        if ($failure) {
+            self::recordFailure($execution, $failure['category']);
+
+            return;
+        }
+
+        self::recordSuccess($execution);
+    }
+
+    /**
+     * Recovers stopped containers first, then cleans the partial S3 upload. Each step runs even when the other one
+     * fails; the first failure is rethrown after both steps ran.
+     */
     public static function recover(ScheduledVolumeBackupExecution $execution): void
+    {
+        $failure = self::runRecoverySteps($execution);
+
+        if ($failure) {
+            throw $failure['exception'];
+        }
+    }
+
+    /**
+     * @return array{category: string, exception: Throwable}|null The first failure, or null when every step succeeded.
+     */
+    private static function runRecoverySteps(ScheduledVolumeBackupExecution $execution): ?array
     {
         $execution->loadMissing('scheduledVolumeBackup.backupable.resource');
+        $failure = null;
 
         if ($execution->stop_recovery_pending) {
-            self::recoverContainers($execution);
+            try {
+                self::recoverContainers($execution);
+            } catch (Throwable $exception) {
+                $failure = ['category' => self::categorizeContainerFailure($exception), 'exception' => $exception];
+            }
         }
 
         if ($execution->s3_cleanup_pending) {
-            self::cleanupS3Upload($execution);
+            try {
+                self::cleanupS3Upload($execution);
+            } catch (Throwable $exception) {
+                $failure ??= ['category' => self::categorizeS3Failure($exception), 'exception' => $exception];
+            }
         }
+
+        return $failure;
+    }
+
+    /**
+     * Returns the minutes to wait before the next automatic attempt after the given number of failed attempts.
+     */
+    public static function backoffMinutes(int $attempts): int
+    {
+        $exponent = max(0, min($attempts - 1, 16));
+
+        return (int) min(self::BACKOFF_BASE_MINUTES * (2 ** $exponent), self::BACKOFF_CAP_MINUTES);
+    }
+
+    public static function dispatchCacheKey(int $executionId): string
+    {
+        return "volume-backup-recovery-dispatched:{$executionId}";
+    }
+
+    private static function recordSuccess(ScheduledVolumeBackupExecution $execution): void
+    {
+        $execution->update([
+            'recovery_attempts' => 0,
+            'recovery_last_attempt_at' => null,
+            'recovery_next_retry_at' => null,
+            'recovery_error' => null,
+            'recovery_needs_attention' => false,
+        ]);
+    }
+
+    private static function recordFailure(ScheduledVolumeBackupExecution $execution, string $category): void
+    {
+        $recorded = $execution->getConnection()->transaction(function () use ($execution, $category): ?ScheduledVolumeBackupExecution {
+            $current = ScheduledVolumeBackupExecution::query()->whereKey($execution->id)->lockForUpdate()->first();
+            if (! $current) {
+                return null;
+            }
+
+            $attempts = $current->recovery_attempts + 1;
+            $needsAttention = $attempts >= self::MAX_ATTEMPTS;
+            $current->update([
+                'recovery_attempts' => $attempts,
+                'recovery_last_attempt_at' => now(),
+                'recovery_next_retry_at' => $needsAttention ? null : now()->addMinutes(self::backoffMinutes($attempts)),
+                'recovery_error' => $category,
+                'recovery_needs_attention' => $needsAttention,
+            ]);
+
+            return $current;
+        });
+
+        if (! $recorded) {
+            return;
+        }
+
+        $execution->setRawAttributes($recorded->getAttributes(), true);
+
+        Log::channel('scheduled-errors')->warning('Volume backup recovery failed', [
+            'execution_id' => $recorded->id,
+            'execution_uuid' => $recorded->uuid,
+            'category' => $category,
+            'attempts' => $recorded->recovery_attempts,
+            'needs_attention' => $recorded->recovery_needs_attention,
+            'next_retry_at' => $recorded->recovery_next_retry_at?->toIso8601String(),
+        ]);
+    }
+
+    private static function categorizeS3Failure(Throwable $exception): string
+    {
+        for ($current = $exception; $current; $current = $current->getPrevious()) {
+            if ($current instanceof S3Exception) {
+                return match ($current->getAwsErrorCode()) {
+                    'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'AccessDenied' => self::CATEGORY_S3_AUTH,
+                    'NoSuchBucket' => self::CATEGORY_S3_BUCKET,
+                    default => self::CATEGORY_UNKNOWN,
+                };
+            }
+        }
+
+        return self::CATEGORY_UNKNOWN;
+    }
+
+    private static function categorizeContainerFailure(Throwable $exception): string
+    {
+        if ($exception->getCode() === 255) {
+            return self::CATEGORY_SERVER_UNREACHABLE;
+        }
+
+        $message = strtolower($exception->getMessage());
+        foreach (self::UNREACHABLE_PATTERNS as $pattern) {
+            if (str_contains($message, $pattern)) {
+                return self::CATEGORY_SERVER_UNREACHABLE;
+            }
+        }
+
+        return $exception instanceof RuntimeException ? self::CATEGORY_REMOTE_COMMAND : self::CATEGORY_UNKNOWN;
     }
 
     private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void

app/Livewire/Project/Shared/Storages/VolumeBackups.php

@@ -4,6 +4,7 @@ namespace App\Livewire\Project\Shared\Storages;
 
 use App\Actions\Shared\DeleteScheduledVolumeBackup;
 use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
 use App\Models\LocalFileVolume;
 use App\Models\LocalPersistentVolume;
 use App\Models\S3Storage;
@@ -13,6 +14,7 @@ use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Routing\Redirector;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
 use Livewire\Component;
 use Livewire\WithPagination;
 use Throwable;
@@ -278,6 +280,8 @@ class VolumeBackups extends Component
 
         $deletedCount = $this->backup?->executions()
             ->where('local_storage_deleted', true)
+            ->where('stop_recovery_pending', false)
+            ->where('s3_cleanup_pending', false)
             ->where(fn ($query) => $query
                 ->where('s3_storage_deleted', true)
                 ->orWhereNull('s3_uploaded')
@@ -290,6 +294,33 @@ class VolumeBackups extends Component
         );
     }
 
+    public function retryRecovery(int $executionId): void
+    {
+        $this->authorize('update', $this->resource);
+
+        $execution = $this->backup?->executions()->whereKey($executionId)->first();
+        if (! $execution || ! $execution->hasPendingRecovery()) {
+            $this->dispatch('error', 'No pending recovery found for this backup execution.');
+
+            return;
+        }
+
+        $execution->update([
+            'recovery_attempts' => 0,
+            'recovery_next_retry_at' => null,
+            'recovery_needs_attention' => false,
+        ]);
+        Cache::forget(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id));
+        VolumeBackupRecoveryJob::dispatch($execution);
+        auditLog('ui.volume_backup.recovery_retried', [
+            'team_id' => $this->resource->team()?->id,
+            'resource_uuid' => $this->resource->uuid,
+            'backup_uuid' => $this->backup->uuid,
+            'execution_uuid' => $execution->uuid,
+        ]);
+        $this->dispatch('success', 'Backup recovery queued.');
+    }
+
     public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string
     {
         $this->authorize('update', $this->resource);

app/Models/ScheduledVolumeBackupExecution.php

@@ -21,6 +21,11 @@ class ScheduledVolumeBackupExecution extends BaseModel
         'local_storage_deleted',
         's3_storage_deleted',
         's3_uploaded',
+        'recovery_attempts',
+        'recovery_last_attempt_at',
+        'recovery_next_retry_at',
+        'recovery_error',
+        'recovery_needs_attention',
     ];
 
     protected function casts(): array
@@ -34,9 +39,18 @@ class ScheduledVolumeBackupExecution extends BaseModel
             'local_storage_deleted' => 'boolean',
             's3_storage_deleted' => 'boolean',
             's3_uploaded' => 'boolean',
+            'recovery_attempts' => 'integer',
+            'recovery_last_attempt_at' => 'datetime',
+            'recovery_next_retry_at' => 'datetime',
+            'recovery_needs_attention' => 'boolean',
         ];
     }
 
+    public function hasPendingRecovery(): bool
+    {
+        return $this->stop_recovery_pending || $this->s3_cleanup_pending;
+    }
+
     public function scheduledVolumeBackup(): BelongsTo
     {
         return $this->belongsTo(ScheduledVolumeBackup::class);

bootstrap/helpers/databases.php

@@ -282,9 +282,16 @@ function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void
         'endpoint' => $s3->endpoint,
         'use_path_style_endpoint' => true,
         'aws_url' => $s3->awsUrl(),
+        'throw' => true,
     ]);
 
-    if (! $disk->delete($filenames)) {
+    try {
+        $deleted = $disk->delete($filenames);
+    } catch (Throwable $exception) {
+        throw new RuntimeException('One or more S3 backup files could not be deleted.', previous: $exception);
+    }
+
+    if (! $deleted) {
         throw new RuntimeException('One or more S3 backup files could not be deleted.');
     }
 }

database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php

@@ -0,0 +1,44 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) {
+            $table->unsignedSmallInteger('recovery_attempts')->default(0);
+            $table->timestampTz('recovery_last_attempt_at')->nullable();
+            $table->timestampTz('recovery_next_retry_at')->nullable();
+            $table->string('recovery_error', 32)->nullable();
+            $table->boolean('recovery_needs_attention')->default(false);
+
+            $table->index(
+                ['stop_recovery_pending', 's3_cleanup_pending', 'recovery_needs_attention', 'recovery_next_retry_at'],
+                'scheduled_volume_executions_recovery_index',
+            );
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) {
+            $table->dropIndex('scheduled_volume_executions_recovery_index');
+            $table->dropColumn([
+                'recovery_attempts',
+                'recovery_last_attempt_at',
+                'recovery_next_retry_at',
+                'recovery_error',
+                'recovery_needs_attention',
+            ]);
+        });
+    }
+};

resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php

@@ -62,6 +62,14 @@
                             if (empty($deleteActions)) {
                                 $deleteActions[] = 'This backup execution record will be deleted.';
                             }
+                            $recoveryPending = $execution->status !== 'running' && $execution->hasPendingRecovery();
+                            $recoveryErrorLabel = match ($execution->recovery_error) {
+                                's3_auth' => 'S3 credentials',
+                                's3_bucket' => 'S3 bucket missing',
+                                'server_unreachable' => 'server unreachable',
+                                'remote_command' => 'remote command failed',
+                                default => 'unknown error',
+                            };
                         @endphp
 
                         <div wire:key="volume-backup-execution-{{ $execution->id }}"
@@ -99,6 +107,14 @@
                                             ? 'neutral'
                                             : ($execution->s3_uploaded ? 'success' : 'error')" />
                                 @endif
+                                @if ($recoveryPending)
+                                    @if ($execution->recovery_needs_attention)
+                                        <x-status-badge :status="'Needs attention ('.$recoveryErrorLabel.')'" type="error" />
+                                    @else
+                                        <x-status-badge status="Recovery pending" type="warning"
+                                            :title="$execution->recovery_error ? 'Last attempt: '.$recoveryErrorLabel : null" />
+                                    @endif
+                                @endif
                             </span>
 
                             <span class="flex items-center justify-end gap-1">
@@ -109,6 +125,14 @@
                                         <x-reicon name="upload" class="size-3.5 rotate-180" />
                                     </button>
                                 @endif
+                                @if ($recoveryPending)
+                                    <button type="button" class="icon-button shrink-0"
+                                        wire:click="retryRecovery({{ $execution->id }})"
+                                        wire:loading.attr="disabled" wire:target="retryRecovery({{ $execution->id }})"
+                                        title="Retry recovery" aria-label="Retry recovery">
+                                        <x-reicon name="refresh" class="size-3.5" />
+                                    </button>
+                                @endif
                                 @if ($execution->status !== 'running')
                                     <x-modal-confirmation title="Confirm Backup Deletion?" isErrorButton
                                         submitAction="deleteBackup({{ $execution->id }})"

tests/Feature/VolumeBackupRecoveryBoundsTest.php

@@ -0,0 +1,384 @@
+<?php
+
+use App\Jobs\ScheduledJobManager;
+use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
+use App\Livewire\Project\Shared\Storages\VolumeBackups;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Models\User;
+use Aws\Command;
+use Aws\S3\Exception\S3Exception;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use League\Flysystem\UnableToDeleteFile;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    config(['broadcasting.default' => 'null']);
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+});
+
+afterEach(function () {
+    Carbon::setTestNow();
+});
+
+/**
+ * @return array{0: Application, 1: LocalPersistentVolume, 2: Server, 3: ScheduledVolumeBackup, 4: S3Storage}
+ */
+function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): array
+{
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'ip' => '203.0.113.20',
+    ]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+    $application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ]);
+    $volume = LocalPersistentVolume::create([
+        'name' => 'app-data',
+        'mount_path' => '/data',
+        'resource_id' => $application->id,
+        'resource_type' => $application->getMorphClass(),
+    ]);
+    $s3 = S3Storage::create([
+        'name' => 'Volume backups',
+        'region' => 'us-east-1',
+        'key' => 'AKIARECOVERYKEY',
+        'secret' => 'recovery-secret',
+        'bucket' => 'bucket',
+        'endpoint' => 'https://s3.example.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        's3_storage_id' => $s3->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        ...$backupAttributes,
+    ]);
+
+    return [$application, $volume, $server, $backup, $s3];
+}
+
+function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution
+{
+    return ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        's3_storage_id' => $backup->s3_storage_id,
+        'status' => 'failed',
+        'filename' => '/data/coolify/backups/volumes/test/partial.tar.gz',
+        's3_cleanup_pending' => true,
+        ...$attributes,
+    ]);
+}
+
+function failRecoveryBoundsS3Delete(string $awsCode): void
+{
+    $s3Exception = new S3Exception(
+        'The AWS Access Key Id AKIARECOVERYKEY you provided does not exist in our records.',
+        new Command('DeleteObject'),
+        ['code' => $awsCode],
+    );
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->andThrow(UnableToDeleteFile::atLocation('partial.tar.gz', '', $s3Exception));
+    Storage::shouldReceive('build')->andReturn($disk);
+}
+
+function signInForRecoveryBounds($testCase, Team $team): User
+{
+    $user = User::factory()->create();
+    $user->teams()->attach($team, ['role' => 'owner']);
+    $testCase->actingAs($user);
+    session(['currentTeam' => $team]);
+
+    return $user;
+}
+
+function makeRecoveryBoundsServerFunctional(Server $server): void
+{
+    $server->settings()->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+        'force_disabled' => false,
+    ]);
+}
+
+it('doubles the recovery backoff and caps it at six hours', function () {
+    expect(collect(range(1, 9))->map(fn (int $attempts) => VolumeBackupRecoveryJob::backoffMinutes($attempts))->all())
+        ->toBe([5, 10, 20, 40, 80, 160, 320, 360, 360]);
+});
+
+it('records a categorized S3 cleanup failure without throwing or leaking the S3 message', function () {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    $execution = createRecoveryBoundsExecution($backup, ['message' => 'Worker timed out']);
+    failRecoveryBoundsS3Delete('InvalidAccessKeyId');
+    $logger = Mockery::spy();
+    Log::shouldReceive('channel')->with('scheduled-errors')->andReturn($logger);
+
+    (new VolumeBackupRecoveryJob($execution))->handle();
+
+    $execution->refresh();
+    expect($execution->s3_cleanup_pending)->toBeTrue()
+        ->and($execution->s3_storage_deleted)->toBeFalse()
+        ->and($execution->recovery_attempts)->toBe(1)
+        ->and($execution->recovery_error)->toBe('s3_auth')
+        ->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue()
+        ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(5)))->toBeTrue()
+        ->and($execution->recovery_needs_attention)->toBeFalse()
+        ->and($execution->message)->toBe('Worker timed out');
+    $logger->shouldHaveReceived('warning')->once()->with(
+        'Volume backup recovery failed',
+        Mockery::on(fn (array $context): bool => $context['execution_uuid'] === $execution->uuid
+            && $context['category'] === 's3_auth'
+            && ! str_contains(json_encode($context), 'AKIA')),
+    );
+});
+
+it('categorizes a missing S3 bucket and grows the backoff on each failure', function () {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    $execution = createRecoveryBoundsExecution($backup);
+    failRecoveryBoundsS3Delete('NoSuchBucket');
+
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+    $execution->refresh();
+    expect($execution->recovery_error)->toBe('s3_bucket')
+        ->and($execution->recovery_attempts)->toBe(2)
+        ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(10)))->toBeTrue();
+});
+
+it('needs attention after the maximum attempts and stops automatic recovery', function () {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    config(['constants.coolify.self_hosted' => true]);
+    Queue::fake();
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    $execution = createRecoveryBoundsExecution($backup, [
+        'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS - 1,
+    ]);
+    failRecoveryBoundsS3Delete('AccessDenied');
+
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+    $execution->refresh();
+    expect($execution->recovery_needs_attention)->toBeTrue()
+        ->and($execution->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS)
+        ->and($execution->recovery_next_retry_at)->toBeNull();
+
+    Carbon::setTestNow('2026-10-04 12:00:00');
+    (new ScheduledJobManager)->handle();
+
+    Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
+});
+
+it('waits for the backoff before dispatching recovery again', function () {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    config(['constants.coolify.self_hosted' => true]);
+    Queue::fake();
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    createRecoveryBoundsExecution($backup, [
+        'recovery_attempts' => 3,
+        'recovery_next_retry_at' => now()->addMinutes(20),
+    ]);
+
+    (new ScheduledJobManager)->handle();
+    Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
+
+    Carbon::setTestNow('2026-10-03 12:20:00');
+    (new ScheduledJobManager)->handle();
+    Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);
+});
+
+it('cleans the S3 upload even when container recovery fails', function () {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    config(['constants.ssh.max_retries' => 1, 'constants.ssh.mux_enabled' => false]);
+    Storage::fake('ssh-keys');
+    Storage::fake('ssh-mux');
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['stop_during_backup' => true]);
+    $execution = createRecoveryBoundsExecution($backup, ['stop_recovery_pending' => true]);
+    Process::fake([
+        '*' => Process::result(errorOutput: 'ssh: connect to host 203.0.113.20 port 22: Connection refused', exitCode: 255),
+    ]);
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->once()->andReturnTrue();
+    $filesystem = Mockery::mock(Storage::getFacadeRoot());
+    $filesystem->shouldReceive('build')->once()->andReturn($disk);
+    Storage::swap($filesystem);
+
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+    $execution->refresh();
+    expect($execution->stop_recovery_pending)->toBeTrue()
+        ->and($execution->s3_cleanup_pending)->toBeFalse()
+        ->and($execution->s3_storage_deleted)->toBeTrue()
+        ->and($execution->recovery_error)->toBe('server_unreachable')
+        ->and($execution->recovery_attempts)->toBe(1);
+});
+
+it('resets the recovery state after a successful attempt', function () {
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    $execution = createRecoveryBoundsExecution($backup, [
+        'recovery_attempts' => 4,
+        'recovery_error' => 's3_auth',
+        'recovery_last_attempt_at' => now()->subHour(),
+        'recovery_next_retry_at' => now()->subMinute(),
+    ]);
+    $disk = Mockery::mock();
+    $disk->shouldReceive('delete')->once()->andReturnTrue();
+    Storage::shouldReceive('build')->once()->andReturn($disk);
+
+    (new VolumeBackupRecoveryJob($execution))->handle();
+
+    $execution->refresh();
+    expect($execution->s3_cleanup_pending)->toBeFalse()
+        ->and($execution->s3_storage_deleted)->toBeTrue()
+        ->and($execution->recovery_attempts)->toBe(0)
+        ->and($execution->recovery_error)->toBeNull()
+        ->and($execution->recovery_next_retry_at)->toBeNull();
+});
+
+it('only blocks new scheduled backups while stopped containers await recovery', function (array $pending, bool $dispatched) {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    config(['constants.coolify.self_hosted' => true]);
+    Queue::fake();
+    [, , $server, $backup] = createRecoveryBoundsBackup(Team::factory()->create(), [
+        'frequency' => '* * * * *',
+        'enabled' => true,
+    ]);
+    makeRecoveryBoundsServerFunctional($server);
+    createRecoveryBoundsExecution($backup, $pending);
+    Cache::forget("scheduled-volume-backup:{$backup->id}");
+
+    (new ScheduledJobManager)->handle();
+
+    $dispatched
+        ? Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup))
+        : Queue::assertNotPushed(VolumeBackupJob::class);
+})->with([
+    'only S3 cleanup pending' => [['s3_cleanup_pending' => true, 'recovery_needs_attention' => true], true],
+    'stopped containers pending' => [['stop_recovery_pending' => true, 's3_cleanup_pending' => false], false],
+    'stopped containers need attention' => [['stop_recovery_pending' => true, 'recovery_needs_attention' => true], false],
+]);
+
+it('keeps executions with pending recovery during retention cleanup', function () {
+    Process::fake([
+        '*du -b*' => '128',
+        '*' => '',
+    ]);
+    [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['save_s3' => false]);
+    $pending = createRecoveryBoundsExecution($backup, [
+        'local_storage_deleted' => true,
+        's3_uploaded' => null,
+    ]);
+    $deletable = createRecoveryBoundsExecution($backup, [
+        'filename' => null,
+        'local_storage_deleted' => true,
+        's3_uploaded' => null,
+        's3_cleanup_pending' => false,
+    ]);
+
+    (new VolumeBackupJob($backup->fresh()))->handle();
+
+    expect($pending->fresh())->not->toBeNull()
+        ->and($deletable->fresh())->toBeNull();
+});
+
+it('keeps executions with pending recovery when cleaning up deleted entries', function () {
+    $team = Team::factory()->create();
+    signInForRecoveryBounds($this, $team);
+    [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);
+    $pending = createRecoveryBoundsExecution($backup, [
+        'local_storage_deleted' => true,
+        's3_uploaded' => false,
+    ]);
+    $deletable = createRecoveryBoundsExecution($backup, [
+        'local_storage_deleted' => true,
+        's3_uploaded' => false,
+        's3_cleanup_pending' => false,
+    ]);
+
+    Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
+        ->call('cleanupDeleted');
+
+    expect($pending->fresh())->not->toBeNull()
+        ->and($deletable->fresh())->toBeNull();
+});
+
+it('shows the recovery state and retries recovery manually', function () {
+    Queue::fake();
+    $team = Team::factory()->create();
+    signInForRecoveryBounds($this, $team);
+    [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);
+    $execution = createRecoveryBoundsExecution($backup, [
+        'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,
+        'recovery_error' => 's3_auth',
+        'recovery_needs_attention' => true,
+    ]);
+    Cache::put(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(5));
+
+    Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 'executions'])
+        ->assertSee('Needs attention (S3 credentials)')
+        ->assertSee('Retry recovery')
+        ->call('retryRecovery', $execution->id)
+        ->assertDispatched('success')
+        ->assertSee('Recovery pending');
+
+    $execution->refresh();
+    expect($execution->recovery_attempts)->toBe(0)
+        ->and($execution->recovery_needs_attention)->toBeFalse()
+        ->and($execution->recovery_next_retry_at)->toBeNull()
+        ->and(Cache::has(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id)))->toBeFalse();
+    Queue::assertPushed(VolumeBackupRecoveryJob::class, fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution));
+});
+
+it('prevents another team from retrying recovery', function () {
+    Queue::fake();
+    [$application, $volume, , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+    $execution = createRecoveryBoundsExecution($backup, [
+        'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,
+        'recovery_needs_attention' => true,
+    ]);
+    $otherTeam = Team::factory()->create();
+    signInForRecoveryBounds($this, $otherTeam);
+    [$otherApplication, $otherVolume] = createRecoveryBoundsBackup($otherTeam);
+
+    Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
+        ->assertForbidden();
+    Livewire::test(VolumeBackups::class, ['storage' => $otherVolume, 'resource' => $otherApplication])
+        ->call('retryRecovery', $execution->id)
+        ->assertDispatched('error');
+
+    expect($execution->fresh()->recovery_needs_attention)->toBeTrue()
+        ->and($execution->fresh()->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS);
+    Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
+});