From c3b32322eb827ff229a9620418ab3c139f9cf35b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:25:11 +0200 Subject: [PATCH] fix(volume-backups): back off failed container and S3 recovery Recovery attempts now back off exponentially, stop after 8 attempts, and mark the execution as needing attention with a failure category. The scheduler dispatches at most 100 recovery jobs per run, and a pending S3 cleanup no longer blocks new backups. S3 delete errors now raise with their cause. --- app/Jobs/ScheduledJobManager.php | 30 +- app/Jobs/VolumeBackupJob.php | 2 + app/Jobs/VolumeBackupRecoveryJob.php | 192 ++++++++- .../Project/Shared/Storages/VolumeBackups.php | 31 ++ app/Models/ScheduledVolumeBackupExecution.php | 14 + bootstrap/helpers/databases.php | 9 +- ...heduled_volume_backup_executions_table.php | 44 ++ .../volume-backups/executions.blade.php | 24 ++ .../VolumeBackupRecoveryBoundsTest.php | 384 ++++++++++++++++++ 9 files changed, 717 insertions(+), 13 deletions(-) create mode 100644 database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php create mode 100644 tests/Feature/VolumeBackupRecoveryBoundsTest.php diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php index d3af90a996..e1411412fb 100644 --- a/app/Jobs/ScheduledJobManager.php +++ b/app/Jobs/ScheduledJobManager.php @@ -38,6 +38,11 @@ class ScheduledJobManager implements ShouldQueue */ private const VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES = 5; + /** + * Bounds the volume backup recovery jobs dispatched in one scheduler run. + */ + private const VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES = 100; + /** * The schedule types that run as separate dispatchers, with the delivery job types they own. * Each type has its own overlap lock, so a slow type cannot make another type skip a run. @@ -485,16 +490,29 @@ class ScheduledJobManager implements ShouldQueue private function recoverStoppedVolumeBackupContainers(): void { + $dispatched = 0; + ScheduledVolumeBackupExecution::query() ->where(fn (Builder $query) => $query ->where('stop_recovery_pending', true) ->orWhere('s3_cleanup_pending', true)) - ->chunkById(self::CHUNK_SIZE, function ($executions): void { + ->where('recovery_needs_attention', false) + ->where(fn (Builder $query) => $query + ->whereNull('recovery_next_retry_at') + ->orWhere('recovery_next_retry_at', '<=', now())) + ->chunkById(self::CHUNK_SIZE, function ($executions) use (&$dispatched): bool { foreach ($executions as $execution) { - if (Cache::add("volume-backup-recovery-dispatched:{$execution->id}", true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) { + if ($dispatched >= self::VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES) { + return false; + } + + if (Cache::add(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) { VolumeBackupRecoveryJob::dispatch($execution); + $dispatched++; } } + + return true; }); } @@ -503,13 +521,9 @@ class ScheduledJobManager implements ShouldQueue try { $server = $backup->server(); - if ($backup->executions() - ->where(fn (Builder $query) => $query - ->where('stop_recovery_pending', true) - ->orWhere('s3_cleanup_pending', true)) - ->exists()) { + if ($backup->executions()->where('stop_recovery_pending', true)->exists()) { $this->skippedCount++; - $this->logSkip('volume_backup', 'container_recovery_pending', [ + $this->logSkip('volume_backup', 'stopped_container_recovery_pending', [ 'backup_id' => $backup->id, 'team_id' => $backup->team_id, ]); diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index 78f091bbe2..f142a65941 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -488,6 +488,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $this->backup->executions() ->where('local_storage_deleted', true) + ->where('stop_recovery_pending', false) + ->where('s3_cleanup_pending', false) ->where(function (Builder $query): void { $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded'); }) diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php index c136c45820..7f475d24ce 100644 --- a/app/Jobs/VolumeBackupRecoveryJob.php +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\ScheduledVolumeBackupExecution; +use Aws\S3\Exception\S3Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; @@ -12,13 +13,15 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; +use RuntimeException; +use Throwable; class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; /** - * ScheduledJobManager dispatches pending recovery again, so this job does not retry by itself. + * ScheduledJobManager dispatches pending recovery again after a backoff, so this job does not retry by itself. */ public int $tries = 1; @@ -26,6 +29,34 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou public int $uniqueFor = 300; + public const MAX_ATTEMPTS = 8; + + public const BACKOFF_BASE_MINUTES = 5; + + public const BACKOFF_CAP_MINUTES = 360; + + public const CATEGORY_S3_AUTH = 's3_auth'; + + public const CATEGORY_S3_BUCKET = 's3_bucket'; + + public const CATEGORY_SERVER_UNREACHABLE = 'server_unreachable'; + + public const CATEGORY_REMOTE_COMMAND = 'remote_command'; + + public const CATEGORY_UNKNOWN = 'unknown'; + + private const UNREACHABLE_PATTERNS = [ + 'ssh connection failed', + 'connection refused', + 'connection timed out', + 'operation timed out', + 'no route to host', + 'network is unreachable', + 'could not resolve hostname', + 'permission denied (publickey', + 'host key verification failed', + ]; + public function __construct(public ScheduledVolumeBackupExecution $execution) { $this->onQueue(crons_queue()); @@ -51,20 +82,173 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou public function handle(): void { - self::recover($this->execution); + self::recoverWithBounds($this->execution); } + /** + * Runs one recovery attempt and records its outcome instead of throwing, so a persistent failure backs off + * and eventually needs attention rather than creating a failed job every few minutes. + */ + public static function recoverWithBounds(ScheduledVolumeBackupExecution $execution): void + { + $current = $execution->fresh(); + if (! $current) { + return; + } + + $execution->setRawAttributes($current->getAttributes(), true); + + if (! $execution->hasPendingRecovery() || $execution->recovery_needs_attention) { + return; + } + + try { + $failure = self::runRecoverySteps($execution); + } catch (Throwable $exception) { + $failure = ['category' => self::CATEGORY_UNKNOWN, 'exception' => $exception]; + } + + if ($failure) { + self::recordFailure($execution, $failure['category']); + + return; + } + + self::recordSuccess($execution); + } + + /** + * Recovers stopped containers first, then cleans the partial S3 upload. Each step runs even when the other one + * fails; the first failure is rethrown after both steps ran. + */ public static function recover(ScheduledVolumeBackupExecution $execution): void + { + $failure = self::runRecoverySteps($execution); + + if ($failure) { + throw $failure['exception']; + } + } + + /** + * @return array{category: string, exception: Throwable}|null The first failure, or null when every step succeeded. + */ + private static function runRecoverySteps(ScheduledVolumeBackupExecution $execution): ?array { $execution->loadMissing('scheduledVolumeBackup.backupable.resource'); + $failure = null; if ($execution->stop_recovery_pending) { - self::recoverContainers($execution); + try { + self::recoverContainers($execution); + } catch (Throwable $exception) { + $failure = ['category' => self::categorizeContainerFailure($exception), 'exception' => $exception]; + } } if ($execution->s3_cleanup_pending) { - self::cleanupS3Upload($execution); + try { + self::cleanupS3Upload($execution); + } catch (Throwable $exception) { + $failure ??= ['category' => self::categorizeS3Failure($exception), 'exception' => $exception]; + } } + + return $failure; + } + + /** + * Returns the minutes to wait before the next automatic attempt after the given number of failed attempts. + */ + public static function backoffMinutes(int $attempts): int + { + $exponent = max(0, min($attempts - 1, 16)); + + return (int) min(self::BACKOFF_BASE_MINUTES * (2 ** $exponent), self::BACKOFF_CAP_MINUTES); + } + + public static function dispatchCacheKey(int $executionId): string + { + return "volume-backup-recovery-dispatched:{$executionId}"; + } + + private static function recordSuccess(ScheduledVolumeBackupExecution $execution): void + { + $execution->update([ + 'recovery_attempts' => 0, + 'recovery_last_attempt_at' => null, + 'recovery_next_retry_at' => null, + 'recovery_error' => null, + 'recovery_needs_attention' => false, + ]); + } + + private static function recordFailure(ScheduledVolumeBackupExecution $execution, string $category): void + { + $recorded = $execution->getConnection()->transaction(function () use ($execution, $category): ?ScheduledVolumeBackupExecution { + $current = ScheduledVolumeBackupExecution::query()->whereKey($execution->id)->lockForUpdate()->first(); + if (! $current) { + return null; + } + + $attempts = $current->recovery_attempts + 1; + $needsAttention = $attempts >= self::MAX_ATTEMPTS; + $current->update([ + 'recovery_attempts' => $attempts, + 'recovery_last_attempt_at' => now(), + 'recovery_next_retry_at' => $needsAttention ? null : now()->addMinutes(self::backoffMinutes($attempts)), + 'recovery_error' => $category, + 'recovery_needs_attention' => $needsAttention, + ]); + + return $current; + }); + + if (! $recorded) { + return; + } + + $execution->setRawAttributes($recorded->getAttributes(), true); + + Log::channel('scheduled-errors')->warning('Volume backup recovery failed', [ + 'execution_id' => $recorded->id, + 'execution_uuid' => $recorded->uuid, + 'category' => $category, + 'attempts' => $recorded->recovery_attempts, + 'needs_attention' => $recorded->recovery_needs_attention, + 'next_retry_at' => $recorded->recovery_next_retry_at?->toIso8601String(), + ]); + } + + private static function categorizeS3Failure(Throwable $exception): string + { + for ($current = $exception; $current; $current = $current->getPrevious()) { + if ($current instanceof S3Exception) { + return match ($current->getAwsErrorCode()) { + 'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'AccessDenied' => self::CATEGORY_S3_AUTH, + 'NoSuchBucket' => self::CATEGORY_S3_BUCKET, + default => self::CATEGORY_UNKNOWN, + }; + } + } + + return self::CATEGORY_UNKNOWN; + } + + private static function categorizeContainerFailure(Throwable $exception): string + { + if ($exception->getCode() === 255) { + return self::CATEGORY_SERVER_UNREACHABLE; + } + + $message = strtolower($exception->getMessage()); + foreach (self::UNREACHABLE_PATTERNS as $pattern) { + if (str_contains($message, $pattern)) { + return self::CATEGORY_SERVER_UNREACHABLE; + } + } + + return $exception instanceof RuntimeException ? self::CATEGORY_REMOTE_COMMAND : self::CATEGORY_UNKNOWN; } private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php index b4ae3ead3a..0299c98c8b 100644 --- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -4,6 +4,7 @@ namespace App\Livewire\Project\Shared\Storages; use App\Actions\Shared\DeleteScheduledVolumeBackup; use App\Jobs\VolumeBackupJob; +use App\Jobs\VolumeBackupRecoveryJob; use App\Models\LocalFileVolume; use App\Models\LocalPersistentVolume; use App\Models\S3Storage; @@ -13,6 +14,7 @@ use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Http\RedirectResponse; use Illuminate\Routing\Redirector; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Cache; use Livewire\Component; use Livewire\WithPagination; use Throwable; @@ -278,6 +280,8 @@ class VolumeBackups extends Component $deletedCount = $this->backup?->executions() ->where('local_storage_deleted', true) + ->where('stop_recovery_pending', false) + ->where('s3_cleanup_pending', false) ->where(fn ($query) => $query ->where('s3_storage_deleted', true) ->orWhereNull('s3_uploaded') @@ -290,6 +294,33 @@ class VolumeBackups extends Component ); } + public function retryRecovery(int $executionId): void + { + $this->authorize('update', $this->resource); + + $execution = $this->backup?->executions()->whereKey($executionId)->first(); + if (! $execution || ! $execution->hasPendingRecovery()) { + $this->dispatch('error', 'No pending recovery found for this backup execution.'); + + return; + } + + $execution->update([ + 'recovery_attempts' => 0, + 'recovery_next_retry_at' => null, + 'recovery_needs_attention' => false, + ]); + Cache::forget(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id)); + VolumeBackupRecoveryJob::dispatch($execution); + auditLog('ui.volume_backup.recovery_retried', [ + 'team_id' => $this->resource->team()?->id, + 'resource_uuid' => $this->resource->uuid, + 'backup_uuid' => $this->backup->uuid, + 'execution_uuid' => $execution->uuid, + ]); + $this->dispatch('success', 'Backup recovery queued.'); + } + public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string { $this->authorize('update', $this->resource); diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php index 5287344467..eec799218e 100644 --- a/app/Models/ScheduledVolumeBackupExecution.php +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -21,6 +21,11 @@ class ScheduledVolumeBackupExecution extends BaseModel 'local_storage_deleted', 's3_storage_deleted', 's3_uploaded', + 'recovery_attempts', + 'recovery_last_attempt_at', + 'recovery_next_retry_at', + 'recovery_error', + 'recovery_needs_attention', ]; protected function casts(): array @@ -34,9 +39,18 @@ class ScheduledVolumeBackupExecution extends BaseModel 'local_storage_deleted' => 'boolean', 's3_storage_deleted' => 'boolean', 's3_uploaded' => 'boolean', + 'recovery_attempts' => 'integer', + 'recovery_last_attempt_at' => 'datetime', + 'recovery_next_retry_at' => 'datetime', + 'recovery_needs_attention' => 'boolean', ]; } + public function hasPendingRecovery(): bool + { + return $this->stop_recovery_pending || $this->s3_cleanup_pending; + } + public function scheduledVolumeBackup(): BelongsTo { return $this->belongsTo(ScheduledVolumeBackup::class); diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php index a4407c064a..f4a97be5ce 100644 --- a/bootstrap/helpers/databases.php +++ b/bootstrap/helpers/databases.php @@ -282,9 +282,16 @@ function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void 'endpoint' => $s3->endpoint, 'use_path_style_endpoint' => true, 'aws_url' => $s3->awsUrl(), + 'throw' => true, ]); - if (! $disk->delete($filenames)) { + try { + $deleted = $disk->delete($filenames); + } catch (Throwable $exception) { + throw new RuntimeException('One or more S3 backup files could not be deleted.', previous: $exception); + } + + if (! $deleted) { throw new RuntimeException('One or more S3 backup files could not be deleted.'); } } diff --git a/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php b/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php new file mode 100644 index 0000000000..f10f334912 --- /dev/null +++ b/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php @@ -0,0 +1,44 @@ +unsignedSmallInteger('recovery_attempts')->default(0); + $table->timestampTz('recovery_last_attempt_at')->nullable(); + $table->timestampTz('recovery_next_retry_at')->nullable(); + $table->string('recovery_error', 32)->nullable(); + $table->boolean('recovery_needs_attention')->default(false); + + $table->index( + ['stop_recovery_pending', 's3_cleanup_pending', 'recovery_needs_attention', 'recovery_next_retry_at'], + 'scheduled_volume_executions_recovery_index', + ); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) { + $table->dropIndex('scheduled_volume_executions_recovery_index'); + $table->dropColumn([ + 'recovery_attempts', + 'recovery_last_attempt_at', + 'recovery_next_retry_at', + 'recovery_error', + 'recovery_needs_attention', + ]); + }); + } +}; diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php index b6dd4ffd4d..b4bf4465ec 100644 --- a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php +++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php @@ -62,6 +62,14 @@ if (empty($deleteActions)) { $deleteActions[] = 'This backup execution record will be deleted.'; } + $recoveryPending = $execution->status !== 'running' && $execution->hasPendingRecovery(); + $recoveryErrorLabel = match ($execution->recovery_error) { + 's3_auth' => 'S3 credentials', + 's3_bucket' => 'S3 bucket missing', + 'server_unreachable' => 'server unreachable', + 'remote_command' => 'remote command failed', + default => 'unknown error', + }; @endphp
s3_uploaded ? 'success' : 'error')" /> @endif + @if ($recoveryPending) + @if ($execution->recovery_needs_attention) + + @else + + @endif + @endif @@ -109,6 +125,14 @@ @endif + @if ($recoveryPending) + + @endif @if ($execution->status !== 'running') 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +/** + * @return array{0: Application, 1: LocalPersistentVolume, 2: Server, 3: ScheduledVolumeBackup, 4: S3Storage} + */ +function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): array +{ + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => $privateKey->id, + 'ip' => '203.0.113.20', + ]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + $s3 = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'AKIARECOVERYKEY', + 'secret' => 'recovery-secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $s3->id, + 'frequency' => 'daily', + 'save_s3' => true, + ...$backupAttributes, + ]); + + return [$application, $volume, $server, $backup, $s3]; +} + +function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution +{ + return ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $backup->s3_storage_id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/partial.tar.gz', + 's3_cleanup_pending' => true, + ...$attributes, + ]); +} + +function failRecoveryBoundsS3Delete(string $awsCode): void +{ + $s3Exception = new S3Exception( + 'The AWS Access Key Id AKIARECOVERYKEY you provided does not exist in our records.', + new Command('DeleteObject'), + ['code' => $awsCode], + ); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->andThrow(UnableToDeleteFile::atLocation('partial.tar.gz', '', $s3Exception)); + Storage::shouldReceive('build')->andReturn($disk); +} + +function signInForRecoveryBounds($testCase, Team $team): User +{ + $user = User::factory()->create(); + $user->teams()->attach($team, ['role' => 'owner']); + $testCase->actingAs($user); + session(['currentTeam' => $team]); + + return $user; +} + +function makeRecoveryBoundsServerFunctional(Server $server): void +{ + $server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); +} + +it('doubles the recovery backoff and caps it at six hours', function () { + expect(collect(range(1, 9))->map(fn (int $attempts) => VolumeBackupRecoveryJob::backoffMinutes($attempts))->all()) + ->toBe([5, 10, 20, 40, 80, 160, 320, 360, 360]); +}); + +it('records a categorized S3 cleanup failure without throwing or leaking the S3 message', function () { + Carbon::setTestNow('2026-10-03 12:00:00'); + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create()); + $execution = createRecoveryBoundsExecution($backup, ['message' => 'Worker timed out']); + failRecoveryBoundsS3Delete('InvalidAccessKeyId'); + $logger = Mockery::spy(); + Log::shouldReceive('channel')->with('scheduled-errors')->andReturn($logger); + + (new VolumeBackupRecoveryJob($execution))->handle(); + + $execution->refresh(); + expect($execution->s3_cleanup_pending)->toBeTrue() + ->and($execution->s3_storage_deleted)->toBeFalse() + ->and($execution->recovery_attempts)->toBe(1) + ->and($execution->recovery_error)->toBe('s3_auth') + ->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue() + ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(5)))->toBeTrue() + ->and($execution->recovery_needs_attention)->toBeFalse() + ->and($execution->message)->toBe('Worker timed out'); + $logger->shouldHaveReceived('warning')->once()->with( + 'Volume backup recovery failed', + Mockery::on(fn (array $context): bool => $context['execution_uuid'] === $execution->uuid + && $context['category'] === 's3_auth' + && ! str_contains(json_encode($context), 'AKIA')), + ); +}); + +it('categorizes a missing S3 bucket and grows the backoff on each failure', function () { + Carbon::setTestNow('2026-10-03 12:00:00'); + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create()); + $execution = createRecoveryBoundsExecution($backup); + failRecoveryBoundsS3Delete('NoSuchBucket'); + + VolumeBackupRecoveryJob::recoverWithBounds($execution); + VolumeBackupRecoveryJob::recoverWithBounds($execution); + + $execution->refresh(); + expect($execution->recovery_error)->toBe('s3_bucket') + ->and($execution->recovery_attempts)->toBe(2) + ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(10)))->toBeTrue(); +}); + +it('needs attention after the maximum attempts and stops automatic recovery', function () { + Carbon::setTestNow('2026-10-03 12:00:00'); + config(['constants.coolify.self_hosted' => true]); + Queue::fake(); + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create()); + $execution = createRecoveryBoundsExecution($backup, [ + 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS - 1, + ]); + failRecoveryBoundsS3Delete('AccessDenied'); + + VolumeBackupRecoveryJob::recoverWithBounds($execution); + VolumeBackupRecoveryJob::recoverWithBounds($execution); + + $execution->refresh(); + expect($execution->recovery_needs_attention)->toBeTrue() + ->and($execution->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS) + ->and($execution->recovery_next_retry_at)->toBeNull(); + + Carbon::setTestNow('2026-10-04 12:00:00'); + (new ScheduledJobManager)->handle(); + + Queue::assertNotPushed(VolumeBackupRecoveryJob::class); +}); + +it('waits for the backoff before dispatching recovery again', function () { + Carbon::setTestNow('2026-10-03 12:00:00'); + config(['constants.coolify.self_hosted' => true]); + Queue::fake(); + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create()); + createRecoveryBoundsExecution($backup, [ + 'recovery_attempts' => 3, + 'recovery_next_retry_at' => now()->addMinutes(20), + ]); + + (new ScheduledJobManager)->handle(); + Queue::assertNotPushed(VolumeBackupRecoveryJob::class); + + Carbon::setTestNow('2026-10-03 12:20:00'); + (new ScheduledJobManager)->handle(); + Queue::assertPushed(VolumeBackupRecoveryJob::class, 1); +}); + +it('cleans the S3 upload even when container recovery fails', function () { + Carbon::setTestNow('2026-10-03 12:00:00'); + config(['constants.ssh.max_retries' => 1, 'constants.ssh.mux_enabled' => false]); + Storage::fake('ssh-keys'); + Storage::fake('ssh-mux'); + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['stop_during_backup' => true]); + $execution = createRecoveryBoundsExecution($backup, ['stop_recovery_pending' => true]); + Process::fake([ + '*' => Process::result(errorOutput: 'ssh: connect to host 203.0.113.20 port 22: Connection refused', exitCode: 255), + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + $filesystem = Mockery::mock(Storage::getFacadeRoot()); + $filesystem->shouldReceive('build')->once()->andReturn($disk); + Storage::swap($filesystem); + + VolumeBackupRecoveryJob::recoverWithBounds($execution); + + $execution->refresh(); + expect($execution->stop_recovery_pending)->toBeTrue() + ->and($execution->s3_cleanup_pending)->toBeFalse() + ->and($execution->s3_storage_deleted)->toBeTrue() + ->and($execution->recovery_error)->toBe('server_unreachable') + ->and($execution->recovery_attempts)->toBe(1); +}); + +it('resets the recovery state after a successful attempt', function () { + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create()); + $execution = createRecoveryBoundsExecution($backup, [ + 'recovery_attempts' => 4, + 'recovery_error' => 's3_auth', + 'recovery_last_attempt_at' => now()->subHour(), + 'recovery_next_retry_at' => now()->subMinute(), + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build')->once()->andReturn($disk); + + (new VolumeBackupRecoveryJob($execution))->handle(); + + $execution->refresh(); + expect($execution->s3_cleanup_pending)->toBeFalse() + ->and($execution->s3_storage_deleted)->toBeTrue() + ->and($execution->recovery_attempts)->toBe(0) + ->and($execution->recovery_error)->toBeNull() + ->and($execution->recovery_next_retry_at)->toBeNull(); +}); + +it('only blocks new scheduled backups while stopped containers await recovery', function (array $pending, bool $dispatched) { + Carbon::setTestNow('2026-10-03 12:00:00'); + config(['constants.coolify.self_hosted' => true]); + Queue::fake(); + [, , $server, $backup] = createRecoveryBoundsBackup(Team::factory()->create(), [ + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + makeRecoveryBoundsServerFunctional($server); + createRecoveryBoundsExecution($backup, $pending); + Cache::forget("scheduled-volume-backup:{$backup->id}"); + + (new ScheduledJobManager)->handle(); + + $dispatched + ? Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup)) + : Queue::assertNotPushed(VolumeBackupJob::class); +})->with([ + 'only S3 cleanup pending' => [['s3_cleanup_pending' => true, 'recovery_needs_attention' => true], true], + 'stopped containers pending' => [['stop_recovery_pending' => true, 's3_cleanup_pending' => false], false], + 'stopped containers need attention' => [['stop_recovery_pending' => true, 'recovery_needs_attention' => true], false], +]); + +it('keeps executions with pending recovery during retention cleanup', function () { + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['save_s3' => false]); + $pending = createRecoveryBoundsExecution($backup, [ + 'local_storage_deleted' => true, + 's3_uploaded' => null, + ]); + $deletable = createRecoveryBoundsExecution($backup, [ + 'filename' => null, + 'local_storage_deleted' => true, + 's3_uploaded' => null, + 's3_cleanup_pending' => false, + ]); + + (new VolumeBackupJob($backup->fresh()))->handle(); + + expect($pending->fresh())->not->toBeNull() + ->and($deletable->fresh())->toBeNull(); +}); + +it('keeps executions with pending recovery when cleaning up deleted entries', function () { + $team = Team::factory()->create(); + signInForRecoveryBounds($this, $team); + [$application, $volume, , $backup] = createRecoveryBoundsBackup($team); + $pending = createRecoveryBoundsExecution($backup, [ + 'local_storage_deleted' => true, + 's3_uploaded' => false, + ]); + $deletable = createRecoveryBoundsExecution($backup, [ + 'local_storage_deleted' => true, + 's3_uploaded' => false, + 's3_cleanup_pending' => false, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('cleanupDeleted'); + + expect($pending->fresh())->not->toBeNull() + ->and($deletable->fresh())->toBeNull(); +}); + +it('shows the recovery state and retries recovery manually', function () { + Queue::fake(); + $team = Team::factory()->create(); + signInForRecoveryBounds($this, $team); + [$application, $volume, , $backup] = createRecoveryBoundsBackup($team); + $execution = createRecoveryBoundsExecution($backup, [ + 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS, + 'recovery_error' => 's3_auth', + 'recovery_needs_attention' => true, + ]); + Cache::put(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(5)); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 'executions']) + ->assertSee('Needs attention (S3 credentials)') + ->assertSee('Retry recovery') + ->call('retryRecovery', $execution->id) + ->assertDispatched('success') + ->assertSee('Recovery pending'); + + $execution->refresh(); + expect($execution->recovery_attempts)->toBe(0) + ->and($execution->recovery_needs_attention)->toBeFalse() + ->and($execution->recovery_next_retry_at)->toBeNull() + ->and(Cache::has(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id)))->toBeFalse(); + Queue::assertPushed(VolumeBackupRecoveryJob::class, fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution)); +}); + +it('prevents another team from retrying recovery', function () { + Queue::fake(); + [$application, $volume, , $backup] = createRecoveryBoundsBackup(Team::factory()->create()); + $execution = createRecoveryBoundsExecution($backup, [ + 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS, + 'recovery_needs_attention' => true, + ]); + $otherTeam = Team::factory()->create(); + signInForRecoveryBounds($this, $otherTeam); + [$otherApplication, $otherVolume] = createRecoveryBoundsBackup($otherTeam); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertForbidden(); + Livewire::test(VolumeBackups::class, ['storage' => $otherVolume, 'resource' => $otherApplication]) + ->call('retryRecovery', $execution->id) + ->assertDispatched('error'); + + expect($execution->fresh()->recovery_needs_attention)->toBeTrue() + ->and($execution->fresh()->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS); + Queue::assertNotPushed(VolumeBackupRecoveryJob::class); +});