diff --git a/.ai/lessons.md b/.ai/lessons.md index 5cde3ef34e..7b94e29b3b 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -12,6 +12,7 @@ - SSH retries replay the full command batch. Only exit code 255 is an SSH transport failure; other exit codes come from the remote command, so do not retry them. Keep batches safe to run twice (for example `docker rm -f || true` before `docker run --name `). - Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open. - Browser `click('Text')` can match a hidden element first (for example inactive modal steps) and wait forever; use `button:visible:has-text("Text")`. +- A new worktree has no `vendor/` or `public/build/`. Run `composer install --ignore-platform-req=ext-zip` and copy `public/build` from the main checkout before page-rendering tests; otherwise they fail with a Vite manifest error. - Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only. - Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`. diff --git a/app/Actions/Development/StartDevelopmentInstanceDatabase.php b/app/Actions/Development/StartDevelopmentInstanceDatabase.php new file mode 100644 index 0000000000..41b8428bd4 --- /dev/null +++ b/app/Actions/Development/StartDevelopmentInstanceDatabase.php @@ -0,0 +1,33 @@ + config('database.connections.pgsql.username'), + 'POSTGRES_PASSWORD' => config('database.connections.pgsql.password'), + 'POSTGRES_DB' => config('database.connections.pgsql.database'), + ])->map(fn ($value, string $key): string => '-e '.escapeshellarg("{$key}={$value}"))->implode(' '); + + instant_remote_process([ + 'if ! docker inspect coolify-db >/dev/null 2>&1; then', + "docker run -d --name coolify-db --restart unless-stopped {$environment} postgres:15-alpine", + 'fi', + 'docker start coolify-db', + ], $server, disableMultiplexing: true); + } +} diff --git a/app/Actions/Shared/DeleteScheduledVolumeBackup.php b/app/Actions/Shared/DeleteScheduledVolumeBackup.php index d56cce9384..afeb44fce6 100644 --- a/app/Actions/Shared/DeleteScheduledVolumeBackup.php +++ b/app/Actions/Shared/DeleteScheduledVolumeBackup.php @@ -4,6 +4,7 @@ namespace App\Actions\Shared; use App\Jobs\VolumeBackupJob; use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use Illuminate\Support\Facades\Cache; use Lorisleiva\Actions\Concerns\AsAction; @@ -52,23 +53,10 @@ class DeleteScheduledVolumeBackup } if ($deleteS3Archives) { - $s3Executions = $backup->executions() - ->with('s3') - ->where('s3_uploaded', true) - ->where('s3_storage_deleted', false) - ->get(); - - foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { - $s3 = $executions->first()->s3; - if (! $s3) { - throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); - } - - $filenames = $executions->pluck('filename')->filter()->all(); - if ($filenames !== []) { - deleteBackupsS3($filenames, $s3); - } - } + $backup->executions() + ->whereHas('s3Replicas', fn ($query) => $query->where('s3_uploaded', true)->where('s3_storage_deleted', false)) + ->get() + ->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->deleteS3Copies()); } $backup->delete(); diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php index 082ad83ae2..26fc463ff7 100644 --- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php +++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php @@ -3,7 +3,9 @@ namespace App\Console\Commands; use App\Actions\Development\SeedDevelopmentQemuServer; +use App\Actions\Development\StartDevelopmentInstanceDatabase; use Illuminate\Console\Command; +use Throwable; class SeedDevelopmentQemuServerCommand extends Command { @@ -22,6 +24,15 @@ class SeedDevelopmentQemuServerCommand extends Command $server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost'), $this->option('ip') ?: null, (int) $this->option('port')); $this->info("Seeded {$server->name} at {$server->ip}."); + if ($this->option('as-localhost')) { + try { + StartDevelopmentInstanceDatabase::run($server); + $this->info('Started the coolify-db container for instance backups.'); + } catch (Throwable $e) { + $this->warn("Could not start the coolify-db container for instance backups: {$e->getMessage()}"); + } + } + return self::SUCCESS; } } diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index d71b64563b..b6fe02d8bc 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -32,6 +32,8 @@ use Illuminate\Database\Eloquent\Collection as EloquentCollection; use Illuminate\Database\Eloquent\Model; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; +use Illuminate\Support\Arr; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\DB; use OpenApi\Attributes as OA; @@ -152,6 +154,58 @@ class DatabasesController extends Controller } } + /** + * Resolves the S3 destinations of a backup request, primary first. `s3_storage_uuids` lists every destination; + * `s3_storage_uuid` alone selects a single destination, and sent with `s3_storage_uuids` it must be in the list and + * becomes the primary destination. + * + * @return Collection|JsonResponse|null Null when the request does not select destinations. + */ + private function requestedBackupS3Storages(Request $request, int|string $teamId): Collection|JsonResponse|null + { + $hasList = $request->has('s3_storage_uuids'); + if (! $hasList && ! $request->filled('s3_storage_uuid')) { + return null; + } + + $uuids = collect($hasList ? $request->input('s3_storage_uuids') : [$request->input('s3_storage_uuid')])->unique()->values(); + $primaryUuid = $request->input('s3_storage_uuid') ?? $uuids->first(); + $errorField = $hasList ? 's3_storage_uuids' : 's3_storage_uuid'; + + if (filled($primaryUuid) && ! $uuids->contains($primaryUuid)) { + return response()->json([ + 'message' => 'Validation failed.', + 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid must be one of the s3_storage_uuids.']], + ], 422); + } + + $storages = S3Storage::ownedByCurrentTeamAPI($teamId)->whereIn('uuid', $uuids->all())->get(); + if ($storages->count() !== $uuids->count()) { + return response()->json([ + 'message' => 'Validation failed.', + 'errors' => [$errorField => ['The selected S3 storage is invalid for this team.']], + ], 422); + } + + return $storages->sortBy(fn (S3Storage $storage): int => $storage->uuid === $primaryUuid ? 0 : 1)->values(); + } + + /** + * Adds the S3 destination UUIDs (primary first) to a serialized backup without exposing the storages themselves. + */ + private function withBackupS3StorageUuids(ScheduledDatabaseBackup $backup): ScheduledDatabaseBackup + { + $storages = $backup->s3Storages->sortBy(fn (S3Storage $storage): int => $storage->id === $backup->s3_storage_id ? 0 : 1); + if ($storages->isEmpty() && $backup->s3) { + $storages = collect([$backup->s3]); + } + + $backup->setAttribute('s3_storage_uuid', $backup->s3?->uuid); + $backup->setAttribute('s3_storage_uuids', $storages->pluck('uuid')->values()->all()); + + return $backup->unsetRelation('s3Storages')->unsetRelation('s3'); + } + private function removeSensitiveData($database, bool $loadNestedServerSecrets = false) { $database->makeHidden([ @@ -302,9 +356,10 @@ class DatabasesController extends Controller $databaseIds = $databases->pluck('id')->toArray(); - $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('latest_log') + $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['latest_log', 's3', 's3Storages']) ->whereIn('database_id', $databaseIds) ->get() + ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup)) ->groupBy(fn (ScheduledDatabaseBackup $backup) => $backup->database_type.':'.$backup->database_id); $databases = $databases->map(function ($database) use ($backupConfigs) { @@ -340,7 +395,7 @@ class DatabasesController extends Controller responses: [ new OA\Response( response: 200, - description: 'Get all backups for a database', + description: 'Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).', content: new OA\JsonContent( type: 'string', example: 'Content is very complex. Will be implemented later.', @@ -376,7 +431,8 @@ class DatabasesController extends Controller $this->authorize('view', $database); - $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('executions')->where('database_id', $database->id)->get(); + $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['executions', 's3', 's3Storages'])->where('database_id', $database->id)->get() + ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup)); return response()->json($backupConfig); } @@ -873,7 +929,8 @@ class DatabasesController extends Controller 'frequency' => ['type' => 'string', 'description' => 'Backup frequency (cron expression or: every_minute, hourly, daily, weekly, monthly, yearly)'], 'enabled' => ['type' => 'boolean', 'description' => 'Whether the backup is enabled', 'default' => true], 'save_s3' => ['type' => 'boolean', 'description' => 'Whether to save backups to S3', 'default' => false], - 's3_storage_uuid' => ['type' => 'string', 'description' => 'S3 storage UUID (required if save_s3 is true)'], + 's3_storage_uuid' => ['type' => 'string', 'description' => 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true.'], + 's3_storage_uuids' => ['type' => 'array', 'items' => ['type' => 'string'], 'description' => 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'], 'databases_to_backup' => ['type' => 'string', 'description' => 'Comma separated list of databases to backup'], 'dump_all' => ['type' => 'boolean', 'description' => 'Whether to dump all databases', 'default' => false], 'backup_now' => ['type' => 'boolean', 'description' => 'Whether to trigger backup immediately after creation'], @@ -921,7 +978,7 @@ class DatabasesController extends Controller )] public function create_backup(Request $request) { - $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days']; + $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 's3_storage_uuids', 'timeout', 'missing_backup_notification_days']; $teamId = getTeamIdFromToken(); if (is_null($teamId)) { @@ -941,6 +998,8 @@ class DatabasesController extends Controller 'dump_all' => 'boolean', 'backup_now' => 'boolean|nullable', 's3_storage_uuid' => 'string|exists:s3_storages,uuid|nullable', + 's3_storage_uuids' => 'array', + 's3_storage_uuids.*' => 'string|distinct', 'databases_to_backup' => 'string|nullable', 'database_backup_retention_amount_locally' => 'integer|min:0', 'database_backup_retention_days_locally' => 'integer|min:0', @@ -986,24 +1045,17 @@ class DatabasesController extends Controller ], 422); } - // Validate S3 storage if save_s3 is true - if ($request->boolean('save_s3') && ! $request->filled('s3_storage_uuid')) { + $s3Storages = $this->requestedBackupS3Storages($request, $teamId); + if ($s3Storages instanceof JsonResponse) { + return $s3Storages; + } + if ($request->boolean('save_s3') && blank($s3Storages)) { return response()->json([ 'message' => 'Validation failed.', - 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid field is required when save_s3 is true.']], + 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid or s3_storage_uuids field is required when save_s3 is true.']], ], 422); } - if ($request->filled('s3_storage_uuid')) { - $existsInTeam = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $request->s3_storage_uuid)->exists(); - if (! $existsInTeam) { - return response()->json([ - 'message' => 'Validation failed.', - 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']], - ], 422); - } - } - // Check for extra fields $extraFields = array_diff(array_keys($request->all()), $backupConfigFields, ['backup_now']); if (! empty($extraFields)) { @@ -1018,20 +1070,9 @@ class DatabasesController extends Controller ], 422); } - $backupData = $request->only($backupConfigFields); - - // Convert s3_storage_uuid to s3_storage_id - if (isset($backupData['s3_storage_uuid'])) { - $s3Storage = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $backupData['s3_storage_uuid'])->first(); - if ($s3Storage) { - $backupData['s3_storage_id'] = $s3Storage->id; - } elseif ($request->boolean('save_s3')) { - return response()->json([ - 'message' => 'Validation failed.', - 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']], - ], 422); - } - unset($backupData['s3_storage_uuid']); + $backupData = Arr::except($request->only($backupConfigFields), ['s3_storage_uuid', 's3_storage_uuids']); + if ($s3Storages !== null) { + $backupData['s3_storage_id'] = $s3Storages->first()?->id; } // Set default databases_to_backup based on database type if not provided @@ -1070,6 +1111,9 @@ class DatabasesController extends Controller } $backupConfig = ScheduledDatabaseBackup::create($backupData); + if ($s3Storages !== null) { + $backupConfig->syncS3Storages($s3Storages->pluck('id')->all()); + } // Trigger immediate backup if requested if ($request->backup_now) { @@ -1129,7 +1173,8 @@ class DatabasesController extends Controller type: 'object', properties: [ 'save_s3' => ['type' => 'boolean', 'description' => 'Whether data is saved in s3 or not'], - 's3_storage_uuid' => ['type' => 'string', 'description' => 'S3 storage UUID'], + 's3_storage_uuid' => ['type' => 'string', 'description' => 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay.'], + 's3_storage_uuids' => ['type' => 'array', 'items' => ['type' => 'string'], 'description' => 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'], 'backup_now' => ['type' => 'boolean', 'description' => 'Whether to take a backup now or not'], 'enabled' => ['type' => 'boolean', 'description' => 'Whether the backup is enabled or not'], 'databases_to_backup' => ['type' => 'string', 'description' => 'Comma separated list of databases to backup'], @@ -1172,7 +1217,7 @@ class DatabasesController extends Controller )] public function update_backup(Request $request) { - $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days']; + $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 's3_storage_uuids', 'timeout', 'missing_backup_notification_days']; $teamId = getTeamIdFromToken(); if (is_null($teamId)) { @@ -1189,6 +1234,8 @@ class DatabasesController extends Controller 'enabled' => 'boolean', 'dump_all' => 'boolean', 's3_storage_uuid' => 'string|exists:s3_storages,uuid|nullable', + 's3_storage_uuids' => 'array', + 's3_storage_uuids.*' => 'string|distinct', 'databases_to_backup' => 'string|nullable', 'frequency' => 'string', 'database_backup_retention_amount_locally' => 'integer|min:0', @@ -1236,20 +1283,9 @@ class DatabasesController extends Controller } } - if ($request->boolean('save_s3') && ! $request->filled('s3_storage_uuid')) { - return response()->json([ - 'message' => 'Validation failed.', - 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid field is required when save_s3 is true.']], - ], 422); - } - if ($request->filled('s3_storage_uuid')) { - $existsInTeam = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $request->s3_storage_uuid)->exists(); - if (! $existsInTeam) { - return response()->json([ - 'message' => 'Validation failed.', - 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']], - ], 422); - } + $s3Storages = $this->requestedBackupS3Storages($request, $teamId); + if ($s3Storages instanceof JsonResponse) { + return $s3Storages; } $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->where('database_id', $database->id) @@ -1259,6 +1295,14 @@ class DatabasesController extends Controller return response()->json(['message' => 'Backup config not found.'], 404); } + $saveToS3 = $request->has('save_s3') ? $request->boolean('save_s3') : (bool) $backupConfig->save_s3; + if (($request->boolean('save_s3') && $s3Storages === null) || ($saveToS3 && $s3Storages?->isEmpty())) { + return response()->json([ + 'message' => 'Validation failed.', + 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid or s3_storage_uuids field is required when save_s3 is true.']], + ], 422); + } + $extraFields = array_diff(array_keys($request->all()), $backupConfigFields, ['backup_now']); if (! empty($extraFields)) { $errors = $validator->errors(); @@ -1272,20 +1316,9 @@ class DatabasesController extends Controller ], 422); } - $backupData = $request->only($backupConfigFields); - - // Convert s3_storage_uuid to s3_storage_id - if (isset($backupData['s3_storage_uuid'])) { - $s3Storage = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $backupData['s3_storage_uuid'])->first(); - if ($s3Storage) { - $backupData['s3_storage_id'] = $s3Storage->id; - } elseif ($request->boolean('save_s3')) { - return response()->json([ - 'message' => 'Validation failed.', - 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']], - ], 422); - } - unset($backupData['s3_storage_uuid']); + $backupData = Arr::except($request->only($backupConfigFields), ['s3_storage_uuid', 's3_storage_uuids']); + if ($s3Storages !== null) { + $backupData['s3_storage_id'] = $s3Storages->first()?->id; } // Validate databases_to_backup input @@ -1300,7 +1333,19 @@ class DatabasesController extends Controller } } + $storageIds = $s3Storages?->pluck('id'); + // An older client sends only s3_storage_uuid. When it is already a destination, the other destinations stay. + if ($storageIds !== null && ! $request->has('s3_storage_uuids')) { + $currentStorageIds = $backupConfig->selectedS3Storages()->pluck('id'); + if ($currentStorageIds->contains($storageIds->first())) { + $storageIds = $currentStorageIds; + } + } + $backupConfig->update($backupData); + if ($storageIds !== null) { + $backupConfig->syncS3Storages($storageIds->all()); + } if ($request->backup_now) { dispatch(new DatabaseBackupJob($backupConfig)); @@ -2938,9 +2983,10 @@ class DatabasesController extends Controller if ($execution->filename) { deleteBackupsLocally($execution->filename, $database->destination->server); - if ($deleteS3 && $backup->s3) { - deleteBackupsS3($execution->filename, $backup->s3); - } + } + + if ($deleteS3) { + $execution->deleteS3Copies(); } $execution->delete(); @@ -3073,9 +3119,10 @@ class DatabasesController extends Controller if ($execution->filename) { deleteBackupsLocally($execution->filename, $database->destination->server); - if ($deleteS3 && $backup->s3) { - deleteBackupsS3($execution->filename, $backup->s3); - } + } + + if ($deleteS3) { + $execution->deleteS3Copies(); } $execution->delete(); @@ -5195,7 +5242,7 @@ class DatabasesController extends Controller } foreach ($database->scheduledBackups()->get() as $backup) { - $backup->replicate([ + $newBackup = $backup->replicate([ 'id', 'created_at', 'updated_at', @@ -5205,7 +5252,9 @@ class DatabasesController extends Controller 'database_id' => $newDatabase->id, 'database_type' => $newDatabase->getMorphClass(), 'team_id' => $teamId, - ])->save(); + ]); + $newBackup->save(); + $backup->copyS3StoragesTo($newBackup); } foreach ($database->environment_variables()->get() as $environmentVariable) { diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.php index 7411f8a472..01f3657434 100644 --- a/app/Http/Controllers/Api/ServicesController.php +++ b/app/Http/Controllers/Api/ServicesController.php @@ -3399,7 +3399,7 @@ class ServicesController extends Controller } foreach ($sourceDatabase->scheduledBackups()->get() as $backup) { - $backup->replicate([ + $newBackup = $backup->replicate([ 'id', 'created_at', 'updated_at', @@ -3408,7 +3408,9 @@ class ServicesController extends Controller 'database_id' => $database->id, 'database_type' => $database->getMorphClass(), 'team_id' => $teamId, - ])->save(); + ]); + $newBackup->save(); + $backup->copyS3StoragesTo($newBackup); } } diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.php index e3b48a367a..a199f530db 100644 --- a/app/Http/Controllers/Api/VolumeBackupsController.php +++ b/app/Http/Controllers/Api/VolumeBackupsController.php @@ -14,6 +14,7 @@ use App\Models\Service; use Illuminate\Database\Eloquent\Model; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; +use Illuminate\Support\Collection; use Illuminate\Support\MessageBag; use OpenApi\Attributes as OA; use RuntimeException; @@ -27,7 +28,8 @@ use RuntimeException; new OA\Property(property: 'save_s3', type: 'boolean', default: false), new OA\Property(property: 'disable_local_backup', type: 'boolean', default: false), new OA\Property(property: 'stop_during_backup', type: 'boolean', default: false), - new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true), + new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true, description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.'), + new OA\Property(property: 's3_storage_uuids', type: 'array', items: new OA\Items(type: 'string'), description: 'UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.'), new OA\Property(property: 'retention_amount_locally', type: 'integer', default: 7, minimum: 0, maximum: 10000), new OA\Property(property: 'retention_days_locally', type: 'integer', default: 0, maximum: 2147483647, minimum: 0), new OA\Property(property: 'retention_max_storage_locally', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0), @@ -53,7 +55,8 @@ use RuntimeException; new OA\Property(property: 'save_s3', type: 'boolean'), new OA\Property(property: 'disable_local_backup', type: 'boolean'), new OA\Property(property: 'stop_during_backup', type: 'boolean'), - new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true), + new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true, description: 'Primary S3 destination.'), + new OA\Property(property: 's3_storage_uuids', type: 'array', items: new OA\Items(type: 'string'), description: 'Every S3 destination, primary first.'), new OA\Property(property: 'retention_amount_locally', type: 'integer'), new OA\Property(property: 'retention_days_locally', type: 'integer'), new OA\Property(property: 'retention_max_storage_locally', type: 'number', format: 'float'), @@ -165,7 +168,7 @@ class VolumeBackupsController extends Controller return response()->json(['message' => 'Storage not found.'], 404); } - ['errors' => $errors, 's3Storage' => $s3Storage, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId); + ['errors' => $errors, 's3Storages' => $s3Storages, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId); if ($errors->isNotEmpty()) { return response()->json([ @@ -174,11 +177,11 @@ class VolumeBackupsController extends Controller ], 422); } - return $this->persistSchedule($request, $storage, $teamId, $s3Storage, $saveToS3, $resourceType, $resource); + return $this->persistSchedule($request, $storage, $teamId, $s3Storages, $saveToS3, $resourceType, $resource); } /** - * @return array{errors: MessageBag, s3Storage: S3Storage|null, saveToS3: bool} + * @return array{errors: MessageBag, s3Storages: Collection, saveToS3: bool} */ private function validateUpsertRequest( Request $request, @@ -192,6 +195,8 @@ class VolumeBackupsController extends Controller 'disable_local_backup' => 'boolean', 'stop_during_backup' => 'boolean', 's3_storage_uuid' => 'nullable|string', + 's3_storage_uuids' => 'array', + 's3_storage_uuids.*' => 'string|distinct', 'retention_amount_locally' => 'integer|min:0|max:10000', 'retention_days_locally' => 'integer|min:0|max:2147483647', 'retention_max_storage_locally' => 'numeric|min:0|max:9999999999', @@ -209,6 +214,7 @@ class VolumeBackupsController extends Controller 'disable_local_backup', 'stop_during_backup', 's3_storage_uuid', + 's3_storage_uuids', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', @@ -232,16 +238,24 @@ class VolumeBackupsController extends Controller $errors->add('disable_local_backup', 'Local backups can only be disabled when S3 backups are enabled.'); } - $s3Storage = null; - if ($saveToS3) { - $s3Storage = S3Storage::query() + $s3Storages = collect(); + if ($saveToS3 && ! $errors->hasAny(['s3_storage_uuid', 's3_storage_uuids', 's3_storage_uuids.*'])) { + $hasList = $request->has('s3_storage_uuids'); + $uuids = collect($hasList ? $request->input('s3_storage_uuids') : [$request->input('s3_storage_uuid')])->filter()->unique()->values(); + $primaryUuid = $request->input('s3_storage_uuid') ?? $uuids->first(); + + $s3Storages = S3Storage::query() ->where('team_id', $teamId) ->where('is_usable', true) - ->where('uuid', $request->input('s3_storage_uuid')) - ->first(); + ->whereIn('uuid', $uuids->all()) + ->get() + ->sortBy(fn (S3Storage $s3Storage): int => $s3Storage->uuid === $primaryUuid ? 0 : 1) + ->values(); - if (! $s3Storage) { - $errors->add('s3_storage_uuid', 'Select a usable S3 storage owned by your team.'); + if ($hasList && filled($primaryUuid) && ! $uuids->contains($primaryUuid)) { + $errors->add('s3_storage_uuid', 'The s3_storage_uuid must be one of the s3_storage_uuids.'); + } elseif ($uuids->isEmpty() || $s3Storages->count() !== $uuids->count()) { + $errors->add($hasList ? 's3_storage_uuids' : 's3_storage_uuid', 'Select a usable S3 storage owned by your team.'); } } @@ -251,7 +265,7 @@ class VolumeBackupsController extends Controller return [ 'errors' => $errors, - 's3Storage' => $s3Storage, + 's3Storages' => $s3Storages, 'saveToS3' => $saveToS3, ]; } @@ -260,7 +274,7 @@ class VolumeBackupsController extends Controller Request $request, LocalPersistentVolume|LocalFileVolume $storage, int|string $teamId, - ?S3Storage $s3Storage, + Collection $s3Storages, bool $saveToS3, string $resourceType, Model $resource, @@ -272,7 +286,7 @@ class VolumeBackupsController extends Controller 'save_s3' => $saveToS3, 'disable_local_backup' => $saveToS3 && $request->boolean('disable_local_backup'), 'stop_during_backup' => $request->boolean('stop_during_backup'), - 's3_storage_id' => $s3Storage?->id, + 's3_storage_id' => $s3Storages->first()?->id, 'retention_amount_locally' => $request->integer('retention_amount_locally', 7), 'retention_days_locally' => $request->integer('retention_days_locally'), 'retention_max_storage_locally' => $request->float('retention_max_storage_locally'), @@ -288,6 +302,7 @@ class VolumeBackupsController extends Controller } $backup = $storage->scheduledBackups()->updateOrCreate([], $attributes); + $backup->syncS3Storages($s3Storages->pluck('id')->all()); $created = $backup->wasRecentlyCreated; auditLog('api.volume_backup.schedule_set', [ @@ -298,7 +313,7 @@ class VolumeBackupsController extends Controller 'backup_uuid' => $backup->uuid, ]); - return response()->json($this->responseData($backup, $storage, $s3Storage, $created), $created ? 201 : 200); + return response()->json($this->responseData($backup, $storage, $s3Storages, $created), $created ? 201 : 200); } #[OA\Delete( @@ -431,7 +446,7 @@ class VolumeBackupsController extends Controller private function responseData( ScheduledVolumeBackup $backup, LocalPersistentVolume|LocalFileVolume $storage, - ?S3Storage $s3Storage, + Collection $s3Storages, bool $created, ): array { return [ @@ -444,7 +459,8 @@ class VolumeBackupsController extends Controller 'save_s3' => $backup->save_s3, 'disable_local_backup' => $backup->disable_local_backup, 'stop_during_backup' => $backup->stop_during_backup, - 's3_storage_uuid' => $s3Storage?->uuid, + 's3_storage_uuid' => $s3Storages->first()?->uuid, + 's3_storage_uuids' => $s3Storages->pluck('uuid')->all(), 'retention_amount_locally' => $backup->retention_amount_locally, 'retention_days_locally' => $backup->retention_days_locally, 'retention_max_storage_locally' => $backup->retention_max_storage_locally, diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 92d24bab3d..eb0959865b 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -83,8 +83,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue public ?string $mongo_root_password = null; - public ?S3Storage $s3 = null; - public $timeout = 3600 + self::WORKER_TIMEOUT_MARGIN_SECONDS; public ?string $backup_log_uuid = null; @@ -129,11 +127,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue if (data_get($this->backup, 'database_type') === ServiceDatabase::class) { $this->database = data_get($this->backup, 'database'); $this->server = $this->database->service->server; - $this->s3 = $this->backup->s3; } else { $this->database = data_get($this->backup, 'database'); $this->server = $this->database->destination->server; - $this->s3 = $this->backup->s3; } if (is_null($this->server)) { throw new \Exception('Server not found?!'); @@ -329,6 +325,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $this->backup_location = null; $this->backup_output = null; $this->error_output = null; + $this->s3_uploaded = false; // Generate unique UUID for each database backup execution $attempts = 0; @@ -484,10 +481,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $localStorageDeleted = false; if ($this->backup->save_s3 && $localBackupSucceeded) { try { - $this->upload_to_s3(); + $s3UploadError = $this->uploadToS3Destinations(); - // If local backup is disabled, delete the local file immediately after S3 upload - if ($this->backup->disable_local_backup) { + // If local backup is disabled, delete the local file only when every destination has a copy + if ($s3UploadError === null && $this->backup->disable_local_backup) { deleteBackupsLocally($this->backup_location, $this->server); $localStorageDeleted = true; } @@ -843,9 +840,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue return devHostDockerPath($this->server, $this->backup_location); } - private function upload_to_s3(): void + /** + * Uploads the backup file to every selected S3 destination and records one replica per destination. + * + * @return string|null The failed destinations and their errors, or null when every upload succeeded. + */ + private function uploadToS3Destinations(): ?string { - if (is_null($this->s3)) { + $storages = $this->backup->selectedS3Storages(); + if ($storages->isEmpty()) { $previousS3StorageId = $this->backup->s3_storage_id; $this->backup->update([ @@ -856,13 +859,43 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($previousS3StorageId ?? 'null').'). S3 backup has been disabled for this schedule.'); } + $failures = []; + foreach ($storages as $storage) { + $uploaded = false; + $uploadError = null; + try { + $this->upload_to_s3($storage); + $uploaded = true; + } catch (Throwable $e) { + $uploadError = $e->getMessage(); + $failures[] = "{$storage->name}: {$uploadError}"; + } + + try { + $this->backup_log->s3Replicas()->create([ + 's3_storage_id' => $storage->id, + 's3_uploaded' => $uploaded, + 'message' => $uploadError, + ]); + } catch (Throwable $e) { + $failures[] = "{$storage->name}: {$e->getMessage()}"; + } + } + + $this->backup_log->refreshS3Summary(); + $this->s3_uploaded = $failures === []; + + return $failures === [] ? null : implode("\n", $failures); + } + + protected function upload_to_s3(S3Storage $s3): void + { try { - $key = $this->s3->key; - $secret = $this->s3->secret; - // $region = $this->s3->region; - $bucket = $this->s3->bucket; - $endpoint = $this->s3->endpoint; - $this->s3->testConnection(shouldSave: true); + $key = $s3->key; + $secret = $s3->secret; + $bucket = $s3->bucket; + $endpoint = $s3->endpoint; + $s3->testConnection(shouldSave: true); if (data_get($this->backup, 'database_type') === ServiceDatabase::class) { $network = $this->database->service->destination->network; } else { @@ -883,7 +916,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $escapedSecret = escapeshellarg($secret); $escapedBackupLocation = escapeshellarg($this->backup_location); $escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/"); - $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $this->s3->trustedInternalHosts())) + $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $s3->trustedInternalHosts())) ->map(fn (string $resolveOption): string => '--resolve '.escapeshellarg($resolveOption)) ->implode(' '); $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions; @@ -892,9 +925,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}"; instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true); - $this->s3_uploaded = true; } catch (Throwable $e) { - $this->s3_uploaded = false; $this->add_to_error_output($e->getMessage()); throw $e; } finally { diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index d635894b09..db226e5675 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -4,9 +4,11 @@ namespace App\Jobs; use App\Events\BackupCreated; use App\Models\LocalPersistentVolume; +use App\Models\S3Storage; use App\Models\ScheduledVolumeBackup; use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; +use App\Models\VolumeBackupS3Replica; use App\Notifications\VolumeBackup\BackupFailed; use App\Notifications\VolumeBackup\BackupSuccess; use App\Rules\SafeWebhookUrl; @@ -16,7 +18,6 @@ use Carbon\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; -use Illuminate\Database\Eloquent\Builder; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; @@ -71,16 +72,21 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue throw new \RuntimeException('The storage backup resource, team, or server no longer exists.'); } + $destinations = $this->backup->save_s3 ? $this->backup->selectedS3Storages() : collect(); $this->execution = $this->backup->executions()->create([ - 's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null, + 's3_storage_id' => $destinations->first()?->id, ]); + foreach ($destinations as $s3) { + $this->execution->s3Replicas()->create(['s3_storage_id' => $s3->id]); + } BackupCreated::dispatch($team->id); $backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid; $filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz'; $backupLocation = $backupDirectory.'/'.$filename; $this->execution->update(['filename' => $backupLocation]); - $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup; + // Streaming writes no local archive, so it can upload to one destination only. + $streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1; try { $source = $this->backup->sourcePath(); @@ -96,7 +102,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume ."; if ($streamToS3) { $this->execution->update(['local_storage_deleted' => true]); - $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image); + $archiveCommand = $this->streamToS3Command($destinations->first(), $archiveScript, $backupLocation, $source, $containerName, $image); $this->execution->update(['s3_cleanup_pending' => true]); } else { $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName) @@ -141,24 +147,27 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } $warning = null; - $s3Uploaded = null; + $s3Uploaded = false; $s3CleanupPending = false; $localStorageDeleted = $streamToS3; if ($streamToS3) { $s3Uploaded = true; + $this->execution->s3Replicas()->update(['s3_uploaded' => true]); + $this->execution->refreshS3Summary(); $this->execution->update(['s3_cleanup_pending' => false]); + } elseif ($this->backup->save_s3 && $destinations->isEmpty()) { + $warning = 'S3 upload failed: The selected S3 storage no longer exists.'; } elseif ($this->backup->save_s3) { $s3CleanupPending = true; $this->execution->update(['s3_cleanup_pending' => true]); + $uploadErrors = $this->uploadToDestinations($backupLocation, $backupDirectory, $server); + $s3Uploaded = $uploadErrors === []; - try { - $this->uploadToS3($backupLocation, $backupDirectory, $server); - $s3Uploaded = true; + if ($s3Uploaded) { $s3CleanupPending = false; - } catch (Throwable $exception) { - $s3Uploaded = false; - $warning = 'S3 upload failed: '.$exception->getMessage(); + } else { + $warning = 'S3 upload failed: '.implode(' ', $uploadErrors); try { VolumeBackupRecoveryJob::cleanupS3Upload($this->execution); @@ -184,7 +193,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue 'message' => $warning, 'size' => $size, 'filename' => $backupLocation, - 's3_uploaded' => $s3Uploaded, 's3_cleanup_pending' => $s3CleanupPending, 'local_storage_deleted' => $localStorageDeleted, ]); @@ -220,6 +228,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } } + $this->execution->s3Replicas()->whereNull('s3_uploaded')->update(['s3_uploaded' => false]); + $this->execution->refreshS3Summary(); $s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending; $this->execution->update([ @@ -347,16 +357,37 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } } - private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void + /** + * Uploads the local archive to every destination of the execution and records the result on each copy. + * + * @return array One error message per failed destination. + */ + private function uploadToDestinations(string $backupLocation, string $backupDirectory, Server $server): array { - $s3 = $this->backup->s3; + $errors = []; - if (! $s3) { - $this->backup->update(['save_s3' => false, 's3_storage_id' => null]); + foreach ($this->execution->s3Replicas()->with('s3')->get() as $replica) { + /** @var VolumeBackupS3Replica $replica */ + try { + if (! $replica->s3) { + throw new \RuntimeException('The selected S3 storage no longer exists.'); + } - throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.'); + $this->uploadToS3($replica->s3, $backupLocation, $backupDirectory, $server); + $replica->update(['s3_uploaded' => true, 'message' => null]); + } catch (Throwable $exception) { + $replica->update(['s3_uploaded' => false, 'message' => $exception->getMessage()]); + $errors[] = ($replica->s3?->name ?? 'Deleted storage').': '.$exception->getMessage(); + } } + $this->execution->refreshS3Summary(); + + return $errors; + } + + private function uploadToS3(S3Storage $s3, string $backupLocation, string $backupDirectory, Server $server): void + { $s3->testConnection(shouldSave: true); $containerName = 'volume-upload-'.$this->execution->uuid; $image = coolifyHelperImage().':'.getHelperVersion(); @@ -385,16 +416,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } } - private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string + private function streamToS3Command(S3Storage $s3, string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string { - $s3 = $this->backup->s3; - - if (! $s3) { - $this->backup->update(['save_s3' => false, 's3_storage_id' => null]); - - throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.'); - } - $s3->testConnection(shouldSave: true); $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts())) ->map(fn (string $option): string => '--resolve '.escapeshellarg($option)) @@ -462,36 +485,45 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue } } - if ($this->backup->save_s3 && $this->backup->s3 && $this->hasRetentionLimits( + if ($this->backup->save_s3 && $this->hasRetentionLimits( $this->backup->retention_amount_s3, $this->backup->retention_days_s3, $this->backup->retention_max_storage_s3, )) { + $liveCopies = fn ($query) => $query->where('s3_uploaded', true)->where('s3_storage_deleted', false); $s3Executions = $this->backup->executions() - ->with('s3') + ->with(['s3Replicas' => fn ($query) => $liveCopies($query)->with('s3')]) ->where('status', 'success') - ->where('s3_uploaded', true) - ->where('s3_storage_deleted', false) + ->whereHas('s3Replicas', $liveCopies) ->get(); - $s3Executions = $this->executionsOutsideRetention( - $s3Executions, - $this->backup->retention_amount_s3, - $this->backup->retention_days_s3, - $this->backup->retention_max_storage_s3, - ); - foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { - $s3 = $executions->first()->s3; + // Each destination keeps its own copies within the same retention settings. + foreach ($s3Executions->flatMap->s3Replicas->pluck('s3_storage_id')->unique() as $storageId) { + $executions = $this->executionsOutsideRetention( + $s3Executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->s3Replicas->contains('s3_storage_id', $storageId))->values(), + $this->backup->retention_amount_s3, + $this->backup->retention_days_s3, + $this->backup->retention_max_storage_s3, + ); + if ($executions->isEmpty()) { + continue; + } + + $replicas = $executions->map(fn (ScheduledVolumeBackupExecution $execution): VolumeBackupS3Replica => $execution->s3Replicas->firstWhere('s3_storage_id', $storageId)); + $s3 = $replicas->first()->s3; if (! $s3) { - throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + VolumeBackupS3Replica::query()->whereKey($replicas->pluck('id')->all())->update(['s3_storage_deleted' => true]); + $executions->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->refreshS3Summary()); + + continue; } $filenames = $executions->pluck('filename')->filter()->all(); if ($filenames !== []) { deleteBackupsS3($filenames, $s3); - $this->backup->executions()->whereKey($executions->pluck('id')->all()) - ->update(['s3_storage_deleted' => true]); } + VolumeBackupS3Replica::query()->whereKey($replicas->pluck('id')->all())->update(['s3_storage_deleted' => true]); + $executions->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->refreshS3Summary()); } } @@ -499,9 +531,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue ->where('local_storage_deleted', true) ->where('stop_recovery_pending', false) ->where('s3_cleanup_pending', false) - ->where(function (Builder $query): void { - $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded'); - }) + ->withoutLiveS3Copies() ->delete(); } diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php index cda0fdd046..4965df1379 100644 --- a/app/Jobs/VolumeBackupRecoveryJob.php +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -295,12 +295,19 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou ]); } + /** + * Deletes the partial upload from every destination whose copy did not finish uploading. Every destination is + * tried; the first failure is rethrown afterwards and keeps the cleanup pending. + */ public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void { - $execution->loadMissing('s3'); - $s3 = $execution->s3; + $replicas = $execution->s3Replicas() + ->with('s3') + ->where('s3_storage_deleted', false) + ->where(fn ($query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false)) + ->get(); - if (! $s3 || blank($execution->filename)) { + if ($replicas->isEmpty() || blank($execution->filename)) { self::skipRecovery( $execution, ['s3_cleanup_pending' => false], @@ -310,11 +317,25 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou return; } - deleteBackupsS3($execution->filename, $s3); - $execution->update([ - 's3_cleanup_pending' => false, - 's3_storage_deleted' => true, - ]); + $failure = null; + foreach ($replicas as $replica) { + try { + if ($replica->s3) { + deleteBackupsS3($execution->filename, $replica->s3); + } + $replica->update(['s3_storage_deleted' => true]); + } catch (Throwable $exception) { + $failure ??= $exception; + } + } + + $execution->refreshS3Summary(); + + if ($failure) { + throw $failure; + } + + $execution->update(['s3_cleanup_pending' => false]); } /** diff --git a/app/Livewire/Project/Application/Backup/Index.php b/app/Livewire/Project/Application/Backup/Index.php index fe7e2eac5d..7ca097db96 100644 --- a/app/Livewire/Project/Application/Backup/Index.php +++ b/app/Livewire/Project/Application/Backup/Index.php @@ -31,7 +31,7 @@ class Index extends Component public function render(): View { $backups = ScheduledVolumeBackup::query() - ->with(['backupable', 'latestExecution', 's3']) + ->with(['backupable', 'latestExecution', 's3', 's3Storages']) ->withCount('executions') ->forApplication($this->application) ->latest() diff --git a/app/Livewire/Project/CloneMe.php b/app/Livewire/Project/CloneMe.php index 8f2ff9af40..b310c248ee 100644 --- a/app/Livewire/Project/CloneMe.php +++ b/app/Livewire/Project/CloneMe.php @@ -279,6 +279,7 @@ class CloneMe extends Component 'team_id' => $teamId, ]); $newBackup->save(); + $backup->copyS3StoragesTo($newBackup); } $environmentVaribles = $database->environment_variables()->get(); @@ -469,6 +470,7 @@ class CloneMe extends Component 'team_id' => $teamId, ]); $newBackup->save(); + $backup->copyS3StoragesTo($newBackup); } } diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.php index 59dd1a29c0..a7c49eeb58 100644 --- a/app/Livewire/Project/Database/BackupEdit.php +++ b/app/Livewire/Project/Database/BackupEdit.php @@ -9,6 +9,7 @@ use App\Models\ServiceDatabase; use App\Models\StandalonePostgresql; use App\Traits\ListensToTeamChannel; use Exception; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Auth; @@ -76,8 +77,9 @@ class BackupEdit extends Component #[Validate(['required', 'boolean'])] public bool $disableLocalBackup = false; - #[Validate(['nullable', 'integer'])] - public ?int $s3StorageId = null; + /** @var array */ + #[Validate(['array'])] + public array $s3StorageIds = []; #[Validate(['nullable', 'string'])] public ?string $databasesToBackup = null; @@ -113,6 +115,10 @@ class BackupEdit extends Component { try { $this->authorize('view', $this->backup->database); + $this->availableS3Storages = S3Storage::query() + ->whereKey($this->availableS3StorageIds()->all()) + ->orderBy('name') + ->get(); $this->parameters = get_route_parameters(); $this->syncData(); $this->refreshStatus(); @@ -148,7 +154,6 @@ class BackupEdit extends Component $this->backup->database_backup_retention_max_storage_s3 = $this->databaseBackupRetentionMaxStorageS3; $this->backup->save_s3 = $this->saveS3; $this->backup->disable_local_backup = $this->disableLocalBackup; - $this->backup->s3_storage_id = $this->s3StorageId; // Validate databases_to_backup to prevent command injection // Handles all formats including MongoDB's "db:col1,col2|db2:col3" @@ -160,10 +165,16 @@ class BackupEdit extends Component $this->backup->dump_all = $this->dumpAll; $this->backup->timeout = $this->timeout; $this->backup->missing_backup_notification_days = $this->missingBackupNotificationDays; - $this->customValidate(); + $storageIds = $this->customValidate(); $changedFields = auditChangedFields($this->backup); $this->backup->save(); + $previousStorageIds = $this->backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all(); + $this->backup->syncS3Storages($storageIds->all()); + if ($previousStorageIds !== $storageIds->sort()->values()->all()) { + $changedFields[] = 's3_storages'; + } + return $changedFields; } else { $this->backupEnabled = $this->backup->enabled; @@ -177,7 +188,8 @@ class BackupEdit extends Component $this->databaseBackupRetentionMaxStorageS3 = $this->backup->database_backup_retention_max_storage_s3; $this->saveS3 = $this->backup->save_s3; $this->disableLocalBackup = $this->backup->disable_local_backup ?? false; - $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3StorageIds()->first(); + $selectedIds = $this->backup->selectedS3Storages()->pluck('id'); + $this->s3StorageIds = ($selectedIds->isNotEmpty() ? $selectedIds : $this->availableS3StorageIds()->take(1))->all(); $this->databasesToBackup = $this->backup->databases_to_backup; $this->dumpAll = $this->backup->dump_all; $this->timeout = $this->backup->timeout; @@ -216,10 +228,10 @@ class BackupEdit extends Component if ($this->delete_associated_backups_locally && $server) { deleteBackupsLocally($filenames, $server); } + } - if ($this->delete_associated_backups_s3 && $this->backup->s3) { - deleteBackupsS3($filenames, $this->backup->s3); - } + if ($this->delete_associated_backups_s3) { + $this->backup->executions()->each(fn ($execution) => $execution->deleteS3Copies()); } $backupUuid = $this->backup->uuid; @@ -333,14 +345,14 @@ class BackupEdit extends Component } } - public function updatedS3StorageId(): void + public function updatedS3StorageIds(): void { $this->instantSave(); } public function toggleS3(): void { - if (! $this->saveS3 && $this->availableS3StorageIds()->isEmpty()) { + if (! $this->saveS3 && $this->selectedAvailableS3StorageIds()->isEmpty()) { $this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.'); return; @@ -351,21 +363,22 @@ class BackupEdit extends Component $this->instantSave(); } - private function customValidate() + /** + * @return Collection The usable S3 storage IDs to keep as destinations. + */ + private function customValidate(): Collection { - if (! is_numeric($this->backup->s3_storage_id)) { - $this->backup->s3_storage_id = null; - } + // Only usable S3 storages of the database's team can be destinations; others are dropped. + $storageIds = $this->selectedAvailableS3StorageIds(); + $this->s3StorageIds = $storageIds->all(); - // S3 backup cannot be enabled without a valid S3 storage owned by the team - $availableS3Ids = $this->availableS3StorageIds(); - if ($availableS3Ids->isEmpty()) { - $this->backup->s3_storage_id = $this->s3StorageId = null; - if ($this->backup->save_s3) { - $this->backup->save_s3 = $this->saveS3 = false; - } - } elseif (! $availableS3Ids->contains($this->backup->s3_storage_id)) { - $this->backup->s3_storage_id = $this->s3StorageId = $availableS3Ids->first(); + if ($this->availableS3StorageIds()->isEmpty()) { + $this->backup->save_s3 = $this->saveS3 = false; + } elseif ($this->backup->save_s3 && $storageIds->isEmpty()) { + $message = 'Select at least one usable S3 storage.'; + $this->addError('s3StorageIds', $message); + + throw new Exception($message); } // Validate that disable_local_backup can only be true when S3 backup is enabled @@ -378,6 +391,8 @@ class BackupEdit extends Component throw new Exception('Invalid Cron / Human expression'); } $this->validate(); + + return $storageIds; } /** @@ -399,6 +414,20 @@ class BackupEdit extends Component ]); } + /** + * @return Collection + */ + private function selectedAvailableS3StorageIds(): Collection + { + $availableIds = $this->availableS3StorageIds(); + + return collect($this->s3StorageIds) + ->map(fn ($id): int => (int) $id) + ->filter(fn (int $id): bool => $availableIds->contains($id)) + ->unique() + ->values(); + } + private function availableS3StorageIds(): Collection { $database = $this->backup->database; @@ -409,9 +438,12 @@ class BackupEdit extends Component return collect(); } + // Current destinations stay selectable while unusable, so saving other settings does not remove them. + $selectedIds = $this->backup->selectedS3Storages()->pluck('id')->all(); + return S3Storage::query() ->where('team_id', $teamId) - ->where('is_usable', true) + ->where(fn (Builder $query) => $query->where('is_usable', true)->orWhereIn('id', $selectedIds)) ->pluck('id'); } @@ -432,7 +464,7 @@ class BackupEdit extends Component return view('livewire.project.database.backup-edit', [ 'checkboxes' => [ ['id' => 'delete_associated_backups_locally', 'label' => __('database.delete_backups_locally')], - ['id' => 'delete_associated_backups_s3', 'label' => 'All backups will be permanently deleted (associated with this backup job) from the selected S3 Storage.'], + ['id' => 'delete_associated_backups_s3', 'label' => 'All backups will be permanently deleted (associated with this backup job) from every S3 storage of this schedule.'], // ['id' => 'delete_associated_backups_sftp', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected SFTP Storage.'] ], ]); diff --git a/app/Livewire/Project/Database/BackupExecutions.php b/app/Livewire/Project/Database/BackupExecutions.php index 6f17e2fcf5..f9d4658688 100644 --- a/app/Livewire/Project/Database/BackupExecutions.php +++ b/app/Livewire/Project/Database/BackupExecutions.php @@ -109,12 +109,8 @@ class BackupExecutions extends Component deleteBackupsLocally($execution->filename, $server, throwError: true); } - if ($deleteFromS3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) { - if (! $execution->scheduledDatabaseBackup->s3) { - throw new \RuntimeException('The S3 storage is unavailable.'); - } - - deleteBackupsS3($execution->filename, $execution->scheduledDatabaseBackup->s3); + if ($deleteFromS3) { + $execution->deleteS3Copies(); } $execution->delete(); diff --git a/app/Livewire/Project/Database/ScheduledBackups.php b/app/Livewire/Project/Database/ScheduledBackups.php index ece99b45f4..2071337b4b 100644 --- a/app/Livewire/Project/Database/ScheduledBackups.php +++ b/app/Livewire/Project/Database/ScheduledBackups.php @@ -73,7 +73,7 @@ class ScheduledBackups extends Component public function render(): View { - $this->database->loadMissing('scheduledBackups.s3'); + $this->database->loadMissing(['scheduledBackups.s3', 'scheduledBackups.s3Storages']); return view('livewire.project.database.scheduled-backups'); } diff --git a/app/Livewire/Project/Service/BackupExecutions.php b/app/Livewire/Project/Service/BackupExecutions.php index 81be0ff8ad..b319f66412 100644 --- a/app/Livewire/Project/Service/BackupExecutions.php +++ b/app/Livewire/Project/Service/BackupExecutions.php @@ -114,7 +114,7 @@ class BackupExecutions extends Component private function formatExecutions(Collection $rows): Collection { $databaseExecutions = ScheduledDatabaseBackupExecution::query() - ->with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.s3']) + ->with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.s3', 'scheduledDatabaseBackup.s3Storages']) ->whereIn('id', $rows->where('type', 'database')->pluck('id')) ->get()->keyBy('id'); $volumeExecutions = ScheduledVolumeBackupExecution::query() @@ -128,11 +128,16 @@ class BackupExecutions extends Component $isDatabase = $row->type === 'database'; $execution = $isDatabase ? $databaseExecutions->get($row->id) : $volumeExecutions->get($row->id); $schedule = $isDatabase ? $execution->scheduledDatabaseBackup : $execution->scheduledVolumeBackup; - $storage = $isDatabase ? ($schedule->save_s3 ? $schedule->s3 : null) : $execution->s3; - if ($storage?->team_id !== $serviceTeamId) { - $storage = null; - } - $storageLabel = $storage ? $storage->name.' (bucket: '.$storage->bucket.')' : 'Unavailable'; + $storages = match (true) { + ! $isDatabase => collect([$execution->s3]), + ! $schedule->save_s3 => collect(), + $schedule->s3Storages->isNotEmpty() => $schedule->s3Storages, + default => collect([$schedule->s3]), + }; + $storageLabel = $storages + ->filter(fn ($storage) => $storage?->team_id === $serviceTeamId) + ->map(fn ($storage) => $storage->name.' (bucket: '.$storage->bucket.')') + ->implode(', ') ?: 'Unavailable'; if ($isDatabase && ! $schedule->save_s3) { $storageLabel = 'Not configured'; } elseif (! $isDatabase && ! $execution->s3_storage_id && ! $execution->s3_uploaded && ! $execution->s3_storage_deleted) { diff --git a/app/Livewire/Project/Service/VolumeBackup/Index.php b/app/Livewire/Project/Service/VolumeBackup/Index.php index 60a78e5c15..64dbbb8df2 100644 --- a/app/Livewire/Project/Service/VolumeBackup/Index.php +++ b/app/Livewire/Project/Service/VolumeBackup/Index.php @@ -118,7 +118,7 @@ class Index extends Component ); $databaseBackups = ScheduledDatabaseBackup::query() - ->with(['database', 'latest_log', 's3']) + ->with(['database', 'latest_log', 's3', 's3Storages']) ->withCount('executions') ->where('database_type', (new ServiceDatabase)->getMorphClass()) ->whereHasMorph('database', [ServiceDatabase::class], fn ($query) => $query->where('service_id', $this->service->id)) @@ -126,7 +126,7 @@ class Index extends Component ->get(); $backups = ScheduledVolumeBackup::query() - ->with(['backupable.resource', 'latestExecution', 's3']) + ->with(['backupable.resource', 'latestExecution', 's3', 's3Storages']) ->withCount('executions') ->forService($this->service) ->latest() diff --git a/app/Livewire/Project/Shared/ResourceOperations.php b/app/Livewire/Project/Shared/ResourceOperations.php index 2b6f5b4587..a459b988e0 100644 --- a/app/Livewire/Project/Shared/ResourceOperations.php +++ b/app/Livewire/Project/Shared/ResourceOperations.php @@ -232,6 +232,7 @@ class ResourceOperations extends Component 'team_id' => $this->resourceTeamId(), ]); $newBackup->save(); + $backup->copyS3StoragesTo($newBackup); } $environmentVaribles = $this->resource->environment_variables()->get(); diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php index d741bcc0e4..083585fee6 100644 --- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -11,6 +11,7 @@ use App\Models\LocalPersistentVolume; use App\Models\S3Storage; use App\Models\ScheduledVolumeBackup; use App\Models\Service; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Http\RedirectResponse; use Illuminate\Routing\Redirector; @@ -43,7 +44,8 @@ class VolumeBackups extends Component public bool $stopDuringBackup = false; - public ?int $s3StorageId = null; + /** @var array */ + public array $s3StorageIds = []; public int $retentionAmountLocally = 7; @@ -88,7 +90,7 @@ class VolumeBackups extends Component 'saveToS3' => ['required', 'boolean'], 'disableLocalBackup' => ['required', 'boolean'], 'stopDuringBackup' => ['required', 'boolean'], - 's3StorageId' => ['nullable', 'integer'], + 's3StorageIds' => ['array'], 'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'], 'retentionDaysLocally' => ['required', 'integer', 'min:0'], 'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'], @@ -103,12 +105,8 @@ class VolumeBackups extends Component public function mount(): void { $this->authorize('view', $this->resource); - $this->availableS3Storages = S3Storage::query() - ->where('team_id', $this->resourceTeamId()) - ->where('is_usable', true) - ->orderBy('name') - ->get(); $this->backup = $this->storage->scheduledBackups()->first(); + $this->availableS3Storages = $this->selectableS3Storages()->orderBy('name')->get(); $server = $this->backup?->server() ?? data_get($this->resource, 'destination.server'); $this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone'); @@ -118,7 +116,8 @@ class VolumeBackups extends Component $this->saveToS3 = $this->backup->save_s3; $this->disableLocalBackup = $this->backup->disable_local_backup; $this->stopDuringBackup = $this->backup->stop_during_backup; - $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id; + $selectedIds = $this->backup->selectedS3Storages()->pluck('id'); + $this->s3StorageIds = ($selectedIds->isNotEmpty() ? $selectedIds : $this->availableS3Storages->take(1)->pluck('id'))->all(); $this->retentionAmountLocally = $this->backup->retention_amount_locally; $this->retentionDaysLocally = $this->backup->retention_days_locally; $this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally; @@ -128,7 +127,7 @@ class VolumeBackups extends Component $this->timeout = $this->backup->timeout; $this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days; } else { - $this->s3StorageId = $this->availableS3Storages->first()?->id; + $this->s3StorageIds = $this->availableS3Storages->take(1)->pluck('id')->all(); } } @@ -149,30 +148,24 @@ class VolumeBackups extends Component $this->save(); } - public function updatedS3StorageId(): void + public function updatedS3StorageIds(): void { $this->authorize('update', $this->resource); - if (! $this->hasValidS3Storage()) { - $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.'); - - return; - } - - $this->resetErrorBag('s3StorageId'); if (! $this->validateSettings()) { return; } + $this->resetErrorBag('s3StorageIds'); $this->backup = $this->persistBackup($this->enabled); - $this->dispatch('success', 'S3 storage updated.'); + $this->dispatch('success', 'S3 storages updated.'); } public function toggleS3(): void { $this->authorize('update', $this->resource); - if (! $this->saveToS3 && ! $this->hasValidS3Storage()) { + if (! $this->saveToS3 && $this->selectedS3StorageIds()->isEmpty()) { $this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.'); return; @@ -302,6 +295,7 @@ class VolumeBackups extends Component ->where('s3_storage_deleted', true) ->orWhereNull('s3_uploaded') ->orWhere('s3_uploaded', false)) + ->withoutLiveS3Copies() ->delete() ?? 0; $this->dispatch( @@ -366,12 +360,8 @@ class VolumeBackups extends Component deleteBackupsLocally($execution->filename, $server, throwError: true); } - if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) { - if (! $execution->s3) { - throw new \RuntimeException('The S3 storage is unavailable.'); - } - - deleteBackupsS3($execution->filename, $execution->s3); + if ($this->delete_backup_s3) { + $execution->deleteS3Copies(); } $execution->delete(); @@ -388,7 +378,7 @@ class VolumeBackups extends Component public function render() { - $executions = $this->backup?->executions()->paginate($this->perPage); + $executions = $this->backup?->executions()->with('s3Replicas.s3')->paginate($this->perPage); return view('livewire.project.shared.storages.volume-backups', [ 'executions' => $executions ?? collect(), @@ -410,8 +400,8 @@ class VolumeBackups extends Component return false; } - if ($this->saveToS3 && ! $this->hasValidS3Storage()) { - $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.'); + if ($this->saveToS3 && $this->selectedS3StorageIds()->isEmpty()) { + $this->addError('s3StorageIds', 'Select at least one usable S3 storage owned by your team.'); return false; } @@ -431,7 +421,6 @@ class VolumeBackups extends Component 'save_s3' => $this->saveToS3, 'disable_local_backup' => $this->disableLocalBackup, 'stop_during_backup' => $this->stopDuringBackup, - 's3_storage_id' => $this->hasValidS3Storage() ? $this->s3StorageId : $this->backup?->s3_storage_id, 'retention_amount_locally' => $this->retentionAmountLocally, 'retention_days_locally' => $this->retentionDaysLocally, 'retention_max_storage_locally' => $this->retentionMaxStorageLocally, @@ -443,6 +432,15 @@ class VolumeBackups extends Component ]); $changedFields = auditChangedFields($backup); $backup->save(); + + $storageIds = $this->selectedS3StorageIds(); + $previousStorageIds = $backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all(); + $backup->syncS3Storages($storageIds->all()); + $this->s3StorageIds = $storageIds->all(); + if (! $backup->wasRecentlyCreated && $previousStorageIds !== $storageIds->sort()->values()->all()) { + $changedFields[] = 's3_storages'; + } + $this->auditScheduleSet($backup, $changedFields); return $backup; @@ -485,14 +483,33 @@ class VolumeBackups extends Component ]; } - private function hasValidS3Storage(): bool + /** + * The submitted destinations that are selectable S3 storages of the resource's team. Others are dropped. + * + * @return Collection + */ + private function selectedS3StorageIds(): Collection { - return $this->s3StorageId !== null - && S3Storage::query() - ->whereKey($this->s3StorageId) - ->where('team_id', $this->resourceTeamId()) - ->where('is_usable', true) - ->exists(); + $submittedIds = collect($this->s3StorageIds)->map(fn ($id): int => (int) $id)->filter()->unique(); + + return $this->selectableS3Storages() + ->whereKey($submittedIds->all()) + ->pluck('id') + ->sortBy(fn (int $id): int|false => $submittedIds->search($id)) + ->values(); + } + + /** + * Usable storages of the resource's team, plus the current destinations. A destination that is unusable for a + * while stays selected, so saving other settings does not remove it. + */ + private function selectableS3Storages(): Builder + { + $selectedIds = $this->backup?->selectedS3Storages()->pluck('id')->all() ?? []; + + return S3Storage::query() + ->where('team_id', $this->resourceTeamId()) + ->where(fn (Builder $query) => $query->where('is_usable', true)->orWhereIn('id', $selectedIds)); } private function resourceTeamId(): int diff --git a/app/Livewire/Storage/Resources.php b/app/Livewire/Storage/Resources.php index 95e7917515..306a6a4a5f 100644 --- a/app/Livewire/Storage/Resources.php +++ b/app/Livewire/Storage/Resources.php @@ -9,6 +9,7 @@ use App\Models\ScheduledVolumeBackup; use App\Models\Service; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; @@ -26,7 +27,7 @@ class Resources extends Component { $this->authorize('view', $this->storage); - $backups = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id) + $backups = $this->backupsUsingStorage(ScheduledDatabaseBackup::query()) ->where('save_s3', true) ->get(); @@ -34,8 +35,7 @@ class Resources extends Component $this->selectedStorages[$backup->id] = $this->storage->id; } - ScheduledVolumeBackup::query() - ->where('s3_storage_id', $this->storage->id) + $this->backupsUsingStorage(ScheduledVolumeBackup::query()) ->where('save_s3', true) ->each(function (ScheduledVolumeBackup $backup): void { $this->selectedVolumeStorages[$backup->id] = $this->storage->id; @@ -46,29 +46,26 @@ class Resources extends Component { $this->authorize('update', $this->storage); - $backup = ScheduledDatabaseBackup::where('id', $backupId) - ->where('s3_storage_id', $this->storage->id) + $backup = $this->backupsUsingStorage(ScheduledDatabaseBackup::query()) + ->whereKey($backupId) ->firstOrFail(); - $backup->fill([ - 'save_s3' => false, - 's3_storage_id' => null, - ]); - $changedFields = auditChangedFields($backup); - $backup->save(); + $changedFields = $this->replaceStorage($backup, null); $this->auditDatabaseBackupUpdated($backup, $changedFields); unset($this->selectedStorages[$backupId]); - $this->dispatch('success', 'S3 disabled.', 'S3 backup has been disabled for this schedule.'); + $this->dispatch('success', 'Storage removed.', $backup->save_s3 + ? 'This storage was removed from the schedule destinations.' + : 'S3 backup has been disabled for this schedule.'); } public function moveBackup(int $backupId): void { $this->authorize('update', $this->storage); - $backup = ScheduledDatabaseBackup::where('id', $backupId) - ->where('s3_storage_id', $this->storage->id) + $backup = $this->backupsUsingStorage(ScheduledDatabaseBackup::query()) + ->whereKey($backupId) ->firstOrFail(); $newStorageId = $this->selectedStorages[$backupId] ?? null; @@ -90,9 +87,7 @@ class Resources extends Component $this->authorize('update', $newStorage); - $backup->s3_storage_id = $newStorage->id; - $changedFields = auditChangedFields($backup); - $backup->save(); + $changedFields = $this->replaceStorage($backup, $newStorage->id); $this->auditDatabaseBackupUpdated($backup, $changedFields); unset($this->selectedStorages[$backupId]); @@ -104,31 +99,26 @@ class Resources extends Component { $this->authorize('update', $this->storage); - $backup = ScheduledVolumeBackup::query() - ->where('id', $backupId) - ->where('s3_storage_id', $this->storage->id) + $backup = $this->backupsUsingStorage(ScheduledVolumeBackup::query()) + ->whereKey($backupId) ->firstOrFail(); - $backup->fill([ - 'save_s3' => false, - 's3_storage_id' => null, - ]); - $changedFields = auditChangedFields($backup); - $backup->save(); + $changedFields = $this->replaceStorage($backup, null); $this->auditVolumeBackupUpdated($backup, $changedFields); unset($this->selectedVolumeStorages[$backupId]); - $this->dispatch('success', 'S3 disabled.', 'S3 backup has been disabled for this schedule.'); + $this->dispatch('success', 'Storage removed.', $backup->save_s3 + ? 'This storage was removed from the schedule destinations.' + : 'S3 backup has been disabled for this schedule.'); } public function moveVolumeBackup(int $backupId): void { $this->authorize('update', $this->storage); - $backup = ScheduledVolumeBackup::query() - ->where('id', $backupId) - ->where('s3_storage_id', $this->storage->id) + $backup = $this->backupsUsingStorage(ScheduledVolumeBackup::query()) + ->whereKey($backupId) ->firstOrFail(); $newStorageId = $this->selectedVolumeStorages[$backupId] ?? null; @@ -151,9 +141,7 @@ class Resources extends Component $this->authorize('update', $newStorage); - $backup->s3_storage_id = $newStorage->id; - $changedFields = auditChangedFields($backup); - $backup->save(); + $changedFields = $this->replaceStorage($backup, $newStorage->id); $this->auditVolumeBackupUpdated($backup, $changedFields); unset($this->selectedVolumeStorages[$backupId]); @@ -161,6 +149,49 @@ class Resources extends Component $this->dispatch('success', 'Backup moved.', "Moved to {$newStorage->name}."); } + /** + * Schedules that have this storage among their destinations. + * + * @template TModel of ScheduledDatabaseBackup|ScheduledVolumeBackup + * + * @param Builder $query + * @return Builder + */ + private function backupsUsingStorage(Builder $query): Builder + { + return $query->usingS3Storage($this->storage->id); + } + + /** + * Replaces this storage in the destinations of a schedule, or removes it when no replacement is given. S3 backups + * turn off when no destination remains. + * + * @return array Names of the changed backup fields. + */ + private function replaceStorage(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup, ?int $newStorageId): array + { + $storageIds = $backup->selectedS3Storages() + ->pluck('id') + ->map(fn (int $id): ?int => $id === $this->storage->id ? $newStorageId : $id) + ->filter() + ->unique() + ->values(); + $previousPrimaryId = $backup->s3_storage_id; + + $backup->syncS3Storages($storageIds->all()); + + $changedFields = ['s3_storages']; + if ($backup->s3_storage_id !== $previousPrimaryId) { + $changedFields[] = 's3_storage_id'; + } + if ($storageIds->isEmpty() && $backup->save_s3) { + $backup->update(['save_s3' => false]); + $changedFields[] = 'save_s3'; + } + + return $changedFields; + } + /** * @param array $changedFields */ @@ -211,7 +242,7 @@ class Resources extends Component public function render() { - $backups = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id) + $backups = $this->backupsUsingStorage(ScheduledDatabaseBackup::query()) ->where('save_s3', true) ->with('database') ->get() @@ -221,8 +252,7 @@ class Resources extends Component ->orderBy('name') ->get(['id', 'name', 'is_usable']); - $volumeBackups = ScheduledVolumeBackup::query() - ->where('s3_storage_id', $this->storage->id) + $volumeBackups = $this->backupsUsingStorage(ScheduledVolumeBackup::query()) ->where('save_s3', true) ->with('backupable.resource') ->get(); diff --git a/app/Livewire/Storage/Show.php b/app/Livewire/Storage/Show.php index 17abd19e51..b7ddfcf5e2 100644 --- a/app/Livewire/Storage/Show.php +++ b/app/Livewire/Storage/Show.php @@ -32,8 +32,10 @@ class Show extends Component return $this->redirectRoute('storage.index', navigate: true); } $this->currentRoute = request()->route()->getName(); - $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count() - + ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count(); + $this->backupCount = collect([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class]) + ->sum(fn (string $backupModel): int => $backupModel::query() + ->usingS3Storage($this->storage->id) + ->count()); } public function delete() diff --git a/app/Mcp/Tools/ListDatabaseBackups.php b/app/Mcp/Tools/ListDatabaseBackups.php index 7c65130935..92bee0da01 100644 --- a/app/Mcp/Tools/ListDatabaseBackups.php +++ b/app/Mcp/Tools/ListDatabaseBackups.php @@ -43,6 +43,7 @@ class ListDatabaseBackups extends Tool $backups = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId) ->where('database_id', $database->id) ->where('database_type', $database->getMorphClass()) + ->with(['s3', 's3Storages']) ->get() ->map(fn ($backup) => $this->scrubSensitive([ 'uuid' => $backup->uuid, @@ -51,6 +52,7 @@ class ListDatabaseBackups extends Tool 'database_backup_retention_amount_locally' => $backup->database_backup_retention_amount_locally ?? null, 'save_s3' => $backup->save_s3 ?? null, 's3_storage_uuid' => $backup->s3?->uuid ?? null, + 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->all(), 'created_at' => $backup->created_at, 'updated_at' => $backup->updated_at, ])) diff --git a/app/Models/DatabaseBackupS3Replica.php b/app/Models/DatabaseBackupS3Replica.php new file mode 100644 index 0000000000..1dee319c2b --- /dev/null +++ b/app/Models/DatabaseBackupS3Replica.php @@ -0,0 +1,38 @@ + 'boolean', + 's3_storage_deleted' => 'boolean', + ]; + } + + public function execution(): BelongsTo + { + return $this->belongsTo(ScheduledDatabaseBackupExecution::class, 'execution_id'); + } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } +} diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index 3c0d9e7e95..3ee7d5f269 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -6,10 +6,12 @@ use App\Rules\SafeWebhookUrl; use App\Rules\ValidS3BucketName; use App\Traits\Auditable; use App\Traits\HasSafeStringAttribute; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Notifications\Messages\MailMessage; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Validator; @@ -67,19 +69,48 @@ class S3Storage extends BaseModel }); static::deleting(function (S3Storage $storage) { - ScheduledDatabaseBackup::where('s3_storage_id', $storage->id)->update([ - 'save_s3' => false, - 's3_storage_id' => null, - ]); - ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id) - ->update([ - 's3_storage_deleted' => true, - 's3_cleanup_pending' => false, - ]); - ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([ - 'save_s3' => false, - 's3_storage_id' => null, - ]); + $storage->detachFromBackups(); + }); + } + + /** + * Removes this storage from every backup schedule and marks the copies in it as deleted. A schedule moves its + * primary destination to a remaining storage, or stops S3 backups when none remains. + */ + public function detachFromBackups(): void + { + DB::transaction(function (): void { + foreach ([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class] as $backupModel) { + $backupModel::query() + ->usingS3Storage($this->id) + ->lazyById() + ->each(function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): void { + $backup->syncS3Storages($backup->s3Storages()->whereKeyNot($this->id)->pluck('s3_storages.id')->all()); + if ($backup->s3_storage_id === null && $backup->save_s3) { + $backup->update(['save_s3' => false]); + } + }); + } + + foreach ([DatabaseBackupS3Replica::class, VolumeBackupS3Replica::class] as $replicaModel) { + $replicaModel::query() + ->with('execution') + ->where('s3_storage_id', $this->id) + ->where('s3_storage_deleted', false) + ->lazyById() + ->each(function (DatabaseBackupS3Replica|VolumeBackupS3Replica $replica): void { + $replica->update(['s3_storage_deleted' => true]); + $replica->execution?->refreshS3Summary(); + }); + } + + ScheduledVolumeBackupExecution::query() + ->where('s3_cleanup_pending', true) + ->whereHas('s3Replicas', fn (Builder $query) => $query->where('s3_storage_id', $this->id)) + ->whereDoesntHave('s3Replicas', fn (Builder $query) => $query + ->where('s3_storage_deleted', false) + ->where(fn (Builder $query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false))) + ->update(['s3_cleanup_pending' => false]); }); } diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.php index 53ae084349..3c31c0fbc7 100644 --- a/app/Models/ScheduledDatabaseBackup.php +++ b/app/Models/ScheduledDatabaseBackup.php @@ -2,12 +2,16 @@ namespace App\Models; +use App\Traits\HasS3Destinations; +use Illuminate\Database\Eloquent\Relations\BelongsToMany; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\Relations\HasOne; use Illuminate\Database\Eloquent\Relations\MorphTo; class ScheduledDatabaseBackup extends BaseModel { + use HasS3Destinations; + protected function casts(): array { return [ @@ -78,6 +82,11 @@ class ScheduledDatabaseBackup extends BaseModel return $this->belongsTo(S3Storage::class, 's3_storage_id'); } + public function s3Storages(): BelongsToMany + { + return $this->belongsToMany(S3Storage::class, 'scheduled_database_backup_s3_storage')->withTimestamps(); + } + public function get_last_days_backup_status($days = 7) { return $this->hasMany(ScheduledDatabaseBackupExecution::class)->where('created_at', '>=', now()->subDays($days))->get(); @@ -85,7 +94,7 @@ class ScheduledDatabaseBackup extends BaseModel public function executionsPaginated(int $skip = 0, int $take = 10) { - $executions = $this->hasMany(ScheduledDatabaseBackupExecution::class)->orderBy('created_at', 'desc'); + $executions = $this->hasMany(ScheduledDatabaseBackupExecution::class)->with('s3Replicas.s3')->orderBy('created_at', 'desc'); $count = $executions->count(); $executions = $executions->skip($skip)->take($take)->get(); diff --git a/app/Models/ScheduledDatabaseBackupExecution.php b/app/Models/ScheduledDatabaseBackupExecution.php index 1a479772cf..eb43a1a2e3 100644 --- a/app/Models/ScheduledDatabaseBackupExecution.php +++ b/app/Models/ScheduledDatabaseBackupExecution.php @@ -2,10 +2,14 @@ namespace App\Models; +use App\Traits\HasS3Replicas; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Database\Eloquent\Relations\HasMany; class ScheduledDatabaseBackupExecution extends BaseModel { + use HasS3Replicas; + protected static function booted(): void { static::created(function (ScheduledDatabaseBackupExecution $execution): void { @@ -42,4 +46,9 @@ class ScheduledDatabaseBackupExecution extends BaseModel { return $this->belongsTo(ScheduledDatabaseBackup::class); } + + public function s3Replicas(): HasMany + { + return $this->hasMany(DatabaseBackupS3Replica::class, 'execution_id'); + } } diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.php index 9ad676d953..0c4443b2e3 100644 --- a/app/Models/ScheduledVolumeBackup.php +++ b/app/Models/ScheduledVolumeBackup.php @@ -2,15 +2,19 @@ namespace App\Models; +use App\Traits\HasS3Destinations; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Database\Eloquent\Relations\BelongsToMany; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\Relations\HasOne; use Illuminate\Database\Eloquent\Relations\MorphTo; class ScheduledVolumeBackup extends BaseModel { + use HasS3Destinations; + public const int DEFAULT_TIMEOUT = 36000; protected $fillable = [ @@ -123,6 +127,11 @@ class ScheduledVolumeBackup extends BaseModel return $this->belongsTo(S3Storage::class, 's3_storage_id'); } + public function s3Storages(): BelongsToMany + { + return $this->belongsToMany(S3Storage::class, 'scheduled_volume_backup_s3_storage')->withTimestamps(); + } + public function executions(): HasMany { return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest(); diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php index 664efa11c1..82dfe0ee3b 100644 --- a/app/Models/ScheduledVolumeBackupExecution.php +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -2,10 +2,14 @@ namespace App\Models; +use App\Traits\HasS3Replicas; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Database\Eloquent\Relations\HasMany; class ScheduledVolumeBackupExecution extends BaseModel { + use HasS3Replicas; + protected $fillable = [ 'uuid', 'scheduled_volume_backup_id', @@ -56,4 +60,9 @@ class ScheduledVolumeBackupExecution extends BaseModel { return $this->belongsTo(S3Storage::class, 's3_storage_id'); } + + public function s3Replicas(): HasMany + { + return $this->hasMany(VolumeBackupS3Replica::class, 'execution_id'); + } } diff --git a/app/Models/VolumeBackupS3Replica.php b/app/Models/VolumeBackupS3Replica.php new file mode 100644 index 0000000000..779ee7571c --- /dev/null +++ b/app/Models/VolumeBackupS3Replica.php @@ -0,0 +1,38 @@ + 'boolean', + 's3_storage_deleted' => 'boolean', + ]; + } + + public function execution(): BelongsTo + { + return $this->belongsTo(ScheduledVolumeBackupExecution::class, 'execution_id'); + } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } +} diff --git a/app/Services/ServerTransfer/ServerTransferExporter.php b/app/Services/ServerTransfer/ServerTransferExporter.php index 8db6e5afd1..db7ddf556e 100644 --- a/app/Services/ServerTransfer/ServerTransferExporter.php +++ b/app/Services/ServerTransfer/ServerTransferExporter.php @@ -16,6 +16,7 @@ use App\Models\Project; use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledTask; +use App\Models\ScheduledVolumeBackup; use App\Models\Server; use App\Models\Service; use App\Models\SharedEnvironmentVariable; @@ -202,22 +203,22 @@ class ServerTransferExporter } } - $collectVolumeS3 = function ($resource) use (&$s3Ids): void { + $collectBackupS3 = function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup) use (&$s3Ids): void { + $s3Ids->push($backup->s3_storage_id, ...$backup->s3Storages()->pluck('s3_storages.id')->all()); + }; + + $collectVolumeS3 = function ($resource) use ($collectBackupS3): void { if (method_exists($resource, 'persistentStorages')) { foreach ($resource->persistentStorages as $volume) { foreach ($volume->scheduledBackups as $vb) { - if ($vb->s3_storage_id) { - $s3Ids->push($vb->s3_storage_id); - } + $collectBackupS3($vb); } } } if (method_exists($resource, 'fileStorages')) { foreach ($resource->fileStorages as $file) { foreach ($file->scheduledBackups as $vb) { - if ($vb->s3_storage_id) { - $s3Ids->push($vb->s3_storage_id); - } + $collectBackupS3($vb); } } } @@ -226,9 +227,7 @@ class ServerTransferExporter foreach ($databases as $database) { if (method_exists($database, 'scheduledBackups')) { foreach ($database->scheduledBackups as $backup) { - if ($backup->s3_storage_id) { - $s3Ids->push($backup->s3_storage_id); - } + $collectBackupS3($backup); } } $database->loadMissing(['persistentStorages.scheduledBackups', 'fileStorages.scheduledBackups']); @@ -257,9 +256,7 @@ class ServerTransferExporter } foreach ($service->databases as $serviceDb) { foreach ($serviceDb->scheduledBackups as $backup) { - if ($backup->s3_storage_id) { - $s3Ids->push($backup->s3_storage_id); - } + $collectBackupS3($backup); } $collectVolumeS3($serviceDb); } @@ -981,6 +978,7 @@ class ServerTransferExporter 'timeout' => $backup->timeout, 'disable_local_backup' => (bool) ($backup->disable_local_backup ?? false), 's3_storage_uuid' => $backup->s3?->uuid, + 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->values()->all(), 'had_s3_storage' => filled($backup->s3_storage_id), ]; } @@ -1193,6 +1191,7 @@ class ServerTransferExporter 'retention_max_storage_s3' => $backup->retention_max_storage_s3, 'timeout' => $backup->timeout, 's3_storage_uuid' => $backup->s3?->uuid, + 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->values()->all(), 'had_s3_storage' => filled($backup->s3_storage_id), ]; } diff --git a/app/Services/ServerTransfer/ServerTransferImporter.php b/app/Services/ServerTransfer/ServerTransferImporter.php index 90fe5954d0..d1714d4e70 100644 --- a/app/Services/ServerTransfer/ServerTransferImporter.php +++ b/app/Services/ServerTransfer/ServerTransferImporter.php @@ -1237,13 +1237,10 @@ class ServerTransferImporter $uuid = new_public_id(); } - $s3Id = null; - $s3Uuid = data_get($backup, 's3_storage_uuid'); - if ($s3Uuid && isset($this->s3StorageMap[$s3Uuid])) { - $s3Id = $this->s3StorageMap[$s3Uuid]->id; - } + $s3Ids = $this->importedBackupS3StorageIds($backup); + $s3Id = $s3Ids[0] ?? null; - ScheduledDatabaseBackup::create([ + $importedBackup = ScheduledDatabaseBackup::create([ 'uuid' => $uuid, 'team_id' => $teamId, 'description' => data_get($backup, 'description'), @@ -1264,9 +1261,26 @@ class ServerTransferImporter 'database_type' => $database->getMorphClass(), 'database_id' => $database->id, ]); + $importedBackup->syncS3Storages($s3Ids); } } + /** + * Maps the exported S3 destinations of a backup to imported storages, primary first. Older bundles only have + * `s3_storage_uuid`. + * + * @param array $backup + * @return list + */ + private function importedBackupS3StorageIds(array $backup): array + { + $uuids = collect(data_get($backup, 's3_storage_uuids', [])) + ->prepend(data_get($backup, 's3_storage_uuid')) + ->filter(fn ($uuid): bool => is_string($uuid) && isset($this->s3StorageMap[$uuid])); + + return $uuids->map(fn (string $uuid): int => $this->s3StorageMap[$uuid]->id)->unique()->values()->all(); + } + /** * @param array $payload */ @@ -1612,13 +1626,10 @@ class ServerTransferImporter $uuid = new_public_id(); } - $s3Id = null; - $s3Uuid = data_get($backupPayload, 's3_storage_uuid'); - if ($s3Uuid && isset($this->s3StorageMap[$s3Uuid])) { - $s3Id = $this->s3StorageMap[$s3Uuid]->id; - } + $s3Ids = $this->importedBackupS3StorageIds($backupPayload); + $s3Id = $s3Ids[0] ?? null; - ScheduledVolumeBackup::create([ + $importedBackup = ScheduledVolumeBackup::create([ 'uuid' => $uuid, 'backupable_type' => $volume->getMorphClass(), 'backupable_id' => $volume->id, @@ -1637,6 +1648,7 @@ class ServerTransferImporter 'retention_max_storage_s3' => data_get($backupPayload, 'retention_max_storage_s3'), 'timeout' => data_get($backupPayload, 'timeout'), ]); + $importedBackup->syncS3Storages($s3Ids); $count++; } diff --git a/app/Traits/HasS3Destinations.php b/app/Traits/HasS3Destinations.php new file mode 100644 index 0000000000..59668f3961 --- /dev/null +++ b/app/Traits/HasS3Destinations.php @@ -0,0 +1,68 @@ + The primary destination comes first. + */ + public function selectedS3Storages(): Collection + { + $storages = $this->relationLoaded('s3Storages') + ? $this->s3Storages->sortBy('name') + : $this->s3Storages()->orderBy('s3_storages.name')->get(); + + if ($storages->isEmpty()) { + return $this->s3 ? collect([$this->s3]) : collect(); + } + + return $storages + ->sortBy(fn (S3Storage $storage): int => $storage->id === $this->s3_storage_id ? 0 : 1) + ->values(); + } + + /** + * Schedules that send backups to the given storage. + */ + public function scopeUsingS3Storage(Builder $query, int $storageId): Builder + { + return $query->where(fn (Builder $query) => $query + ->where('s3_storage_id', $storageId) + ->orWhereHas('s3Storages', fn (Builder $query) => $query->whereKey($storageId))); + } + + /** + * Gives a cloned schedule the same destinations, limited to storages of the clone's team. + */ + public function copyS3StoragesTo(self $copy): void + { + $copy->s3Storages()->sync( + $this->s3Storages()->where('s3_storages.team_id', $copy->team_id)->pluck('s3_storages.id')->all(), + ); + } + + /** + * Replaces the selected destinations. The current primary destination stays primary when it is still selected. + * + * @param array $storageIds + */ + public function syncS3Storages(array $storageIds): void + { + $storageIds = collect($storageIds)->map(fn ($id): int => (int) $id)->filter()->unique()->values(); + + $this->s3Storages()->sync($storageIds->all()); + $this->s3_storage_id = $storageIds->contains($this->s3_storage_id) ? $this->s3_storage_id : $storageIds->first(); + $this->save(); + } +} diff --git a/app/Traits/HasS3Replicas.php b/app/Traits/HasS3Replicas.php new file mode 100644 index 0000000000..81e935e174 --- /dev/null +++ b/app/Traits/HasS3Replicas.php @@ -0,0 +1,75 @@ +s3Replicas() + ->with('s3') + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->get(); + + try { + foreach ($replicas as $replica) { + if (! $replica->s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + if (filled($this->filename)) { + deleteBackupsS3($this->filename, $replica->s3); + } + $replica->update(['s3_storage_deleted' => true]); + } + } finally { + $this->refreshS3Summary(); + } + } + + public function hasLiveS3Copies(): bool + { + return $this->s3Replicas()->where('s3_uploaded', true)->where('s3_storage_deleted', false)->exists(); + } + + /** + * Writes the replica states to the summary columns: uploaded when every copy uploaded, deleted when every uploaded + * copy is deleted. + */ + public function refreshS3Summary(): void + { + $replicas = $this->s3Replicas()->get(); + if ($replicas->isEmpty()) { + return; + } + + $uploaded = $replicas->where('s3_uploaded', true); + + $this->update([ + 's3_uploaded' => $replicas->every(fn ($replica): bool => $replica->s3_uploaded === true), + 's3_storage_deleted' => $uploaded->isNotEmpty() && $uploaded->every(fn ($replica): bool => $replica->s3_storage_deleted), + ]); + } + + /** + * Executions that have no S3 copy left to keep. + */ + public function scopeWithoutLiveS3Copies(Builder $query): Builder + { + return $query->whereDoesntHave('s3Replicas', fn (Builder $query) => $query + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false)); + } +} diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php index f4a97be5ce..966b2de3e6 100644 --- a/bootstrap/helpers/databases.php +++ b/bootstrap/helpers/databases.php @@ -1,7 +1,9 @@ executions) { - // Delete old local backups (only if local backup is NOT disabled) - // Note: When disable_local_backup is enabled, each execution already marks its own - // local_storage_deleted status at the time of backup, so we don't need to retroactively - // update old executions - if (! $backup->disable_local_backup) { - $localBackupsToDelete = deleteOldBackupsLocally($backup); - if ($localBackupsToDelete->isNotEmpty()) { - $backup->executions() - ->whereIn('id', $localBackupsToDelete->pluck('id')) - ->update(['local_storage_deleted' => true]); - } - } - } - - if ($backup->save_s3 && $backup->executions) { - $s3BackupsToDelete = deleteOldBackupsFromS3($backup); - if ($s3BackupsToDelete->isNotEmpty()) { - $backup->executions() - ->whereIn('id', $s3BackupsToDelete->pluck('id')) - ->update(['s3_storage_deleted' => true]); - } - } - - // Delete execution records where all backup copies are gone - // Case 1: Both local and S3 backups are deleted + // When local backups are disabled, retain the local file until an S3 copy is available. + $localBackupsToDelete = deleteOldBackupsLocally($backup); + if ($localBackupsToDelete->isNotEmpty()) { $backup->executions() - ->where('local_storage_deleted', true) - ->where('s3_storage_deleted', true) - ->delete(); - - // Case 2: Local backup is deleted and S3 was never used (s3_uploaded is null) - $backup->executions() - ->where('local_storage_deleted', true) - ->whereNull('s3_uploaded') - ->delete(); - - } catch (Exception $e) { - throw $e; + ->whereIn('id', $localBackupsToDelete->pluck('id')) + ->update(['local_storage_deleted' => true]); } + + if ($backup->save_s3) { + deleteOldBackupsFromS3($backup); + } + + // Delete execution records when no local file and no S3 copy is left + $backup->executions() + ->where('local_storage_deleted', true) + ->withoutLiveS3Copies() + ->delete(); } -function deleteOldBackupsLocally($backup): Collection +/** + * Successful executions that fall outside the retention settings, newest execution first. + * + * @param Collection $executions Sorted newest first. + * @return Collection + */ +function backupExecutionsOutsideRetention(Collection $executions, ?int $retentionAmount, ?int $retentionDays, int|float|null $maxStorageGB): Collection { - if (! $backup || ! $backup->executions) { - return collect(); - } - - $successfulBackups = $backup->executions() - ->where('status', 'success') - ->where('local_storage_deleted', false) - ->orderBy('created_at', 'desc') - ->get(); - - if ($successfulBackups->isEmpty()) { - return collect(); - } - - $retentionAmount = $backup->database_backup_retention_amount_locally; - $retentionDays = $backup->database_backup_retention_days_locally; - $maxStorageGB = $backup->database_backup_retention_max_storage_locally; - - if ($retentionAmount === 0 && $retentionDays === 0 && $maxStorageGB === 0) { + if ($executions->isEmpty()) { return collect(); } $backupsToDelete = collect(); if ($retentionAmount > 0) { - $byAmount = $successfulBackups->skip($retentionAmount); - $backupsToDelete = $backupsToDelete->merge($byAmount); + $backupsToDelete = $backupsToDelete->merge($executions->skip($retentionAmount)); } if ($retentionDays > 0) { - $oldestAllowedDate = $successfulBackups->first()->created_at->clone()->utc()->subDays($retentionDays); - $oldBackups = $successfulBackups->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate); - $backupsToDelete = $backupsToDelete->merge($oldBackups); + $oldestAllowedDate = $executions->first()->created_at->clone()->utc()->subDays($retentionDays); + $backupsToDelete = $backupsToDelete->merge( + $executions->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate) + ); } if ($maxStorageGB > 0) { $maxStorageBytes = $maxStorageGB * pow(1024, 3); $totalSize = 0; - $backupsOverLimit = collect(); - $backupsToCheck = $successfulBackups->skip(1); - - foreach ($backupsToCheck as $backupExecution) { + foreach ($executions->skip(1) as $backupExecution) { $totalSize += (int) $backupExecution->size; if ($totalSize > $maxStorageBytes) { - $backupsOverLimit = $successfulBackups->filter( + $backupsToDelete = $backupsToDelete->merge($executions->filter( fn ($b) => $b->created_at->utc() <= $backupExecution->created_at->utc() - )->skip(1); + )->skip(1)); break; } } - - $backupsToDelete = $backupsToDelete->merge($backupsOverLimit); } - $backupsToDelete = $backupsToDelete->unique('id'); - $processedBackups = collect(); + return $backupsToDelete->unique('id')->values(); +} + +function deleteOldBackupsLocally($backup): Collection +{ + if (! $backup) { + return collect(); + } + + $successfulBackups = $backup->executions() + ->where('status', 'success') + ->where('local_storage_deleted', false) + ->when($backup->disable_local_backup, fn ($query) => $query + ->whereHas('s3Replicas', fn ($query) => $query + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false))) + ->orderBy('created_at', 'desc') + ->orderBy('id', 'desc') + ->get(); + + $backupsToDelete = backupExecutionsOutsideRetention( + $successfulBackups, + $backup->database_backup_retention_amount_locally, + $backup->database_backup_retention_days_locally, + $backup->database_backup_retention_max_storage_locally, + ); + + if ($backupsToDelete->isEmpty()) { + return collect(); + } - $server = null; if ($backup->database_type === ServiceDatabase::class) { $server = $backup->database->service->server; } else { @@ -433,89 +421,81 @@ function deleteOldBackupsLocally($backup): Collection ->pluck('filename') ->all(); - if (! empty($filesToDelete)) { - deleteBackupsLocally($filesToDelete, $server); - $processedBackups = $backupsToDelete; + if (empty($filesToDelete)) { + return collect(); } - return $processedBackups; + deleteBackupsLocally($filesToDelete, $server); + + return $backupsToDelete; } +/** + * Applies the S3 retention settings to each destination separately. Copies in a storage that no longer exists are + * marked as deleted. + * + * @return Collection The executions that lost at least one S3 copy. + */ function deleteOldBackupsFromS3($backup): Collection { - if (! $backup || ! $backup->executions || ! $backup->s3) { + if (! $backup) { return collect(); } - $successfulBackups = $backup->executions() - ->where('status', 'success') + $liveReplicas = DatabaseBackupS3Replica::query() + ->with('s3') + ->whereHas('execution', fn ($query) => $query + ->where('scheduled_database_backup_id', $backup->id) + ->where('status', 'success')) + ->where('s3_uploaded', true) ->where('s3_storage_deleted', false) - ->orderBy('created_at', 'desc') ->get(); - if ($successfulBackups->isEmpty()) { - return collect(); - } + $deletedReplicas = $liveReplicas->filter(fn (DatabaseBackupS3Replica $replica) => ! $replica->s3); - $retentionAmount = $backup->database_backup_retention_amount_s3; - $retentionDays = $backup->database_backup_retention_days_s3; - $maxStorageGB = $backup->database_backup_retention_max_storage_s3; + foreach ($liveReplicas->filter(fn (DatabaseBackupS3Replica $replica) => $replica->s3)->groupBy('s3_storage_id') as $replicas) { + $successfulBackups = $backup->executions() + ->whereIn('id', $replicas->pluck('execution_id')) + ->orderBy('created_at', 'desc') + ->orderBy('id', 'desc') + ->get(); - if ($retentionAmount === 0 && $retentionDays === 0 && $maxStorageGB === 0) { - return collect(); - } + $backupsToDelete = backupExecutionsOutsideRetention( + $successfulBackups, + $backup->database_backup_retention_amount_s3, + $backup->database_backup_retention_days_s3, + $backup->database_backup_retention_max_storage_s3, + ); - $backupsToDelete = collect(); + $filesToDelete = $backupsToDelete + ->filter(fn ($execution) => ! empty($execution->filename)) + ->pluck('filename') + ->all(); - if ($retentionAmount > 0) { - $byAmount = $successfulBackups->skip($retentionAmount); - $backupsToDelete = $backupsToDelete->merge($byAmount); - } - - if ($retentionDays > 0) { - $oldestAllowedDate = $successfulBackups->first()->created_at->clone()->utc()->subDays($retentionDays); - $oldBackups = $successfulBackups->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate); - $backupsToDelete = $backupsToDelete->merge($oldBackups); - } - - if ($maxStorageGB > 0) { - $maxStorageBytes = $maxStorageGB * pow(1024, 3); - $totalSize = 0; - $backupsOverLimit = collect(); - - $backupsToCheck = $successfulBackups->skip(1); - - foreach ($backupsToCheck as $backupExecution) { - $totalSize += (int) $backupExecution->size; - if ($totalSize > $maxStorageBytes) { - $backupsOverLimit = $successfulBackups->filter( - fn ($b) => $b->created_at->utc() <= $backupExecution->created_at->utc() - )->skip(1); - break; - } + if (empty($filesToDelete)) { + continue; } - $backupsToDelete = $backupsToDelete->merge($backupsOverLimit); - } - - $backupsToDelete = $backupsToDelete->unique('id'); - $processedBackups = collect(); - - $filesToDelete = $backupsToDelete - ->filter(fn ($execution) => ! empty($execution->filename)) - ->pluck('filename') - ->all(); - - if (! empty($filesToDelete)) { try { - deleteBackupsS3($filesToDelete, $backup->s3); - $processedBackups = $backupsToDelete; + deleteBackupsS3($filesToDelete, $replicas->first()->s3); + $deletedReplicas = $deletedReplicas->merge($replicas->whereIn('execution_id', $backupsToDelete->pluck('id'))); } catch (Throwable $e) { report($e); } } - return $processedBackups; + if ($deletedReplicas->isEmpty()) { + return collect(); + } + + DatabaseBackupS3Replica::query() + ->whereKey($deletedReplicas->pluck('id')) + ->update(['s3_storage_deleted' => true]); + + $executions = $backup->executions()->whereIn('id', $deletedReplicas->pluck('execution_id')->unique())->get(); + $executions->each(fn (ScheduledDatabaseBackupExecution $execution) => $execution->refreshS3Summary()); + + return $executions; } function isPublicPortAlreadyUsed(Server $server, int $port, ?string $id = null): bool diff --git a/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php b/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php new file mode 100644 index 0000000000..39aff70ad2 --- /dev/null +++ b/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php @@ -0,0 +1,126 @@ +id(); + $table->foreignId('scheduled_database_backup_id') + ->constrained(indexName: 'db_backup_s3_storage_backup_fk') + ->cascadeOnDelete(); + $table->foreignId('s3_storage_id') + ->constrained(indexName: 'db_backup_s3_storage_storage_fk') + ->cascadeOnDelete(); + $table->timestamps(); + $table->unique(['scheduled_database_backup_id', 's3_storage_id'], 'db_backup_s3_storage_unique'); + }); + + Schema::create('scheduled_volume_backup_s3_storage', function (Blueprint $table) { + $table->id(); + $table->foreignId('scheduled_volume_backup_id') + ->constrained(indexName: 'volume_backup_s3_storage_backup_fk') + ->cascadeOnDelete(); + $table->foreignId('s3_storage_id') + ->constrained(indexName: 'volume_backup_s3_storage_storage_fk') + ->cascadeOnDelete(); + $table->timestamps(); + $table->unique(['scheduled_volume_backup_id', 's3_storage_id'], 'volume_backup_s3_storage_unique'); + }); + + Schema::create('database_backup_s3_replicas', function (Blueprint $table) { + $table->id(); + $table->foreignId('execution_id') + ->index() + ->constrained('scheduled_database_backup_executions', indexName: 'db_backup_s3_replicas_execution_fk') + ->cascadeOnDelete(); + $table->foreignId('s3_storage_id') + ->nullable() + ->constrained(indexName: 'db_backup_s3_replicas_storage_fk') + ->nullOnDelete(); + $table->boolean('s3_uploaded')->nullable(); + $table->boolean('s3_storage_deleted')->default(false); + $table->text('message')->nullable(); + $table->timestamps(); + }); + + Schema::create('volume_backup_s3_replicas', function (Blueprint $table) { + $table->id(); + $table->foreignId('execution_id') + ->index() + ->constrained('scheduled_volume_backup_executions', indexName: 'volume_backup_s3_replicas_execution_fk') + ->cascadeOnDelete(); + $table->foreignId('s3_storage_id') + ->nullable() + ->constrained(indexName: 'volume_backup_s3_replicas_storage_fk') + ->nullOnDelete(); + $table->boolean('s3_uploaded')->nullable(); + $table->boolean('s3_storage_deleted')->default(false); + $table->text('message')->nullable(); + $table->timestamps(); + }); + + $this->backfill(); + } + + public function down(): void + { + Schema::dropIfExists('volume_backup_s3_replicas'); + Schema::dropIfExists('database_backup_s3_replicas'); + Schema::dropIfExists('scheduled_volume_backup_s3_storage'); + Schema::dropIfExists('scheduled_database_backup_s3_storage'); + } + + /** + * Copies the single S3 destination of existing schedules and executions into the new tables. + */ + private function backfill(): void + { + $now = now(); + + DB::table('scheduled_database_backup_s3_storage')->insertUsing( + ['scheduled_database_backup_id', 's3_storage_id', 'created_at', 'updated_at'], + DB::table('scheduled_database_backups') + ->join('s3_storages', 's3_storages.id', '=', 'scheduled_database_backups.s3_storage_id') + ->select('scheduled_database_backups.id', 'scheduled_database_backups.s3_storage_id') + ->selectRaw('?, ?', [$now, $now]), + ); + + DB::table('scheduled_volume_backup_s3_storage')->insertUsing( + ['scheduled_volume_backup_id', 's3_storage_id', 'created_at', 'updated_at'], + DB::table('scheduled_volume_backups') + ->whereNotNull('s3_storage_id') + ->select('id', 's3_storage_id') + ->selectRaw('?, ?', [$now, $now]), + ); + + // Database executions have no storage column: the copy went to the storage of the schedule. + DB::table('database_backup_s3_replicas')->insertUsing( + ['execution_id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted', 'created_at', 'updated_at'], + DB::table('scheduled_database_backup_executions') + ->join('scheduled_database_backups', 'scheduled_database_backups.id', '=', 'scheduled_database_backup_executions.scheduled_database_backup_id') + ->leftJoin('s3_storages', 's3_storages.id', '=', 'scheduled_database_backups.s3_storage_id') + ->whereNotNull('scheduled_database_backup_executions.s3_uploaded') + ->select( + 'scheduled_database_backup_executions.id', + 's3_storages.id', + 'scheduled_database_backup_executions.s3_uploaded', + 'scheduled_database_backup_executions.s3_storage_deleted', + ) + ->selectRaw('?, ?', [$now, $now]), + ); + + DB::table('volume_backup_s3_replicas')->insertUsing( + ['execution_id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted', 'created_at', 'updated_at'], + DB::table('scheduled_volume_backup_executions') + ->whereNotNull('s3_storage_id') + ->select('id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted') + ->selectRaw('?, ?', [$now, $now]), + ); + } +}; diff --git a/openapi.json b/openapi.json index 192d6f49b9..e418f93489 100644 --- a/openapi.json +++ b/openapi.json @@ -86,9 +86,9 @@ "properties": { "pull_request_id": { "type": "integer", + "example": 42, "maximum": 2147483647, - "minimum": 1, - "example": 42 + "minimum": 1 }, "pull_request_html_url": { "type": [ @@ -6857,7 +6857,7 @@ ], "responses": { "200": { - "description": "Get all backups for a database", + "description": "Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).", "content": { "application\/json": { "schema": { @@ -6927,7 +6927,14 @@ }, "s3_storage_uuid": { "type": "string", - "description": "S3 storage UUID (required if save_s3 is true)" + "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true." + }, + "s3_storage_uuids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination." }, "databases_to_backup": { "type": "string", @@ -7530,7 +7537,14 @@ }, "s3_storage_uuid": { "type": "string", - "description": "S3 storage UUID" + "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay." + }, + "s3_storage_uuids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination." }, "backup_now": { "type": "boolean", @@ -10878,8 +10892,8 @@ }, "network": { "type": "string", - "maxLength": 255, - "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$" + "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$", + "maxLength": 255 }, "type": { "type": "string", @@ -23541,8 +23555,8 @@ "properties": { "frequency": { "type": "string", - "maxLength": 255, - "example": "0 2 * * *" + "example": "0 2 * * *", + "maxLength": 255 }, "enabled": { "type": "boolean", @@ -23561,11 +23575,19 @@ "default": false }, "s3_storage_uuid": { + "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.", "type": [ "string", "null" ] }, + "s3_storage_uuids": { + "description": "UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.", + "type": "array", + "items": { + "type": "string" + } + }, "retention_amount_locally": { "type": "integer", "default": 7, @@ -23674,11 +23696,19 @@ "type": "boolean" }, "s3_storage_uuid": { + "description": "Primary S3 destination.", "type": [ "string", "null" ] }, + "s3_storage_uuids": { + "description": "Every S3 destination, primary first.", + "type": "array", + "items": { + "type": "string" + } + }, "retention_amount_locally": { "type": "integer" }, diff --git a/openapi.yaml b/openapi.yaml index 5983cbe38d..a7e4087848 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -62,9 +62,9 @@ paths: properties: pull_request_id: type: integer + example: 42 maximum: 2147483647 minimum: 1 - example: 42 pull_request_html_url: type: [string, 'null'] example: 'https://github.com/org/repo/pull/42' @@ -4401,7 +4401,7 @@ paths: type: string responses: '200': - description: 'Get all backups for a database' + description: 'Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).' content: application/json: schema: @@ -4452,7 +4452,11 @@ paths: default: false s3_storage_uuid: type: string - description: 'S3 storage UUID (required if save_s3 is true)' + description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true.' + s3_storage_uuids: + type: array + items: { type: string } + description: 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.' databases_to_backup: type: string description: 'Comma separated list of databases to backup' @@ -4875,7 +4879,11 @@ paths: description: 'Whether data is saved in s3 or not' s3_storage_uuid: type: string - description: 'S3 storage UUID' + description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay.' + s3_storage_uuids: + type: array + items: { type: string } + description: 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.' backup_now: type: boolean description: 'Whether to take a backup now or not' @@ -7099,8 +7107,8 @@ paths: maxLength: 255 network: type: string - maxLength: 255 pattern: '^[a-zA-Z0-9][a-zA-Z0-9._-]*$' + maxLength: 255 type: type: string enum: [standalone, swarm] @@ -14947,8 +14955,8 @@ components: properties: frequency: type: string - maxLength: 255 example: '0 2 * * *' + maxLength: 255 enabled: type: boolean default: true @@ -14962,9 +14970,15 @@ components: type: boolean default: false s3_storage_uuid: + description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.' type: - string - 'null' + s3_storage_uuids: + description: 'UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.' + type: array + items: + type: string retention_amount_locally: type: integer default: 7 @@ -15052,9 +15066,15 @@ components: stop_during_backup: type: boolean s3_storage_uuid: + description: 'Primary S3 destination.' type: - string - 'null' + s3_storage_uuids: + description: 'Every S3 destination, primary first.' + type: array + items: + type: string retention_amount_locally: type: integer retention_days_locally: diff --git a/resources/views/components/status-badge.blade.php b/resources/views/components/status-badge.blade.php index 5aa02454bd..a8f49e8bf5 100644 --- a/resources/views/components/status-badge.blade.php +++ b/resources/views/components/status-badge.blade.php @@ -4,6 +4,7 @@ 'type' => 'neutral', 'as' => 'span', 'dynamic' => false, + 'pulse' => false, // ping the dot for a state that is still in progress ]) @php @@ -22,7 +23,12 @@ @if ($dynamic) {{ $slot }} @else - + + @if ($pulse) + + @endif + + {{ collect([$label, $status])->filter()->join(' ') }} @endif @@ -31,7 +37,12 @@ @if ($dynamic) {{ $slot }} @else - + + @if ($pulse) + + @endif + + {{ collect([$label, $status])->filter()->join(' ') }} @endif @@ -40,7 +51,12 @@ @if ($dynamic) {{ $slot }} @else - + + @if ($pulse) + + @endif + + {{ collect([$label, $status])->filter()->join(' ') }} @endif diff --git a/resources/views/livewire/project/application/backup/index.blade.php b/resources/views/livewire/project/application/backup/index.blade.php index 7e89bbcd07..785b855cae 100644 --- a/resources/views/livewire/project/application/backup/index.blade.php +++ b/resources/views/livewire/project/application/backup/index.blade.php @@ -181,6 +181,7 @@ @foreach ($backups as $backup) @php $latestExecution = $backup->latestExecution; + $s3StorageNames = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name'); $status = $latestExecution?->status; $statusLabel = match ($status) { 'running' => 'In progress', @@ -208,9 +209,9 @@ {{ $backup->targetType() }} {{ $backup->frequency }} - - + + {{ $latestExecution?->finished_at?->diffForHumans() ?? ($status === 'running' ? 'Running now' : 'Never') }} diff --git a/resources/views/livewire/project/database/backup-edit/s3.blade.php b/resources/views/livewire/project/database/backup-edit/s3.blade.php index dcfb7c3815..4f0f7b43fd 100644 --- a/resources/views/livewire/project/database/backup-edit/s3.blade.php +++ b/resources/views/livewire/project/database/backup-edit/s3.blade.php @@ -25,11 +25,12 @@ @endif
- +
- @if ($executionStatus === 'running') - - @endif + :type="$executionStatusType" :pulse="$executionStatus === 'running'" />
{{ data_get($execution, 'database_name', 'N/A') }} @@ -112,7 +112,22 @@ - @if (data_get($execution, 's3_uploaded') !== null) + @foreach ($s3Replicas as $replica) + + @endforeach + @if ($s3Replicas->isEmpty() && data_get($execution, 's3_uploaded') !== null) name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase()) - || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())" + || @js(strtolower($s3StorageNames)).includes(search.toLowerCase())" class="data-table-row scheduled-backups-table-grid border-b border-neutral-200 last:border-b-0 dark:border-white/[0.06]">
@@ -96,8 +97,8 @@ @endif
-
- {{ $backup->save_s3 ? ($backup->s3?->name ?? 'Unavailable') : 'Local only' }} +
+ {{ $backup->save_s3 ? ($s3StorageNames ?: 'Unavailable') : 'Local only' }}
'neutral', }; $databaseBackupId = 'database:'.$databaseBackup->id; - $databaseS3 = $databaseBackup->s3?->team_id === $serviceTeamId ? $databaseBackup->s3 : null; - $databaseS3Tooltip = ! $databaseBackup->save_s3 ? 'S3 storage: Not configured' : ($databaseS3 ? 'S3 storage: '.$databaseS3->name.' (bucket: '.$databaseS3->bucket.')' : 'S3 storage: Unavailable'); + $databaseS3s = ($databaseBackup->s3Storages->isNotEmpty() ? $databaseBackup->s3Storages : collect([$databaseBackup->s3])->filter())->where('team_id', $serviceTeamId); + $databaseS3Tooltip = ! $databaseBackup->save_s3 ? 'S3 storage: Not configured' : ($databaseS3s->isNotEmpty() ? 'S3 storage: '.$databaseS3s->map(fn ($s3) => $s3->name.' (bucket: '.$s3->bucket.')')->join(', ') : 'S3 storage: Unavailable'); @endphp
{{ $databaseBackup->frequency }} - {{ $latestExecution?->finished_at?->diffForHumans() ?? ($status === 'running' ? 'Running now' : 'Never') }} @@ -308,8 +308,8 @@ @foreach ($backups as $backup) @php $latestExecution = $backup->latestExecution; - $volumeS3 = $backup->s3?->team_id === $serviceTeamId ? $backup->s3 : null; - $volumeS3Tooltip = ! $backup->save_s3 ? 'S3 storage: Not configured' : ($volumeS3 ? 'S3 storage: '.$volumeS3->name.' (bucket: '.$volumeS3->bucket.')' : 'S3 storage: Unavailable'); + $volumeS3s = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->where('team_id', $serviceTeamId); + $volumeS3Tooltip = ! $backup->save_s3 ? 'S3 storage: Not configured' : ($volumeS3s->isNotEmpty() ? 'S3 storage: '.$volumeS3s->map(fn ($s3) => $s3->name.' (bucket: '.$s3->bucket.')')->join(', ') : 'S3 storage: Unavailable'); $status = $latestExecution?->status; $statusLabel = match ($status) { 'running' => 'In progress', @@ -338,8 +338,8 @@ {{ $backup->frequency }} - diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php index b4bf4465ec..041fb4470d 100644 --- a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php +++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php @@ -48,15 +48,19 @@ 'failed' => 'error', default => 'neutral', }; + $s3Replicas = $execution->s3Replicas; + $hasLiveS3Copies = $s3Replicas->isNotEmpty() + ? $s3Replicas->contains(fn ($replica) => $replica->s3_uploaded === true && ! $replica->s3_storage_deleted) + : $execution->s3_uploaded === true && ! $execution->s3_storage_deleted; $executionCheckboxes = []; $deleteActions = []; if (! $execution->local_storage_deleted) { $deleteActions[] = 'This backup will be permanently deleted from local storage.'; } - if ($execution->s3_uploaded === true && ! $execution->s3_storage_deleted) { + if ($hasLiveS3Copies) { $executionCheckboxes[] = [ 'id' => 'delete_backup_s3', - 'label' => 'Delete the selected backup permanently from S3 Storage', + 'label' => 'Delete the selected backup permanently from every S3 storage', ]; } if (empty($deleteActions)) { @@ -75,7 +79,7 @@
- + @@ -98,7 +102,22 @@ - @if ($execution->s3_uploaded !== null) + @foreach ($s3Replicas as $replica) + + @endforeach + @if ($s3Replicas->isEmpty() && $execution->s3_uploaded !== null) - + Move
- + :confirmWithPassword="false" step2ButtonText="Remove storage" + :actions="['Backups from this schedule will no longer be uploaded to this storage. S3 backup turns off when no other storage remains.']"> @@ -148,10 +150,10 @@
- + :confirmWithPassword="false" step2ButtonText="Remove storage" + :actions="['Backups from this schedule will no longer be uploaded to this storage. S3 backup turns off when no other storage remains.']"> diff --git a/tests/Feature/Api/BackupS3DestinationsApiTest.php b/tests/Feature/Api/BackupS3DestinationsApiTest.php new file mode 100644 index 0000000000..5428dd51b7 --- /dev/null +++ b/tests/Feature/Api/BackupS3DestinationsApiTest.php @@ -0,0 +1,292 @@ +set('app.maintenance.store', 'array'); + InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]); + + $this->team = Team::factory()->create(); + $user = User::factory()->create(); + $this->team->members()->attach($user->id, ['role' => 'owner']); + + $plainTextToken = Str::random(40); + $token = $user->tokens()->create([ + 'name' => 's3-destinations-test', + 'token' => hash('sha256', $plainTextToken), + 'abilities' => ['*'], + 'team_id' => $this->team->id, + ]); + $this->headers = ['Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken]; + + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + + $this->database = StandalonePostgresql::create([ + 'name' => 'db', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'app', + 'image' => 'postgres:16', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + $this->application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $this->volume = LocalPersistentVolume::create([ + 'name' => 'data', + 'mount_path' => '/data', + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]); + + $this->firstS3 = createBackupDestinationS3($this->team, 'first'); + $this->secondS3 = createBackupDestinationS3($this->team, 'second'); + $this->otherTeamS3 = createBackupDestinationS3(Team::factory()->create(), 'foreign'); +}); + +function createBackupDestinationS3(Team $team, string $name): S3Storage +{ + return S3Storage::create([ + 'name' => $name, + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => $name.'-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); +} + +/** + * @return list + */ +function backupDestinationUuids(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): array +{ + return $backup->refresh()->selectedS3Storages()->pluck('uuid')->all(); +} + +describe('database backups', function () { + test('creates a backup with several destinations and lists them primary first', function () { + $response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid], + 's3_storage_uuid' => $this->secondS3->uuid, + ])->assertCreated(); + + $backup = ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->sole(); + expect($backup->s3_storage_id)->toBe($this->secondS3->id) + ->and(backupDestinationUuids($backup))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]); + + $this->withHeaders($this->headers)->getJson("/api/v1/databases/{$this->database->uuid}/backups") + ->assertOk() + ->assertJsonPath('0.s3_storage_uuid', $this->secondS3->uuid) + ->assertJsonPath('0.s3_storage_uuids', [$this->secondS3->uuid, $this->firstS3->uuid]) + ->assertJsonMissingPath('0.s3_storages') + ->assertJsonMissingPath('0.s3'); + }); + + test('uses the first listed destination as primary when s3_storage_uuid is not sent', function () { + $response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuids' => [$this->secondS3->uuid, $this->firstS3->uuid], + ])->assertCreated(); + + expect(ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->sole()->s3_storage_id)->toBe($this->secondS3->id); + }); + + test('a single s3_storage_uuid that is already a destination becomes primary and keeps the others', function () { + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => 'daily', + 'save_s3' => true, + 'database_id' => $this->database->id, + 'database_type' => $this->database->getMorphClass(), + 'team_id' => $this->team->id, + ]); + $backup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]); + + $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [ + 'save_s3' => true, + 's3_storage_uuid' => $this->secondS3->uuid, + ])->assertOk(); + + expect(backupDestinationUuids($backup))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]) + ->and($backup->s3_storage_id)->toBe($this->secondS3->id); + }); + + test('a single new s3_storage_uuid replaces every destination', function () { + $thirdS3 = createBackupDestinationS3($this->team, 'third'); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => 'daily', + 'save_s3' => true, + 'database_id' => $this->database->id, + 'database_type' => $this->database->getMorphClass(), + 'team_id' => $this->team->id, + ]); + $backup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]); + + $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [ + 's3_storage_uuid' => $thirdS3->uuid, + ])->assertOk(); + + expect(backupDestinationUuids($backup))->toBe([$thirdS3->uuid]) + ->and($backup->s3_storage_id)->toBe($thirdS3->id); + }); + + test('updates the destination list', function () { + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => 'daily', + 'database_id' => $this->database->id, + 'database_type' => $this->database->getMorphClass(), + 'team_id' => $this->team->id, + ]); + + $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [ + 'save_s3' => true, + 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid], + ])->assertOk(); + + expect(backupDestinationUuids($backup))->toBe([$this->firstS3->uuid, $this->secondS3->uuid]) + ->and((bool) $backup->save_s3)->toBeTrue(); + }); + + test('rejects an S3 storage of another team without changing the backup', function () { + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $this->firstS3->id, + 'database_id' => $this->database->id, + 'database_type' => $this->database->getMorphClass(), + 'team_id' => $this->team->id, + ]); + $backup->syncS3Storages([$this->firstS3->id]); + + $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [ + 's3_storage_uuids' => [$this->firstS3->uuid, $this->otherTeamS3->uuid], + ])->assertUnprocessable() + ->assertJsonValidationErrors(['s3_storage_uuids']) + ->assertDontSee($this->otherTeamS3->name); + + expect(backupDestinationUuids($backup))->toBe([$this->firstS3->uuid]); + }); + + test('rejects invalid destination combinations', function (array $payload, string $errorField) { + $payload = array_map(fn ($value) => is_array($value) + ? array_map(fn (string $name) => $this->{$name}->uuid, $value) + : (is_string($value) && property_exists($this, $value) ? $this->{$value}->uuid : $value), $payload); + + $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", ['frequency' => 'daily', ...$payload]) + ->assertUnprocessable() + ->assertJsonValidationErrors([$errorField]); + + expect(ScheduledDatabaseBackup::count())->toBe(0); + })->with([ + 'primary outside the list' => [['s3_storage_uuids' => ['firstS3'], 's3_storage_uuid' => 'secondS3'], 's3_storage_uuid'], + 'save_s3 without destinations' => [['save_s3' => true, 's3_storage_uuids' => []], 's3_storage_uuid'], + 'foreign storage in the list' => [['save_s3' => true, 's3_storage_uuids' => ['firstS3', 'otherTeamS3']], 's3_storage_uuids'], + ]); + + test('rejects clearing the destinations of a backup that saves to S3', function () { + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $this->firstS3->id, + 'database_id' => $this->database->id, + 'database_type' => $this->database->getMorphClass(), + 'team_id' => $this->team->id, + ]); + + $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [ + 's3_storage_uuids' => [], + ])->assertUnprocessable(); + + expect($backup->refresh()->s3_storage_id)->toBe($this->firstS3->id); + }); +}); + +describe('volume backups', function () { + test('sets several destinations and returns them primary first', function () { + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid], + 's3_storage_uuid' => $this->secondS3->uuid, + ]) + ->assertCreated() + ->assertJsonPath('s3_storage_uuid', $this->secondS3->uuid) + ->assertJsonPath('s3_storage_uuids', [$this->secondS3->uuid, $this->firstS3->uuid]); + + expect(backupDestinationUuids(ScheduledVolumeBackup::sole()))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]); + }); + + test('a single s3_storage_uuid replaces every destination', function () { + $url = "/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups"; + $this->withHeaders($this->headers)->putJson($url, [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid], + ])->assertCreated(); + + $this->withHeaders($this->headers)->putJson($url, [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuid' => $this->secondS3->uuid, + ])->assertOk()->assertJsonPath('s3_storage_uuids', [$this->secondS3->uuid]); + + expect(backupDestinationUuids(ScheduledVolumeBackup::sole()))->toBe([$this->secondS3->uuid]); + }); + + test('rejects an S3 storage of another team', function () { + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuids' => [$this->firstS3->uuid, $this->otherTeamS3->uuid], + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors(['s3_storage_uuids']) + ->assertDontSee($this->otherTeamS3->name); + + expect(ScheduledVolumeBackup::count())->toBe(0); + }); + + test('rejects a primary destination outside the list', function () { + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuids' => [$this->firstS3->uuid], + 's3_storage_uuid' => $this->secondS3->uuid, + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors(['s3_storage_uuid']); + }); +}); diff --git a/tests/Feature/BackupEditValidationTest.php b/tests/Feature/BackupEditValidationTest.php index 66643ffc46..3479e8db11 100644 --- a/tests/Feature/BackupEditValidationTest.php +++ b/tests/Feature/BackupEditValidationTest.php @@ -424,7 +424,7 @@ it('defaults to the first available storage when multiple storages are available ]); Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) - ->assertSet('s3StorageId', $firstS3->id) + ->assertSet('s3StorageIds', [$firstS3->id]) ->set('saveS3', true) ->call('instantSave') ->assertDispatched('success'); @@ -461,7 +461,7 @@ it('accepts a root team S3 storage for the instance database backup', function ( Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => collect([$s3])]) ->set('saveS3', true) - ->set('s3StorageId', $s3->id) + ->set('s3StorageIds', [$s3->id]) ->call('instantSave') ->assertDispatched('success'); @@ -548,7 +548,7 @@ it('saves selected S3 storage immediately when it changes', function () { ]); Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) - ->set('s3StorageId', $secondS3->id) + ->set('s3StorageIds', [$secondS3->id]) ->assertDispatched('success'); $backup->refresh(); diff --git a/tests/Feature/BackupRetentionAndStaleDetectionTest.php b/tests/Feature/BackupRetentionAndStaleDetectionTest.php index 97e45599ff..8e235a6501 100644 --- a/tests/Feature/BackupRetentionAndStaleDetectionTest.php +++ b/tests/Feature/BackupRetentionAndStaleDetectionTest.php @@ -276,6 +276,8 @@ test('database S3 retention remains best effort when deletion fails', function ( ]); ScheduledDatabaseBackupExecution::whereKey($newestExecution->id)->update(['created_at' => now()]); ScheduledDatabaseBackupExecution::whereKey($oldExecution->id)->update(['created_at' => now()->subDay()]); + $newestExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]); + $oldExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]); $disk = Mockery::mock(); $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse(); Storage::shouldReceive('build')->once()->andReturn($disk); diff --git a/tests/Feature/BackupS3DestinationsFormTest.php b/tests/Feature/BackupS3DestinationsFormTest.php new file mode 100644 index 0000000000..211360408a --- /dev/null +++ b/tests/Feature/BackupS3DestinationsFormTest.php @@ -0,0 +1,322 @@ + $name, + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => str($name)->slug()->toString(), + 'endpoint' => 'https://s3.example.com', + 'is_usable' => true, + 'team_id' => $team->id, + ]); +} + +/** + * @return array{0: Application, 1: LocalPersistentVolume, 2: StandalonePostgresql} + */ +function createResourcesForDestinationsFormTest(Team $team): array +{ + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + $database = StandalonePostgresql::create([ + 'name' => 'pg-destinations', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + return [$application, $volume, $database]; +} + +function createDatabaseBackupForDestinationsFormTest(StandalonePostgresql $database, array $overrides = []): ScheduledDatabaseBackup +{ + return ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => $database->getMorphClass(), + 'database_id' => $database->id, + 'team_id' => $database->team()->id, + ...$overrides, + ]); +} + +beforeEach(function () { + Server::flushIdentityMap(); + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + + $this->team = Team::factory()->create(); + $this->otherTeam = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->user->teams()->attach($this->team, ['role' => 'admin']); + $this->user->teams()->attach($this->otherTeam, ['role' => 'owner']); + $this->actingAs($this->user); + session(['currentTeam' => $this->team]); + + [$this->application, $this->volume, $this->database] = createResourcesForDestinationsFormTest($this->team); + $this->firstStorage = createS3StorageForDestinationsFormTest($this->team, 'First S3'); + $this->secondStorage = createS3StorageForDestinationsFormTest($this->team, 'Second S3'); + $this->foreignStorage = createS3StorageForDestinationsFormTest($this->otherTeam, 'Foreign S3'); +}); + +it('saves two S3 destinations for a database backup schedule', function () { + $backup = createDatabaseBackupForDestinationsFormTest($this->database); + + Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3']) + ->assertSee('First S3') + ->assertSee('Second S3') + ->assertSet('s3StorageIds', [$this->firstStorage->id]) + ->set('saveS3', true) + ->set('s3StorageIds', [(string) $this->firstStorage->id, (string) $this->secondStorage->id]) + ->call('submit') + ->assertHasNoErrors() + ->assertDispatched('success'); + + $backup->refresh(); + expect($backup->save_s3)->toBeTruthy() + ->and($backup->s3_storage_id)->toBe($this->firstStorage->id) + ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all()) + ->toBe([$this->firstStorage->id, $this->secondStorage->id]); +}); + +it('saves two S3 destinations for a volume backup schedule', function () { + Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application]) + ->set('frequency', 'daily') + ->set('saveToS3', true) + ->set('s3StorageIds', [(string) $this->secondStorage->id, (string) $this->firstStorage->id]) + ->assertHasNoErrors() + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + expect($backup->save_s3)->toBeTrue() + ->and($backup->s3_storage_id)->toBe($this->secondStorage->id) + ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all()) + ->toBe([$this->firstStorage->id, $this->secondStorage->id]); +}); + +it('drops S3 storages of another team after a team switch', function () { + $backup = createDatabaseBackupForDestinationsFormTest($this->database, [ + 'save_s3' => true, + 's3_storage_id' => $this->firstStorage->id, + ]); + $backup->syncS3Storages([$this->firstStorage->id]); + + $databaseComponent = Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]); + $volumeComponent = Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application]); + session(['currentTeam' => $this->otherTeam]); + + $databaseComponent->set('s3StorageIds', [$this->foreignStorage->id]) + ->assertHasErrors('s3StorageIds'); + expect($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->firstStorage->id]); + + $databaseComponent->set('s3StorageIds', [$this->secondStorage->id, $this->foreignStorage->id]) + ->assertHasNoErrors(); + expect($backup->fresh()->s3_storage_id)->toBe($this->secondStorage->id) + ->and($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->secondStorage->id]); + + $volumeComponent->set('frequency', 'daily') + ->set('saveToS3', true) + ->set('s3StorageIds', [$this->foreignStorage->id]) + ->assertHasErrors('s3StorageIds'); + expect(ScheduledVolumeBackup::query()->count())->toBe(0); + + $volumeComponent->set('s3StorageIds', [$this->firstStorage->id, $this->foreignStorage->id]) + ->assertHasNoErrors(); + expect(ScheduledVolumeBackup::query()->sole()->s3Storages()->pluck('s3_storages.id')->all()) + ->toBe([$this->firstStorage->id]); +}); + +it('rejects S3 backups without a destination', function () { + $backup = createDatabaseBackupForDestinationsFormTest($this->database, [ + 'save_s3' => true, + 's3_storage_id' => $this->firstStorage->id, + ]); + $backup->syncS3Storages([$this->firstStorage->id]); + + Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) + ->set('s3StorageIds', []) + ->call('submit') + ->assertHasErrors('s3StorageIds') + ->assertDispatched('error'); + + expect($backup->fresh()->save_s3)->toBeTruthy() + ->and($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->firstStorage->id]); + + Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application]) + ->set('frequency', 'daily') + ->set('saveToS3', true) + ->set('s3StorageIds', []) + ->call('save') + ->assertHasErrors('s3StorageIds'); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('moves and removes one destination of a multi-destination schedule from the storage page', function () { + $thirdStorage = createS3StorageForDestinationsFormTest($this->team, 'Third S3'); + $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]); + $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + $volumeBackup = ScheduledVolumeBackup::create([ + 'backupable_type' => $this->volume->getMorphClass(), + 'backupable_id' => $this->volume->id, + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'enabled' => true, + 'save_s3' => true, + ]); + $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + + Livewire::test(StorageResources::class, ['storage' => $this->secondStorage]) + ->assertSet("selectedStorages.{$databaseBackup->id}", $this->secondStorage->id) + ->assertSet("selectedVolumeStorages.{$volumeBackup->id}", $this->secondStorage->id) + ->assertSee('pg-destinations') + ->assertSee('app-data'); + + Livewire::test(StorageResources::class, ['storage' => $this->firstStorage]) + ->set("selectedStorages.{$databaseBackup->id}", $thirdStorage->id) + ->call('moveBackup', $databaseBackup->id) + ->set("selectedVolumeStorages.{$volumeBackup->id}", $thirdStorage->id) + ->call('moveVolumeBackup', $volumeBackup->id); + + foreach ([$databaseBackup, $volumeBackup] as $backup) { + $backup->refresh(); + expect($backup->s3_storage_id)->toBe($thirdStorage->id) + ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all()) + ->toBe([$this->secondStorage->id, $thirdStorage->id]); + } + + Livewire::test(StorageResources::class, ['storage' => $this->secondStorage]) + ->call('disableS3', $databaseBackup->id) + ->call('disableVolumeS3', $volumeBackup->id); + + foreach ([$databaseBackup, $volumeBackup] as $backup) { + $backup->refresh(); + expect($backup->save_s3)->toBeTruthy() + ->and($backup->s3_storage_id)->toBe($thirdStorage->id) + ->and($backup->s3Storages()->pluck('s3_storages.id')->all())->toBe([$thirdStorage->id]); + } + + Livewire::test(StorageResources::class, ['storage' => $thirdStorage]) + ->call('disableS3', $databaseBackup->id) + ->call('disableVolumeS3', $volumeBackup->id); + + foreach ([$databaseBackup, $volumeBackup] as $backup) { + $backup->refresh(); + expect($backup->save_s3)->toBeFalsy() + ->and($backup->s3_storage_id)->toBeNull() + ->and($backup->s3Storages()->count())->toBe(0); + } +}); + +it('shows the copy status of each destination in the execution lists', function () { + $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]); + $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + $databaseExecution = ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $databaseBackup->id, + 'status' => 'success', + 'filename' => '/backups/dump.dmp', + 's3_uploaded' => false, + ]); + $volumeBackup = ScheduledVolumeBackup::create([ + 'backupable_type' => $this->volume->getMorphClass(), + 'backupable_id' => $this->volume->id, + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + $volumeExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $volumeBackup->id, + 'status' => 'success', + 'filename' => '/backups/volume.tar.gz', + 's3_uploaded' => false, + ]); + foreach ([$databaseExecution, $volumeExecution] as $execution) { + $execution->s3Replicas()->create(['s3_storage_id' => $this->firstStorage->id, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $this->secondStorage->id, 's3_uploaded' => false, 'message' => 'Access denied']); + } + + Livewire::test(BackupExecutions::class, ['backup' => $databaseBackup]) + ->assertSeeInOrder(['First S3 Available', 'Second S3 Failed']) + ->assertSee('Access denied') + ->assertSee('Delete the selected backup permanently from every S3 storage'); + + Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application, 'section' => 'executions']) + ->assertSeeInOrder(['First S3', 'Second S3 failed']) + ->assertSee('Delete the selected backup permanently from every S3 storage'); +}); + +it('keeps a selected destination that is unusable for a while when other settings are saved', function () { + $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]); + $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + $volumeBackup = ScheduledVolumeBackup::create([ + 'backupable_type' => $this->volume->getMorphClass(), + 'backupable_id' => $this->volume->id, + 'team_id' => $this->team->id, + 'frequency' => '0 0 * * *', + 'save_s3' => true, + ]); + $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + $this->secondStorage->update(['is_usable' => false]); + + Livewire::test(BackupEdit::class, ['backup' => $databaseBackup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3']) + ->assertSee('Second S3 (unavailable)') + ->set('timeout', 7200) + ->call('submit') + ->assertHasNoErrors(); + Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application, 'section' => 's3']) + ->assertSee('Second S3 (unavailable)') + ->set('stopDuringBackup', true) + ->call('save') + ->assertHasNoErrors(); + + expect($databaseBackup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all()) + ->toBe([$this->firstStorage->id, $this->secondStorage->id]) + ->and($volumeBackup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all()) + ->toBe([$this->firstStorage->id, $this->secondStorage->id]); +}); diff --git a/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php b/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php new file mode 100644 index 0000000000..0191aa09d7 --- /dev/null +++ b/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php @@ -0,0 +1,434 @@ + $name, + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => strtolower($name).'-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); +} + +function multiS3Execution(ScheduledDatabaseBackup $backup, string $name, array $attributes = []): ScheduledDatabaseBackupExecution +{ + $execution = ScheduledDatabaseBackupExecution::create([ + 'uuid' => $name, + 'database_name' => 'app', + 'filename' => "/backup/{$name}.dmp", + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + ...$attributes, + ]); + + return $execution; +} + +/** + * Records the files that deleteBackupsS3() removes, keyed by bucket. + */ +function fakeMultiS3Deletes(): Collection +{ + $deleted = collect(); + Storage::shouldReceive('build')->andReturnUsing(function (array $config) use ($deleted) { + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->andReturnUsing(function (array $files) use ($config, $deleted) { + $deleted->put($config['bucket'], [...$deleted->get($config['bucket'], []), ...$files]); + + return true; + }); + + return $disk; + }); + + return $deleted; +} + +/** + * Runs a backup with two S3 destinations, failing the upload to the storages named in $failingStorages. + * + * @return array{0: ScheduledDatabaseBackupExecution, 1: Collection} + */ +function runMultiS3BackupJob(array $failingStorages): array +{ + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]); + $database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']); + $first = multiS3Storage($team, 'First'); + $second = multiS3Storage($team, 'Second'); + $backup = new ScheduledDatabaseBackup; + $backup->forceFill([ + 'enabled' => true, + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $first->id, + 'frequency' => 'daily', + 'databases_to_backup' => 'app', + 'database_id' => $database->id, + 'database_type' => $database->getMorphClass(), + 'team_id' => $team->id, + ])->save(); + $backup->syncS3Storages([$first->id, $second->id]); + $commands = collect(); + Process::fake(function ($process) use ($commands) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $commands->push($command); + + return Process::result(output: str_contains($command, 'du -b') ? '128' : ''); + }); + + $job = Mockery::mock(DatabaseBackupJob::class.'[upload_to_s3]', [$backup->fresh()])->shouldAllowMockingProtectedMethods(); + $job->shouldReceive('upload_to_s3')->andReturnUsing(function (S3Storage $storage) use ($failingStorages) { + if (in_array($storage->name, $failingStorages, true)) { + throw new RuntimeException('access denied'); + } + }); + $job->handle(); + + return [ScheduledDatabaseBackupExecution::query()->sole(), $commands]; +} + +it('applies S3 retention to each destination separately', function () { + $team = Team::factory()->create(); + $first = multiS3Storage($team, 'First'); + $second = multiS3Storage($team, 'Second'); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'database_backup_retention_amount_s3' => 1, + ]); + $backup->syncS3Storages([$first->id, $second->id]); + $old = multiS3Execution($backup, 'old'); + $new = multiS3Execution($backup, 'new', ['s3_uploaded' => false]); + ScheduledDatabaseBackupExecution::whereKey($old->id)->update(['created_at' => now()->subDay()]); + $old->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $old->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]); + $new->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $new->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => false, 'message' => 'denied']); + $deleted = fakeMultiS3Deletes(); + + removeOldBackups($backup); + + expect($deleted->all())->toBe(['first-bucket' => ['/backup/old.dmp']]) + ->and($old->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeTrue() + ->and($old->s3Replicas()->where('s3_storage_id', $second->id)->sole()->s3_storage_deleted)->toBeFalse() + ->and($old->fresh()->s3_storage_deleted)->toBeFalse() + ->and($backup->executions()->count())->toBe(2); +}); + +it('deletes the execution record when its last S3 copy expires', function () { + $team = Team::factory()->create(); + $first = multiS3Storage($team, 'First'); + $second = multiS3Storage($team, 'Second'); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'database_backup_retention_amount_s3' => 1, + ]); + $backup->syncS3Storages([$first->id, $second->id]); + $old = multiS3Execution($backup, 'old'); + $new = multiS3Execution($backup, 'new'); + ScheduledDatabaseBackupExecution::whereKey($old->id)->update(['created_at' => now()->subDay()]); + foreach ([$old, $new] as $execution) { + $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]); + } + $deleted = fakeMultiS3Deletes(); + + removeOldBackups($backup); + + expect($deleted->all())->toBe(['first-bucket' => ['/backup/old.dmp'], 'second-bucket' => ['/backup/old.dmp']]) + ->and($old->fresh())->toBeNull() + ->and(DatabaseBackupS3Replica::where('execution_id', $old->id)->exists())->toBeFalse() + ->and($new->fresh())->not->toBeNull(); +}); + +it('marks copies in a deleted storage as gone without calling S3', function () { + $team = Team::factory()->create(); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + ]); + $execution = multiS3Execution($backup, 'orphaned'); + $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => true]); + Storage::shouldReceive('build')->never(); + + removeOldBackups($backup); + + expect($execution->fresh())->toBeNull(); +}); + +it('keeps an execution whose local file still exists after its S3 copies expire', function () { + $team = Team::factory()->create(); + $storage = multiS3Storage($team, 'First'); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + ]); + $execution = multiS3Execution($backup, 'local-copy', ['local_storage_deleted' => false]); + $execution->s3Replicas()->create(['s3_storage_id' => $storage->id, 's3_uploaded' => true, 's3_storage_deleted' => true]); + + removeOldBackups($backup); + + expect($execution->fresh())->not->toBeNull(); +}); + +it('deletes the S3 copies of an execution from every destination that has one', function () { + $team = Team::factory()->create(); + $first = multiS3Storage($team, 'First'); + $second = multiS3Storage($team, 'Second'); + $failed = multiS3Storage($team, 'Failed'); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + ]); + $execution = multiS3Execution($backup, 'manual', ['s3_uploaded' => false]); + $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $failed->id, 's3_uploaded' => false]); + $deleted = fakeMultiS3Deletes(); + + $execution->deleteS3Copies(); + + expect($deleted->keys()->sort()->values()->all())->toBe(['first-bucket', 'second-bucket']) + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($execution->hasLiveS3Copies())->toBeFalse(); +}); + +describe('backup job', function () { + beforeEach(function () { + Storage::fake('ssh-keys'); + Storage::fake('ssh-mux'); + Event::fake([BackupCreated::class]); + Notification::fake(); + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + }); + + it('uploads to every destination and deletes the local file when all uploads succeed', function () { + [$execution, $commands] = runMultiS3BackupJob([]); + + expect($execution->status)->toBe('success') + ->and($execution->s3_uploaded)->toBeTrue() + ->and($execution->local_storage_deleted)->toBeTrue() + ->and($execution->s3Replicas()->pluck('s3_uploaded')->all())->toBe([true, true]) + ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeTrue(); + }); + + it('continues to later destinations and keeps the local file when a replica write fails', function (array $failingStorages) { + $event = 'eloquent.creating: '.DatabaseBackupS3Replica::class; + Event::listen($event, function (DatabaseBackupS3Replica $replica): void { + if (S3Storage::find($replica->s3_storage_id)?->name === 'First') { + throw new RuntimeException('replica persistence failed'); + } + }); + + try { + [$execution, $commands] = runMultiS3BackupJob($failingStorages); + } finally { + Event::forget($event); + } + + $replica = $execution->s3Replicas()->with('s3')->sole(); + expect($replica->s3->name)->toBe('Second') + ->and($replica->s3_uploaded)->toBeTrue() + ->and($execution->s3_storage_deleted)->toBeFalse() + ->and($execution->s3_uploaded)->toBeFalse() + ->and($execution->local_storage_deleted)->toBeFalse() + ->and($execution->message)->toContain('First: replica persistence failed') + ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeFalse(); + + if ($failingStorages !== []) { + expect($execution->message)->toContain('First: access denied'); + } + })->with([ + 'successful upload' => [[]], + 'failed upload' => [['First']], + ]); + + it('keeps the local file and names the failed destination when one upload fails', function () { + [$execution, $commands] = runMultiS3BackupJob(['Second']); + + $replicas = $execution->s3Replicas()->with('s3')->get()->keyBy(fn ($replica) => $replica->s3->name); + expect($execution->status)->toBe('success') + ->and($execution->s3_uploaded)->toBeFalse() + ->and($execution->local_storage_deleted)->toBeFalse() + ->and($execution->message)->toContain('Second: access denied') + ->and($replicas['First']->s3_uploaded)->toBeTrue() + ->and($replicas['Second']->s3_uploaded)->toBeFalse() + ->and($replicas['Second']->message)->toBe('access denied') + ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeFalse(); + }); +}); + +it('deletes an execution from every S3 destination through the API', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + $sshKeys = Storage::fake('ssh-keys'); + $sshMux = Storage::fake('ssh-mux'); + Process::fake(); + $team = Team::factory()->create(); + $user = User::factory()->create(); + $team->members()->attach($user, ['role' => 'owner']); + session(['currentTeam' => $team]); + $token = $user->createToken('test-token', ['*'])->plainTextToken; + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]); + $database = create_standalone_postgresql($environment->id, $destination); + $first = multiS3Storage($team, 'First'); + $second = multiS3Storage($team, 'Second'); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'database_type' => $database->getMorphClass(), + 'database_id' => $database->id, + 'team_id' => $team->id, + ]); + $backup->syncS3Storages([$first->id, $second->id]); + $execution = multiS3Execution($backup, 'api', ['local_storage_deleted' => false]); + $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]); + $deleted = fakeMultiS3Deletes(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshKeys); + Storage::shouldReceive('disk')->with('ssh-mux')->andReturn($sshMux); + + $this->withHeaders(['Authorization' => 'Bearer '.$token]) + ->deleteJson("/api/v1/databases/{$database->uuid}/backups/{$backup->uuid}/executions/{$execution->uuid}?delete_s3=true") + ->assertOk(); + + expect($deleted->all())->toBe(['first-bucket' => ['/backup/api.dmp'], 'second-bucket' => ['/backup/api.dmp']]) + ->and($execution->fresh())->toBeNull(); +}); + +it('copies the destinations of the same team to a cloned schedule', function () { + $team = Team::factory()->create(); + $otherTeam = Team::factory()->create(); + $first = multiS3Storage($team, 'First'); + $second = multiS3Storage($team, 'Second'); + $foreign = multiS3Storage($otherTeam, 'Foreign'); + $attributes = [ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + ]; + $backup = ScheduledDatabaseBackup::create($attributes); + $backup->syncS3Storages([$first->id, $second->id]); + $backup->s3Storages()->attach($foreign->id); + $copy = ScheduledDatabaseBackup::create($attributes); + + $backup->copyS3StoragesTo($copy); + + expect($copy->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())->toBe([$first->id, $second->id]); +}); + +it('requires a live S3 copy for local retention only when local backups are disabled', function (bool $disabled, ?array $replica, bool $deleted) { + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]); + $database = create_standalone_postgresql($environment->id, $destination); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => $disabled, + 'database_backup_retention_amount_locally' => 1, + 'database_type' => $database->getMorphClass(), + 'database_id' => $database->id, + 'team_id' => $team->id, + ]); + $older = multiS3Execution($backup, 'older', ['local_storage_deleted' => false, 'created_at' => now()->subDay()]); + $newer = multiS3Execution($backup, 'newer', ['local_storage_deleted' => false]); + if ($replica !== null) { + $storage = multiS3Storage($team, 'Retention'); + foreach ([$older, $newer] as $execution) { + $execution->s3Replicas()->create(['s3_storage_id' => $storage->id, ...$replica]); + } + } + $commands = collect(); + Process::fake(function ($process) use ($commands) { + $commands->push(is_array($process->command) ? implode(' ', $process->command) : $process->command); + + return Process::result(); + }); + + removeOldBackups($backup); + + expect($commands->contains(fn (string $command): bool => str_contains($command, '/backup/older.dmp')))->toBe($deleted); + if ($deleted && $replica === null) { + $this->assertModelMissing($older); + } else { + expect($older->fresh()->local_storage_deleted)->toBe($deleted); + } + expect($newer->fresh()->local_storage_deleted)->toBeFalse(); +})->with([ + 'no replica' => [true, null, false], + 'failed upload' => [true, ['s3_uploaded' => false], false], + 'deleted replica' => [true, ['s3_uploaded' => true, 's3_storage_deleted' => true], false], + 'live replica' => [true, ['s3_uploaded' => true, 's3_storage_deleted' => false], true], + 'normal local retention' => [false, null, true], +]); diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.php index ad6d1d76d0..864536cfa7 100644 --- a/tests/Feature/DevelopmentQemuVmTest.php +++ b/tests/Feature/DevelopmentQemuVmTest.php @@ -173,16 +173,41 @@ it('can change the localhost qemu profile without adding a server', function () it('seeds localhost through the container command', function () { $this->seed(ServerSeeder::class); + Process::fake(); expect(Artisan::call('dev:qemu:seed', [ 'profile' => 'ubuntu-root', '--as-localhost' => true, ]))->toBe(Command::SUCCESS) ->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10'); + + Process::assertRan(fn ($process) => str_contains($process->command, 'docker run -d --name coolify-db') + && str_contains($process->command, 'POSTGRES_DB=')); +}); + +it('starts the instance database container with sudo on a non-root localhost vm', function () { + $this->seed(ServerSeeder::class); + Process::fake(); + + Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-non-root', '--as-localhost' => true]); + + Process::assertRan(fn ($process) => str_contains($process->command, 'if ! sudo docker inspect coolify-db') + && str_contains($process->command, 'sudo docker run -d --name coolify-db') + && str_contains($process->command, 'sudo docker start coolify-db')); +}); + +it('still seeds localhost when the instance database container cannot start', function () { + $this->seed(ServerSeeder::class); + Process::fake(fn () => Process::result(errorOutput: 'Cannot connect to the Docker daemon', exitCode: 1)); + + expect(Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-root', '--as-localhost' => true]))->toBe(Command::SUCCESS) + ->and(Artisan::output())->toContain('Could not start the coolify-db container') + ->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10'); }); it('seeds localhost with the forwarded ssh address of a lima vm', function () { $this->seed(ServerSeeder::class); + Process::fake(); expect(Artisan::call('dev:qemu:seed', [ 'profile' => 'debian-non-root', diff --git a/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php b/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php new file mode 100644 index 0000000000..2cbac04a38 --- /dev/null +++ b/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php @@ -0,0 +1,137 @@ + $name, + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => str($name)->slug()->toString(), + 'endpoint' => 'https://s3.example.com', + 'is_usable' => true, + 'team_id' => $team->id, + ]); +} + +beforeEach(function () { + Server::flushIdentityMap(); + $this->rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]); + $server = Server::factory()->create([ + 'id' => 0, + 'team_id' => 0, + 'ip' => '127.0.0.1', + 'private_key_id' => PrivateKey::factory()->create(['team_id' => 0])->id, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]); + StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail()->forceFill(['id' => 0])->save(); + InstanceSettings::forceCreate(['id' => 0]); + Once::flush(); + + $this->database = new StandalonePostgresql; + $this->database->forceFill([ + 'id' => 0, + 'name' => 'coolify-db', + 'postgres_user' => 'coolify', + 'postgres_password' => 'password', + 'postgres_db' => 'coolify', + 'status' => 'running', + 'destination_type' => StandaloneDocker::class, + 'destination_id' => 0, + ])->save(); + $this->backup = ScheduledDatabaseBackup::create([ + 'enabled' => true, + 'save_s3' => false, + 'frequency' => '0 0 * * *', + 'database_id' => 0, + 'database_type' => StandalonePostgresql::class, + 'team_id' => 0, + ]); + + $this->firstStorage = instanceBackupS3Storage($this->rootTeam, 'First S3'); + $this->secondStorage = instanceBackupS3Storage($this->rootTeam, 'Second S3'); + $this->otherTeam = Team::factory()->create(); + $this->foreignStorage = instanceBackupS3Storage($this->otherTeam, 'Foreign S3'); + + $user = User::factory()->create(); + $this->rootTeam->members()->attach($user->id, ['role' => 'admin']); + $user->teams()->attach($this->otherTeam->id, ['role' => 'owner']); + $this->actingAs($user); + session(['currentTeam' => ['id' => 0]]); +}); + +it('offers every root team S3 storage on the instance backup page', function () { + Livewire::test(SettingsBackup::class) + ->assertOk() + ->assertSee('First S3') + ->assertSee('Second S3') + ->assertDontSee('Foreign S3'); +}); + +it('saves several S3 destinations for the instance backup and drops storages of other teams', function () { + // The session team can change in another tab after the page loads. + session(['currentTeam' => ['id' => $this->otherTeam->id]]); + + Livewire::test(BackupEdit::class, ['backup' => $this->backup->fresh(), 'availableS3Storages' => collect(), 'section' => 's3']) + ->set('saveS3', true) + ->set('s3StorageIds', [$this->firstStorage->id, $this->secondStorage->id, $this->foreignStorage->id]) + ->call('submit') + ->assertHasNoErrors() + ->assertDispatched('success'); + + $backup = $this->backup->fresh(); + + expect($backup->save_s3)->toBeTruthy() + ->and($backup->s3_storage_id)->toBe($this->firstStorage->id) + ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all()) + ->toBe([$this->firstStorage->id, $this->secondStorage->id]); +}); + +it('uploads the instance backup to every selected destination', function () { + $this->backup->forceFill(['save_s3' => true, 'databases_to_backup' => 'coolify'])->save(); + $this->backup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + Process::fake(fn ($process) => Process::result(output: str_contains(implode(' ', (array) $process->command), 'du -b') ? '128' : '')); + + $uploadedTo = collect(); + $job = Mockery::mock(DatabaseBackupJob::class.'[upload_to_s3]', [$this->backup->fresh()])->shouldAllowMockingProtectedMethods(); + $job->shouldReceive('upload_to_s3')->andReturnUsing(fn (S3Storage $storage) => $uploadedTo->push($storage->id)); + $job->handle(); + + $execution = ScheduledDatabaseBackupExecution::query()->sole(); + + expect($uploadedTo->sort()->values()->all())->toBe([$this->firstStorage->id, $this->secondStorage->id]) + ->and($execution->status)->toBe('success') + ->and($execution->s3_uploaded)->toBeTrue() + ->and($execution->s3Replicas()->where('s3_uploaded', true)->pluck('s3_storage_id')->sort()->values()->all()) + ->toBe([$this->firstStorage->id, $this->secondStorage->id]); +}); + +it('links the instance backup from the storage resources page', function () { + $this->backup->forceFill(['save_s3' => true])->save(); + $this->backup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]); + + Livewire::test(StorageResources::class, ['storage' => $this->secondStorage]) + ->assertSee('coolify-db') + ->assertSeeHtml(route('settings.backup')); +}); diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.php index a19958608f..d0ba825511 100644 --- a/tests/Feature/Jobs/DatabaseBackupJobTest.php +++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php @@ -52,7 +52,7 @@ test('scheduled database backup casts full dump selection to boolean', function ->and((new ScheduledDatabaseBackup(['dump_all' => '1']))->dump_all)->toBeTrue(); }); -test('upload_to_s3 throws exception and disables s3 when storage is null', function () { +test('s3 upload throws exception and disables s3 when no destination exists', function () { $backup = ScheduledDatabaseBackup::create([ 'frequency' => '0 0 * * *', 'save_s3' => true, @@ -64,11 +64,7 @@ test('upload_to_s3 throws exception and disables s3 when storage is null', funct $job = new DatabaseBackupJob($backup); - $reflection = new ReflectionClass($job); - $s3Property = $reflection->getProperty('s3'); - $s3Property->setValue($job, null); - - $method = $reflection->getMethod('upload_to_s3'); + $method = (new ReflectionClass($job))->getMethod('uploadToS3Destinations'); expect(fn () => $method->invoke($job)) ->toThrow(Exception::class, 'S3 storage configuration is missing or has been deleted'); @@ -78,7 +74,7 @@ test('upload_to_s3 throws exception and disables s3 when storage is null', funct expect($backup->s3_storage_id)->toBeNull(); }); -test('upload_to_s3 exception message reports the previous s3 storage id', function () { +test('s3 upload exception message reports the previous s3 storage id', function () { $backup = ScheduledDatabaseBackup::create([ 'frequency' => '0 0 * * *', 'save_s3' => true, @@ -90,10 +86,7 @@ test('upload_to_s3 exception message reports the previous s3 storage id', functi $job = new DatabaseBackupJob($backup); - $reflection = new ReflectionClass($job); - $reflection->getProperty('s3')->setValue($job, null); - - expect(fn () => $reflection->getMethod('upload_to_s3')->invoke($job)) + expect(fn () => (new ReflectionClass($job))->getMethod('uploadToS3Destinations')->invoke($job)) ->toThrow(Exception::class, 'S3 storage ID: 12345'); $backup->refresh(); @@ -101,7 +94,7 @@ test('upload_to_s3 exception message reports the previous s3 storage id', functi expect($backup->s3_storage_id)->toBeNull(); }); -test('upload_to_s3 exception message reports null when no previous s3 storage id exists', function () { +test('s3 upload exception message reports null when no previous s3 storage id exists', function () { $backup = ScheduledDatabaseBackup::create([ 'frequency' => '0 0 * * *', 'save_s3' => true, @@ -113,10 +106,7 @@ test('upload_to_s3 exception message reports null when no previous s3 storage id $job = new DatabaseBackupJob($backup); - $reflection = new ReflectionClass($job); - $reflection->getProperty('s3')->setValue($job, null); - - expect(fn () => $reflection->getMethod('upload_to_s3')->invoke($job)) + expect(fn () => (new ReflectionClass($job))->getMethod('uploadToS3Destinations')->invoke($job)) ->toThrow(Exception::class, 'S3 storage ID: null'); }); @@ -281,7 +271,7 @@ test('retention cleanup failure does not fail a successful database backup', fun 'local_storage_deleted' => true, 'created_at' => now()->subDay(), ]); - ScheduledDatabaseBackupExecution::create([ + $newExecution = ScheduledDatabaseBackupExecution::create([ 'uuid' => 'new-backup', 'database_name' => 'database', 'filename' => '/backup/new.dmp', @@ -290,6 +280,8 @@ test('retention cleanup failure does not fail a successful database backup', fun 's3_uploaded' => true, 'local_storage_deleted' => true, ]); + $oldExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]); + $newExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]); $disk = Mockery::mock(); $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse(); Storage::shouldReceive('build')->once()->andReturn($disk); diff --git a/tests/Feature/Mcp/McpReadToolsTest.php b/tests/Feature/Mcp/McpReadToolsTest.php index 21df936c94..f45757b8d1 100644 --- a/tests/Feature/Mcp/McpReadToolsTest.php +++ b/tests/Feature/Mcp/McpReadToolsTest.php @@ -10,6 +10,7 @@ use App\Models\EnvironmentVariable; use App\Models\GithubApp; use App\Models\InstanceSettings; use App\Models\Project; +use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledDatabaseBackupExecution; use App\Models\ScheduledTask; @@ -217,6 +218,36 @@ test('database backup tools scope schedules by database type and id', function ( expect($response->json('result.isError'))->toBeTrue(); }); +test('list_database_backups returns every S3 destination primary first', function () { + $postgres = StandalonePostgresql::create([ + 'name' => 'multi-s3-db', + 'postgres_password' => 'password', + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + [$first, $second] = collect(['first', 'second'])->map(fn (string $name) => S3Storage::create([ + 'name' => $name, 'region' => 'us-east-1', 'key' => 'key', 'secret' => 'secret', + 'bucket' => $name, 'endpoint' => 'https://s3.example.com', 'team_id' => $this->team->id, + ]))->all(); + $backup = ScheduledDatabaseBackup::create([ + 'team_id' => $this->team->id, + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 's3_storage_id' => $second->id, + 'database_id' => $postgres->id, + 'database_type' => $postgres->getMorphClass(), + ]); + $backup->syncS3Storages([$first->id, $second->id]); + + $response = mcpReadCall('list_database_backups', ['uuid' => $postgres->uuid]); + $response->assertOk(); + + $listed = mcpReadJson($response)['data']['backups'][0]; + expect($listed['s3_storage_uuid'])->toBe($second->uuid) + ->and($listed['s3_storage_uuids'])->toBe([$second->uuid, $first->uuid]); +}); + test('list_backup_executions omits messages without sensitive read and redacts when included', function () { $postgres = StandalonePostgresql::create([ 'name' => 'backup-msg-db', diff --git a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php index 5563e87769..47b215d5f0 100644 --- a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php +++ b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php @@ -182,7 +182,7 @@ it('authorizes every volume backup form control', function (string $path, array [ '/]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$resource")[^>]*>\s*Enable S3\s*<\/x-forms\.button>/s', '/]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$resource")[^>]*>\s*Disable S3\s*<\/x-forms\.button>/s', - '/fresh()->s3_storage_id)->toBe($ownS3->id) ->and($backup->fresh()->save_s3)->toBeTruthy(); - $component->set('s3StorageId', $foreignS3->id)->call('instantSave'); + $component->set('s3StorageIds', [$foreignS3->id])->call('instantSave'); expect($backup->fresh()->s3_storage_id)->toBe($ownS3->id); }); diff --git a/tests/Feature/ScheduleAuditTest.php b/tests/Feature/ScheduleAuditTest.php index 281fdd42b4..6c700eae69 100644 --- a/tests/Feature/ScheduleAuditTest.php +++ b/tests/Feature/ScheduleAuditTest.php @@ -300,12 +300,12 @@ test('disabling or moving S3 storage for backups records audit events', function $volumeEvent = scheduleAuditEvent('ui.volume_backup.schedule_set'); expect($databaseEvent->resource_uuid)->toBe($database->uuid) - ->and($databaseEvent->metadata['changed_fields'])->toEqualCanonicalizing(['save_s3', 's3_storage_id']) + ->and($databaseEvent->metadata['changed_fields'])->toEqualCanonicalizing(['save_s3', 's3_storage_id', 's3_storages']) ->and($volumeEvent->metadata['resource_type'])->toBe('application') ->and($volumeEvent->metadata['resource_uuid'])->toBe($this->application->uuid) ->and($volumeEvent->metadata['storage_uuid'])->toBe($volume->uuid) ->and($volumeEvent->metadata['backup_uuid'])->toBe($volumeBackup->uuid) - ->and($volumeEvent->metadata['changed_fields'])->toBe(['s3_storage_id']); + ->and($volumeEvent->metadata['changed_fields'])->toEqualCanonicalizing(['s3_storage_id', 's3_storages']); }); test('configuring the instance database backup records an audit event', function () { diff --git a/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php b/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php new file mode 100644 index 0000000000..3eadab443a --- /dev/null +++ b/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php @@ -0,0 +1,386 @@ + 'null']); + defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203); +}); + +/** + * @return array{0: ScheduledVolumeBackup, 1: S3Storage, 2: S3Storage, 3: Server} + */ +function createMultiDestinationVolumeBackup(array $backupAttributes = []): array +{ + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + $team = Team::factory()->create(); + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => $privateKey->id, + 'ip' => '203.0.113.10', + ]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + $storages = collect(['first', 'second'])->map(fn (string $name): S3Storage => S3Storage::create([ + 'name' => "{$name} storage", + 'region' => 'us-east-1', + 'key' => "{$name}-key", + 'secret' => 'secret', + 'bucket' => "{$name}-bucket", + 'endpoint' => 'https://s3.amazonaws.com', + 'team_id' => $team->id, + 'is_usable' => true, + ])); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + ...$backupAttributes, + ]); + $backup->syncS3Storages($storages->pluck('id')->all()); + + return [$backup->fresh(), $storages[0], $storages[1], $server]; +} + +function fakeMultiDestinationS3Disks(): void +{ + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(FilesystemAdapter::class); + $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]); + $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk); +} + +it('archives locally and uploads to every destination instead of streaming', function () { + [$backup, $first, $second] = createMultiDestinationVolumeBackup(['disable_local_backup' => true]); + fakeMultiDestinationS3Disks(); + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + Process::assertNotRan(fn ($process) => str_contains($process->command, 'mc pipe')); + Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I') && str_contains($process->command, ' > ')); + Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') && str_contains($process->command, 'temporary/first-bucket/')); + Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') && str_contains($process->command, 'temporary/second-bucket/')); + Process::assertRan(fn ($process) => preg_match("/rm -f '[^']+\\.tar\\.gz'/", $process->command) === 1); + expect($execution->status)->toBe('success') + ->and($execution->s3_storage_id)->toBe($first->id) + ->and($execution->s3_uploaded)->toBeTrue() + ->and($execution->local_storage_deleted)->toBeTrue() + ->and($execution->s3Replicas()->orderBy('s3_storage_id')->get()->map->only(['s3_storage_id', 's3_uploaded'])->all())->toBe([ + ['s3_storage_id' => $first->id, 's3_uploaded' => true], + ['s3_storage_id' => $second->id, 's3_uploaded' => true], + ]); +}); + +it('keeps the local archive and cleans the partial upload when one destination fails', function () { + [$backup, $first, $second] = createMultiDestinationVolumeBackup(['disable_local_backup' => true]); + fakeMultiDestinationS3Disks(); + Process::fake([ + '*du -b*' => '128', + '*temporary/second-bucket/*' => Process::result(errorOutput: 'access denied', exitCode: 1), + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + $firstReplica = $execution->s3Replicas()->where('s3_storage_id', $first->id)->sole(); + $secondReplica = $execution->s3Replicas()->where('s3_storage_id', $second->id)->sole(); + Process::assertNotRan(fn ($process) => preg_match("/rm -f '[^']+\\.tar\\.gz'/", $process->command) === 1); + expect($execution->status)->toBe('success') + ->and($execution->message)->toContain('S3 upload failed: second storage:') + ->and($execution->s3_uploaded)->toBeFalse() + ->and($execution->s3_cleanup_pending)->toBeFalse() + ->and($execution->local_storage_deleted)->toBeFalse() + ->and($firstReplica->s3_uploaded)->toBeTrue() + ->and($firstReplica->s3_storage_deleted)->toBeFalse() + ->and($secondReplica->s3_uploaded)->toBeFalse() + ->and($secondReplica->s3_storage_deleted)->toBeTrue() + ->and($secondReplica->message)->not->toBeEmpty(); +}); + +it('applies S3 retention to each destination separately', function () { + [$backup, $first, $second, $server] = createMultiDestinationVolumeBackup([ + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0, + 'retention_amount_s3' => 1, + 'retention_days_s3' => 0, + 'retention_max_storage_s3' => 0, + ]); + $createExecution = function (string $name, int $daysAgo, array $uploadedTo) use ($backup, $first, $second): ScheduledVolumeBackupExecution { + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $first->id, + 'status' => 'success', + 'filename' => "/data/coolify/backups/volumes/test/{$name}.tar.gz", + 'local_storage_deleted' => true, + 'created_at' => now()->subDays($daysAgo), + ]); + foreach ([$first, $second] as $storage) { + $execution->s3Replicas()->create([ + 's3_storage_id' => $storage->id, + 's3_uploaded' => in_array($storage->id, $uploadedTo, true), + ]); + } + $execution->refreshS3Summary(); + + return $execution; + }; + $oldest = $createExecution('oldest', 3, [$first->id]); + $older = $createExecution('older', 2, [$first->id, $second->id]); + $latest = $createExecution('latest', 1, [$first->id]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete') + ->once() + ->with(['/data/coolify/backups/volumes/test/older.tar.gz', '/data/coolify/backups/volumes/test/oldest.tar.gz']) + ->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'first-key')) + ->andReturn($disk); + $job = new VolumeBackupJob($backup); + + (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job, $server); + + expect($oldest->fresh())->toBeNull() + ->and($older->fresh())->not->toBeNull() + ->and($older->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeTrue() + ->and($older->s3Replicas()->where('s3_storage_id', $second->id)->sole()->s3_storage_deleted)->toBeFalse() + ->and($older->fresh()->s3_storage_deleted)->toBeFalse() + ->and($latest->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeFalse(); +}); + +it('continues retention and execution cleanup when a replica storage is missing', function () { + $this->freezeTime(); + [$backup, , $second, $server] = createMultiDestinationVolumeBackup([ + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0, + 'retention_amount_s3' => 0, + 'retention_days_s3' => 7, + 'retention_max_storage_s3' => 0, + ]); + $executions = collect([false, true])->map(function (bool $localDeleted) use ($backup, $second): ScheduledVolumeBackupExecution { + $execution = $backup->executions()->create([ + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/'.($localDeleted ? 'deleted' : 'local').'.tar.gz', + 'local_storage_deleted' => $localDeleted, + ]); + $execution->forceFill(['created_at' => now()->subDays(10)])->save(); + $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]); + $execution->refreshS3Summary(); + + return $execution; + }); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(Mockery::on( + fn (array $filenames): bool => count($filenames) === 2 + && in_array($executions[0]->filename, $filenames, true) + && in_array($executions[1]->filename, $filenames, true), + ))->andReturnTrue(); + Storage::shouldReceive('build')->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key')) + ->andReturn($disk); + $job = new VolumeBackupJob($backup); + + (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job, $server); + + expect($executions[0]->fresh()->s3_storage_deleted)->toBeTrue() + ->and($executions[0]->s3Replicas()->where('s3_storage_deleted', false)->count())->toBe(0); + $this->assertModelMissing($executions[1]); +}); + +it('cleans an interrupted upload from every destination whose copy did not finish', function () { + [$backup, $first, $second] = createMultiDestinationVolumeBackup(); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $first->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $uploaded = $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $interrupted = $execution->s3Replicas()->create(['s3_storage_id' => $second->id]); + $withoutStorage = $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => false]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/interrupted.tar.gz'])->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key')) + ->andReturn($disk); + + VolumeBackupRecoveryJob::cleanupS3Upload($execution); + + expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() + ->and($uploaded->fresh()->s3_storage_deleted)->toBeFalse() + ->and($interrupted->fresh()->s3_storage_deleted)->toBeTrue() + ->and($withoutStorage->fresh()->s3_storage_deleted)->toBeTrue(); +}); + +it('keeps the cleanup pending when one destination cannot delete the partial upload', function () { + [$backup, $first, $second] = createMultiDestinationVolumeBackup(); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $first->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $failing = $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => false]); + $cleaned = $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => false]); + $failingDisk = Mockery::mock(); + $failingDisk->shouldReceive('delete')->once()->andReturnFalse(); + $cleanedDisk = Mockery::mock(); + $cleanedDisk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build')->with(Mockery::on(fn (array $config): bool => $config['key'] === 'first-key'))->andReturn($failingDisk); + Storage::shouldReceive('build')->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))->andReturn($cleanedDisk); + + expect(fn () => VolumeBackupRecoveryJob::cleanupS3Upload($execution))->toThrow(RuntimeException::class); + + expect($execution->fresh()->s3_cleanup_pending)->toBeTrue() + ->and($failing->fresh()->s3_storage_deleted)->toBeFalse() + ->and($cleaned->fresh()->s3_storage_deleted)->toBeTrue(); +}); + +it('moves the primary destination to a remaining storage when the primary storage is deleted', function () { + [$backup, $first, $second] = createMultiDestinationVolumeBackup(); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $first->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + ]); + $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]); + $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]); + $execution->refreshS3Summary(); + + expect($backup->s3_storage_id)->toBe($first->id); + + $first->delete(); + + $backup->refresh(); + expect($backup->save_s3)->toBeTrue() + ->and($backup->s3_storage_id)->toBe($second->id) + ->and($backup->s3Storages()->pluck('s3_storages.id')->all())->toBe([$second->id]) + ->and($execution->fresh()->s3_storage_deleted)->toBeFalse() + ->and($execution->hasLiveS3Copies())->toBeTrue(); +}); + +it('preserves the schedule and uploaded replica when its S3 storage is unavailable', function () { + [$backup] = createMultiDestinationVolumeBackup(); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'local_storage_deleted' => true, + ]); + $replica = $execution->s3Replicas()->create([ + 's3_storage_id' => null, + 's3_uploaded' => true, + 's3_storage_deleted' => false, + ]); + $execution->refreshS3Summary(); + + expect(fn () => (new DeleteScheduledVolumeBackup)->handle($backup, deleteLocalArchives: false)) + ->toThrow(RuntimeException::class, 'The S3 storage used by an existing backup is unavailable.'); + + expect($backup->fresh())->not->toBeNull(); + expect($replica->fresh()->s3_storage_deleted)->toBeFalse(); + expect($execution->fresh()->s3_storage_deleted)->toBeFalse(); + expect($execution->hasLiveS3Copies())->toBeTrue(); +}); + +it('rolls back all backup detachment changes when an execution update fails', function () { + [$backup, $storage] = createMultiDestinationVolumeBackup(); + $backup->syncS3Storages([$storage->id]); + $databaseBackup = ScheduledDatabaseBackup::create([ + 'frequency' => 'daily', + 'save_s3' => true, + 'database_type' => 'App\\Models\\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $storage->team_id, + ]); + $databaseBackup->syncS3Storages([$storage->id]); + $databaseExecution = $databaseBackup->executions()->create([ + 'uuid' => 'rollback-database', + 'database_name' => 'app', + 'status' => 'success', + 's3_uploaded' => true, + ]); + $databaseReplica = $databaseExecution->s3Replicas()->create([ + 's3_storage_id' => $storage->id, + 's3_uploaded' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $storage->id, + 'status' => 'failed', + 's3_cleanup_pending' => true, + ]); + $replica = $execution->s3Replicas()->create([ + 's3_storage_id' => $storage->id, + 's3_uploaded' => false, + ]); + Event::listen('eloquent.updating: '.ScheduledVolumeBackupExecution::class, function (): void { + throw new RuntimeException('Execution update failed'); + }); + + expect(fn () => $storage->delete())->toThrow(RuntimeException::class, 'Execution update failed'); + + $this->assertModelExists($storage); + foreach ([$databaseBackup, $backup] as $schedule) { + expect($schedule->fresh()->s3_storage_id)->toBe($storage->id) + ->and((bool) $schedule->fresh()->save_s3)->toBeTrue() + ->and($schedule->s3Storages()->pluck('s3_storages.id')->all())->toBe([$storage->id]); + } + expect($databaseReplica->fresh()->s3_storage_deleted)->toBeFalse() + ->and($databaseExecution->fresh()->s3_storage_deleted)->toBeFalse() + ->and($replica->fresh()->s3_storage_deleted)->toBeFalse() + ->and($execution->fresh()->s3_cleanup_pending)->toBeTrue(); +}); diff --git a/tests/Feature/VolumeBackupRecoveryBoundsTest.php b/tests/Feature/VolumeBackupRecoveryBoundsTest.php index ff3ed2d031..08d90ab774 100644 --- a/tests/Feature/VolumeBackupRecoveryBoundsTest.php +++ b/tests/Feature/VolumeBackupRecoveryBoundsTest.php @@ -93,7 +93,7 @@ function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): a function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution { - return ScheduledVolumeBackupExecution::create([ + $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, 's3_storage_id' => $backup->s3_storage_id, 'status' => 'failed', @@ -101,6 +101,9 @@ function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $att 's3_cleanup_pending' => true, ...$attributes, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $backup->s3_storage_id]); + + return $execution; } function failRecoveryBoundsS3Delete(string $awsCode): void @@ -164,7 +167,7 @@ it('records a categorized S3 cleanup failure without throwing or leaking the S3 $execution->refresh(); expect($execution->s3_cleanup_pending)->toBeTrue() - ->and($execution->s3_storage_deleted)->toBeFalse() + ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeFalse() ->and($execution->recovery_error)->toBe('s3_auth') ->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue() ->and($execution->recovery_needs_attention)->toBeTrue() @@ -261,7 +264,7 @@ it('cleans the S3 upload even when container recovery fails', function () { $execution->refresh(); expect($execution->stop_recovery_pending)->toBeTrue() ->and($execution->s3_cleanup_pending)->toBeFalse() - ->and($execution->s3_storage_deleted)->toBeTrue() + ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue() ->and($execution->recovery_error)->toBe('server_unreachable') ->and($execution->recovery_needs_attention)->toBeFalse(); Notification::assertNothingSent(); @@ -281,7 +284,7 @@ it('resets the recovery state after a successful attempt', function () { $execution->refresh(); expect($execution->s3_cleanup_pending)->toBeFalse() - ->and($execution->s3_storage_deleted)->toBeTrue() + ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue() ->and($execution->recovery_error)->toBeNull() ->and($execution->recovery_last_attempt_at)->toBeNull() ->and($execution->recovery_needs_attention)->toBeFalse(); diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php index d247a7a4e6..66eadf98ca 100644 --- a/tests/Feature/VolumeBackupTest.php +++ b/tests/Feature/VolumeBackupTest.php @@ -867,6 +867,7 @@ it('deletes an individual S3 archive from the storage recorded on its execution' 'local_storage_deleted' => true, 's3_uploaded' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]); $disk = Mockery::mock(); $disk->shouldReceive('delete') ->once() @@ -981,7 +982,7 @@ it('persists S3 settings the first time when a volume backup schedule does not e 'resource' => $application, 'section' => 's3', ]) - ->assertSet('s3StorageId', $s3Storage->id) + ->assertSet('s3StorageIds', [$s3Storage->id]) ->call('toggleS3') ->assertSet('saveToS3', true) ->assertDispatched('success'); @@ -1013,7 +1014,7 @@ it('persists the selected S3 storage when a volume backup schedule does not exis 'resource' => $application, 'section' => 's3', ]) - ->set('s3StorageId', $s3Storage->id) + ->set('s3StorageIds', [$s3Storage->id]) ->assertDispatched('success'); $backup = ScheduledVolumeBackup::query()->sole(); @@ -1136,7 +1137,7 @@ it('does not change S3 storage when another volume backup setting is invalid', f 'section' => 's3', ]) ->set('frequency', 'not a valid schedule') - ->set('s3StorageId', $secondS3Storage->id) + ->set('s3StorageIds', [$secondS3Storage->id]) ->assertHasErrors('frequency') ->assertNotDispatched('success'); @@ -1254,18 +1255,18 @@ it('lists and accepts only S3 storages of the resource team when the session tea $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]); expect($component->get('availableS3Storages')->pluck('id')->all())->toBe([$resourceTeamStorage->id]) - ->and($component->get('s3StorageId'))->toBe($resourceTeamStorage->id); + ->and($component->get('s3StorageIds'))->toBe([$resourceTeamStorage->id]); $component->set('frequency', 'daily') ->set('saveToS3', true) - ->set('s3StorageId', $otherTeamStorage->id) - ->assertHasErrors('s3StorageId') + ->set('s3StorageIds', [$otherTeamStorage->id]) + ->assertHasErrors('s3StorageIds') ->call('save') - ->assertHasErrors('s3StorageId'); + ->assertHasErrors('s3StorageIds'); expect(ScheduledVolumeBackup::query()->count())->toBe(0); - $component->set('s3StorageId', $resourceTeamStorage->id) + $component->set('s3StorageIds', [$resourceTeamStorage->id]) ->call('save') ->assertHasNoErrors(); @@ -1295,9 +1296,9 @@ it('only accepts a usable S3 storage owned by the current team', function () { Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) ->set('frequency', 'daily') ->set('saveToS3', true) - ->set('s3StorageId', $foreignStorage->id) + ->set('s3StorageIds', [$foreignStorage->id]) ->call('save') - ->assertHasErrors('s3StorageId'); + ->assertHasErrors('s3StorageIds'); expect(ScheduledVolumeBackup::query()->count())->toBe(0); }); @@ -1322,7 +1323,7 @@ it('saves the database-style S3 backup controls immediately', function () { ]); $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) - ->assertSet('s3StorageId', $s3Storage->id) + ->assertSet('s3StorageIds', [$s3Storage->id]) ->set('saveToS3', true) ->call('instantSave') ->set('disableLocalBackup', true) @@ -1378,7 +1379,7 @@ it('saves the selected volume backup S3 storage immediately while S3 is disabled ]); Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 's3']) - ->set('s3StorageId', $secondS3Storage->id) + ->set('s3StorageIds', [$secondS3Storage->id]) ->assertDispatched('success'); $backup = ScheduledVolumeBackup::query()->sole(); @@ -1486,6 +1487,7 @@ it('disables S3 volume backups when the storage is deleted', function () { 's3_uploaded' => true, 's3_cleanup_pending' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id, 's3_uploaded' => true]); $s3Storage->delete(); @@ -1530,6 +1532,7 @@ it('marks historical executions deleted when their recorded S3 storage is remove 's3_uploaded' => true, 's3_cleanup_pending' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]); $originalStorage->delete(); @@ -1704,7 +1707,7 @@ it('deletes S3 archives from the storage recorded on each execution', function ( 'frequency' => 'daily', 'save_s3' => true, ]); - ScheduledVolumeBackupExecution::create([ + $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, 's3_storage_id' => $originalStorage->id, 'status' => 'success', @@ -1712,6 +1715,7 @@ it('deletes S3 archives from the storage recorded on each execution', function ( 'local_storage_deleted' => true, 's3_uploaded' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]); $disk = Mockery::mock(); $disk->shouldReceive('delete') ->once() @@ -1977,7 +1981,8 @@ it('keeps the upload destination on the volume backup execution', function () { $execution = ScheduledVolumeBackupExecution::query()->sole(); expect($execution->s3_storage_id)->toBe($s3Storage->id) - ->and($execution->s3->is($s3Storage))->toBeTrue(); + ->and($execution->s3->is($s3Storage))->toBeTrue() + ->and($execution->s3Replicas()->sole()->s3_storage_id)->toBe($s3Storage->id); }); it('streams S3-only volume backups without creating or copying a local archive', function () { @@ -2032,6 +2037,7 @@ it('streams S3-only volume backups without creating or copying a local archive', expect($execution->status)->toBe('success') ->and($execution->size)->toBe(128) ->and($execution->s3_uploaded)->toBeTrue() + ->and($execution->s3Replicas()->sole()->s3_uploaded)->toBeTrue() ->and($execution->local_storage_deleted)->toBeTrue(); }); @@ -2240,15 +2246,26 @@ it('removes retained S3 archives from the storage recorded on each execution', f 'status' => 'success', 'filename' => '/data/coolify/backups/volumes/test/old.tar.gz', 's3_uploaded' => true, + 'created_at' => now()->subDays(2), + ]); + $oldExecution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]); + $retainedOriginalExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/retained-original.tar.gz', + 's3_uploaded' => true, 'created_at' => now()->subDay(), ]); - ScheduledVolumeBackupExecution::create([ + $retainedOriginalExecution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]); + $newExecution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, 's3_storage_id' => $newStorage->id, 'status' => 'success', 'filename' => '/data/coolify/backups/volumes/test/new.tar.gz', 's3_uploaded' => true, ]); + $newExecution->s3Replicas()->create(['s3_storage_id' => $newStorage->id, 's3_uploaded' => true]); $disk = Mockery::mock(); $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue(); Storage::shouldReceive('build') @@ -2260,7 +2277,10 @@ it('removes retained S3 archives from the storage recorded on each execution', f $method->invoke($job, $server); - expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue(); + expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($oldExecution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue() + ->and($retainedOriginalExecution->fresh()->s3_storage_deleted)->toBeFalse() + ->and($newExecution->fresh()->s3_storage_deleted)->toBeFalse(); }); it('does not query execution history when local retention is unlimited', function () { @@ -2514,6 +2534,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id]); $disk = Mockery::mock(); $disk->shouldReceive('delete')->once()->andReturnTrue(); Storage::shouldReceive('build')->once()->andReturn($disk); @@ -2523,7 +2544,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc $job->handle(); expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() - ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); + ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue(); }); it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () { @@ -2562,6 +2583,7 @@ it('cleans an interrupted upload from the execution S3 storage after a schedule 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id]); $disk = Mockery::mock(); $disk->shouldReceive('delete')->once()->andReturnTrue(); Storage::shouldReceive('build') @@ -2572,7 +2594,7 @@ it('cleans an interrupted upload from the execution S3 storage after a schedule VolumeBackupRecoveryJob::cleanupS3Upload($execution); expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() - ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); + ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue(); }); it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () { @@ -2603,6 +2625,7 @@ it('keeps the S3 key tracked when interrupted upload cleanup must be retried', f 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true, ]); + $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id]); $disk = Mockery::mock(); $disk->shouldReceive('delete')->once()->andReturnFalse(); Storage::shouldReceive('build')->once()->andReturn($disk); diff --git a/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php b/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php new file mode 100644 index 0000000000..479367ee70 --- /dev/null +++ b/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php @@ -0,0 +1,144 @@ + 0, 'is_api_enabled' => true, 'fqdn' => 'https://coolify-a.test']); + + $this->team = Team::factory()->create(); + $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => $privateKey->id, + 'ip' => '10.56.0.10', + ]); + $destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = $project->environments()->first() + ?? Environment::factory()->create(['project_id' => $project->id]); + + $this->application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => $this->application->uuid.'-data', + 'mount_path' => '/app/data', + 'resource_type' => $this->application->getMorphClass(), + 'resource_id' => $this->application->id, + ]); + + StandalonePostgresql::withoutEvents(function () use ($environment, $destination) { + $database = new StandalonePostgresql; + $database->forceFill([ + 'name' => 'app-db', + 'postgres_user' => 'postgres', + 'postgres_password' => 'secret', + 'postgres_db' => 'app', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $database->uuid = new_public_id(); + $database->save(); + $this->database = $database; + }); + + $this->firstS3 = S3Storage::create([ + 'name' => 'first', 'region' => 'us-east-1', 'key' => 'k1', 'secret' => 's1', + 'bucket' => 'first-bucket', 'endpoint' => 'https://first.example.com', 'team_id' => $this->team->id, + ]); + $this->secondS3 = S3Storage::create([ + 'name' => 'second', 'region' => 'us-east-1', 'key' => 'k2', 'secret' => 's2', + 'bucket' => 'second-bucket', 'endpoint' => 'https://second.example.com', 'team_id' => $this->team->id, + ]); + + $this->databaseBackup = ScheduledDatabaseBackup::create([ + 'team_id' => $this->team->id, + 'save_s3' => true, + 'frequency' => '0 2 * * *', + 's3_storage_id' => $this->secondS3->id, + 'database_type' => $this->database->getMorphClass(), + 'database_id' => $this->database->id, + ]); + $this->databaseBackup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]); + + $this->volumeBackup = $volume->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'save_s3' => true, + 'frequency' => '0 3 * * *', + 's3_storage_id' => $this->secondS3->id, + ]); + $this->volumeBackup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]); +}); + +/** + * @return list Buckets of the backup's destinations, primary first. + */ +function transferredBackupBuckets(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): array +{ + return $backup->selectedS3Storages()->pluck('bucket')->all(); +} + +function importTransferBundleIntoNewTeam(array $bundle, $context): Team +{ + $targetTeam = Team::factory()->create(); + $context->application->forceDelete(); + $context->database->forceDelete(); + $context->server->forceDelete(); + + app(ServerTransferImporter::class)->import($bundle, teamId: $targetTeam->id); + + return $targetTeam; +} + +test('transfer keeps every S3 destination of database and volume backups', function () { + $bundle = app(ServerTransferExporter::class)->export($this->server); + + expect(collect($bundle['s3_storages'])->pluck('uuid')->sort()->values()->all()) + ->toEqual(collect([$this->firstS3->uuid, $this->secondS3->uuid])->sort()->values()->all()); + + $targetTeam = importTransferBundleIntoNewTeam($bundle, $this); + + $databaseBackup = ScheduledDatabaseBackup::where('team_id', $targetTeam->id)->sole(); + $volumeBackup = ScheduledVolumeBackup::where('team_id', $targetTeam->id)->sole(); + + expect(transferredBackupBuckets($databaseBackup))->toBe(['second-bucket', 'first-bucket']) + ->and($databaseBackup->save_s3)->toBeTruthy() + ->and($databaseBackup->s3->team_id)->toBe($targetTeam->id) + ->and(transferredBackupBuckets($volumeBackup))->toBe(['second-bucket', 'first-bucket']) + ->and($volumeBackup->save_s3)->toBeTrue() + ->and($volumeBackup->s3->team_id)->toBe($targetTeam->id); +}); + +test('transfer imports bundles that only have the primary S3 destination', function () { + $bundle = app(ServerTransferExporter::class)->export($this->server); + data_forget($bundle, 'projects.*.environments.*.databases.*.scheduled_backups.*.s3_storage_uuids'); + data_forget($bundle, 'volume_backups.*.s3_storage_uuids'); + expect(data_get($bundle, 'volume_backups.0'))->not->toHaveKey('s3_storage_uuids'); + + $targetTeam = importTransferBundleIntoNewTeam($bundle, $this); + + expect(transferredBackupBuckets(ScheduledDatabaseBackup::where('team_id', $targetTeam->id)->sole()))->toBe(['second-bucket']) + ->and(transferredBackupBuckets(ScheduledVolumeBackup::where('team_id', $targetTeam->id)->sole()))->toBe(['second-bucket']); +});