diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index 4b59fe7911..0359bbc2ab 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -96,9 +96,10 @@ class StartDatabaseImport throw new DatabaseImportException('The uploaded backup contains disallowed PostgreSQL restore directives.'); } $serverPath = "/tmp/database-import-{$operation}"; + $magic = bin2hex((string) file_get_contents($local, length: 6)); instant_scp($local, $serverPath, $server); $source->uploadId ? Storage::deleteDirectory(dirname($staged)) : Storage::delete($staged); - $commandList[] = 'docker cp '.escapeshellarg($serverPath).' '.escapeshellarg("{$container}:{$containerPath}"); + $commandList = [...$commandList, ...$this->copyIntoContainer($serverPath, $magic, $server, $operation, $container, $containerPath, $cleanup)]; $commandList[] = 'rm -f '.escapeshellarg($serverPath); $cleanup['serverTmpPath'] = $serverPath; } elseif ($source->type === 'server') { @@ -107,7 +108,8 @@ class StartDatabaseImport if ($size < 1 || $size > self::MAX_BYTES) { throw new DatabaseImportException('The backup file is empty or exceeds the 10 GiB limit.'); } - $commandList[] = 'docker cp '.escapeshellarg($source->path).' '.escapeshellarg("{$container}:{$containerPath}"); + $magic = (string) instant_remote_process(['head -c 6 -- '.escapeshellarg($source->path).' | od -An -tx1'], $server); + $commandList = [...$commandList, ...$this->copyIntoContainer($source->path, $magic, $server, $operation, $container, $containerPath, $cleanup)]; } else { $storage = S3Storage::ownedByCurrentTeamAPI($teamId) ->where(fn ($query) => $query->whereUuid($source->s3StorageUuid)->orWhere('id', ctype_digit((string) $source->s3StorageUuid) ? (int) $source->s3StorageUuid : -1)) @@ -122,18 +124,15 @@ class StartDatabaseImport throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.'); } $helper = "s3-restore-{$operation}"; - $serverPath = "/tmp/s3-restore-{$operation}"; $this->startS3HelperWithEnv($storage, $server, $helper, $network); $sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}"); $commandList = [ 'docker exec '.escapeshellarg($helper).' sh -c '.escapeshellarg('mc alias set s3temp "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"'), 'docker exec '.escapeshellarg($helper).' mc cp '.$sourceArg.' /tmp/restore', - 'docker cp '.escapeshellarg("{$helper}:/tmp/restore").' '.escapeshellarg($serverPath), - 'docker cp '.escapeshellarg($serverPath).' '.escapeshellarg("{$container}:{$containerPath}"), + ...$this->streamFromHelper($helper, $container, $containerPath), 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true', - 'rm -f '.escapeshellarg($serverPath), ]; - $cleanup += ['containerName' => $helper, 'serverTmpPath' => $serverPath]; + $cleanup['containerName'] = $helper; } if ($safety = $this->commands->buildPostgresSafetyCommand($resource, $container, $containerPath)) { @@ -176,6 +175,59 @@ class StartDatabaseImport } } + /** + * Copies a backup into the database container. bz2, xz and zip backups go through + * the Coolify helper image, because database images do not ship those tools. + * + * @param array $cleanup + * @return list + */ + private function copyIntoContainer(string $path, string $magic, Server $server, string $operation, string $container, string $containerPath, array &$cleanup): array + { + $magic = strtolower((string) preg_replace('/[^0-9a-f]/i', '', $magic)); + if (! preg_match('/^(425a68|fd377a585a00|504b0304)/', $magic)) { + return ['docker cp '.escapeshellarg($path).' '.escapeshellarg("{$container}:{$containerPath}")]; + } + + $helper = "backup-decompress-{$operation}"; + $image = escapeshellarg(coolifyHelperImage().':'.getHelperVersion()); + + try { + instant_remote_process([ + 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true', + 'docker run -d --network none --name '.escapeshellarg($helper).' '.$image.' sleep 86400', + ], $server); + } catch (Throwable) { + instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false); + + throw new DatabaseImportException('Unable to start the backup decompression helper.'); + } + $cleanup['containerName'] = $helper; + + return [ + 'docker cp '.escapeshellarg($path).' '.escapeshellarg("{$helper}:/tmp/restore"), + ...$this->streamFromHelper($helper, $container, $containerPath), + 'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true', + ]; + } + + /** + * Prepares /tmp/restore in a helper container, then streams it into the database + * container. Preparing first keeps decompression failures from being lost in a pipe. + * + * @return list + */ + private function streamFromHelper(string $helper, string $container, string $containerPath): array + { + $target = escapeshellarg($containerPath); + $writer = "cat > {$target} && [ -s {$target} ] || { echo 'The backup is empty or could not be read.' >&2; exit 1; }"; + + return [ + 'docker exec '.escapeshellarg($helper).' sh -c '.escapeshellarg($this->commands->buildNormalizeScript('/tmp/restore', '/tmp/restore.prepared')), + 'docker exec '.escapeshellarg($helper).' cat /tmp/restore.prepared | docker exec -i '.escapeshellarg($container).' sh -c '.escapeshellarg($writer), + ]; + } + private function startS3HelperWithEnv(S3Storage $storage, Server $server, string $helper, string $network): void { $image = escapeshellarg(coolifyHelperImage().':'.getHelperVersion()); @@ -188,7 +240,7 @@ class StartDatabaseImport .' -e S3_ENDPOINT='.escapeshellarg((string) $storage->endpoint) .' -e S3_ACCESS_KEY='.escapeshellarg((string) $storage->key) .' -e S3_SECRET_KEY='.escapeshellarg((string) $storage->secret) - .' '.$image.' sleep 3600', + .' '.$image.' sleep 86400', ], $server); } catch (Throwable) { instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false); diff --git a/app/Livewire/Project/Database/ImportForm.php b/app/Livewire/Project/Database/ImportForm.php index e5a359b30d..f13f2e1313 100644 --- a/app/Livewire/Project/Database/ImportForm.php +++ b/app/Livewire/Project/Database/ImportForm.php @@ -10,9 +10,6 @@ use App\Models\ServiceDatabase; use App\Models\StandaloneClickhouse; use App\Models\StandaloneDragonfly; use App\Models\StandaloneKeydb; -use App\Models\StandaloneMariadb; -use App\Models\StandaloneMysql; -use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; use App\Rules\SafeWebhookUrl; use App\Support\DatabaseImport\DatabaseImportCommandBuilder; @@ -168,14 +165,6 @@ class ImportForm extends Component public ?int $activityId = null; - public string $postgresqlRestoreCommand = 'pg_restore --exit-on-error -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; - - public string $mysqlRestoreCommand = 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE'; - - public string $mariadbRestoreCommand = 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE'; - - public string $mongodbRestoreCommand = 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive='; - // S3 Restore properties public array $availableS3Storages = []; @@ -219,83 +208,29 @@ class ImportForm extends Component $this->parameters = get_route_parameters(); $this->getContainers(); $this->loadAvailableS3Storages(); + $this->refreshRestoreCommandText(); } - public function updatedDumpAll($value) + public function updatedDumpAll(): void { - $morphClass = $this->resource->getMorphClass(); - - // Handle ServiceDatabase by checking the database type - if ($morphClass === ServiceDatabase::class) { - $dbType = $this->resource->databaseType(); - if (str_contains($dbType, 'mysql')) { - $morphClass = 'mysql'; - } elseif (str_contains($dbType, 'mariadb')) { - $morphClass = 'mariadb'; - } elseif (str_contains($dbType, 'postgres')) { - $morphClass = 'postgresql'; - } - } - - switch ($morphClass) { - case StandaloneMariadb::class: - case 'mariadb': - if ($value === true) { - $this->mariadbRestoreCommand = <<<'EOD' -for pid in $(mariadb -u root -p$MARIADB_ROOT_PASSWORD -N -e "SELECT id FROM information_schema.processlist WHERE user != 'root';"); do - mariadb -u root -p$MARIADB_ROOT_PASSWORD -e "KILL $pid" 2>/dev/null || true -done && \ -mariadb -u root -p$MARIADB_ROOT_PASSWORD -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`',schema_name,'\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | mariadb -u root -p$MARIADB_ROOT_PASSWORD && \ -mariadb -u root -p$MARIADB_ROOT_PASSWORD -e "CREATE DATABASE IF NOT EXISTS \`${MARIADB_DATABASE:-default}\`;" && \ -(gunzip -cf $tmpPath 2>/dev/null || cat $tmpPath) | sed -e '/^CREATE DATABASE/d' -e '/^USE \`mysql\`/d' | mariadb -u root -p$MARIADB_ROOT_PASSWORD ${MARIADB_DATABASE:-default} -EOD; - $this->restoreCommandText = $this->mariadbRestoreCommand.' && (gunzip -cf 2>/dev/null || cat ) | mariadb -u root -p$MARIADB_ROOT_PASSWORD ${MARIADB_DATABASE:-default}'; - } else { - $this->mariadbRestoreCommand = 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE'; - } - break; - case StandaloneMysql::class: - case 'mysql': - if ($value === true) { - $this->mysqlRestoreCommand = <<<'EOD' -for pid in $(mysql -u root -p$MYSQL_ROOT_PASSWORD -N -e "SELECT id FROM information_schema.processlist WHERE user != 'root';"); do - mysql -u root -p$MYSQL_ROOT_PASSWORD -e "KILL $pid" 2>/dev/null || true -done && \ -mysql -u root -p$MYSQL_ROOT_PASSWORD -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`',schema_name,'\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | mysql -u root -p$MYSQL_ROOT_PASSWORD && \ -mysql -u root -p$MYSQL_ROOT_PASSWORD -e "CREATE DATABASE IF NOT EXISTS \`${MYSQL_DATABASE:-default}\`;" && \ -(gunzip -cf $tmpPath 2>/dev/null || cat $tmpPath) | sed -e '/^CREATE DATABASE/d' -e '/^USE \`mysql\`/d' | mysql -u root -p$MYSQL_ROOT_PASSWORD ${MYSQL_DATABASE:-default} -EOD; - $this->restoreCommandText = $this->mysqlRestoreCommand.' && (gunzip -cf 2>/dev/null || cat ) | mysql -u root -p$MYSQL_ROOT_PASSWORD ${MYSQL_DATABASE:-default}'; - } else { - $this->mysqlRestoreCommand = 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE'; - } - break; - case StandalonePostgresql::class: - case 'postgresql': - if ($value === true) { - $this->postgresqlRestoreCommand = <<<'EOD' -psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && \ -psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} && \ -createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} -EOD; - $this->restoreCommandText = $this->postgresqlRestoreCommand.' && (gunzip -cf 2>/dev/null || cat ) | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; - } else { - $this->syncPostgresqlRestoreCommand(); - } - break; - } - + $this->refreshRestoreCommandText(); } public function updatedReplaceExisting(): void { - $this->syncPostgresqlRestoreCommand(); + $this->refreshRestoreCommandText(); } - private function syncPostgresqlRestoreCommand(): void + /** + * Shows the exact script the import runs, so the confirmation matches the restore. + */ + private function refreshRestoreCommandText(): void { - $replaceExisting = $this->replaceExisting ? ' --clean --if-exists' : ''; - $this->postgresqlRestoreCommand = 'pg_restore --exit-on-error'.$replaceExisting.' -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'; + $commands = app(DatabaseImportCommandBuilder::class); + + $this->restoreCommandText = $this->resource && $commands->supports($this->resource) + ? $commands->buildRestoreCommand($this->resource, '', $this->dumpAll, $this->replaceExisting) + : ''; } public function getContainers() diff --git a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php index 311a4634a5..ffb28ba6fd 100644 --- a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php +++ b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php @@ -7,23 +7,60 @@ use InvalidArgumentException; class DatabaseImportCommandBuilder { + /** + * Shell helpers shared by every restore script. The backup inside the database + * container is plain or gzip-compressed (bz2, xz and zip are decompressed by + * {@see self::buildNormalizeScript()} before it arrives). Every script decides the + * backup format before it changes anything in the database. + */ + private const PRELUDE = <<<'SH' +fail() { echo "$1" >&2; exit 1; } +is_gzip() { [ "$(head -c 2 "$backup" | od -An -tx1 | tr -d ' \n')" = 1f8b ]; } +stream() { if is_gzip; then gunzip -c "$backup"; else cat "$backup"; fi; } +header() { stream | head -c "$1" | od -An -tx1 | tr -d ' \n'; } +is_tar() { [ "$(stream | head -c 262 | tail -c 5)" = ustar ]; } +is_text() { [ "$(stream | head -c 65536 | tr -d '\000' | wc -c | tr -d ' ')" = "$(stream | head -c 65536 | wc -c | tr -d ' ')" ]; } +extract_tar() { work=$(mktemp -d) && trap 'rm -rf "$work"' EXIT && stream | tar -xf - -C "$work" || fail 'The tar backup cannot be extracted.'; } +use_single_tar_member() { [ "$(find "$work" -type f | wc -l | tr -d ' ')" = 1 ] && backup=$(find "$work" -type f); } + +SH; + public function buildRestoreCommand(object $resource, string $path, bool $dumpAll, bool $replaceExisting = false): string { - $path = escapeshellarg($path); - - return match ($this->databaseType($resource)) { - 'postgresql' => $dumpAll - ? 'psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} && createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} && (gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}' - : 'pg_restore --exit-on-error'.($replaceExisting ? ' --clean --if-exists' : '').' -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$path, - 'mysql' => $dumpAll - ? $this->mysqlDumpAll('mysql', 'MYSQL', $path) - : '(gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE', - 'mariadb' => $dumpAll - ? $this->mysqlDumpAll('mariadb', 'MARIADB', $path) - : '(gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE', - 'mongodb' => 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive='.$path, + $script = match ($this->databaseType($resource)) { + 'postgresql' => $dumpAll ? $this->postgresqlDumpAll() : $this->postgresqlSingle($replaceExisting), + 'mysql' => $this->mysql('mysql', 'MYSQL', $dumpAll), + 'mariadb' => $this->mysql('mariadb', 'MARIADB', $dumpAll), + 'mongodb' => $this->mongodb($replaceExisting), default => throw new InvalidArgumentException('Database import is not supported for this database type.'), }; + + return 'backup='.escapeshellarg($path)."\n".self::PRELUDE.$script; + } + + /** + * Prepares a backup inside the Coolify helper image. bz2, xz and single-file zip + * backups are decompressed because database images do not ship those tools; + * plain and gzip backups are moved unchanged. Exits non-zero on any failure. + */ + public function buildNormalizeScript(string $source, string $target): string + { + $source = escapeshellarg($source); + $target = escapeshellarg($target); + + return << "\$out" || { echo 'The bz2 backup cannot be decompressed.' >&2; exit 1; } ;; + fd377a585a00) unxz -c "\$f" > "\$out" || { echo 'The xz backup cannot be decompressed.' >&2; exit 1; } ;; + 504b0304*) + d=\$(mktemp -d) && unzip -q "\$f" -d "\$d" || { echo 'The zip backup cannot be extracted.' >&2; exit 1; } + [ "\$(find "\$d" -type f | wc -l | tr -d ' ')" = 1 ] || { echo 'A zip backup must contain exactly one file.' >&2; exit 1; } + mv "\$(find "\$d" -type f)" "\$out" ;; + *) mv "\$f" "\$out" ;; +esac +SH; } public function buildPostgresRestoreScanScript(object $resource, string $path): ?string @@ -51,7 +88,8 @@ class DatabaseImportCommandBuilder return << "\$inspect"; then @@ -101,11 +139,120 @@ SH; }; } - private function mysqlDumpAll(string $binary, string $prefix, string $path): string + /** + * pg_dump custom and tar archives are restored with pg_restore, SQL dumps with psql. + * pg_restore cannot read gzip files, so both clients receive the backup on stdin. + * SQL cannot replace single objects, so replacing recreates the target database. + */ + private function postgresqlSingle(bool $replaceExisting): string { - $rootPassword = '${'.$prefix.'_ROOT_PASSWORD}'; - $database = '${'.$prefix.'_DATABASE:-default}'; + $clean = $replaceExisting ? ' --clean --if-exists' : ''; + $sqlNotice = $replaceExisting ? <<<'SH' - return "for pid in \$({$binary} -u root -p{$rootPassword} -N -e \"SELECT id FROM information_schema.processlist WHERE user != 'root';\"); do {$binary} -u root -p{$rootPassword} -e \"KILL \$pid\" 2>/dev/null || true; done && {$binary} -u root -p{$rootPassword} -N -e \"SELECT CONCAT('DROP DATABASE IF EXISTS \\`',schema_name,'\\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');\" | {$binary} -u root -p{$rootPassword} && {$binary} -u root -p{$rootPassword} -e \"CREATE DATABASE IF NOT EXISTS \\`{$database}\\`;\" && (gunzip -cf {$path} 2>/dev/null || cat {$path}) | {$binary} -u root -p{$rootPassword} {$database}"; + echo 'SQL backups cannot replace single objects. The database is recreated before the restore.' + echo "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db' AND pid <> pg_backend_pid();" | psql -v db="$db" -U $POSTGRES_USER -d template1 >/dev/null || exit 1 + dropdb --maintenance-db=template1 -U $POSTGRES_USER --if-exists "$db" || exit 1 + createdb -U $POSTGRES_USER "$db" || exit 1 +SH : ''; + + return <</dev/null 2>&1 || fail 'pg_restore cannot read this backup archive. Nothing was changed.' +elif is_text; then + kind=sql +else + fail 'Unsupported PostgreSQL backup format. Use a pg_dump archive (custom or tar format) or an SQL file. Nothing was changed.' +fi +psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" || exit 1 +psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} || exit 1 +createdb -U ${POSTGRES_USER} "$db" || exit 1 +if [ "$kind" = archive ]; then + stream | pg_restore -U ${POSTGRES_USER} -d "$db" +else + stream | psql -U ${POSTGRES_USER} -d "$db" +fi +SH; + } + + /** + * MySQL and MariaDB restore SQL dumps. A tar backup must wrap exactly one dump. + * The all-databases mode checks the backup before it drops any database. + */ + private function mysql(string $binary, string $prefix, bool $dumpAll): string + { + $preflight = <<<'SH' +if is_tar; then + extract_tar + use_single_tar_member || fail 'A tar backup must contain exactly one SQL dump. Nothing was changed.' +fi +is_text || fail 'Unsupported backup format. Use an SQL dump. Nothing was changed.' + +SH; + + if (! $dumpAll) { + return $preflight.<</dev/null || true; done +{$root} -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \\`',schema_name,'\\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | {$root} || exit 1 +{$root} -e "CREATE DATABASE IF NOT EXISTS \\`{$database}\\`;" || exit 1 +stream | {$root} {$database} +SH; + } + + /** + * MongoDB restores mongodump archives (plain or gzip) and dump directories packed + * as tar. Replacing existing data drops each restored collection first. + */ + private function mongodb(bool $replaceExisting): string + { + $drop = $replaceExisting ? ' --drop' : ''; + + return <<
- @if ($resourceDbType === 'standalone-postgresql') - @if ($dumpAll) + @if ($resourceDbType === 'standalone-postgresql' && $dumpAll) The backup replaces PostgreSQL administrator role passwords, including the destination administrator password. If the administrator password changes, update it in Coolify's database configuration after the restore. - - @else - - @endif - @elseif ($resourceDbType === 'standalone-mysql') - @if ($dumpAll) - - @else - - @endif - @elseif ($resourceDbType === 'standalone-mariadb') - @if ($dumpAll) - - @else - - @endif @endif +
+
+ @endif + @if ($resourceDbType === 'standalone-mongodb') +
+
@endif diff --git a/tests/Feature/DatabaseRestoreDialogTest.php b/tests/Feature/DatabaseRestoreDialogTest.php index f373afe6e2..485ed2e26e 100644 --- a/tests/Feature/DatabaseRestoreDialogTest.php +++ b/tests/Feature/DatabaseRestoreDialogTest.php @@ -13,7 +13,7 @@ test('postgresql dump all restore warns that administrator passwords are overwri $view = file_get_contents(resource_path('views/livewire/project/database/import-form.blade.php')); expect($view) - ->toContain('@if ($dumpAll)') + ->toContain("@if (\$resourceDbType === 'standalone-postgresql' && \$dumpAll)") ->toContain('Full restore overwrites administrator passwords') ->toContain('The backup replaces PostgreSQL administrator role passwords, including the destination administrator password.') ->toContain("If the administrator password changes, update it in Coolify's database configuration after the restore."); @@ -24,8 +24,8 @@ test('postgresql single database restore offers explicit object replacement', fu expect($view) ->toContain('Replace objects that already exist') - ->toContain('wire:model="postgresqlRestoreCommand"') - ->toContain('label="Import command" readonly'); + ->toContain('wire:model="restoreCommandText"') + ->toContain('readonly label="Import command"'); }); test('restore confirmation dialogs warn that existing objects can block the import', function () { diff --git a/tests/Feature/StartDatabaseImportBackupPreparationTest.php b/tests/Feature/StartDatabaseImportBackupPreparationTest.php new file mode 100644 index 0000000000..3fa0c42f55 --- /dev/null +++ b/tests/Feature/StartDatabaseImportBackupPreparationTest.php @@ -0,0 +1,159 @@ +set('cache.default', 'array'); + config()->set('constants.ssh.mux_enabled', false); + InstanceSettings::forceCreate(['id' => 0]); + + $this->team = Team::factory()->create(); + $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => $this->privateKey->id, + 'user' => 'root', + ]); + $this->destination = StandaloneDocker::firstOrCreate( + ['server_id' => $this->server->id, 'network' => 'coolify'], + ['uuid' => (string) Str::uuid(), 'name' => 'docker'] + ); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); + $this->database = StandalonePostgresql::create([ + 'uuid' => (string) Str::uuid(), + 'name' => 'db', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'db', + 'image' => 'postgres:17', + 'status' => 'running', + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + + Queue::fake(); +}); + +function importServerBackup(object $test, string $path, string $magicHex): string +{ + Process::fake([ + '*stat -c %s*' => Process::result('2048'), + '*od -An -tx1*' => Process::result(' '.implode(' ', str_split($magicHex, 2))), + '*' => Process::result(''), + ]); + + $activity = app(StartDatabaseImport::class)->handle( + $test->database, + new DatabaseImportSource('server', path: $path), + $test->team->id, + ); + + return (string) $activity->getExtraProperty('command'); +} + +test('plain and gzip server backups are copied into the database container unchanged', function (string $path, string $magicHex) { + $command = importServerBackup($this, $path, $magicHex); + + expect($command) + ->toContain("docker cp '{$path}' '{$this->database->uuid}:/tmp/restore_") + ->not->toContain('backup-decompress-'); +})->with([ + 'plain SQL' => ['/srv/backups/app.sql', '2d2d20506f73'], + 'gzip' => ['/srv/backups/app.sql.gz', '1f8b08000000'], + 'custom archive' => ['/srv/backups/app.dmp', '5047444d5001'], +]); + +test('bz2, xz, and zip server backups are prepared in the helper image', function (string $path, string $magicHex) { + $command = importServerBackup($this, $path, $magicHex); + + expect($command) + ->toContain("docker cp '{$path}' 'backup-decompress-") + ->toContain('bunzip2 -c') + ->toContain("cat /tmp/restore.prepared | docker exec -i '{$this->database->uuid}' sh -c") + ->toContain("docker rm -f 'backup-decompress-") + ->not->toContain("docker cp '{$path}' '{$this->database->uuid}:"); + + Process::assertRan(fn ($process) => str_contains($process->command, "docker run -d --network none --name 'backup-decompress-")); + Queue::assertPushed(CoolifyTask::class, fn (CoolifyTask $job) => str_starts_with((string) ($job->call_event_data['containerName'] ?? ''), 'backup-decompress-')); +})->with([ + 'bz2' => ['/srv/backups/app.sql.bz2', '425a68393141'], + 'xz' => ['/srv/backups/app.sql.xz', 'fd377a585a00'], + 'zip' => ['/srv/backups/app.zip', '504b03041400'], +]); + +test('uploaded backups use the same preparation as server backups', function (string $contents, bool $usesHelper) { + Storage::fake(); + Storage::put("upload/{$this->database->uuid}/restore", $contents); + Process::fake(); + + $activity = app(StartDatabaseImport::class)->handle( + $this->database, + new DatabaseImportSource('upload'), + $this->team->id, + ); + $command = (string) $activity->getExtraProperty('command'); + + expect(str_contains($command, "'backup-decompress-"))->toBe($usesHelper) + ->and(str_contains($command, "docker cp '/tmp/database-import-"))->toBeTrue(); +})->with([ + 'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", false], + 'gzip SQL' => [gzencode("SELECT 1;\n"), false], + 'bz2 SQL' => ["BZh91AY&SY\x00\x00", true], +]); + +test('s3 backups are prepared in the s3 helper and streamed into the database container', function () { + $storage = S3Storage::create([ + 'name' => 'Import S3', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://8.8.8.8', + 'is_usable' => true, + 'team_id' => $this->team->id, + ]); + $disk = Mockery::mock(FilesystemAdapter::class); + $disk->shouldReceive('exists')->once()->with('backups/restore.sql.xz')->andReturn(true); + $disk->shouldReceive('size')->once()->with('backups/restore.sql.xz')->andReturn(1024); + $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial(); + $filesystem->shouldReceive('build')->once()->andReturn($disk); + Storage::swap($filesystem); + Process::fake(); + + $activity = app(StartDatabaseImport::class)->handle( + $this->database, + new DatabaseImportSource('s3', path: 'backups/restore.sql.xz', s3StorageUuid: $storage->uuid), + $this->team->id, + ); + $command = (string) $activity->getExtraProperty('command'); + + expect($command) + ->toContain('unxz -c') + ->toContain("cat /tmp/restore.prepared | docker exec -i '{$this->database->uuid}' sh -c") + ->not->toContain('/tmp/s3-restore-'); + + Queue::assertPushed(CoolifyTask::class, fn (CoolifyTask $job) => ! array_key_exists('serverTmpPath', $job->call_event_data) + && str_starts_with((string) $job->call_event_data['containerName'], 's3-restore-')); +}); diff --git a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php index 87b37b6668..b2964a8209 100644 --- a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php +++ b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php @@ -41,9 +41,10 @@ test('decompresses gzip backups for single-database mysql and mariadb restores', $command = $builder->buildRestoreCommand(importResource($class, $type), '/tmp/restore file.sql.gz', false); - expect($command)->toBe( - "(gunzip -cf '/tmp/restore file.sql.gz' 2>/dev/null || cat '/tmp/restore file.sql.gz') | {$client}" - ); + expect($command) + ->toStartWith("backup='/tmp/restore file.sql.gz'\n") + ->toContain('is_gzip() { [ "$(head -c 2 "$backup" | od -An -tx1 | tr -d \' \n\')" = 1f8b ]; }') + ->toEndWith("stream | {$client}"); })->with([ 'mysql' => [StandaloneMysql::class, null, 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE'], 'mariadb' => [StandaloneMariadb::class, null, 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE'], @@ -60,7 +61,7 @@ test('builds dump-all commands and postgres safety scan', function () { expect($builder->buildRestoreCommand($postgres, '/tmp/dump.sql.gz', true)) ->toContain('pg_terminate_backend') - ->toContain("gunzip -cf '/tmp/dump.sql.gz'") + ->toStartWith("backup='/tmp/dump.sql.gz'\n") ->and($safety) ->toContain('COPY ... PROGRAM') ->toContain('docker exec postgres-safe') @@ -72,6 +73,16 @@ test('builds dump-all commands and postgres safety scan', function () { ->toContain("tr '\\n\\r\\t'"); }); +test('restores dump-all PostgreSQL custom archives with pg_restore from the decompressed stream', function () { + $command = (new DatabaseImportCommandBuilder)->buildRestoreCommand(importResource(StandalonePostgresql::class), '/tmp/backup.dump.gz', true); + + // pg_restore cannot read gzip files, so it must receive the decompressed archive on stdin. + expect($command) + ->toContain('stream | pg_restore -U ${POSTGRES_USER} -d "$db"') + ->toContain('stream | psql -U ${POSTGRES_USER} -d "$db"') + ->not->toContain("-d \"\$db\" '/tmp/backup.dump.gz'"); +}); + test('postgres safety command is null for non-postgres databases', function () { $builder = new DatabaseImportCommandBuilder; @@ -89,7 +100,7 @@ test('dump-all mysql and mariadb commands use valid shell parameter expansions', expect($command) ->toContain($binary) - ->toContain("gunzip -cf '/tmp/dump.sql.gz'") + ->toStartWith("backup='/tmp/dump.sql.gz'\n") ->toContain('-p'.$rootPassword) ->toContain('CREATE DATABASE IF NOT EXISTS \`'.$database.'\`') ->and(substr_count($command, $rootPassword))->toBe(6) diff --git a/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php b/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php new file mode 100644 index 0000000000..15ce9c2222 --- /dev/null +++ b/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php @@ -0,0 +1,506 @@ +deleteDirectory($dir); +} + +function restoreScriptHasTool(string $tool): bool +{ + exec('command -v '.escapeshellarg($tool).' >/dev/null 2>&1', $output, $exitCode); + + return $exitCode === 0; +} + +/** + * @param list $command + * @param array $environment + * @return array{exit: int, stdout: string, stderr: string} + */ +function restoreScriptProcess(array $command, string $cwd, array $environment = []): array +{ + $process = proc_open( + $command, + [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + $pipes, + $cwd, + $environment + ['PATH' => getenv('PATH') ?: '/usr/bin:/bin'], + ); + + expect($process)->not->toBeFalse(); + + fclose($pipes[0]); + $stdout = (string) stream_get_contents($pipes[1]); + $stderr = (string) stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + + return ['exit' => proc_close($process), 'stdout' => $stdout, 'stderr' => $stderr]; +} + +/** + * @param array $files relative path => contents + */ +function restoreScriptTar(array $files): string +{ + $dir = restoreScriptTempDir(); + + try { + foreach ($files as $name => $contents) { + $path = $dir.'/src/'.$name; + if (! is_dir(dirname($path))) { + mkdir(dirname($path), 0777, true); + } + file_put_contents($path, $contents); + } + + $result = restoreScriptProcess(['tar', '-cf', $dir.'/out.tar', '-C', $dir.'/src', ...array_keys($files)], $dir); + expect($result['exit'])->toBe(0, $result['stderr']); + + return (string) file_get_contents($dir.'/out.tar'); + } finally { + restoreScriptRemoveDir($dir); + } +} + +function restoreScriptCompress(string $tool, string $contents): string +{ + $dir = restoreScriptTempDir(); + + try { + file_put_contents($dir.'/input', $contents); + $result = restoreScriptProcess([$tool, '-c', $dir.'/input'], $dir); + expect($result['exit'])->toBe(0, $result['stderr']); + + return $result['stdout']; + } finally { + restoreScriptRemoveDir($dir); + } +} + +/** + * Builds an uncompressed (stored) zip archive, so the fixture does not depend on ext-zip. + * + * @param array $files file name => contents + */ +function restoreScriptZip(array $files): string +{ + $entries = ''; + $directory = ''; + + foreach ($files as $name => $contents) { + $crc = crc32($contents); + $size = strlen($contents); + $offset = strlen($entries); + $entries .= pack('VvvvvvVVVvv', 0x04034B50, 20, 0, 0, 0, 0x21, $crc, $size, $size, strlen($name), 0).$name.$contents; + $directory .= pack('VvvvvvvVVVvvvvvVV', 0x02014B50, 20, 20, 0, 0, 0, 0x21, $crc, $size, $size, strlen($name), 0, 0, 0, 0, 0, $offset).$name; + } + + return $entries.$directory.pack('VvvvvVVv', 0x06054B50, 0, 0, count($files), count($files), strlen($directory), strlen($entries), 0); +} + +function restoreScriptFixture(string $name): string +{ + $pgCustom = "PGDMP\x01\x0e\x00\x04\x08\x01\x01\x01".str_repeat("\x00\x01toc", 64); + $pgSql = "-- PostgreSQL database dump\n\nCREATE TABLE items (id integer);\n"; + $mysqlSql = "-- MySQL dump 10.13\n\nCREATE TABLE `items` (`id` int);\nINSERT INTO `items` VALUES (1);\n"; + $mongoArchive = "\x6d\xe2\x99\x81".str_repeat("\x01\x00archive", 32); + $bson = pack('V', 22)."\x02name\x00".pack('V', 5)."item\x00\x00"; + + return match ($name) { + 'pg-custom' => $pgCustom, + 'pg-custom-gz' => gzencode($pgCustom), + 'pg-tar' => restoreScriptTar(['toc.dat' => $pgCustom, '3001.dat' => "1\n\\.\n"]), + 'pg-tar-gz' => gzencode(restoreScriptTar(['toc.dat' => $pgCustom, '3001.dat' => "1\n\\.\n"])), + 'pg-sql' => $pgSql, + 'pg-sql-gz' => gzencode($pgSql), + 'pg-cluster' => "--\n-- PostgreSQL database cluster dump\n--\n\nCREATE ROLE app;\n", + 'garbage' => str_repeat("\x00\xff\x10\x80binary\x00", 64), + 'mysql-sql' => $mysqlSql, + 'mysql-sql-gz' => gzencode($mysqlSql), + 'mysql-tar' => restoreScriptTar(['backup.sql' => $mysqlSql]), + 'mysql-tar-two' => restoreScriptTar(['app.sql' => $mysqlSql, 'other.sql' => $mysqlSql]), + 'mysql-cluster' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\nUSE `mysql`;\nINSERT INTO `user` VALUES ();\n", + 'mysql-two-databases' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\nCREATE DATABASE `other`;\nUSE `other`;\nCREATE TABLE `notes` (`t` text);\n", + 'mysql-one-database' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\n", + 'mongo-archive' => $mongoArchive, + 'mongo-archive-gz' => gzencode($mongoArchive), + 'mongo-dump-tar' => restoreScriptTar(['dump/app/items.bson' => $bson, 'dump/app/items.metadata.json' => '{"indexes":[]}']), + 'mongo-dump-gz-tar' => restoreScriptTar(['dump/app/items.bson.gz' => gzencode($bson), 'dump/app/items.metadata.json.gz' => gzencode('{"indexes":[]}')]), + 'mongo-archive-tar' => restoreScriptTar(['app.archive' => $mongoArchive]), + 'mongo-notes-tar' => restoreScriptTar(['notes.txt' => 'hello', 'readme.txt' => 'world']), + 'mongo-bson' => $bson, + }; +} + +function restoreScriptResource(string $engine): object +{ + $class = match ($engine) { + 'postgresql' => StandalonePostgresql::class, + 'mysql' => StandaloneMysql::class, + 'mariadb' => StandaloneMariadb::class, + 'mongodb' => StandaloneMongodb::class, + }; + + $resource = Mockery::mock($class); + $resource->shouldReceive('getMorphClass')->andReturn($class); + + return $resource; +} + +/** + * @return array{exit: int, stdout: string, stderr: string, calls: list, dir: string, backup: string, leftovers: list} + */ +function restoreScriptRun(string $engine, string $contents, bool $dumpAll = false, bool $replaceExisting = false): array +{ + $dir = restoreScriptTempDir(); + + try { + $log = $dir.'/calls.log'; + file_put_contents($log, ''); + mkdir($dir.'/bin'); + + foreach (['pg_restore', 'psql', 'dropdb', 'createdb', 'mysql', 'mariadb', 'mongorestore'] as $client) { + $escapedLog = escapeshellarg($log); + file_put_contents($dir.'/bin/'.$client, <</dev/null +fi +printf '%s|%s|%s\\n' '{$client}' "\$*" "\$header" >> {$escapedLog} +exit 0 +SH); + chmod($dir.'/bin/'.$client, 0755); + } + + $backup = $dir.'/backup file'; + file_put_contents($backup, $contents); + + $script = (new DatabaseImportCommandBuilder)->buildRestoreCommand(restoreScriptResource($engine), $backup, $dumpAll, $replaceExisting); + + $result = restoreScriptProcess(['sh', '-c', $script], $dir, [ + 'PATH' => $dir.'/bin'.PATH_SEPARATOR.(getenv('PATH') ?: '/usr/bin:/bin'), + 'TMPDIR' => $dir, + 'POSTGRES_USER' => 'postgres', + 'POSTGRES_DB' => 'app', + 'MYSQL_USER' => 'app_user', + 'MYSQL_PASSWORD' => 'app_pass', + 'MYSQL_DATABASE' => 'app', + 'MYSQL_ROOT_PASSWORD' => 'root_pass', + 'MARIADB_USER' => 'app_user', + 'MARIADB_PASSWORD' => 'app_pass', + 'MARIADB_DATABASE' => 'app', + 'MARIADB_ROOT_PASSWORD' => 'root_pass', + 'MONGO_INITDB_ROOT_USERNAME' => 'root', + 'MONGO_INITDB_ROOT_PASSWORD' => 'mongo_pass', + ]); + + $lines = array_values(array_filter(explode("\n", (string) file_get_contents($log)), fn (string $line): bool => $line !== '')); + $calls = array_map(function (string $line): array { + [$name, $args, $header] = array_pad(explode('|', $line, 3), 3, ''); + + return ['name' => $name, 'args' => $args, 'header' => $header]; + }, $lines); + + return $result + [ + 'calls' => $calls, + 'dir' => $dir, + 'backup' => $backup, + 'leftovers' => glob($dir.'/tmp.*') ?: [], + ]; + } finally { + restoreScriptRemoveDir($dir); + } +} + +/** + * @param list $calls + * @param list}> $expected client, stdin header, argument fragments + */ +function restoreScriptExpectCalls(array $calls, array $expected): void +{ + expect(array_map(fn (array $call): string => $call['name'].'|'.$call['header'], $calls)) + ->toBe(array_map(fn (array $call): string => $call[0].'|'.$call[1], $expected)); + + foreach ($expected as $index => $call) { + foreach ($call[2] as $fragment) { + expect($calls[$index]['args'])->toContain($fragment); + } + } +} + +/** + * @return array{exit: int, stdout: string, stderr: string, output: ?string, sourceExists: bool} + */ +function restoreScriptNormalize(string $contents): array +{ + $dir = restoreScriptTempDir(); + + try { + $source = $dir.'/uploaded backup'; + $target = $dir.'/normalized backup'; + file_put_contents($source, $contents); + + $result = restoreScriptProcess( + ['sh', '-c', (new DatabaseImportCommandBuilder)->buildNormalizeScript($source, $target)], + $dir, + ['TMPDIR' => $dir], + ); + + return $result + [ + 'output' => is_file($target) ? (string) file_get_contents($target) : null, + 'sourceExists' => file_exists($source), + ]; + } finally { + restoreScriptRemoveDir($dir); + } +} + +test('restores single PostgreSQL archives with pg_restore and SQL with psql', function (string $fixture, bool $replaceExisting, array $expectedCalls) { + $run = restoreScriptRun('postgresql', restoreScriptFixture($fixture), false, $replaceExisting); + + expect($run['exit'])->toBe(0, $run['stderr']); + restoreScriptExpectCalls($run['calls'], $expectedCalls); +})->with([ + 'custom archive' => ['pg-custom', false, [['pg_restore', 'PGDMP', ['--exit-on-error -U postgres -d app']]]], + 'custom archive replacing existing objects' => ['pg-custom', true, [['pg_restore', 'PGDMP', ['--exit-on-error --clean --if-exists -U postgres -d app']]]], + 'gzip custom archive' => ['pg-custom-gz', false, [['pg_restore', 'PGDMP', ['--exit-on-error -U postgres -d app']]]], + 'gzip custom archive replacing existing objects' => ['pg-custom-gz', true, [['pg_restore', 'PGDMP', ['--exit-on-error --clean --if-exists -U postgres -d app']]]], + 'tar archive' => ['pg-tar', false, [['pg_restore', 'toc.d', ['--exit-on-error -U postgres -d app']]]], + 'gzip tar archive' => ['pg-tar-gz', false, [['pg_restore', 'toc.d', ['--exit-on-error -U postgres -d app']]]], + 'plain SQL' => ['pg-sql', false, [['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']]]], + 'gzip SQL' => ['pg-sql-gz', false, [['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']]]], + // SQL cannot replace single objects, so replacing recreates the target database first. + 'plain SQL replacing existing objects' => ['pg-sql', true, [ + ['psql', 'SELEC', ['-v db=app -U postgres -d template1']], + ['dropdb', '', ['--maintenance-db=template1 -U postgres --if-exists app']], + ['createdb', '', ['-U postgres app']], + ['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']], + ]], + 'gzip SQL replacing existing objects' => ['pg-sql-gz', true, [ + ['psql', 'SELEC', ['-v db=app -U postgres -d template1']], + ['dropdb', '', ['--maintenance-db=template1 -U postgres --if-exists app']], + ['createdb', '', ['-U postgres app']], + ['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']], + ]], +]); + +test('rejects unsupported single PostgreSQL backups before calling any client', function (string $fixture, string $message) { + $run = restoreScriptRun('postgresql', restoreScriptFixture($fixture)); + + expect($run['exit'])->toBe(1) + ->and($run['stderr'])->toContain($message) + ->and($run['calls'])->toBe([]); +})->with([ + 'cluster SQL dump' => ['pg-cluster', 'This backup contains all databases.'], + 'binary garbage' => ['garbage', 'Unsupported PostgreSQL backup format'], +]); + +test('restores PostgreSQL backups containing all databases after checking the format', function (string $fixture, array $restoreCall) { + $run = restoreScriptRun('postgresql', restoreScriptFixture($fixture), dumpAll: true); + + $recreate = [ + ['psql', '', ['-U postgres -c', 'pg_terminate_backend']], + ['psql', '', ['-U postgres -t -c', 'SELECT datname FROM pg_database']], + ['createdb', '', ['-U postgres app']], + ]; + $inspect = $restoreCall[0] === 'pg_restore' ? [['pg_restore', 'PGDMP', ['-l']]] : []; + + expect($run['exit'])->toBe(0, $run['stderr']); + restoreScriptExpectCalls($run['calls'], [...$inspect, ...$recreate, $restoreCall]); +})->with([ + 'custom archive' => ['pg-custom', ['pg_restore', 'PGDMP', ['-U postgres -d app']]], + 'gzip custom archive' => ['pg-custom-gz', ['pg_restore', 'PGDMP', ['-U postgres -d app']]], + 'plain SQL' => ['pg-sql', ['psql', '-- Po', ['-U postgres -d app']]], + 'gzip SQL' => ['pg-sql-gz', ['psql', '-- Po', ['-U postgres -d app']]], +]); + +test('rejects unsupported PostgreSQL backups containing all databases before dropping anything', function () { + $run = restoreScriptRun('postgresql', restoreScriptFixture('garbage'), dumpAll: true); + + expect($run['exit'])->toBe(1) + ->and($run['stderr'])->toContain('Unsupported PostgreSQL backup format')->toContain('Nothing was changed.') + ->and($run['calls'])->toBe([]); +}); + +test('restores single MySQL and MariaDB SQL backups', function (string $engine, string $fixture) { + $run = restoreScriptRun($engine, restoreScriptFixture($fixture)); + + expect($run['exit'])->toBe(0, $run['stderr']) + ->and($run['leftovers'])->toBe([]); + restoreScriptExpectCalls($run['calls'], [[$engine, '-- My', ['-u app_user -papp_pass app']]]); +})->with(['mysql', 'mariadb'])->with([ + 'SQL' => 'mysql-sql', + 'gzip SQL' => 'mysql-sql-gz', + 'tar with one SQL file' => 'mysql-tar', + 'dump of one database made with --databases' => 'mysql-one-database', +]); + +test('restores MySQL and MariaDB backups containing all databases after checking the format', function (string $engine, string $fixture) { + $run = restoreScriptRun($engine, restoreScriptFixture($fixture), dumpAll: true); + + expect($run['exit'])->toBe(0, $run['stderr']) + ->and($run['leftovers'])->toBe([]); + restoreScriptExpectCalls($run['calls'], [ + [$engine, '', ['-u root -proot_pass -N -e', 'information_schema.processlist']], + [$engine, '', ['-u root -proot_pass -N -e', 'DROP DATABASE IF EXISTS']], + [$engine, '', ['-u root -proot_pass']], + [$engine, '', ['-u root -proot_pass -e CREATE DATABASE IF NOT EXISTS `app`;']], + [$engine, '-- My', ['-u root -proot_pass app']], + ]); + expect(end($run['calls'])['args'])->toBe('-u root -proot_pass app'); +})->with(['mysql', 'mariadb'])->with([ + 'SQL' => 'mysql-sql', + 'gzip SQL' => 'mysql-sql-gz', + 'tar with one SQL file' => 'mysql-tar', + 'SQL that uses the mysql schema' => 'mysql-cluster', +]); + +test('rejects unsupported MySQL and MariaDB backups before calling any client', function (string $engine, string $fixture, bool $dumpAll, string $message) { + $run = restoreScriptRun($engine, restoreScriptFixture($fixture), $dumpAll); + + expect($run['exit'])->toBe(1) + ->and($run['stderr'])->toContain($message) + ->and($run['calls'])->toBe([]); +})->with(['mysql', 'mariadb'])->with([ + 'tar with two files' => ['mysql-tar-two', false, 'A tar backup must contain exactly one SQL dump.'], + 'tar with two files, all databases' => ['mysql-tar-two', true, 'A tar backup must contain exactly one SQL dump.'], + 'binary garbage' => ['garbage', false, 'Unsupported backup format.'], + 'binary garbage, all databases' => ['garbage', true, 'Unsupported backup format.'], + 'all-databases dump without dump-all' => ['mysql-cluster', false, 'This backup contains more than one database.'], + 'dump of two databases without dump-all' => ['mysql-two-databases', false, 'This backup contains more than one database.'], +]); + +test('restores mongodump archives', function (string $fixture, bool $replaceExisting, bool $gzip) { + $run = restoreScriptRun('mongodb', restoreScriptFixture($fixture), false, $replaceExisting); + + expect($run['exit'])->toBe(0, $run['stderr']); + restoreScriptExpectCalls($run['calls'], [['mongorestore', '', [ + '--authenticationDatabase=admin --username root --password mongo_pass', + '--archive='.$run['backup'], + ]]]); + + $args = $run['calls'][0]['args']; + expect(str_contains($args, '--gzip'))->toBe($gzip) + ->and(str_contains($args, '--drop'))->toBe($replaceExisting); +})->with([ + 'plain archive' => ['mongo-archive', false, false], + 'plain archive replacing existing collections' => ['mongo-archive', true, false], + 'gzip archive' => ['mongo-archive-gz', false, true], + 'gzip archive replacing existing collections' => ['mongo-archive-gz', true, true], +]); + +test('restores mongodump directories packed as tar', function (string $fixture, bool $gzip) { + $run = restoreScriptRun('mongodb', restoreScriptFixture($fixture)); + + expect($run['exit'])->toBe(0, $run['stderr']) + ->and($run['calls'])->toHaveCount(1) + ->and($run['calls'][0]['name'])->toBe('mongorestore') + ->and($run['calls'][0]['args'])->toMatch('#--dir='.preg_quote($run['dir'], '#').'/tmp\.[^/ ]+/dump$#') + ->and(str_contains($run['calls'][0]['args'], '--gzip'))->toBe($gzip) + ->and($run['calls'][0]['args'])->not->toContain('--archive') + ->and($run['leftovers'])->toBe([]); +})->with([ + 'bson files' => ['mongo-dump-tar', false], + 'gzip bson files' => ['mongo-dump-gz-tar', true], +]); + +test('restores a mongodump archive wrapped in tar', function () { + $run = restoreScriptRun('mongodb', restoreScriptFixture('mongo-archive-tar')); + + expect($run['exit'])->toBe(0, $run['stderr']) + ->and($run['calls'])->toHaveCount(1) + ->and($run['calls'][0]['name'])->toBe('mongorestore') + ->and($run['calls'][0]['args'])->toMatch('#--archive='.preg_quote($run['dir'], '#').'/tmp\.[^/ ]+/app\.archive$#') + ->and($run['calls'][0]['args'])->not->toContain('--gzip') + ->and($run['leftovers'])->toBe([]); +}); + +test('rejects unsupported MongoDB backups before calling mongorestore', function (string $fixture, bool $replaceExisting, string $message) { + $run = restoreScriptRun('mongodb', restoreScriptFixture($fixture), false, $replaceExisting); + + expect($run['exit'])->toBe(1) + ->and($run['stderr'])->toContain($message) + ->and($run['calls'])->toBe([]); +})->with([ + 'single bson file' => ['mongo-bson', false, 'Unsupported MongoDB backup format'], + 'single bson file replacing existing collections' => ['mongo-bson', true, 'Unsupported MongoDB backup format'], + 'tar without a dump directory' => ['mongo-notes-tar', true, 'The tar backup does not contain a mongodump directory.'], +]); + +test('normalize moves plain and gzip backups unchanged', function (string $contents) { + $result = restoreScriptNormalize($contents); + + expect($result['exit'])->toBe(0, $result['stderr']) + ->and($result['output'])->toBe($contents) + ->and($result['sourceExists'])->toBeFalse(); +})->with([ + 'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n"], + 'gzip SQL' => [gzencode("-- PostgreSQL database dump\nSELECT 1;\n")], +]); + +test('normalize decompresses bz2, xz and single-file zip backups', function (string $format, array $tools) { + foreach ($tools as $tool) { + if (! restoreScriptHasTool($tool)) { + $this->markTestSkipped("{$tool} is not installed."); + } + } + + $original = "-- MySQL dump 10.13\nCREATE TABLE `items` (`id` int);\n"; + $compressed = match ($format) { + 'bz2' => restoreScriptCompress('bzip2', $original), + 'xz' => restoreScriptCompress('xz', $original), + 'zip' => restoreScriptZip(['backup.sql' => $original]), + }; + + $result = restoreScriptNormalize($compressed); + + expect($result['exit'])->toBe(0, $result['stderr']) + ->and($result['output'])->toBe($original); +})->with([ + 'bz2' => ['bz2', ['bzip2', 'bunzip2']], + 'xz' => ['xz', ['xz', 'unxz']], + 'zip' => ['zip', ['unzip']], +]); + +test('normalize rejects backups it cannot unpack', function (string $format, string $tool, string $message) { + if (! restoreScriptHasTool($tool)) { + $this->markTestSkipped("{$tool} is not installed."); + } + + $contents = match ($format) { + 'zip with two files' => restoreScriptZip(['app.sql' => 'SELECT 1;', 'other.sql' => 'SELECT 2;']), + 'corrupt bz2' => 'BZh91AY&SY'.str_repeat("\x00\xffcorrupt", 16), + }; + + $result = restoreScriptNormalize($contents); + + expect($result['exit'])->toBe(1) + ->and($result['stderr'])->toContain($message); +})->with([ + 'zip with two files' => ['zip with two files', 'unzip', 'A zip backup must contain exactly one file.'], + 'corrupt bz2' => ['corrupt bz2', 'bunzip2', 'The bz2 backup cannot be decompressed.'], +]); diff --git a/tests/Unit/Livewire/Database/S3RestoreTest.php b/tests/Unit/Livewire/Database/S3RestoreTest.php index a0acf229d8..8716fe5831 100644 --- a/tests/Unit/Livewire/Database/S3RestoreTest.php +++ b/tests/Unit/Livewire/Database/S3RestoreTest.php @@ -17,6 +17,9 @@ function importFormWithResource(string $modelClass): ImportForm } /** + * Runs the dump-all restore command with stub clients. Each entry is the + * client name and the first five bytes it read from stdin. + * * @return list */ function invokedPostgresRestoreClients(string $contents, bool $gzip = false): array @@ -28,13 +31,15 @@ function invokedPostgresRestoreClients(string $contents, bool $gzip = false): ar $logFile = $binDir.'/invoked.log'; file_put_contents($logFile, ''); - foreach (['pg_restore', 'psql'] as $tool) { + foreach (['pg_restore', 'psql', 'dropdb', 'createdb'] as $tool) { $stub = <<> '{$logFile}' +header='' if [ ! -t 0 ]; then + header=\$(head -c 5) cat >/dev/null fi +printf '%s:%s\\n' '{$tool}' "\$header" >> '{$logFile}' exit 0 SH; $path = $binDir.'/'.$tool; @@ -49,7 +54,6 @@ SH; $component = importFormWithResource('App\Models\StandalonePostgresql'); $component->dumpAll = true; - $component->postgresqlRestoreCommand = ':'; $process = proc_open( ['sh', '-c', $component->buildRestoreCommand($dumpPath)], @@ -89,44 +93,51 @@ SH; test('buildRestoreCommand handles PostgreSQL without dumpAll', function () { $component = importFormWithResource('App\Models\StandalonePostgresql'); $component->dumpAll = false; - $component->postgresqlRestoreCommand = 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB'; $result = $component->buildRestoreCommand('/tmp/test.dump'); - expect($result)->toContain('pg_restore'); - expect($result)->toContain('/tmp/test.dump'); + expect($result)->toContain('pg_restore --exit-on-error'); + expect($result)->toStartWith("backup='/tmp/test.dump'"); }); test('buildRestoreCommand handles PostgreSQL with dumpAll', function () { $component = importFormWithResource('App\Models\StandalonePostgresql'); $component->dumpAll = true; - $component->postgresqlRestoreCommand = 'psql -U $POSTGRES_USER -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && psql -U $POSTGRES_USER -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U $POSTGRES_USER --if-exists {} && createdb -U $POSTGRES_USER postgres'; $result = $component->buildRestoreCommand('/tmp/test.dump'); - expect($result)->toContain("gunzip -cf '/tmp/test.dump'"); - expect($result)->toContain('psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'); + expect($result)->toStartWith("backup='/tmp/test.dump'"); + expect($result)->toContain('stream | psql -U ${POSTGRES_USER} -d "$db"'); }); test('buildRestoreCommand dump-all PostgreSQL restore is pg_restore for PGDMP otherwise psql', function () { $component = importFormWithResource('App\Models\StandalonePostgresql'); $component->dumpAll = true; - $component->postgresqlRestoreCommand = 'psql -U ${POSTGRES_USER} -c "cleanup"'; - $escapedTmpPath = escapeshellarg('/tmp/test.dump'); + $command = $component->buildRestoreCommand('/tmp/test.dump'); - expect($component->buildRestoreCommand('/tmp/test.dump'))->toBe( - 'psql -U ${POSTGRES_USER} -c "cleanup" && if [ "$({ gunzip -cf '.$escapedTmpPath.' 2>/dev/null || cat '.$escapedTmpPath.'; } | head -c 5)" = \'PGDMP\' ]; then pg_restore -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$escapedTmpPath.'; else (gunzip -cf '.$escapedTmpPath.' 2>/dev/null || cat '.$escapedTmpPath.') | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}; fi' - ); + expect($command) + ->toContain('if [ "$(stream | head -c 5)" = PGDMP ] || is_tar; then') + ->toContain('stream | pg_restore -U ${POSTGRES_USER} -d "$db"') + ->toContain('stream | psql -U ${POSTGRES_USER} -d "$db"') + ->and(strpos($command, 'kind=archive'))->toBeLessThan(strpos($command, 'dropdb')); }); -test('dump-all PostgreSQL restore selects the client for the dump format', function (string $contents, bool $gzip, string $client) { - expect(invokedPostgresRestoreClients($contents, $gzip))->toBe([$client]); +test('dump-all PostgreSQL import text shows the client chosen by the dump format', function () { + $component = importFormWithResource('App\Models\StandalonePostgresql'); + $component->updatedDumpAll(true); + + expect($component->restoreCommandText)->toBe($component->buildRestoreCommand('')); +}); + +test('dump-all PostgreSQL restore selects the client for the dump format', function (string $contents, bool $gzip, string $restore, array $preflight) { + // Check the archive, terminate sessions, list databases, recreate the target database, then restore. + expect(invokedPostgresRestoreClients($contents, $gzip))->toBe([...$preflight, 'psql:', 'psql:', 'createdb:', $restore]); })->with([ - 'custom archive' => ['PGDMP'.str_repeat("\0", 16), false, 'pg_restore'], - 'gzip custom archive' => ['PGDMP'.str_repeat("\0", 16), true, 'pg_restore'], - 'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", false, 'psql'], - 'gzip SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", true, 'psql'], + 'custom archive' => ['PGDMP'.str_repeat("\0", 16), false, 'pg_restore:PGDMP', ['pg_restore:PGDMP']], + 'gzip custom archive' => ['PGDMP'.str_repeat("\0", 16), true, 'pg_restore:PGDMP', ['pg_restore:PGDMP']], + 'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", false, 'psql:-- Po', []], + 'gzip SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", true, 'psql:-- Po', []], ]); test('buildRestoreCommand handles MySQL without dumpAll', function () { @@ -135,8 +146,8 @@ test('buildRestoreCommand handles MySQL without dumpAll', function () { $result = $component->buildRestoreCommand('/tmp/test.dump'); - expect($result)->toContain('mysql -u $MYSQL_USER'); - expect($result)->toContain("(gunzip -cf '/tmp/test.dump' 2>/dev/null || cat '/tmp/test.dump') | mysql"); + expect($result)->toStartWith("backup='/tmp/test.dump'"); + expect($result)->toContain('stream | mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE'); expect($result)->not->toContain("< '/tmp/test.dump'"); }); @@ -146,18 +157,18 @@ test('buildRestoreCommand handles MariaDB without dumpAll', function () { $result = $component->buildRestoreCommand('/tmp/test.dump'); - expect($result)->toContain('mariadb -u $MARIADB_USER'); - expect($result)->toContain("(gunzip -cf '/tmp/test.dump' 2>/dev/null || cat '/tmp/test.dump') | mariadb"); + expect($result)->toStartWith("backup='/tmp/test.dump'"); + expect($result)->toContain('stream | mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE'); expect($result)->not->toContain("< '/tmp/test.dump'"); }); test('buildRestoreCommand always appends the MongoDB archive path', function (bool $dumpAll) { $component = importFormWithResource('App\Models\StandaloneMongodb'); $component->dumpAll = $dumpAll; - $component->mongodbRestoreCommand = 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive='; $result = $component->buildRestoreCommand('/tmp/test.dump'); + expect($result)->toStartWith("backup='/tmp/test.dump'"); expect($result)->toContain('mongorestore'); - expect($result)->toContain("--archive='/tmp/test.dump'"); + expect($result)->toContain('restore --gzip --archive="$backup"'); })->with([false, true]); diff --git a/tests/Unit/S3RestoreSecurityTest.php b/tests/Unit/S3RestoreSecurityTest.php index 51c374af53..be1bf3a7b4 100644 --- a/tests/Unit/S3RestoreSecurityTest.php +++ b/tests/Unit/S3RestoreSecurityTest.php @@ -169,11 +169,12 @@ it('quotes dump all restore command temp paths with spaces', function (string $m $escapedTmpPath = escapeshellarg($tmpPath); $restoreCommand = $component->buildRestoreCommand($tmpPath); + // The path appears once, as a quoted assignment; the script only uses "$backup". expect($restoreCommand) - ->toContain("gunzip -cf {$escapedTmpPath}") - ->toContain("cat {$escapedTmpPath}") - ->not->toContain("gunzip -cf {$tmpPath}") - ->not->toContain("cat {$tmpPath}"); + ->toStartWith("backup={$escapedTmpPath}\n") + ->and(substr_count($restoreCommand, $tmpPath))->toBe(1) + ->and($restoreCommand)->toContain('gunzip -c "$backup"') + ->toContain('cat "$backup"'); })->with([ 'mariadb' => StandaloneMariadb::class, 'mysql' => StandaloneMysql::class,