diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index eef0ebe929..b91d63cc92 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -152,7 +152,7 @@ class StartDatabaseImport if ($safety = $this->commands->buildPostgresSafetyCommand($resource, $container, $containerPath)) { $commandList[] = $safety; } - $restore = base64_encode($this->commands->buildRestoreCommand($resource, $containerPath, $source->dumpAll, $source->replaceExisting, $source->keepOwners, $sqliteDatabase)); + $restore = base64_encode($this->commands->buildRestoreCommand($resource, $containerPath, $source->dumpAll, $source->replaceExisting, $source->keepOwners, $sqliteDatabase, $source->restoreMysqlUsers)); $commandList[] = 'echo '.escapeshellarg($restore).' | base64 -d > '.escapeshellarg($scriptPath); $commandList[] = 'chmod +x '.escapeshellarg($scriptPath); $commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}"); diff --git a/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php b/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php index 0028fbc3e4..5c06f77885 100644 --- a/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php +++ b/app/Http/Controllers/Api/Concerns/HandlesDatabaseImportsApi.php @@ -61,7 +61,7 @@ trait HandlesDatabaseImportsApi { $this->authorize('update', $resource); $payload = $request->json()->all() ?: $request->request->all(); - $allowed = ['source', 'upload_id', 's3_storage_uuid', 'path', 'dump_all', 'replace_existing', 'keep_owners', 'sqlite_database']; + $allowed = ['source', 'upload_id', 's3_storage_uuid', 'path', 'dump_all', 'replace_existing', 'keep_owners', 'restore_mysql_users', 'sqlite_database']; $validator = Validator::make($payload, [ 'source' => ['required', Rule::in(['upload', 's3', 'server'])], 'upload_id' => ['required_if:source,upload', 'prohibited_unless:source,upload', 'uuid'], @@ -70,6 +70,7 @@ trait HandlesDatabaseImportsApi 'dump_all' => ['sometimes', 'boolean'], 'replace_existing' => ['sometimes', 'boolean'], 'keep_owners' => ['sometimes', 'boolean'], + 'restore_mysql_users' => ['sometimes', 'boolean'], 'sqlite_database' => ['sometimes', 'string', 'max:255'], ]); $extraFields = array_diff(array_keys($payload), $allowed); @@ -83,7 +84,7 @@ trait HandlesDatabaseImportsApi } try { - $source = new DatabaseImportSource((string) $payload['source'], $payload['upload_id'] ?? null, $payload['path'] ?? null, $payload['s3_storage_uuid'] ?? null, (bool) ($payload['dump_all'] ?? false), (bool) ($payload['replace_existing'] ?? false), (bool) ($payload['keep_owners'] ?? false), isset($payload['sqlite_database']) ? (string) $payload['sqlite_database'] : null); + $source = new DatabaseImportSource((string) $payload['source'], $payload['upload_id'] ?? null, $payload['path'] ?? null, $payload['s3_storage_uuid'] ?? null, (bool) ($payload['dump_all'] ?? false), (bool) ($payload['replace_existing'] ?? false), (bool) ($payload['keep_owners'] ?? false), isset($payload['sqlite_database']) ? (string) $payload['sqlite_database'] : null, (bool) ($payload['restore_mysql_users'] ?? false)); $activity = app(StartDatabaseImport::class)->handle($resource, $source, $teamId); } catch (DatabaseImportException $exception) { return response()->json(['message' => $exception->getMessage()], $exception->status); @@ -95,6 +96,7 @@ trait HandlesDatabaseImportsApi 'source' => $source->type, 'replace_existing' => $source->replaceExisting, 'keep_owners' => $source->keepOwners, + 'restore_mysql_users' => $source->restoreMysqlUsers, 'activity_id' => $activity->id, ]); $url = route($statusRoute, [...$routeParameters, 'activity_id' => $activity->id], false); diff --git a/app/Http/Controllers/Api/OpenApi.php b/app/Http/Controllers/Api/OpenApi.php index 9da513c677..77741ea537 100644 --- a/app/Http/Controllers/Api/OpenApi.php +++ b/app/Http/Controllers/Api/OpenApi.php @@ -16,9 +16,9 @@ use OpenApi\Attributes as OA; new OA\Schema( schema: 'DatabaseImportRequest', oneOf: [ - new OA\Schema(required: ['source', 'upload_id'], additionalProperties: false, properties: [new OA\Property(property: 'source', type: 'string', enum: ['upload']), new OA\Property(property: 'upload_id', type: 'string', format: 'uuid'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false), new OA\Property(property: 'keep_owners', description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.', type: 'boolean', default: false), new OA\Property(property: 'sqlite_database', description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.', type: 'string', example: 'app.db')]), - new OA\Schema(required: ['source', 's3_storage_uuid', 'path'], additionalProperties: false, properties: [new OA\Property(property: 'source', type: 'string', enum: ['s3']), new OA\Property(property: 's3_storage_uuid', type: 'string'), new OA\Property(property: 'path', type: 'string'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false), new OA\Property(property: 'keep_owners', description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.', type: 'boolean', default: false), new OA\Property(property: 'sqlite_database', description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.', type: 'string', example: 'app.db')]), - new OA\Schema(required: ['source', 'path'], additionalProperties: false, properties: [new OA\Property(property: 'source', type: 'string', enum: ['server']), new OA\Property(property: 'path', type: 'string', example: '/var/backups/database.sql.gz'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false), new OA\Property(property: 'keep_owners', description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.', type: 'boolean', default: false), new OA\Property(property: 'sqlite_database', description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.', type: 'string', example: 'app.db')]), + new OA\Schema(required: ['source', 'upload_id'], additionalProperties: false, properties: [new OA\Property(property: 'source', type: 'string', enum: ['upload']), new OA\Property(property: 'upload_id', type: 'string', format: 'uuid'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false), new OA\Property(property: 'keep_owners', description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.', type: 'boolean', default: false), new OA\Property(property: 'restore_mysql_users', description: 'MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.', type: 'boolean', default: false), new OA\Property(property: 'sqlite_database', description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.', type: 'string', example: 'app.db')]), + new OA\Schema(required: ['source', 's3_storage_uuid', 'path'], additionalProperties: false, properties: [new OA\Property(property: 'source', type: 'string', enum: ['s3']), new OA\Property(property: 's3_storage_uuid', type: 'string'), new OA\Property(property: 'path', type: 'string'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false), new OA\Property(property: 'keep_owners', description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.', type: 'boolean', default: false), new OA\Property(property: 'restore_mysql_users', description: 'MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.', type: 'boolean', default: false), new OA\Property(property: 'sqlite_database', description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.', type: 'string', example: 'app.db')]), + new OA\Schema(required: ['source', 'path'], additionalProperties: false, properties: [new OA\Property(property: 'source', type: 'string', enum: ['server']), new OA\Property(property: 'path', type: 'string', example: '/var/backups/database.sql.gz'), new OA\Property(property: 'dump_all', type: 'boolean', default: false), new OA\Property(property: 'replace_existing', description: 'Drop matching PostgreSQL objects before restoring a single-database archive.', type: 'boolean', default: false), new OA\Property(property: 'keep_owners', description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.', type: 'boolean', default: false), new OA\Property(property: 'restore_mysql_users', description: 'MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.', type: 'boolean', default: false), new OA\Property(property: 'sqlite_database', description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.', type: 'string', example: 'app.db')]), ], type: 'object', ), diff --git a/app/Livewire/Project/Database/ImportForm.php b/app/Livewire/Project/Database/ImportForm.php index 96d4366e7b..e16c7e331b 100644 --- a/app/Livewire/Project/Database/ImportForm.php +++ b/app/Livewire/Project/Database/ImportForm.php @@ -165,6 +165,11 @@ class ImportForm extends Component */ public bool $keepOwners = false; + /** + * MySQL and MariaDB all-databases backups: also restore the system databases (users, passwords and privileges). + */ + public bool $restoreMysqlUsers = false; + /** * SQLite: the database file to restore into. Always one of the database's own files. */ @@ -252,6 +257,11 @@ class ImportForm extends Component $this->refreshRestoreCommandText(); } + public function updatedRestoreMysqlUsers(): void + { + $this->refreshRestoreCommandText(); + } + public function updatedSqliteDatabase(): void { if (! in_array($this->sqliteDatabase, $this->sqliteDatabaseFiles, true)) { @@ -281,7 +291,7 @@ class ImportForm extends Component try { $this->restoreCommandText = $this->resource && $commands->supports($this->resource) - ? $commands->buildRestoreCommand($this->resource, '', $this->dumpAll, $this->replaceExisting, $this->keepOwners, $this->sqliteDatabase) + ? $commands->buildRestoreCommand($this->resource, '', $this->dumpAll, $this->replaceExisting, $this->keepOwners, $this->sqliteDatabase, $this->restoreMysqlUsers) : ''; } catch (\InvalidArgumentException) { $this->restoreCommandText = ''; @@ -453,8 +463,8 @@ class ImportForm extends Component try { $this->importRunning = true; $source = Storage::exists("upload/{$this->resourceUuid}/restore") - ? new DatabaseImportSource('upload', dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting, keepOwners: $this->keepOwners, sqliteDatabase: $this->sqliteDatabase) - : new DatabaseImportSource('server', path: $this->customLocation, dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting, keepOwners: $this->keepOwners, sqliteDatabase: $this->sqliteDatabase); + ? new DatabaseImportSource('upload', dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting, keepOwners: $this->keepOwners, sqliteDatabase: $this->sqliteDatabase, restoreMysqlUsers: $this->restoreMysqlUsers) + : new DatabaseImportSource('server', path: $this->customLocation, dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting, keepOwners: $this->keepOwners, sqliteDatabase: $this->sqliteDatabase, restoreMysqlUsers: $this->restoreMysqlUsers); $activity = StartDatabaseImport::run($this->resource, $source, (int) currentTeam()->id); $this->activityId = $activity->id; $this->dispatch('activityMonitor', $activity->id); @@ -614,7 +624,7 @@ class ImportForm extends Component try { $this->importRunning = true; - $source = new DatabaseImportSource('s3', path: $this->s3Path, s3StorageUuid: (string) $this->s3StorageId, dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting, keepOwners: $this->keepOwners, sqliteDatabase: $this->sqliteDatabase); + $source = new DatabaseImportSource('s3', path: $this->s3Path, s3StorageUuid: (string) $this->s3StorageId, dumpAll: $this->dumpAll, replaceExisting: $this->replaceExisting, keepOwners: $this->keepOwners, sqliteDatabase: $this->sqliteDatabase, restoreMysqlUsers: $this->restoreMysqlUsers); $activity = StartDatabaseImport::run($this->resource, $source, (int) currentTeam()->id); $this->activityId = $activity->id; $this->dispatch('activityMonitor', $activity->id); @@ -643,6 +653,6 @@ class ImportForm extends Component public function buildRestoreCommand(string $tmpPath): string { - return app(DatabaseImportCommandBuilder::class)->buildRestoreCommand($this->resource, $tmpPath, $this->dumpAll, $this->replaceExisting, $this->keepOwners, $this->sqliteDatabase); + return app(DatabaseImportCommandBuilder::class)->buildRestoreCommand($this->resource, $tmpPath, $this->dumpAll, $this->replaceExisting, $this->keepOwners, $this->sqliteDatabase, $this->restoreMysqlUsers); } } diff --git a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php index 5b5260c6ce..c8c32a80f8 100644 --- a/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php +++ b/app/Support/DatabaseImport/DatabaseImportCommandBuilder.php @@ -25,17 +25,45 @@ use_single_tar_member() { [ "$(find "$work" -type f | wc -l | tr -d ' ')" = 1 ] SH; + /** + * Streams an all-databases MySQL or MariaDB dump without its system databases. The mysql + * database holds the users and their passwords (user and global_priv), so restoring it + * would give the target the source passwords after its next restart. Everything from a + * switch to a system database (`-- Current Database:` or USE) up to the next switch to + * another database is dropped, as are the CREATE DATABASE statements of system databases. + */ + private const MYSQL_SYSTEM_DATABASE_FILTER = <<<'AWK' +function database(line) { + if (index(line, "`") > 0) { sub(/^[^`]*`/, "", line); sub(/`.*$/, "", line); return tolower(line) } + gsub(/\/\*[^*]*\*\//, " ", line) + line = tolower(line) + sub(/^[ \t]*(use|create[ \t]+(database|schema))[ \t]+/, "", line) + sub(/^if[ \t]+not[ \t]+exists[ \t]+/, "", line) + sub(/[ \t;].*$/, "", line) + return line +} +function is_system(name) { return name == "mysql" || name == "sys" || name == "performance_schema" || name == "information_schema" } +/^-- Current Database: `/ { skip = is_system(database($0)) } +tolower($0) ~ /^[ \t]*use[ \t]/ { skip = is_system(database($0)) } +tolower($0) ~ /^[ \t]*create[ \t]+(database|schema)[ \t]/ { if (is_system(database($0))) { next } skip = 0 } +/^\/\*!40103 SET TIME_ZONE=@OLD_TIME_ZONE/ { skip = 0 } +!skip { print } +AWK; + /** * @param bool $keepOwners PostgreSQL archives: restore object owners and privileges. Off by default, * because roles from another host (for example RDS) usually do not exist here. * @param string|null $sqliteDatabase SQLite: the database file to restore into, validated by the resource. + * @param bool $restoreMysqlUsers MySQL and MariaDB all-databases backups: also restore the system databases + * (users, passwords and privileges). Off by default, because the source + * passwords would replace the ones Coolify stores for this database. */ - public function buildRestoreCommand(object $resource, string $path, bool $dumpAll, bool $replaceExisting = false, bool $keepOwners = false, ?string $sqliteDatabase = null): string + public function buildRestoreCommand(object $resource, string $path, bool $dumpAll, bool $replaceExisting = false, bool $keepOwners = false, ?string $sqliteDatabase = null, bool $restoreMysqlUsers = false): string { $script = match ($this->databaseType($resource)) { 'postgresql' => $dumpAll ? $this->postgresqlDumpAll() : $this->postgresqlSingle($replaceExisting, $keepOwners), - 'mysql' => $this->mysql('mysql', 'MYSQL', $dumpAll), - 'mariadb' => $this->mysql('mariadb', 'MARIADB', $dumpAll), + 'mysql' => $this->mysql('mysql', 'MYSQL', $dumpAll, $restoreMysqlUsers), + 'mariadb' => $this->mysql('mariadb', 'MARIADB', $dumpAll, $restoreMysqlUsers), 'mongodb' => $this->mongodb($replaceExisting), 'sqlite' => $this->sqlite($resource->databaseFilePath($sqliteDatabase)), default => throw new InvalidArgumentException('Database import is not supported for this database type.'), @@ -225,9 +253,10 @@ SH; /** * MySQL and MariaDB restore SQL dumps. A tar backup must wrap exactly one dump. - * The all-databases mode checks the backup before it drops any database. + * The all-databases mode checks the backup before it drops any database and skips + * the system databases unless their users and privileges are restored as well. */ - private function mysql(string $binary, string $prefix, bool $dumpAll): string + private function mysql(string $binary, string $prefix, bool $dumpAll, bool $restoreMysqlUsers = false): string { $preflight = <<<'SH' if is_tar; then @@ -250,11 +279,20 @@ SH; $root = "{$binary} -u root -p\${{$prefix}_ROOT_PASSWORD}"; $database = "\${{$prefix}_DATABASE:-default}"; + if ($restoreMysqlUsers) { + $source = 'stream'; + } else { + // Without pipefail a failing filter would feed an empty restore, so it ends the + // input with an invalid statement and the client fails instead. + $preflight .= "command -v awk >/dev/null 2>&1 || fail 'awk is required to skip the system databases of the backup. Nothing was changed.'\n"; + $source = '{ stream | awk '.escapeshellarg(self::MYSQL_SYSTEM_DATABASE_FILTER)." || echo 'Coolify could not filter the system databases of the backup;'; }"; + } + return $preflight.<</dev/null || true; done {$root} -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \\`',schema_name,'\\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | {$root} || exit 1 {$root} -e "CREATE DATABASE IF NOT EXISTS \\`{$database}\\`;" || exit 1 -stream | {$root} {$database} +{$source} | {$root} {$database} SH; } diff --git a/app/Support/DatabaseImport/DatabaseImportSource.php b/app/Support/DatabaseImport/DatabaseImportSource.php index 445baee03e..e09ba471c6 100644 --- a/app/Support/DatabaseImport/DatabaseImportSource.php +++ b/app/Support/DatabaseImport/DatabaseImportSource.php @@ -15,6 +15,7 @@ readonly class DatabaseImportSource public bool $replaceExisting = false, public bool $keepOwners = false, public ?string $sqliteDatabase = null, + public bool $restoreMysqlUsers = false, ) { if (! in_array($type, ['upload', 's3', 'server'], true)) { throw new InvalidArgumentException('Invalid database import source.'); diff --git a/openapi.json b/openapi.json index dc85c20b75..8d2aefe1ad 100644 --- a/openapi.json +++ b/openapi.json @@ -22441,6 +22441,11 @@ "type": "boolean", "default": false }, + "restore_mysql_users": { + "description": "MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.", + "type": "boolean", + "default": false + }, "sqlite_database": { "description": "SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.", "type": "string", @@ -22483,6 +22488,11 @@ "type": "boolean", "default": false }, + "restore_mysql_users": { + "description": "MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.", + "type": "boolean", + "default": false + }, "sqlite_database": { "description": "SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.", "type": "string", @@ -22522,6 +22532,11 @@ "type": "boolean", "default": false }, + "restore_mysql_users": { + "description": "MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.", + "type": "boolean", + "default": false + }, "sqlite_database": { "description": "SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.", "type": "string", diff --git a/openapi.yaml b/openapi.yaml index 24b50d77bd..fd86ff91b1 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -14237,6 +14237,10 @@ components: description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.' type: boolean default: false + restore_mysql_users: + description: 'MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.' + type: boolean + default: false sqlite_database: description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.' type: string @@ -14268,6 +14272,10 @@ components: description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.' type: boolean default: false + restore_mysql_users: + description: 'MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.' + type: boolean + default: false sqlite_database: description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.' type: string @@ -14297,6 +14305,10 @@ components: description: 'Restore object owners and privileges from a single-database PostgreSQL archive. By default they are skipped, because the roles of another server usually do not exist.' type: boolean default: false + restore_mysql_users: + description: 'MySQL and MariaDB backups containing all databases: also restore the mysql system database (users, passwords, and privileges). By default the system databases are skipped, so the database keeps its own users and passwords.' + type: boolean + default: false sqlite_database: description: 'SQLite: the database file to restore into. Must be one of the database files. Defaults to the file named in the backup file name, else the first file.' type: string diff --git a/resources/views/livewire/project/database/import-form.blade.php b/resources/views/livewire/project/database/import-form.blade.php index 399b5f9cb6..0462b81f0c 100644 --- a/resources/views/livewire/project/database/import-form.blade.php +++ b/resources/views/livewire/project/database/import-form.blade.php @@ -86,6 +86,19 @@ canGate="update" :canResource="$this->resource" /> @endif + @if (in_array($resourceDbType, ['standalone-mysql', 'standalone-mariadb', 'mysql', 'mariadb'], true) && $dumpAll) +
+ +
+ @if ($restoreMysqlUsers) + + The backup replaces all users, passwords, and privileges, including the root password. After the next restart of the database, the passwords from the backup apply, so the credentials Coolify stores for this database, and its health check, may stop working. + Update the passwords in Coolify's database configuration after the restore. + + @endif + @endif @if ($resourceDbType === 'standalone-sqlite' && count($this->sqliteDatabaseFiles) > 0)
$this->destination->id, 'destination_type' => $this->destination->getMorphClass(), ]); + $this->mariadb = create_standalone_mariadb($this->environment->id, $this->destination); + $this->mariadb->update(['status' => 'running:healthy']); Queue::fake(); }); @@ -152,6 +154,33 @@ test('the import form strips owners by default and keeps them when the option is $form->set('filename', 'backup.dump')->set('customLocation', '/backups/backup.dump')->call('runImport'); }); +test('passes the restore users option to the MariaDB all-databases restore script', function (bool $restoreMysqlUsers) { + $script = importRestoreScript($this->mariadb, new DatabaseImportSource('server', path: '/backups/all.sql.gz', dumpAll: true, restoreMysqlUsers: $restoreMysqlUsers)); + + expect(str_contains($script, 'awk'))->toBe(! $restoreMysqlUsers); +})->with(['default' => [false], 'restore users' => [true]]); + +test('the import form skips MySQL users by default and restores them when the option is checked', function () { + $activity = Activity::create(['log_name' => 'default', 'description' => 'queued', 'properties' => ['status' => 'queued']]); + StartDatabaseImport::shouldRun()->once()->withArgs(fn ($resource, DatabaseImportSource $source) => $source->dumpAll && $source->restoreMysqlUsers === true)->andReturn($activity); + + $form = restoreOptionsForm($this->mariadb) + ->assertSet('restoreMysqlUsers', false) + ->assertDontSee('Restore users and privileges'); + + $form->set('dumpAll', true)->assertSee('Restore users and privileges'); + expect($form->get('restoreCommandText'))->toContain('awk'); + + $form->set('restoreMysqlUsers', true)->assertSee('passwords'); + expect($form->get('restoreCommandText'))->not->toContain('awk'); + + $form->set('filename', 'all.sql.gz')->set('customLocation', '/backups/all.sql.gz')->call('runImport'); +}); + +test('the import form does not offer the restore users option for PostgreSQL', function () { + restoreOptionsForm($this->postgres)->set('dumpAll', true)->assertDontSee('Restore users and privileges'); +}); + test('the import form lets the user pick the SQLite database file to restore into', function () { $activity = Activity::create(['log_name' => 'default', 'description' => 'queued', 'properties' => ['status' => 'queued']]); StartDatabaseImport::shouldRun()->once()->withArgs(fn ($resource, DatabaseImportSource $source) => $source->sqliteDatabase === 'cache.db')->andReturn($activity); @@ -198,6 +227,27 @@ test('the API passes keep owners and the SQLite target to the import', function ->assertJsonValidationErrors('keep_owners'); }); +test('the API passes the restore users option to the import', function () { + app('auth')->forgetGuards(); + $headers = ['Authorization' => 'Bearer '.$this->user->createToken('imports', ['*'])->plainTextToken]; + $activity = Activity::create(['log_name' => 'default', 'description' => 'queued', 'properties' => ['status' => 'queued']]); + $action = Mockery::mock(StartDatabaseImport::class); + $action->shouldReceive('handle')->once()->withArgs(fn ($resource, DatabaseImportSource $source) => $resource->is($this->mariadb) && $source->restoreMysqlUsers === true)->andReturn($activity); + $action->shouldReceive('handle')->once()->withArgs(fn ($resource, DatabaseImportSource $source) => $resource->is($this->mariadb) && $source->restoreMysqlUsers === false)->andReturn($activity); + app()->instance(StartDatabaseImport::class, $action); + + $this->withHeaders($headers) + ->postJson("/api/v1/databases/{$this->mariadb->uuid}/imports", ['source' => 'server', 'path' => '/tmp/all.sql.gz', 'dump_all' => true, 'restore_mysql_users' => true]) + ->assertAccepted(); + $this->withHeaders($headers) + ->postJson("/api/v1/databases/{$this->mariadb->uuid}/imports", ['source' => 'server', 'path' => '/tmp/all.sql.gz', 'dump_all' => true]) + ->assertAccepted(); + $this->withHeaders($headers) + ->postJson("/api/v1/databases/{$this->mariadb->uuid}/imports", ['source' => 'server', 'path' => '/tmp/all.sql.gz', 'restore_mysql_users' => 'yes']) + ->assertUnprocessable() + ->assertJsonValidationErrors('restore_mysql_users'); +}); + test('the API rejects a SQLite target that is not a database file', function () { Process::fake(); // API requests authenticate with the token, not with the session user from beforeEach. diff --git a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php index 79e9198c04..9307689c47 100644 --- a/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php +++ b/tests/Unit/DatabaseImport/DatabaseImportCommandBuilderTest.php @@ -152,6 +152,33 @@ test('keeps PostgreSQL owners and privileges when requested', function () { ->not->toContain('--no-acl'); }); +test('skips the system databases of MySQL and MariaDB all-databases backups by default', function (string $class, string $binary) { + $command = (new DatabaseImportCommandBuilder)->buildRestoreCommand(importResource($class), '/tmp/backup.sql', true); + + expect($command)->toContain('command -v awk') + ->toMatch('/stream \| \{?\s*awk /') + ->toContain('"mysql"')->toContain('"sys"')->toContain('"performance_schema"')->toContain('"information_schema"') + ->toContain("| {$binary} -u root"); +})->with([ + 'mysql' => [StandaloneMysql::class, 'mysql'], + 'mariadb' => [StandaloneMariadb::class, 'mariadb'], +]); + +test('restores the system databases of MySQL and MariaDB all-databases backups when requested', function (string $class, string $binary) { + $command = (new DatabaseImportCommandBuilder)->buildRestoreCommand(importResource($class), '/tmp/backup.sql', true, restoreMysqlUsers: true); + + expect($command)->not->toContain('awk') + ->toContain("stream | {$binary} -u root"); +})->with([ + 'mysql' => [StandaloneMysql::class, 'mysql'], + 'mariadb' => [StandaloneMariadb::class, 'mariadb'], +]); + +test('restores single MySQL databases without the system database filter', function () { + expect((new DatabaseImportCommandBuilder)->buildRestoreCommand(importResource(StandaloneMysql::class), '/tmp/backup.sql', false)) + ->not->toContain('awk'); +}); + test('restores SQLite backups into the selected database file', function () { $resource = Mockery::mock(StandaloneSqlite::class); $resource->shouldReceive('getMorphClass')->andReturn(StandaloneSqlite::class); diff --git a/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php b/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php index f4c749ab7b..9a32fbd24e 100644 --- a/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php +++ b/tests/Unit/DatabaseImport/DatabaseImportRestoreScriptTest.php @@ -11,7 +11,7 @@ use Illuminate\Filesystem\Filesystem; * These tests execute the generated scripts with `sh` against stub database clients. * Every stub appends "name|arguments|first 5 bytes of stdin" to a log, so the tests * prove which clients ran, in which order, and that nothing runs before a backup - * format is rejected. + * format is rejected. The full stdin of every call is kept as well. */ function restoreScriptTempDir(): string { @@ -142,6 +142,8 @@ function restoreScriptFixture(string $name): string 'mysql-tar' => restoreScriptTar(['backup.sql' => $mysqlSql]), 'mysql-tar-two' => restoreScriptTar(['app.sql' => $mysqlSql, 'other.sql' => $mysqlSql]), 'mysql-cluster' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\nUSE `mysql`;\nINSERT INTO `user` VALUES ();\n", + 'mysql-all-databases' => mysqlAllDatabasesDump(), + 'mysql-all-databases-gz' => gzencode(mysqlAllDatabasesDump()), 'mysql-two-databases' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\nCREATE DATABASE `other`;\nUSE `other`;\nCREATE TABLE `notes` (`t` text);\n", 'mysql-one-database' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\n", 'mongo-archive' => $mongoArchive, @@ -154,6 +156,67 @@ function restoreScriptFixture(string $name): string }; } +/** + * An all-databases dump in the layout of mysqldump and mariadb-dump: application databases + * around the mysql and sys system databases, which hold the users and their passwords. + */ +function mysqlAllDatabasesDump(): string +{ + return <<<'SQL' +-- MariaDB dump 10.19 +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; + +-- +-- Current Database: `app` +-- + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `app` /*!40100 DEFAULT CHARACTER SET utf8mb4 */; + +USE `app`; +CREATE TABLE `items` (`id` int); +INSERT INTO `items` VALUES (1),(2); + +-- +-- Current Database: `mysql` +-- + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysql` /*!40100 DEFAULT CHARACTER SET utf8mb4 */; + +USE `mysql`; +DROP TABLE IF EXISTS `global_priv`; +INSERT INTO `global_priv` VALUES ('localhost','root','{"authentication_string":"*SOURCE"}'); +INSERT INTO `user` VALUES ('%','app_user','*SOURCE'); +DELIMITER ;; +CREATE PROCEDURE `system_proc`() BEGIN SELECT 1; END ;; +DELIMITER ; + +-- +-- Current Database: `shop` +-- + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `shop` /*!40100 DEFAULT CHARACTER SET utf8mb4 */; + +USE `shop`; +INSERT INTO `orders` VALUES (7); + +-- +-- Current Database: `sys` +-- + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `sys`; + +USE `sys`; +INSERT INTO `sys_config` VALUES ('source'); +USE performance_schema; +INSERT INTO `setup_actors` VALUES ('source'); +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +-- Dump completed + +SQL; +} + function restoreScriptResource(string $engine): object { $class = match ($engine) { @@ -170,9 +233,9 @@ function restoreScriptResource(string $engine): object } /** - * @return array{exit: int, stdout: string, stderr: string, calls: list, dir: string, backup: string, leftovers: list} + * @return array{exit: int, stdout: string, stderr: string, calls: list, dir: string, backup: string, leftovers: list} */ -function restoreScriptRun(string $engine, string $contents, bool $dumpAll = false, bool $replaceExisting = false, bool $keepOwners = false): array +function restoreScriptRun(string $engine, string $contents, bool $dumpAll = false, bool $replaceExisting = false, bool $keepOwners = false, bool $restoreMysqlUsers = false): array { $dir = restoreScriptTempDir(); @@ -180,15 +243,18 @@ function restoreScriptRun(string $engine, string $contents, bool $dumpAll = fals $log = $dir.'/calls.log'; file_put_contents($log, ''); mkdir($dir.'/bin'); + mkdir($dir.'/stdin'); foreach (['pg_restore', 'psql', 'dropdb', 'createdb', 'mysql', 'mariadb', 'mongorestore'] as $client) { $escapedLog = escapeshellarg($log); + $escapedStdin = escapeshellarg($dir.'/stdin'); file_put_contents($dir.'/bin/'.$client, <</dev/null + cat > {$escapedStdin}/\$call + header=\$(head -c 5 {$escapedStdin}/\$call) fi printf '%s|%s|%s\\n' '{$client}' "\$*" "\$header" >> {$escapedLog} exit 0 @@ -199,7 +265,7 @@ SH); $backup = $dir.'/backup file'; file_put_contents($backup, $contents); - $script = (new DatabaseImportCommandBuilder)->buildRestoreCommand(restoreScriptResource($engine), $backup, $dumpAll, $replaceExisting, $keepOwners); + $script = (new DatabaseImportCommandBuilder)->buildRestoreCommand(restoreScriptResource($engine), $backup, $dumpAll, $replaceExisting, $keepOwners, restoreMysqlUsers: $restoreMysqlUsers); $result = restoreScriptProcess(['sh', '-c', $script], $dir, [ 'PATH' => $dir.'/bin'.PATH_SEPARATOR.(getenv('PATH') ?: '/usr/bin:/bin'), @@ -219,11 +285,12 @@ SH); ]); $lines = array_values(array_filter(explode("\n", (string) file_get_contents($log)), fn (string $line): bool => $line !== '')); - $calls = array_map(function (string $line): array { + $calls = array_map(function (string $line, int $index) use ($dir): array { [$name, $args, $header] = array_pad(explode('|', $line, 3), 3, ''); + $stdin = $dir.'/stdin/'.$index; - return ['name' => $name, 'args' => $args, 'header' => $header]; - }, $lines); + return ['name' => $name, 'args' => $args, 'header' => $header, 'stdin' => is_file($stdin) ? (string) file_get_contents($stdin) : '']; + }, $lines, array_keys($lines)); return $result + [ 'calls' => $calls, @@ -392,6 +459,65 @@ test('restores MySQL and MariaDB backups containing all databases after checking 'SQL that uses the mysql schema' => 'mysql-cluster', ]); +test('skips the MySQL and MariaDB system databases of an all-databases backup by default', function (string $engine, string $fixture) { + $run = restoreScriptRun($engine, restoreScriptFixture($fixture), dumpAll: true); + + expect($run['exit'])->toBe(0, $run['stderr']); + $restore = end($run['calls']); + expect($restore['args'])->toBe('-u root -proot_pass app') + ->and($restore['stdin']) + ->toContain('INSERT INTO `items` VALUES (1),(2);') + ->toContain('CREATE DATABASE /*!32312 IF NOT EXISTS*/ `app`') + ->toContain('USE `shop`;') + ->toContain('INSERT INTO `orders` VALUES (7);') + ->toContain('/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;') + ->not->toContain('global_priv') + ->not->toContain('app_user') + ->not->toContain('*SOURCE') + ->not->toContain('system_proc') + ->not->toContain('`mysql`') + ->not->toContain('`sys`') + ->not->toContain('sys_config') + ->not->toContain('setup_actors'); +})->with(['mysql', 'mariadb'])->with([ + 'SQL' => 'mysql-all-databases', + 'gzip SQL' => 'mysql-all-databases-gz', +]); + +test('restores the MySQL and MariaDB system databases of an all-databases backup when requested', function (string $engine) { + $run = restoreScriptRun($engine, restoreScriptFixture('mysql-all-databases-gz'), dumpAll: true, restoreMysqlUsers: true); + + expect($run['exit'])->toBe(0, $run['stderr']) + ->and(end($run['calls'])['stdin'])->toBe(mysqlAllDatabasesDump()); +})->with(['mysql', 'mariadb']); + +test('fails an all-databases MySQL restore before dropping anything when awk is missing', function () { + $dir = restoreScriptTempDir(); + + try { + mkdir($dir.'/bin'); + foreach (['sh', 'head', 'od', 'tr', 'cat', 'gunzip', 'tail', 'wc', 'grep', 'mktemp', 'find'] as $tool) { + exec('command -v '.escapeshellarg($tool), $path); + if ($path !== []) { + symlink(end($path), $dir.'/bin/'.$tool); + } + $path = []; + } + file_put_contents($dir.'/bin/mysql', "#!/bin/sh\necho called >> ".escapeshellarg($dir.'/calls')."\n"); + chmod($dir.'/bin/mysql', 0755); + file_put_contents($dir.'/backup', mysqlAllDatabasesDump()); + + $script = (new DatabaseImportCommandBuilder)->buildRestoreCommand(restoreScriptResource('mysql'), $dir.'/backup', true); + $result = restoreScriptProcess([$dir.'/bin/sh', '-c', $script], $dir, ['PATH' => $dir.'/bin', 'MYSQL_ROOT_PASSWORD' => 'root_pass']); + + expect($result['exit'])->toBe(1) + ->and($result['stderr'])->toContain('Nothing was changed.') + ->and(file_exists($dir.'/calls'))->toBeFalse(); + } finally { + restoreScriptRemoveDir($dir); + } +}); + test('rejects unsupported MySQL and MariaDB backups before calling any client', function (string $engine, string $fixture, bool $dumpAll, string $message) { $run = restoreScriptRun($engine, restoreScriptFixture($fixture), $dumpAll); diff --git a/tests/Unit/DatabaseImportOpenApiTest.php b/tests/Unit/DatabaseImportOpenApiTest.php index bbc9e7705e..7b67e7ee3b 100644 --- a/tests/Unit/DatabaseImportOpenApiTest.php +++ b/tests/Unit/DatabaseImportOpenApiTest.php @@ -66,9 +66,9 @@ test('constrains additional properties on each database import source branch', f expect($schema)->not->toHaveKey('additionalProperties'); $expectedProperties = [ - ['source', 'upload_id', 'dump_all', 'replace_existing', 'keep_owners', 'sqlite_database'], - ['source', 's3_storage_uuid', 'path', 'dump_all', 'replace_existing', 'keep_owners', 'sqlite_database'], - ['source', 'path', 'dump_all', 'replace_existing', 'keep_owners', 'sqlite_database'], + ['source', 'upload_id', 'dump_all', 'replace_existing', 'keep_owners', 'restore_mysql_users', 'sqlite_database'], + ['source', 's3_storage_uuid', 'path', 'dump_all', 'replace_existing', 'keep_owners', 'restore_mysql_users', 'sqlite_database'], + ['source', 'path', 'dump_all', 'replace_existing', 'keep_owners', 'restore_mysql_users', 'sqlite_database'], ]; expect($schema['oneOf'])->toHaveCount(count($expectedProperties)); @@ -80,6 +80,8 @@ test('constrains additional properties on each database import source branch', f ->toMatchArray(['type' => 'boolean', 'default' => false]) ->and($source['properties']['keep_owners']) ->toMatchArray(['type' => 'boolean', 'default' => false]) + ->and($source['properties']['restore_mysql_users']) + ->toMatchArray(['type' => 'boolean', 'default' => false]) ->and($source['properties']['sqlite_database']) ->toMatchArray(['type' => 'string']); }