mirror of
https://github.com/coollabsio/coolify.git
synced 2026-09-27 09:45:58 -04:00
fix(database): restore every supported backup format safely
- Detect the backup format before any database is changed. "All databases" imports no longer drop everything when the file cannot be restored. - PostgreSQL: restore custom and tar archives with pg_restore and SQL with psql in both modes, including gzip; restore the #11481 custom-archive fix. Replacing existing data recreates the database for SQL backups. - MySQL/MariaDB: accept a tar with one dump; reject dumps with more than one database in single mode instead of restoring them partially. - MongoDB: restore plain and gzip archives and dump directories packed as tar; "Replace collections" maps to --drop. - Prepare bz2, xz, and zip backups in the helper image, because database images do not ship those tools; stream S3 backups from the S3 helper. - Show the exact restore script in the import form. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
df692e6278
commit
9e999a2782
@@ -96,9 +96,10 @@ class StartDatabaseImport
|
||||
throw new DatabaseImportException('The uploaded backup contains disallowed PostgreSQL restore directives.');
|
||||
}
|
||||
$serverPath = "/tmp/database-import-{$operation}";
|
||||
$magic = bin2hex((string) file_get_contents($local, length: 6));
|
||||
instant_scp($local, $serverPath, $server);
|
||||
$source->uploadId ? Storage::deleteDirectory(dirname($staged)) : Storage::delete($staged);
|
||||
$commandList[] = 'docker cp '.escapeshellarg($serverPath).' '.escapeshellarg("{$container}:{$containerPath}");
|
||||
$commandList = [...$commandList, ...$this->copyIntoContainer($serverPath, $magic, $server, $operation, $container, $containerPath, $cleanup)];
|
||||
$commandList[] = 'rm -f '.escapeshellarg($serverPath);
|
||||
$cleanup['serverTmpPath'] = $serverPath;
|
||||
} elseif ($source->type === 'server') {
|
||||
@@ -107,7 +108,8 @@ class StartDatabaseImport
|
||||
if ($size < 1 || $size > self::MAX_BYTES) {
|
||||
throw new DatabaseImportException('The backup file is empty or exceeds the 10 GiB limit.');
|
||||
}
|
||||
$commandList[] = 'docker cp '.escapeshellarg($source->path).' '.escapeshellarg("{$container}:{$containerPath}");
|
||||
$magic = (string) instant_remote_process(['head -c 6 -- '.escapeshellarg($source->path).' | od -An -tx1'], $server);
|
||||
$commandList = [...$commandList, ...$this->copyIntoContainer($source->path, $magic, $server, $operation, $container, $containerPath, $cleanup)];
|
||||
} else {
|
||||
$storage = S3Storage::ownedByCurrentTeamAPI($teamId)
|
||||
->where(fn ($query) => $query->whereUuid($source->s3StorageUuid)->orWhere('id', ctype_digit((string) $source->s3StorageUuid) ? (int) $source->s3StorageUuid : -1))
|
||||
@@ -122,18 +124,15 @@ class StartDatabaseImport
|
||||
throw new DatabaseImportException('The S3 backup was not found or exceeds the 10 GiB limit.');
|
||||
}
|
||||
$helper = "s3-restore-{$operation}";
|
||||
$serverPath = "/tmp/s3-restore-{$operation}";
|
||||
$this->startS3HelperWithEnv($storage, $server, $helper, $network);
|
||||
$sourceArg = escapeshellarg("s3temp/{$storage->bucket}/{$key}");
|
||||
$commandList = [
|
||||
'docker exec '.escapeshellarg($helper).' sh -c '.escapeshellarg('mc alias set s3temp "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"'),
|
||||
'docker exec '.escapeshellarg($helper).' mc cp '.$sourceArg.' /tmp/restore',
|
||||
'docker cp '.escapeshellarg("{$helper}:/tmp/restore").' '.escapeshellarg($serverPath),
|
||||
'docker cp '.escapeshellarg($serverPath).' '.escapeshellarg("{$container}:{$containerPath}"),
|
||||
...$this->streamFromHelper($helper, $container, $containerPath),
|
||||
'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
|
||||
'rm -f '.escapeshellarg($serverPath),
|
||||
];
|
||||
$cleanup += ['containerName' => $helper, 'serverTmpPath' => $serverPath];
|
||||
$cleanup['containerName'] = $helper;
|
||||
}
|
||||
|
||||
if ($safety = $this->commands->buildPostgresSafetyCommand($resource, $container, $containerPath)) {
|
||||
@@ -176,6 +175,59 @@ class StartDatabaseImport
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a backup into the database container. bz2, xz and zip backups go through
|
||||
* the Coolify helper image, because database images do not ship those tools.
|
||||
*
|
||||
* @param array<string, mixed> $cleanup
|
||||
* @return list<string>
|
||||
*/
|
||||
private function copyIntoContainer(string $path, string $magic, Server $server, string $operation, string $container, string $containerPath, array &$cleanup): array
|
||||
{
|
||||
$magic = strtolower((string) preg_replace('/[^0-9a-f]/i', '', $magic));
|
||||
if (! preg_match('/^(425a68|fd377a585a00|504b0304)/', $magic)) {
|
||||
return ['docker cp '.escapeshellarg($path).' '.escapeshellarg("{$container}:{$containerPath}")];
|
||||
}
|
||||
|
||||
$helper = "backup-decompress-{$operation}";
|
||||
$image = escapeshellarg(coolifyHelperImage().':'.getHelperVersion());
|
||||
|
||||
try {
|
||||
instant_remote_process([
|
||||
'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
|
||||
'docker run -d --network none --name '.escapeshellarg($helper).' '.$image.' sleep 86400',
|
||||
], $server);
|
||||
} catch (Throwable) {
|
||||
instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false);
|
||||
|
||||
throw new DatabaseImportException('Unable to start the backup decompression helper.');
|
||||
}
|
||||
$cleanup['containerName'] = $helper;
|
||||
|
||||
return [
|
||||
'docker cp '.escapeshellarg($path).' '.escapeshellarg("{$helper}:/tmp/restore"),
|
||||
...$this->streamFromHelper($helper, $container, $containerPath),
|
||||
'docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares /tmp/restore in a helper container, then streams it into the database
|
||||
* container. Preparing first keeps decompression failures from being lost in a pipe.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function streamFromHelper(string $helper, string $container, string $containerPath): array
|
||||
{
|
||||
$target = escapeshellarg($containerPath);
|
||||
$writer = "cat > {$target} && [ -s {$target} ] || { echo 'The backup is empty or could not be read.' >&2; exit 1; }";
|
||||
|
||||
return [
|
||||
'docker exec '.escapeshellarg($helper).' sh -c '.escapeshellarg($this->commands->buildNormalizeScript('/tmp/restore', '/tmp/restore.prepared')),
|
||||
'docker exec '.escapeshellarg($helper).' cat /tmp/restore.prepared | docker exec -i '.escapeshellarg($container).' sh -c '.escapeshellarg($writer),
|
||||
];
|
||||
}
|
||||
|
||||
private function startS3HelperWithEnv(S3Storage $storage, Server $server, string $helper, string $network): void
|
||||
{
|
||||
$image = escapeshellarg(coolifyHelperImage().':'.getHelperVersion());
|
||||
@@ -188,7 +240,7 @@ class StartDatabaseImport
|
||||
.' -e S3_ENDPOINT='.escapeshellarg((string) $storage->endpoint)
|
||||
.' -e S3_ACCESS_KEY='.escapeshellarg((string) $storage->key)
|
||||
.' -e S3_SECRET_KEY='.escapeshellarg((string) $storage->secret)
|
||||
.' '.$image.' sleep 3600',
|
||||
.' '.$image.' sleep 86400',
|
||||
], $server);
|
||||
} catch (Throwable) {
|
||||
instant_remote_process(['docker rm -f '.escapeshellarg($helper).' 2>/dev/null || true'], $server, throwError: false);
|
||||
|
||||
@@ -10,9 +10,6 @@ use App\Models\ServiceDatabase;
|
||||
use App\Models\StandaloneClickhouse;
|
||||
use App\Models\StandaloneDragonfly;
|
||||
use App\Models\StandaloneKeydb;
|
||||
use App\Models\StandaloneMariadb;
|
||||
use App\Models\StandaloneMysql;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\StandaloneRedis;
|
||||
use App\Rules\SafeWebhookUrl;
|
||||
use App\Support\DatabaseImport\DatabaseImportCommandBuilder;
|
||||
@@ -168,14 +165,6 @@ class ImportForm extends Component
|
||||
|
||||
public ?int $activityId = null;
|
||||
|
||||
public string $postgresqlRestoreCommand = 'pg_restore --exit-on-error -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}';
|
||||
|
||||
public string $mysqlRestoreCommand = 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE';
|
||||
|
||||
public string $mariadbRestoreCommand = 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE';
|
||||
|
||||
public string $mongodbRestoreCommand = 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive=';
|
||||
|
||||
// S3 Restore properties
|
||||
public array $availableS3Storages = [];
|
||||
|
||||
@@ -219,83 +208,29 @@ class ImportForm extends Component
|
||||
$this->parameters = get_route_parameters();
|
||||
$this->getContainers();
|
||||
$this->loadAvailableS3Storages();
|
||||
$this->refreshRestoreCommandText();
|
||||
}
|
||||
|
||||
public function updatedDumpAll($value)
|
||||
public function updatedDumpAll(): void
|
||||
{
|
||||
$morphClass = $this->resource->getMorphClass();
|
||||
|
||||
// Handle ServiceDatabase by checking the database type
|
||||
if ($morphClass === ServiceDatabase::class) {
|
||||
$dbType = $this->resource->databaseType();
|
||||
if (str_contains($dbType, 'mysql')) {
|
||||
$morphClass = 'mysql';
|
||||
} elseif (str_contains($dbType, 'mariadb')) {
|
||||
$morphClass = 'mariadb';
|
||||
} elseif (str_contains($dbType, 'postgres')) {
|
||||
$morphClass = 'postgresql';
|
||||
}
|
||||
}
|
||||
|
||||
switch ($morphClass) {
|
||||
case StandaloneMariadb::class:
|
||||
case 'mariadb':
|
||||
if ($value === true) {
|
||||
$this->mariadbRestoreCommand = <<<'EOD'
|
||||
for pid in $(mariadb -u root -p$MARIADB_ROOT_PASSWORD -N -e "SELECT id FROM information_schema.processlist WHERE user != 'root';"); do
|
||||
mariadb -u root -p$MARIADB_ROOT_PASSWORD -e "KILL $pid" 2>/dev/null || true
|
||||
done && \
|
||||
mariadb -u root -p$MARIADB_ROOT_PASSWORD -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`',schema_name,'\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | mariadb -u root -p$MARIADB_ROOT_PASSWORD && \
|
||||
mariadb -u root -p$MARIADB_ROOT_PASSWORD -e "CREATE DATABASE IF NOT EXISTS \`${MARIADB_DATABASE:-default}\`;" && \
|
||||
(gunzip -cf $tmpPath 2>/dev/null || cat $tmpPath) | sed -e '/^CREATE DATABASE/d' -e '/^USE \`mysql\`/d' | mariadb -u root -p$MARIADB_ROOT_PASSWORD ${MARIADB_DATABASE:-default}
|
||||
EOD;
|
||||
$this->restoreCommandText = $this->mariadbRestoreCommand.' && (gunzip -cf <temp_backup_file> 2>/dev/null || cat <temp_backup_file>) | mariadb -u root -p$MARIADB_ROOT_PASSWORD ${MARIADB_DATABASE:-default}';
|
||||
} else {
|
||||
$this->mariadbRestoreCommand = 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE';
|
||||
}
|
||||
break;
|
||||
case StandaloneMysql::class:
|
||||
case 'mysql':
|
||||
if ($value === true) {
|
||||
$this->mysqlRestoreCommand = <<<'EOD'
|
||||
for pid in $(mysql -u root -p$MYSQL_ROOT_PASSWORD -N -e "SELECT id FROM information_schema.processlist WHERE user != 'root';"); do
|
||||
mysql -u root -p$MYSQL_ROOT_PASSWORD -e "KILL $pid" 2>/dev/null || true
|
||||
done && \
|
||||
mysql -u root -p$MYSQL_ROOT_PASSWORD -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`',schema_name,'\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | mysql -u root -p$MYSQL_ROOT_PASSWORD && \
|
||||
mysql -u root -p$MYSQL_ROOT_PASSWORD -e "CREATE DATABASE IF NOT EXISTS \`${MYSQL_DATABASE:-default}\`;" && \
|
||||
(gunzip -cf $tmpPath 2>/dev/null || cat $tmpPath) | sed -e '/^CREATE DATABASE/d' -e '/^USE \`mysql\`/d' | mysql -u root -p$MYSQL_ROOT_PASSWORD ${MYSQL_DATABASE:-default}
|
||||
EOD;
|
||||
$this->restoreCommandText = $this->mysqlRestoreCommand.' && (gunzip -cf <temp_backup_file> 2>/dev/null || cat <temp_backup_file>) | mysql -u root -p$MYSQL_ROOT_PASSWORD ${MYSQL_DATABASE:-default}';
|
||||
} else {
|
||||
$this->mysqlRestoreCommand = 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE';
|
||||
}
|
||||
break;
|
||||
case StandalonePostgresql::class:
|
||||
case 'postgresql':
|
||||
if ($value === true) {
|
||||
$this->postgresqlRestoreCommand = <<<'EOD'
|
||||
psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && \
|
||||
psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} && \
|
||||
createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}
|
||||
EOD;
|
||||
$this->restoreCommandText = $this->postgresqlRestoreCommand.' && (gunzip -cf <temp_backup_file> 2>/dev/null || cat <temp_backup_file>) | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}';
|
||||
} else {
|
||||
$this->syncPostgresqlRestoreCommand();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
$this->refreshRestoreCommandText();
|
||||
}
|
||||
|
||||
public function updatedReplaceExisting(): void
|
||||
{
|
||||
$this->syncPostgresqlRestoreCommand();
|
||||
$this->refreshRestoreCommandText();
|
||||
}
|
||||
|
||||
private function syncPostgresqlRestoreCommand(): void
|
||||
/**
|
||||
* Shows the exact script the import runs, so the confirmation matches the restore.
|
||||
*/
|
||||
private function refreshRestoreCommandText(): void
|
||||
{
|
||||
$replaceExisting = $this->replaceExisting ? ' --clean --if-exists' : '';
|
||||
$this->postgresqlRestoreCommand = 'pg_restore --exit-on-error'.$replaceExisting.' -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}';
|
||||
$commands = app(DatabaseImportCommandBuilder::class);
|
||||
|
||||
$this->restoreCommandText = $this->resource && $commands->supports($this->resource)
|
||||
? $commands->buildRestoreCommand($this->resource, '<temp_backup_file>', $this->dumpAll, $this->replaceExisting)
|
||||
: '';
|
||||
}
|
||||
|
||||
public function getContainers()
|
||||
|
||||
@@ -7,23 +7,60 @@ use InvalidArgumentException;
|
||||
|
||||
class DatabaseImportCommandBuilder
|
||||
{
|
||||
/**
|
||||
* Shell helpers shared by every restore script. The backup inside the database
|
||||
* container is plain or gzip-compressed (bz2, xz and zip are decompressed by
|
||||
* {@see self::buildNormalizeScript()} before it arrives). Every script decides the
|
||||
* backup format before it changes anything in the database.
|
||||
*/
|
||||
private const PRELUDE = <<<'SH'
|
||||
fail() { echo "$1" >&2; exit 1; }
|
||||
is_gzip() { [ "$(head -c 2 "$backup" | od -An -tx1 | tr -d ' \n')" = 1f8b ]; }
|
||||
stream() { if is_gzip; then gunzip -c "$backup"; else cat "$backup"; fi; }
|
||||
header() { stream | head -c "$1" | od -An -tx1 | tr -d ' \n'; }
|
||||
is_tar() { [ "$(stream | head -c 262 | tail -c 5)" = ustar ]; }
|
||||
is_text() { [ "$(stream | head -c 65536 | tr -d '\000' | wc -c | tr -d ' ')" = "$(stream | head -c 65536 | wc -c | tr -d ' ')" ]; }
|
||||
extract_tar() { work=$(mktemp -d) && trap 'rm -rf "$work"' EXIT && stream | tar -xf - -C "$work" || fail 'The tar backup cannot be extracted.'; }
|
||||
use_single_tar_member() { [ "$(find "$work" -type f | wc -l | tr -d ' ')" = 1 ] && backup=$(find "$work" -type f); }
|
||||
|
||||
SH;
|
||||
|
||||
public function buildRestoreCommand(object $resource, string $path, bool $dumpAll, bool $replaceExisting = false): string
|
||||
{
|
||||
$path = escapeshellarg($path);
|
||||
|
||||
return match ($this->databaseType($resource)) {
|
||||
'postgresql' => $dumpAll
|
||||
? 'psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} && createdb -U ${POSTGRES_USER} ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} && (gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}'
|
||||
: 'pg_restore --exit-on-error'.($replaceExisting ? ' --clean --if-exists' : '').' -U $POSTGRES_USER -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$path,
|
||||
'mysql' => $dumpAll
|
||||
? $this->mysqlDumpAll('mysql', 'MYSQL', $path)
|
||||
: '(gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE',
|
||||
'mariadb' => $dumpAll
|
||||
? $this->mysqlDumpAll('mariadb', 'MARIADB', $path)
|
||||
: '(gunzip -cf '.$path.' 2>/dev/null || cat '.$path.') | mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE',
|
||||
'mongodb' => 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive='.$path,
|
||||
$script = match ($this->databaseType($resource)) {
|
||||
'postgresql' => $dumpAll ? $this->postgresqlDumpAll() : $this->postgresqlSingle($replaceExisting),
|
||||
'mysql' => $this->mysql('mysql', 'MYSQL', $dumpAll),
|
||||
'mariadb' => $this->mysql('mariadb', 'MARIADB', $dumpAll),
|
||||
'mongodb' => $this->mongodb($replaceExisting),
|
||||
default => throw new InvalidArgumentException('Database import is not supported for this database type.'),
|
||||
};
|
||||
|
||||
return 'backup='.escapeshellarg($path)."\n".self::PRELUDE.$script;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares a backup inside the Coolify helper image. bz2, xz and single-file zip
|
||||
* backups are decompressed because database images do not ship those tools;
|
||||
* plain and gzip backups are moved unchanged. Exits non-zero on any failure.
|
||||
*/
|
||||
public function buildNormalizeScript(string $source, string $target): string
|
||||
{
|
||||
$source = escapeshellarg($source);
|
||||
$target = escapeshellarg($target);
|
||||
|
||||
return <<<SH
|
||||
f={$source}
|
||||
out={$target}
|
||||
case "\$(head -c 6 "\$f" | od -An -tx1 | tr -d ' \\n')" in
|
||||
425a68*) bunzip2 -c "\$f" > "\$out" || { echo 'The bz2 backup cannot be decompressed.' >&2; exit 1; } ;;
|
||||
fd377a585a00) unxz -c "\$f" > "\$out" || { echo 'The xz backup cannot be decompressed.' >&2; exit 1; } ;;
|
||||
504b0304*)
|
||||
d=\$(mktemp -d) && unzip -q "\$f" -d "\$d" || { echo 'The zip backup cannot be extracted.' >&2; exit 1; }
|
||||
[ "\$(find "\$d" -type f | wc -l | tr -d ' ')" = 1 ] || { echo 'A zip backup must contain exactly one file.' >&2; exit 1; }
|
||||
mv "\$(find "\$d" -type f)" "\$out" ;;
|
||||
*) mv "\$f" "\$out" ;;
|
||||
esac
|
||||
SH;
|
||||
}
|
||||
|
||||
public function buildPostgresRestoreScanScript(object $resource, string $path): ?string
|
||||
@@ -51,7 +88,8 @@ class DatabaseImportCommandBuilder
|
||||
|
||||
return <<<SH
|
||||
header=\$({$contents} | head -c 5)
|
||||
if [ "\$header" = 'PGDMP' ]; then
|
||||
tar_magic=\$({$contents} | head -c 262 | tail -c 5)
|
||||
if [ "\$header" = 'PGDMP' ] || [ "\$tar_magic" = 'ustar' ]; then
|
||||
inspect=\$(mktemp)
|
||||
trap 'rm -f "\$inspect"' EXIT
|
||||
if ! {$contents} > "\$inspect"; then
|
||||
@@ -101,11 +139,120 @@ SH;
|
||||
};
|
||||
}
|
||||
|
||||
private function mysqlDumpAll(string $binary, string $prefix, string $path): string
|
||||
/**
|
||||
* pg_dump custom and tar archives are restored with pg_restore, SQL dumps with psql.
|
||||
* pg_restore cannot read gzip files, so both clients receive the backup on stdin.
|
||||
* SQL cannot replace single objects, so replacing recreates the target database.
|
||||
*/
|
||||
private function postgresqlSingle(bool $replaceExisting): string
|
||||
{
|
||||
$rootPassword = '${'.$prefix.'_ROOT_PASSWORD}';
|
||||
$database = '${'.$prefix.'_DATABASE:-default}';
|
||||
$clean = $replaceExisting ? ' --clean --if-exists' : '';
|
||||
$sqlNotice = $replaceExisting ? <<<'SH'
|
||||
|
||||
return "for pid in \$({$binary} -u root -p{$rootPassword} -N -e \"SELECT id FROM information_schema.processlist WHERE user != 'root';\"); do {$binary} -u root -p{$rootPassword} -e \"KILL \$pid\" 2>/dev/null || true; done && {$binary} -u root -p{$rootPassword} -N -e \"SELECT CONCAT('DROP DATABASE IF EXISTS \\`',schema_name,'\\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');\" | {$binary} -u root -p{$rootPassword} && {$binary} -u root -p{$rootPassword} -e \"CREATE DATABASE IF NOT EXISTS \\`{$database}\\`;\" && (gunzip -cf {$path} 2>/dev/null || cat {$path}) | {$binary} -u root -p{$rootPassword} {$database}";
|
||||
echo 'SQL backups cannot replace single objects. The database is recreated before the restore.'
|
||||
echo "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db' AND pid <> pg_backend_pid();" | psql -v db="$db" -U $POSTGRES_USER -d template1 >/dev/null || exit 1
|
||||
dropdb --maintenance-db=template1 -U $POSTGRES_USER --if-exists "$db" || exit 1
|
||||
createdb -U $POSTGRES_USER "$db" || exit 1
|
||||
SH : '';
|
||||
|
||||
return <<<SH
|
||||
db=\${POSTGRES_DB:-\${POSTGRES_USER:-postgres}}
|
||||
if [ "\$(stream | head -c 5)" = PGDMP ] || is_tar; then
|
||||
stream | pg_restore --exit-on-error{$clean} -U \$POSTGRES_USER -d "\$db"
|
||||
elif ! is_text; then
|
||||
fail 'Unsupported PostgreSQL backup format. Use a pg_dump archive (custom or tar format) or an SQL file.'
|
||||
elif stream | head -c 4096 | grep -q 'PostgreSQL database cluster dump'; then
|
||||
fail 'This backup contains all databases. Select "Backup contains all databases" to restore it.'
|
||||
else{$sqlNotice}
|
||||
stream | psql -v ON_ERROR_STOP=1 -U \$POSTGRES_USER -d "\$db"
|
||||
fi
|
||||
SH;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the backup format before any database is dropped, then recreates the
|
||||
* target database and restores archives with pg_restore and SQL with psql.
|
||||
*/
|
||||
private function postgresqlDumpAll(): string
|
||||
{
|
||||
return <<<'SH'
|
||||
db=${POSTGRES_DB:-${POSTGRES_USER:-postgres}}
|
||||
if [ "$(stream | head -c 5)" = PGDMP ] || is_tar; then
|
||||
kind=archive
|
||||
stream | pg_restore -l >/dev/null 2>&1 || fail 'pg_restore cannot read this backup archive. Nothing was changed.'
|
||||
elif is_text; then
|
||||
kind=sql
|
||||
else
|
||||
fail 'Unsupported PostgreSQL backup format. Use a pg_dump archive (custom or tar format) or an SQL file. Nothing was changed.'
|
||||
fi
|
||||
psql -U ${POSTGRES_USER} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" || exit 1
|
||||
psql -U ${POSTGRES_USER} -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U ${POSTGRES_USER} --if-exists {} || exit 1
|
||||
createdb -U ${POSTGRES_USER} "$db" || exit 1
|
||||
if [ "$kind" = archive ]; then
|
||||
stream | pg_restore -U ${POSTGRES_USER} -d "$db"
|
||||
else
|
||||
stream | psql -U ${POSTGRES_USER} -d "$db"
|
||||
fi
|
||||
SH;
|
||||
}
|
||||
|
||||
/**
|
||||
* MySQL and MariaDB restore SQL dumps. A tar backup must wrap exactly one dump.
|
||||
* The all-databases mode checks the backup before it drops any database.
|
||||
*/
|
||||
private function mysql(string $binary, string $prefix, bool $dumpAll): string
|
||||
{
|
||||
$preflight = <<<'SH'
|
||||
if is_tar; then
|
||||
extract_tar
|
||||
use_single_tar_member || fail 'A tar backup must contain exactly one SQL dump. Nothing was changed.'
|
||||
fi
|
||||
is_text || fail 'Unsupported backup format. Use an SQL dump. Nothing was changed.'
|
||||
|
||||
SH;
|
||||
|
||||
if (! $dumpAll) {
|
||||
return $preflight.<<<SH
|
||||
if [ "$(stream | grep -c '^USE `')" -gt 1 ]; then
|
||||
fail 'This backup contains more than one database. Select "Backup contains all databases" to restore it. Nothing was changed.'
|
||||
fi
|
||||
stream | {$binary} -u \${$prefix}_USER -p\${$prefix}_PASSWORD \${$prefix}_DATABASE
|
||||
SH;
|
||||
}
|
||||
|
||||
$root = "{$binary} -u root -p\${{$prefix}_ROOT_PASSWORD}";
|
||||
$database = "\${{$prefix}_DATABASE:-default}";
|
||||
|
||||
return $preflight.<<<SH
|
||||
for pid in \$({$root} -N -e "SELECT id FROM information_schema.processlist WHERE user != 'root';"); do {$root} -e "KILL \$pid" 2>/dev/null || true; done
|
||||
{$root} -N -e "SELECT CONCAT('DROP DATABASE IF EXISTS \\`',schema_name,'\\`;') FROM information_schema.schemata WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys');" | {$root} || exit 1
|
||||
{$root} -e "CREATE DATABASE IF NOT EXISTS \\`{$database}\\`;" || exit 1
|
||||
stream | {$root} {$database}
|
||||
SH;
|
||||
}
|
||||
|
||||
/**
|
||||
* MongoDB restores mongodump archives (plain or gzip) and dump directories packed
|
||||
* as tar. Replacing existing data drops each restored collection first.
|
||||
*/
|
||||
private function mongodb(bool $replaceExisting): string
|
||||
{
|
||||
$drop = $replaceExisting ? ' --drop' : '';
|
||||
|
||||
return <<<SH
|
||||
restore() { mongorestore --authenticationDatabase=admin --username \$MONGO_INITDB_ROOT_USERNAME --password \$MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017{$drop} "\$@"; }
|
||||
if is_tar; then
|
||||
extract_tar
|
||||
if ! use_single_tar_member; then
|
||||
bson=\$(find "\$work" -type f \\( -name '*.bson' -o -name '*.bson.gz' \\) | head -n 1)
|
||||
[ -n "\$bson" ] || fail 'The tar backup does not contain a mongodump directory. Nothing was changed.'
|
||||
root=\$(dirname "\$(dirname "\$bson")")
|
||||
if find "\$root" -type f -name '*.bson.gz' | grep -q .; then restore --gzip --dir="\$root"; else restore --dir="\$root"; fi
|
||||
exit \$?
|
||||
fi
|
||||
fi
|
||||
[ "\$(header 4)" = 6de29981 ] || fail 'Unsupported MongoDB backup format. Use a mongodump archive or a dump directory packed as tar. Single .bson files are not supported. Nothing was changed.'
|
||||
if is_gzip; then restore --gzip --archive="\$backup"; else restore --archive="\$backup"; fi
|
||||
SH;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,40 +55,16 @@
|
||||
<x-application.settings-section title="Restore configuration"
|
||||
description="Configure how the selected backup is applied to this database.">
|
||||
<div class="space-y-4">
|
||||
@if ($resourceDbType === 'standalone-postgresql')
|
||||
@if ($dumpAll)
|
||||
@if ($resourceDbType === 'standalone-postgresql' && $dumpAll)
|
||||
<x-callout type="warning" title="Full restore overwrites administrator passwords">
|
||||
The backup replaces PostgreSQL administrator role passwords, including the destination administrator password.
|
||||
<span class="mt-1 block">If the administrator password changes, update it in Coolify's database configuration after the restore.</span>
|
||||
</x-callout>
|
||||
<x-forms.textarea rows="6" readonly label="Import command"
|
||||
wire:model="restoreCommandText" canGate="update"
|
||||
:canResource="$this->resource" />
|
||||
@else
|
||||
<x-forms.input label="Import command" readonly
|
||||
helper="Enable replacement below to drop and recreate matching objects from the archive."
|
||||
wire:model="postgresqlRestoreCommand" canGate="update"
|
||||
:canResource="$this->resource" />
|
||||
@endif
|
||||
@elseif ($resourceDbType === 'standalone-mysql')
|
||||
@if ($dumpAll)
|
||||
<x-forms.textarea rows="10" readonly label="Import command"
|
||||
wire:model="restoreCommandText" canGate="update"
|
||||
:canResource="$this->resource" />
|
||||
@else
|
||||
<x-forms.input label="Import command" wire:model="mysqlRestoreCommand"
|
||||
canGate="update" :canResource="$this->resource" />
|
||||
@endif
|
||||
@elseif ($resourceDbType === 'standalone-mariadb')
|
||||
@if ($dumpAll)
|
||||
<x-forms.textarea rows="10" readonly label="Import command"
|
||||
wire:model="restoreCommandText" canGate="update"
|
||||
:canResource="$this->resource" />
|
||||
@else
|
||||
<x-forms.input label="Import command" wire:model="mariadbRestoreCommand"
|
||||
canGate="update" :canResource="$this->resource" />
|
||||
@endif
|
||||
@endif
|
||||
<x-forms.textarea rows="10" readonly label="Import command"
|
||||
helper="Coolify detects the backup format (SQL, archive, gzip, bz2, xz, zip, or tar) before it changes the database."
|
||||
wire:model="restoreCommandText" canGate="update"
|
||||
:canResource="$this->resource" />
|
||||
<div class="max-w-sm">
|
||||
<x-forms.listbox id="dumpAll" label="Backup contents" live :options="[
|
||||
['value' => true, 'label' => 'Backup contains all databases'],
|
||||
@@ -98,7 +74,14 @@
|
||||
@if (in_array($resourceDbType, ['standalone-postgresql', 'postgresql'], true) && ! $dumpAll)
|
||||
<div class="max-w-sm">
|
||||
<x-forms.checkbox id="replaceExisting" label="Replace objects that already exist"
|
||||
helper="Drops matching tables, functions, types, and other PostgreSQL objects from the archive before restoring them."
|
||||
helper="Archive backups: drops matching tables, functions, types, and other PostgreSQL objects before restoring them. SQL backups: recreates the database before the restore."
|
||||
canGate="update" :canResource="$this->resource" />
|
||||
</div>
|
||||
@endif
|
||||
@if ($resourceDbType === 'standalone-mongodb')
|
||||
<div class="max-w-sm">
|
||||
<x-forms.checkbox id="replaceExisting" label="Replace collections that already exist"
|
||||
helper="Drops each collection from the backup before restoring it. Without this option, documents that already exist are skipped."
|
||||
canGate="update" :canResource="$this->resource" />
|
||||
</div>
|
||||
@endif
|
||||
|
||||
@@ -13,7 +13,7 @@ test('postgresql dump all restore warns that administrator passwords are overwri
|
||||
$view = file_get_contents(resource_path('views/livewire/project/database/import-form.blade.php'));
|
||||
|
||||
expect($view)
|
||||
->toContain('@if ($dumpAll)')
|
||||
->toContain("@if (\$resourceDbType === 'standalone-postgresql' && \$dumpAll)")
|
||||
->toContain('Full restore overwrites administrator passwords')
|
||||
->toContain('The backup replaces PostgreSQL administrator role passwords, including the destination administrator password.')
|
||||
->toContain("If the administrator password changes, update it in Coolify's database configuration after the restore.");
|
||||
@@ -24,8 +24,8 @@ test('postgresql single database restore offers explicit object replacement', fu
|
||||
|
||||
expect($view)
|
||||
->toContain('Replace objects that already exist')
|
||||
->toContain('wire:model="postgresqlRestoreCommand"')
|
||||
->toContain('label="Import command" readonly');
|
||||
->toContain('wire:model="restoreCommandText"')
|
||||
->toContain('readonly label="Import command"');
|
||||
});
|
||||
|
||||
test('restore confirmation dialogs warn that existing objects can block the import', function () {
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
<?php
|
||||
|
||||
use App\Actions\Database\StartDatabaseImport;
|
||||
use App\Jobs\CoolifyTask;
|
||||
use App\Models\Environment;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\PrivateKey;
|
||||
use App\Models\Project;
|
||||
use App\Models\S3Storage;
|
||||
use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Support\DatabaseImport\DatabaseImportSource;
|
||||
use Illuminate\Filesystem\FilesystemAdapter;
|
||||
use Illuminate\Filesystem\FilesystemManager;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
config()->set('cache.default', 'array');
|
||||
config()->set('constants.ssh.mux_enabled', false);
|
||||
InstanceSettings::forceCreate(['id' => 0]);
|
||||
|
||||
$this->team = Team::factory()->create();
|
||||
$this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
|
||||
$this->server = Server::factory()->create([
|
||||
'team_id' => $this->team->id,
|
||||
'private_key_id' => $this->privateKey->id,
|
||||
'user' => 'root',
|
||||
]);
|
||||
$this->destination = StandaloneDocker::firstOrCreate(
|
||||
['server_id' => $this->server->id, 'network' => 'coolify'],
|
||||
['uuid' => (string) Str::uuid(), 'name' => 'docker']
|
||||
);
|
||||
$this->project = Project::factory()->create(['team_id' => $this->team->id]);
|
||||
$this->environment = Environment::factory()->create(['project_id' => $this->project->id]);
|
||||
$this->database = StandalonePostgresql::create([
|
||||
'uuid' => (string) Str::uuid(),
|
||||
'name' => 'db',
|
||||
'postgres_user' => 'postgres',
|
||||
'postgres_password' => 'password',
|
||||
'postgres_db' => 'db',
|
||||
'image' => 'postgres:17',
|
||||
'status' => 'running',
|
||||
'environment_id' => $this->environment->id,
|
||||
'destination_id' => $this->destination->id,
|
||||
'destination_type' => $this->destination->getMorphClass(),
|
||||
]);
|
||||
|
||||
Queue::fake();
|
||||
});
|
||||
|
||||
function importServerBackup(object $test, string $path, string $magicHex): string
|
||||
{
|
||||
Process::fake([
|
||||
'*stat -c %s*' => Process::result('2048'),
|
||||
'*od -An -tx1*' => Process::result(' '.implode(' ', str_split($magicHex, 2))),
|
||||
'*' => Process::result(''),
|
||||
]);
|
||||
|
||||
$activity = app(StartDatabaseImport::class)->handle(
|
||||
$test->database,
|
||||
new DatabaseImportSource('server', path: $path),
|
||||
$test->team->id,
|
||||
);
|
||||
|
||||
return (string) $activity->getExtraProperty('command');
|
||||
}
|
||||
|
||||
test('plain and gzip server backups are copied into the database container unchanged', function (string $path, string $magicHex) {
|
||||
$command = importServerBackup($this, $path, $magicHex);
|
||||
|
||||
expect($command)
|
||||
->toContain("docker cp '{$path}' '{$this->database->uuid}:/tmp/restore_")
|
||||
->not->toContain('backup-decompress-');
|
||||
})->with([
|
||||
'plain SQL' => ['/srv/backups/app.sql', '2d2d20506f73'],
|
||||
'gzip' => ['/srv/backups/app.sql.gz', '1f8b08000000'],
|
||||
'custom archive' => ['/srv/backups/app.dmp', '5047444d5001'],
|
||||
]);
|
||||
|
||||
test('bz2, xz, and zip server backups are prepared in the helper image', function (string $path, string $magicHex) {
|
||||
$command = importServerBackup($this, $path, $magicHex);
|
||||
|
||||
expect($command)
|
||||
->toContain("docker cp '{$path}' 'backup-decompress-")
|
||||
->toContain('bunzip2 -c')
|
||||
->toContain("cat /tmp/restore.prepared | docker exec -i '{$this->database->uuid}' sh -c")
|
||||
->toContain("docker rm -f 'backup-decompress-")
|
||||
->not->toContain("docker cp '{$path}' '{$this->database->uuid}:");
|
||||
|
||||
Process::assertRan(fn ($process) => str_contains($process->command, "docker run -d --network none --name 'backup-decompress-"));
|
||||
Queue::assertPushed(CoolifyTask::class, fn (CoolifyTask $job) => str_starts_with((string) ($job->call_event_data['containerName'] ?? ''), 'backup-decompress-'));
|
||||
})->with([
|
||||
'bz2' => ['/srv/backups/app.sql.bz2', '425a68393141'],
|
||||
'xz' => ['/srv/backups/app.sql.xz', 'fd377a585a00'],
|
||||
'zip' => ['/srv/backups/app.zip', '504b03041400'],
|
||||
]);
|
||||
|
||||
test('uploaded backups use the same preparation as server backups', function (string $contents, bool $usesHelper) {
|
||||
Storage::fake();
|
||||
Storage::put("upload/{$this->database->uuid}/restore", $contents);
|
||||
Process::fake();
|
||||
|
||||
$activity = app(StartDatabaseImport::class)->handle(
|
||||
$this->database,
|
||||
new DatabaseImportSource('upload'),
|
||||
$this->team->id,
|
||||
);
|
||||
$command = (string) $activity->getExtraProperty('command');
|
||||
|
||||
expect(str_contains($command, "'backup-decompress-"))->toBe($usesHelper)
|
||||
->and(str_contains($command, "docker cp '/tmp/database-import-"))->toBeTrue();
|
||||
})->with([
|
||||
'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", false],
|
||||
'gzip SQL' => [gzencode("SELECT 1;\n"), false],
|
||||
'bz2 SQL' => ["BZh91AY&SY\x00\x00", true],
|
||||
]);
|
||||
|
||||
test('s3 backups are prepared in the s3 helper and streamed into the database container', function () {
|
||||
$storage = S3Storage::create([
|
||||
'name' => 'Import S3',
|
||||
'region' => 'us-east-1',
|
||||
'key' => 'key',
|
||||
'secret' => 'secret',
|
||||
'bucket' => 'test-bucket',
|
||||
'endpoint' => 'https://8.8.8.8',
|
||||
'is_usable' => true,
|
||||
'team_id' => $this->team->id,
|
||||
]);
|
||||
$disk = Mockery::mock(FilesystemAdapter::class);
|
||||
$disk->shouldReceive('exists')->once()->with('backups/restore.sql.xz')->andReturn(true);
|
||||
$disk->shouldReceive('size')->once()->with('backups/restore.sql.xz')->andReturn(1024);
|
||||
$filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial();
|
||||
$filesystem->shouldReceive('build')->once()->andReturn($disk);
|
||||
Storage::swap($filesystem);
|
||||
Process::fake();
|
||||
|
||||
$activity = app(StartDatabaseImport::class)->handle(
|
||||
$this->database,
|
||||
new DatabaseImportSource('s3', path: 'backups/restore.sql.xz', s3StorageUuid: $storage->uuid),
|
||||
$this->team->id,
|
||||
);
|
||||
$command = (string) $activity->getExtraProperty('command');
|
||||
|
||||
expect($command)
|
||||
->toContain('unxz -c')
|
||||
->toContain("cat /tmp/restore.prepared | docker exec -i '{$this->database->uuid}' sh -c")
|
||||
->not->toContain('/tmp/s3-restore-');
|
||||
|
||||
Queue::assertPushed(CoolifyTask::class, fn (CoolifyTask $job) => ! array_key_exists('serverTmpPath', $job->call_event_data)
|
||||
&& str_starts_with((string) $job->call_event_data['containerName'], 's3-restore-'));
|
||||
});
|
||||
@@ -41,9 +41,10 @@ test('decompresses gzip backups for single-database mysql and mariadb restores',
|
||||
|
||||
$command = $builder->buildRestoreCommand(importResource($class, $type), '/tmp/restore file.sql.gz', false);
|
||||
|
||||
expect($command)->toBe(
|
||||
"(gunzip -cf '/tmp/restore file.sql.gz' 2>/dev/null || cat '/tmp/restore file.sql.gz') | {$client}"
|
||||
);
|
||||
expect($command)
|
||||
->toStartWith("backup='/tmp/restore file.sql.gz'\n")
|
||||
->toContain('is_gzip() { [ "$(head -c 2 "$backup" | od -An -tx1 | tr -d \' \n\')" = 1f8b ]; }')
|
||||
->toEndWith("stream | {$client}");
|
||||
})->with([
|
||||
'mysql' => [StandaloneMysql::class, null, 'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE'],
|
||||
'mariadb' => [StandaloneMariadb::class, null, 'mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE'],
|
||||
@@ -60,7 +61,7 @@ test('builds dump-all commands and postgres safety scan', function () {
|
||||
|
||||
expect($builder->buildRestoreCommand($postgres, '/tmp/dump.sql.gz', true))
|
||||
->toContain('pg_terminate_backend')
|
||||
->toContain("gunzip -cf '/tmp/dump.sql.gz'")
|
||||
->toStartWith("backup='/tmp/dump.sql.gz'\n")
|
||||
->and($safety)
|
||||
->toContain('COPY ... PROGRAM')
|
||||
->toContain('docker exec postgres-safe')
|
||||
@@ -72,6 +73,16 @@ test('builds dump-all commands and postgres safety scan', function () {
|
||||
->toContain("tr '\\n\\r\\t'");
|
||||
});
|
||||
|
||||
test('restores dump-all PostgreSQL custom archives with pg_restore from the decompressed stream', function () {
|
||||
$command = (new DatabaseImportCommandBuilder)->buildRestoreCommand(importResource(StandalonePostgresql::class), '/tmp/backup.dump.gz', true);
|
||||
|
||||
// pg_restore cannot read gzip files, so it must receive the decompressed archive on stdin.
|
||||
expect($command)
|
||||
->toContain('stream | pg_restore -U ${POSTGRES_USER} -d "$db"')
|
||||
->toContain('stream | psql -U ${POSTGRES_USER} -d "$db"')
|
||||
->not->toContain("-d \"\$db\" '/tmp/backup.dump.gz'");
|
||||
});
|
||||
|
||||
test('postgres safety command is null for non-postgres databases', function () {
|
||||
$builder = new DatabaseImportCommandBuilder;
|
||||
|
||||
@@ -89,7 +100,7 @@ test('dump-all mysql and mariadb commands use valid shell parameter expansions',
|
||||
|
||||
expect($command)
|
||||
->toContain($binary)
|
||||
->toContain("gunzip -cf '/tmp/dump.sql.gz'")
|
||||
->toStartWith("backup='/tmp/dump.sql.gz'\n")
|
||||
->toContain('-p'.$rootPassword)
|
||||
->toContain('CREATE DATABASE IF NOT EXISTS \`'.$database.'\`')
|
||||
->and(substr_count($command, $rootPassword))->toBe(6)
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
<?php
|
||||
|
||||
use App\Models\StandaloneMariadb;
|
||||
use App\Models\StandaloneMongodb;
|
||||
use App\Models\StandaloneMysql;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Support\DatabaseImport\DatabaseImportCommandBuilder;
|
||||
use Illuminate\Filesystem\Filesystem;
|
||||
|
||||
/**
|
||||
* These tests execute the generated scripts with `sh` against stub database clients.
|
||||
* Every stub appends "name|arguments|first 5 bytes of stdin" to a log, so the tests
|
||||
* prove which clients ran, in which order, and that nothing runs before a backup
|
||||
* format is rejected.
|
||||
*/
|
||||
function restoreScriptTempDir(): string
|
||||
{
|
||||
$dir = sys_get_temp_dir().'/coolify-restore-script-'.bin2hex(random_bytes(8));
|
||||
mkdir($dir);
|
||||
|
||||
return $dir;
|
||||
}
|
||||
|
||||
function restoreScriptRemoveDir(string $dir): void
|
||||
{
|
||||
(new Filesystem)->deleteDirectory($dir);
|
||||
}
|
||||
|
||||
function restoreScriptHasTool(string $tool): bool
|
||||
{
|
||||
exec('command -v '.escapeshellarg($tool).' >/dev/null 2>&1', $output, $exitCode);
|
||||
|
||||
return $exitCode === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<string> $command
|
||||
* @param array<string, string> $environment
|
||||
* @return array{exit: int, stdout: string, stderr: string}
|
||||
*/
|
||||
function restoreScriptProcess(array $command, string $cwd, array $environment = []): array
|
||||
{
|
||||
$process = proc_open(
|
||||
$command,
|
||||
[0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']],
|
||||
$pipes,
|
||||
$cwd,
|
||||
$environment + ['PATH' => getenv('PATH') ?: '/usr/bin:/bin'],
|
||||
);
|
||||
|
||||
expect($process)->not->toBeFalse();
|
||||
|
||||
fclose($pipes[0]);
|
||||
$stdout = (string) stream_get_contents($pipes[1]);
|
||||
$stderr = (string) stream_get_contents($pipes[2]);
|
||||
fclose($pipes[1]);
|
||||
fclose($pipes[2]);
|
||||
|
||||
return ['exit' => proc_close($process), 'stdout' => $stdout, 'stderr' => $stderr];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, string> $files relative path => contents
|
||||
*/
|
||||
function restoreScriptTar(array $files): string
|
||||
{
|
||||
$dir = restoreScriptTempDir();
|
||||
|
||||
try {
|
||||
foreach ($files as $name => $contents) {
|
||||
$path = $dir.'/src/'.$name;
|
||||
if (! is_dir(dirname($path))) {
|
||||
mkdir(dirname($path), 0777, true);
|
||||
}
|
||||
file_put_contents($path, $contents);
|
||||
}
|
||||
|
||||
$result = restoreScriptProcess(['tar', '-cf', $dir.'/out.tar', '-C', $dir.'/src', ...array_keys($files)], $dir);
|
||||
expect($result['exit'])->toBe(0, $result['stderr']);
|
||||
|
||||
return (string) file_get_contents($dir.'/out.tar');
|
||||
} finally {
|
||||
restoreScriptRemoveDir($dir);
|
||||
}
|
||||
}
|
||||
|
||||
function restoreScriptCompress(string $tool, string $contents): string
|
||||
{
|
||||
$dir = restoreScriptTempDir();
|
||||
|
||||
try {
|
||||
file_put_contents($dir.'/input', $contents);
|
||||
$result = restoreScriptProcess([$tool, '-c', $dir.'/input'], $dir);
|
||||
expect($result['exit'])->toBe(0, $result['stderr']);
|
||||
|
||||
return $result['stdout'];
|
||||
} finally {
|
||||
restoreScriptRemoveDir($dir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an uncompressed (stored) zip archive, so the fixture does not depend on ext-zip.
|
||||
*
|
||||
* @param array<string, string> $files file name => contents
|
||||
*/
|
||||
function restoreScriptZip(array $files): string
|
||||
{
|
||||
$entries = '';
|
||||
$directory = '';
|
||||
|
||||
foreach ($files as $name => $contents) {
|
||||
$crc = crc32($contents);
|
||||
$size = strlen($contents);
|
||||
$offset = strlen($entries);
|
||||
$entries .= pack('VvvvvvVVVvv', 0x04034B50, 20, 0, 0, 0, 0x21, $crc, $size, $size, strlen($name), 0).$name.$contents;
|
||||
$directory .= pack('VvvvvvvVVVvvvvvVV', 0x02014B50, 20, 20, 0, 0, 0, 0x21, $crc, $size, $size, strlen($name), 0, 0, 0, 0, 0, $offset).$name;
|
||||
}
|
||||
|
||||
return $entries.$directory.pack('VvvvvVVv', 0x06054B50, 0, 0, count($files), count($files), strlen($directory), strlen($entries), 0);
|
||||
}
|
||||
|
||||
function restoreScriptFixture(string $name): string
|
||||
{
|
||||
$pgCustom = "PGDMP\x01\x0e\x00\x04\x08\x01\x01\x01".str_repeat("\x00\x01toc", 64);
|
||||
$pgSql = "-- PostgreSQL database dump\n\nCREATE TABLE items (id integer);\n";
|
||||
$mysqlSql = "-- MySQL dump 10.13\n\nCREATE TABLE `items` (`id` int);\nINSERT INTO `items` VALUES (1);\n";
|
||||
$mongoArchive = "\x6d\xe2\x99\x81".str_repeat("\x01\x00archive", 32);
|
||||
$bson = pack('V', 22)."\x02name\x00".pack('V', 5)."item\x00\x00";
|
||||
|
||||
return match ($name) {
|
||||
'pg-custom' => $pgCustom,
|
||||
'pg-custom-gz' => gzencode($pgCustom),
|
||||
'pg-tar' => restoreScriptTar(['toc.dat' => $pgCustom, '3001.dat' => "1\n\\.\n"]),
|
||||
'pg-tar-gz' => gzencode(restoreScriptTar(['toc.dat' => $pgCustom, '3001.dat' => "1\n\\.\n"])),
|
||||
'pg-sql' => $pgSql,
|
||||
'pg-sql-gz' => gzencode($pgSql),
|
||||
'pg-cluster' => "--\n-- PostgreSQL database cluster dump\n--\n\nCREATE ROLE app;\n",
|
||||
'garbage' => str_repeat("\x00\xff\x10\x80binary\x00", 64),
|
||||
'mysql-sql' => $mysqlSql,
|
||||
'mysql-sql-gz' => gzencode($mysqlSql),
|
||||
'mysql-tar' => restoreScriptTar(['backup.sql' => $mysqlSql]),
|
||||
'mysql-tar-two' => restoreScriptTar(['app.sql' => $mysqlSql, 'other.sql' => $mysqlSql]),
|
||||
'mysql-cluster' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\nUSE `mysql`;\nINSERT INTO `user` VALUES ();\n",
|
||||
'mysql-two-databases' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\nCREATE DATABASE `other`;\nUSE `other`;\nCREATE TABLE `notes` (`t` text);\n",
|
||||
'mysql-one-database' => "-- MySQL dump 10.13\n\nCREATE DATABASE `app`;\nUSE `app`;\nCREATE TABLE `items` (`id` int);\n",
|
||||
'mongo-archive' => $mongoArchive,
|
||||
'mongo-archive-gz' => gzencode($mongoArchive),
|
||||
'mongo-dump-tar' => restoreScriptTar(['dump/app/items.bson' => $bson, 'dump/app/items.metadata.json' => '{"indexes":[]}']),
|
||||
'mongo-dump-gz-tar' => restoreScriptTar(['dump/app/items.bson.gz' => gzencode($bson), 'dump/app/items.metadata.json.gz' => gzencode('{"indexes":[]}')]),
|
||||
'mongo-archive-tar' => restoreScriptTar(['app.archive' => $mongoArchive]),
|
||||
'mongo-notes-tar' => restoreScriptTar(['notes.txt' => 'hello', 'readme.txt' => 'world']),
|
||||
'mongo-bson' => $bson,
|
||||
};
|
||||
}
|
||||
|
||||
function restoreScriptResource(string $engine): object
|
||||
{
|
||||
$class = match ($engine) {
|
||||
'postgresql' => StandalonePostgresql::class,
|
||||
'mysql' => StandaloneMysql::class,
|
||||
'mariadb' => StandaloneMariadb::class,
|
||||
'mongodb' => StandaloneMongodb::class,
|
||||
};
|
||||
|
||||
$resource = Mockery::mock($class);
|
||||
$resource->shouldReceive('getMorphClass')->andReturn($class);
|
||||
|
||||
return $resource;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{exit: int, stdout: string, stderr: string, calls: list<array{name: string, args: string, header: string}>, dir: string, backup: string, leftovers: list<string>}
|
||||
*/
|
||||
function restoreScriptRun(string $engine, string $contents, bool $dumpAll = false, bool $replaceExisting = false): array
|
||||
{
|
||||
$dir = restoreScriptTempDir();
|
||||
|
||||
try {
|
||||
$log = $dir.'/calls.log';
|
||||
file_put_contents($log, '');
|
||||
mkdir($dir.'/bin');
|
||||
|
||||
foreach (['pg_restore', 'psql', 'dropdb', 'createdb', 'mysql', 'mariadb', 'mongorestore'] as $client) {
|
||||
$escapedLog = escapeshellarg($log);
|
||||
file_put_contents($dir.'/bin/'.$client, <<<SH
|
||||
#!/bin/sh
|
||||
header=''
|
||||
if [ ! -t 0 ]; then
|
||||
header=\$(head -c 5)
|
||||
cat >/dev/null
|
||||
fi
|
||||
printf '%s|%s|%s\\n' '{$client}' "\$*" "\$header" >> {$escapedLog}
|
||||
exit 0
|
||||
SH);
|
||||
chmod($dir.'/bin/'.$client, 0755);
|
||||
}
|
||||
|
||||
$backup = $dir.'/backup file';
|
||||
file_put_contents($backup, $contents);
|
||||
|
||||
$script = (new DatabaseImportCommandBuilder)->buildRestoreCommand(restoreScriptResource($engine), $backup, $dumpAll, $replaceExisting);
|
||||
|
||||
$result = restoreScriptProcess(['sh', '-c', $script], $dir, [
|
||||
'PATH' => $dir.'/bin'.PATH_SEPARATOR.(getenv('PATH') ?: '/usr/bin:/bin'),
|
||||
'TMPDIR' => $dir,
|
||||
'POSTGRES_USER' => 'postgres',
|
||||
'POSTGRES_DB' => 'app',
|
||||
'MYSQL_USER' => 'app_user',
|
||||
'MYSQL_PASSWORD' => 'app_pass',
|
||||
'MYSQL_DATABASE' => 'app',
|
||||
'MYSQL_ROOT_PASSWORD' => 'root_pass',
|
||||
'MARIADB_USER' => 'app_user',
|
||||
'MARIADB_PASSWORD' => 'app_pass',
|
||||
'MARIADB_DATABASE' => 'app',
|
||||
'MARIADB_ROOT_PASSWORD' => 'root_pass',
|
||||
'MONGO_INITDB_ROOT_USERNAME' => 'root',
|
||||
'MONGO_INITDB_ROOT_PASSWORD' => 'mongo_pass',
|
||||
]);
|
||||
|
||||
$lines = array_values(array_filter(explode("\n", (string) file_get_contents($log)), fn (string $line): bool => $line !== ''));
|
||||
$calls = array_map(function (string $line): array {
|
||||
[$name, $args, $header] = array_pad(explode('|', $line, 3), 3, '');
|
||||
|
||||
return ['name' => $name, 'args' => $args, 'header' => $header];
|
||||
}, $lines);
|
||||
|
||||
return $result + [
|
||||
'calls' => $calls,
|
||||
'dir' => $dir,
|
||||
'backup' => $backup,
|
||||
'leftovers' => glob($dir.'/tmp.*') ?: [],
|
||||
];
|
||||
} finally {
|
||||
restoreScriptRemoveDir($dir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{name: string, args: string, header: string}> $calls
|
||||
* @param list<array{0: string, 1: string, 2: list<string>}> $expected client, stdin header, argument fragments
|
||||
*/
|
||||
function restoreScriptExpectCalls(array $calls, array $expected): void
|
||||
{
|
||||
expect(array_map(fn (array $call): string => $call['name'].'|'.$call['header'], $calls))
|
||||
->toBe(array_map(fn (array $call): string => $call[0].'|'.$call[1], $expected));
|
||||
|
||||
foreach ($expected as $index => $call) {
|
||||
foreach ($call[2] as $fragment) {
|
||||
expect($calls[$index]['args'])->toContain($fragment);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{exit: int, stdout: string, stderr: string, output: ?string, sourceExists: bool}
|
||||
*/
|
||||
function restoreScriptNormalize(string $contents): array
|
||||
{
|
||||
$dir = restoreScriptTempDir();
|
||||
|
||||
try {
|
||||
$source = $dir.'/uploaded backup';
|
||||
$target = $dir.'/normalized backup';
|
||||
file_put_contents($source, $contents);
|
||||
|
||||
$result = restoreScriptProcess(
|
||||
['sh', '-c', (new DatabaseImportCommandBuilder)->buildNormalizeScript($source, $target)],
|
||||
$dir,
|
||||
['TMPDIR' => $dir],
|
||||
);
|
||||
|
||||
return $result + [
|
||||
'output' => is_file($target) ? (string) file_get_contents($target) : null,
|
||||
'sourceExists' => file_exists($source),
|
||||
];
|
||||
} finally {
|
||||
restoreScriptRemoveDir($dir);
|
||||
}
|
||||
}
|
||||
|
||||
test('restores single PostgreSQL archives with pg_restore and SQL with psql', function (string $fixture, bool $replaceExisting, array $expectedCalls) {
|
||||
$run = restoreScriptRun('postgresql', restoreScriptFixture($fixture), false, $replaceExisting);
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr']);
|
||||
restoreScriptExpectCalls($run['calls'], $expectedCalls);
|
||||
})->with([
|
||||
'custom archive' => ['pg-custom', false, [['pg_restore', 'PGDMP', ['--exit-on-error -U postgres -d app']]]],
|
||||
'custom archive replacing existing objects' => ['pg-custom', true, [['pg_restore', 'PGDMP', ['--exit-on-error --clean --if-exists -U postgres -d app']]]],
|
||||
'gzip custom archive' => ['pg-custom-gz', false, [['pg_restore', 'PGDMP', ['--exit-on-error -U postgres -d app']]]],
|
||||
'gzip custom archive replacing existing objects' => ['pg-custom-gz', true, [['pg_restore', 'PGDMP', ['--exit-on-error --clean --if-exists -U postgres -d app']]]],
|
||||
'tar archive' => ['pg-tar', false, [['pg_restore', 'toc.d', ['--exit-on-error -U postgres -d app']]]],
|
||||
'gzip tar archive' => ['pg-tar-gz', false, [['pg_restore', 'toc.d', ['--exit-on-error -U postgres -d app']]]],
|
||||
'plain SQL' => ['pg-sql', false, [['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']]]],
|
||||
'gzip SQL' => ['pg-sql-gz', false, [['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']]]],
|
||||
// SQL cannot replace single objects, so replacing recreates the target database first.
|
||||
'plain SQL replacing existing objects' => ['pg-sql', true, [
|
||||
['psql', 'SELEC', ['-v db=app -U postgres -d template1']],
|
||||
['dropdb', '', ['--maintenance-db=template1 -U postgres --if-exists app']],
|
||||
['createdb', '', ['-U postgres app']],
|
||||
['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']],
|
||||
]],
|
||||
'gzip SQL replacing existing objects' => ['pg-sql-gz', true, [
|
||||
['psql', 'SELEC', ['-v db=app -U postgres -d template1']],
|
||||
['dropdb', '', ['--maintenance-db=template1 -U postgres --if-exists app']],
|
||||
['createdb', '', ['-U postgres app']],
|
||||
['psql', '-- Po', ['-v ON_ERROR_STOP=1 -U postgres -d app']],
|
||||
]],
|
||||
]);
|
||||
|
||||
test('rejects unsupported single PostgreSQL backups before calling any client', function (string $fixture, string $message) {
|
||||
$run = restoreScriptRun('postgresql', restoreScriptFixture($fixture));
|
||||
|
||||
expect($run['exit'])->toBe(1)
|
||||
->and($run['stderr'])->toContain($message)
|
||||
->and($run['calls'])->toBe([]);
|
||||
})->with([
|
||||
'cluster SQL dump' => ['pg-cluster', 'This backup contains all databases.'],
|
||||
'binary garbage' => ['garbage', 'Unsupported PostgreSQL backup format'],
|
||||
]);
|
||||
|
||||
test('restores PostgreSQL backups containing all databases after checking the format', function (string $fixture, array $restoreCall) {
|
||||
$run = restoreScriptRun('postgresql', restoreScriptFixture($fixture), dumpAll: true);
|
||||
|
||||
$recreate = [
|
||||
['psql', '', ['-U postgres -c', 'pg_terminate_backend']],
|
||||
['psql', '', ['-U postgres -t -c', 'SELECT datname FROM pg_database']],
|
||||
['createdb', '', ['-U postgres app']],
|
||||
];
|
||||
$inspect = $restoreCall[0] === 'pg_restore' ? [['pg_restore', 'PGDMP', ['-l']]] : [];
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr']);
|
||||
restoreScriptExpectCalls($run['calls'], [...$inspect, ...$recreate, $restoreCall]);
|
||||
})->with([
|
||||
'custom archive' => ['pg-custom', ['pg_restore', 'PGDMP', ['-U postgres -d app']]],
|
||||
'gzip custom archive' => ['pg-custom-gz', ['pg_restore', 'PGDMP', ['-U postgres -d app']]],
|
||||
'plain SQL' => ['pg-sql', ['psql', '-- Po', ['-U postgres -d app']]],
|
||||
'gzip SQL' => ['pg-sql-gz', ['psql', '-- Po', ['-U postgres -d app']]],
|
||||
]);
|
||||
|
||||
test('rejects unsupported PostgreSQL backups containing all databases before dropping anything', function () {
|
||||
$run = restoreScriptRun('postgresql', restoreScriptFixture('garbage'), dumpAll: true);
|
||||
|
||||
expect($run['exit'])->toBe(1)
|
||||
->and($run['stderr'])->toContain('Unsupported PostgreSQL backup format')->toContain('Nothing was changed.')
|
||||
->and($run['calls'])->toBe([]);
|
||||
});
|
||||
|
||||
test('restores single MySQL and MariaDB SQL backups', function (string $engine, string $fixture) {
|
||||
$run = restoreScriptRun($engine, restoreScriptFixture($fixture));
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr'])
|
||||
->and($run['leftovers'])->toBe([]);
|
||||
restoreScriptExpectCalls($run['calls'], [[$engine, '-- My', ['-u app_user -papp_pass app']]]);
|
||||
})->with(['mysql', 'mariadb'])->with([
|
||||
'SQL' => 'mysql-sql',
|
||||
'gzip SQL' => 'mysql-sql-gz',
|
||||
'tar with one SQL file' => 'mysql-tar',
|
||||
'dump of one database made with --databases' => 'mysql-one-database',
|
||||
]);
|
||||
|
||||
test('restores MySQL and MariaDB backups containing all databases after checking the format', function (string $engine, string $fixture) {
|
||||
$run = restoreScriptRun($engine, restoreScriptFixture($fixture), dumpAll: true);
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr'])
|
||||
->and($run['leftovers'])->toBe([]);
|
||||
restoreScriptExpectCalls($run['calls'], [
|
||||
[$engine, '', ['-u root -proot_pass -N -e', 'information_schema.processlist']],
|
||||
[$engine, '', ['-u root -proot_pass -N -e', 'DROP DATABASE IF EXISTS']],
|
||||
[$engine, '', ['-u root -proot_pass']],
|
||||
[$engine, '', ['-u root -proot_pass -e CREATE DATABASE IF NOT EXISTS `app`;']],
|
||||
[$engine, '-- My', ['-u root -proot_pass app']],
|
||||
]);
|
||||
expect(end($run['calls'])['args'])->toBe('-u root -proot_pass app');
|
||||
})->with(['mysql', 'mariadb'])->with([
|
||||
'SQL' => 'mysql-sql',
|
||||
'gzip SQL' => 'mysql-sql-gz',
|
||||
'tar with one SQL file' => 'mysql-tar',
|
||||
'SQL that uses the mysql schema' => 'mysql-cluster',
|
||||
]);
|
||||
|
||||
test('rejects unsupported MySQL and MariaDB backups before calling any client', function (string $engine, string $fixture, bool $dumpAll, string $message) {
|
||||
$run = restoreScriptRun($engine, restoreScriptFixture($fixture), $dumpAll);
|
||||
|
||||
expect($run['exit'])->toBe(1)
|
||||
->and($run['stderr'])->toContain($message)
|
||||
->and($run['calls'])->toBe([]);
|
||||
})->with(['mysql', 'mariadb'])->with([
|
||||
'tar with two files' => ['mysql-tar-two', false, 'A tar backup must contain exactly one SQL dump.'],
|
||||
'tar with two files, all databases' => ['mysql-tar-two', true, 'A tar backup must contain exactly one SQL dump.'],
|
||||
'binary garbage' => ['garbage', false, 'Unsupported backup format.'],
|
||||
'binary garbage, all databases' => ['garbage', true, 'Unsupported backup format.'],
|
||||
'all-databases dump without dump-all' => ['mysql-cluster', false, 'This backup contains more than one database.'],
|
||||
'dump of two databases without dump-all' => ['mysql-two-databases', false, 'This backup contains more than one database.'],
|
||||
]);
|
||||
|
||||
test('restores mongodump archives', function (string $fixture, bool $replaceExisting, bool $gzip) {
|
||||
$run = restoreScriptRun('mongodb', restoreScriptFixture($fixture), false, $replaceExisting);
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr']);
|
||||
restoreScriptExpectCalls($run['calls'], [['mongorestore', '', [
|
||||
'--authenticationDatabase=admin --username root --password mongo_pass',
|
||||
'--archive='.$run['backup'],
|
||||
]]]);
|
||||
|
||||
$args = $run['calls'][0]['args'];
|
||||
expect(str_contains($args, '--gzip'))->toBe($gzip)
|
||||
->and(str_contains($args, '--drop'))->toBe($replaceExisting);
|
||||
})->with([
|
||||
'plain archive' => ['mongo-archive', false, false],
|
||||
'plain archive replacing existing collections' => ['mongo-archive', true, false],
|
||||
'gzip archive' => ['mongo-archive-gz', false, true],
|
||||
'gzip archive replacing existing collections' => ['mongo-archive-gz', true, true],
|
||||
]);
|
||||
|
||||
test('restores mongodump directories packed as tar', function (string $fixture, bool $gzip) {
|
||||
$run = restoreScriptRun('mongodb', restoreScriptFixture($fixture));
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr'])
|
||||
->and($run['calls'])->toHaveCount(1)
|
||||
->and($run['calls'][0]['name'])->toBe('mongorestore')
|
||||
->and($run['calls'][0]['args'])->toMatch('#--dir='.preg_quote($run['dir'], '#').'/tmp\.[^/ ]+/dump$#')
|
||||
->and(str_contains($run['calls'][0]['args'], '--gzip'))->toBe($gzip)
|
||||
->and($run['calls'][0]['args'])->not->toContain('--archive')
|
||||
->and($run['leftovers'])->toBe([]);
|
||||
})->with([
|
||||
'bson files' => ['mongo-dump-tar', false],
|
||||
'gzip bson files' => ['mongo-dump-gz-tar', true],
|
||||
]);
|
||||
|
||||
test('restores a mongodump archive wrapped in tar', function () {
|
||||
$run = restoreScriptRun('mongodb', restoreScriptFixture('mongo-archive-tar'));
|
||||
|
||||
expect($run['exit'])->toBe(0, $run['stderr'])
|
||||
->and($run['calls'])->toHaveCount(1)
|
||||
->and($run['calls'][0]['name'])->toBe('mongorestore')
|
||||
->and($run['calls'][0]['args'])->toMatch('#--archive='.preg_quote($run['dir'], '#').'/tmp\.[^/ ]+/app\.archive$#')
|
||||
->and($run['calls'][0]['args'])->not->toContain('--gzip')
|
||||
->and($run['leftovers'])->toBe([]);
|
||||
});
|
||||
|
||||
test('rejects unsupported MongoDB backups before calling mongorestore', function (string $fixture, bool $replaceExisting, string $message) {
|
||||
$run = restoreScriptRun('mongodb', restoreScriptFixture($fixture), false, $replaceExisting);
|
||||
|
||||
expect($run['exit'])->toBe(1)
|
||||
->and($run['stderr'])->toContain($message)
|
||||
->and($run['calls'])->toBe([]);
|
||||
})->with([
|
||||
'single bson file' => ['mongo-bson', false, 'Unsupported MongoDB backup format'],
|
||||
'single bson file replacing existing collections' => ['mongo-bson', true, 'Unsupported MongoDB backup format'],
|
||||
'tar without a dump directory' => ['mongo-notes-tar', true, 'The tar backup does not contain a mongodump directory.'],
|
||||
]);
|
||||
|
||||
test('normalize moves plain and gzip backups unchanged', function (string $contents) {
|
||||
$result = restoreScriptNormalize($contents);
|
||||
|
||||
expect($result['exit'])->toBe(0, $result['stderr'])
|
||||
->and($result['output'])->toBe($contents)
|
||||
->and($result['sourceExists'])->toBeFalse();
|
||||
})->with([
|
||||
'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n"],
|
||||
'gzip SQL' => [gzencode("-- PostgreSQL database dump\nSELECT 1;\n")],
|
||||
]);
|
||||
|
||||
test('normalize decompresses bz2, xz and single-file zip backups', function (string $format, array $tools) {
|
||||
foreach ($tools as $tool) {
|
||||
if (! restoreScriptHasTool($tool)) {
|
||||
$this->markTestSkipped("{$tool} is not installed.");
|
||||
}
|
||||
}
|
||||
|
||||
$original = "-- MySQL dump 10.13\nCREATE TABLE `items` (`id` int);\n";
|
||||
$compressed = match ($format) {
|
||||
'bz2' => restoreScriptCompress('bzip2', $original),
|
||||
'xz' => restoreScriptCompress('xz', $original),
|
||||
'zip' => restoreScriptZip(['backup.sql' => $original]),
|
||||
};
|
||||
|
||||
$result = restoreScriptNormalize($compressed);
|
||||
|
||||
expect($result['exit'])->toBe(0, $result['stderr'])
|
||||
->and($result['output'])->toBe($original);
|
||||
})->with([
|
||||
'bz2' => ['bz2', ['bzip2', 'bunzip2']],
|
||||
'xz' => ['xz', ['xz', 'unxz']],
|
||||
'zip' => ['zip', ['unzip']],
|
||||
]);
|
||||
|
||||
test('normalize rejects backups it cannot unpack', function (string $format, string $tool, string $message) {
|
||||
if (! restoreScriptHasTool($tool)) {
|
||||
$this->markTestSkipped("{$tool} is not installed.");
|
||||
}
|
||||
|
||||
$contents = match ($format) {
|
||||
'zip with two files' => restoreScriptZip(['app.sql' => 'SELECT 1;', 'other.sql' => 'SELECT 2;']),
|
||||
'corrupt bz2' => 'BZh91AY&SY'.str_repeat("\x00\xffcorrupt", 16),
|
||||
};
|
||||
|
||||
$result = restoreScriptNormalize($contents);
|
||||
|
||||
expect($result['exit'])->toBe(1)
|
||||
->and($result['stderr'])->toContain($message);
|
||||
})->with([
|
||||
'zip with two files' => ['zip with two files', 'unzip', 'A zip backup must contain exactly one file.'],
|
||||
'corrupt bz2' => ['corrupt bz2', 'bunzip2', 'The bz2 backup cannot be decompressed.'],
|
||||
]);
|
||||
@@ -17,6 +17,9 @@ function importFormWithResource(string $modelClass): ImportForm
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the dump-all restore command with stub clients. Each entry is the
|
||||
* client name and the first five bytes it read from stdin.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
function invokedPostgresRestoreClients(string $contents, bool $gzip = false): array
|
||||
@@ -28,13 +31,15 @@ function invokedPostgresRestoreClients(string $contents, bool $gzip = false): ar
|
||||
$logFile = $binDir.'/invoked.log';
|
||||
file_put_contents($logFile, '');
|
||||
|
||||
foreach (['pg_restore', 'psql'] as $tool) {
|
||||
foreach (['pg_restore', 'psql', 'dropdb', 'createdb'] as $tool) {
|
||||
$stub = <<<SH
|
||||
#!/bin/sh
|
||||
printf '%s\\n' '{$tool}' >> '{$logFile}'
|
||||
header=''
|
||||
if [ ! -t 0 ]; then
|
||||
header=\$(head -c 5)
|
||||
cat >/dev/null
|
||||
fi
|
||||
printf '%s:%s\\n' '{$tool}' "\$header" >> '{$logFile}'
|
||||
exit 0
|
||||
SH;
|
||||
$path = $binDir.'/'.$tool;
|
||||
@@ -49,7 +54,6 @@ SH;
|
||||
|
||||
$component = importFormWithResource('App\Models\StandalonePostgresql');
|
||||
$component->dumpAll = true;
|
||||
$component->postgresqlRestoreCommand = ':';
|
||||
|
||||
$process = proc_open(
|
||||
['sh', '-c', $component->buildRestoreCommand($dumpPath)],
|
||||
@@ -89,44 +93,51 @@ SH;
|
||||
test('buildRestoreCommand handles PostgreSQL without dumpAll', function () {
|
||||
$component = importFormWithResource('App\Models\StandalonePostgresql');
|
||||
$component->dumpAll = false;
|
||||
$component->postgresqlRestoreCommand = 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB';
|
||||
|
||||
$result = $component->buildRestoreCommand('/tmp/test.dump');
|
||||
|
||||
expect($result)->toContain('pg_restore');
|
||||
expect($result)->toContain('/tmp/test.dump');
|
||||
expect($result)->toContain('pg_restore --exit-on-error');
|
||||
expect($result)->toStartWith("backup='/tmp/test.dump'");
|
||||
});
|
||||
|
||||
test('buildRestoreCommand handles PostgreSQL with dumpAll', function () {
|
||||
$component = importFormWithResource('App\Models\StandalonePostgresql');
|
||||
$component->dumpAll = true;
|
||||
$component->postgresqlRestoreCommand = 'psql -U $POSTGRES_USER -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IS NOT NULL AND pid <> pg_backend_pid()" && psql -U $POSTGRES_USER -t -c "SELECT datname FROM pg_database WHERE NOT datistemplate" | xargs -I {} dropdb -U $POSTGRES_USER --if-exists {} && createdb -U $POSTGRES_USER postgres';
|
||||
|
||||
$result = $component->buildRestoreCommand('/tmp/test.dump');
|
||||
|
||||
expect($result)->toContain("gunzip -cf '/tmp/test.dump'");
|
||||
expect($result)->toContain('psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}');
|
||||
expect($result)->toStartWith("backup='/tmp/test.dump'");
|
||||
expect($result)->toContain('stream | psql -U ${POSTGRES_USER} -d "$db"');
|
||||
});
|
||||
|
||||
test('buildRestoreCommand dump-all PostgreSQL restore is pg_restore for PGDMP otherwise psql', function () {
|
||||
$component = importFormWithResource('App\Models\StandalonePostgresql');
|
||||
$component->dumpAll = true;
|
||||
$component->postgresqlRestoreCommand = 'psql -U ${POSTGRES_USER} -c "cleanup"';
|
||||
|
||||
$escapedTmpPath = escapeshellarg('/tmp/test.dump');
|
||||
$command = $component->buildRestoreCommand('/tmp/test.dump');
|
||||
|
||||
expect($component->buildRestoreCommand('/tmp/test.dump'))->toBe(
|
||||
'psql -U ${POSTGRES_USER} -c "cleanup" && if [ "$({ gunzip -cf '.$escapedTmpPath.' 2>/dev/null || cat '.$escapedTmpPath.'; } | head -c 5)" = \'PGDMP\' ]; then pg_restore -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}} '.$escapedTmpPath.'; else (gunzip -cf '.$escapedTmpPath.' 2>/dev/null || cat '.$escapedTmpPath.') | psql -U ${POSTGRES_USER} -d ${POSTGRES_DB:-${POSTGRES_USER:-postgres}}; fi'
|
||||
);
|
||||
expect($command)
|
||||
->toContain('if [ "$(stream | head -c 5)" = PGDMP ] || is_tar; then')
|
||||
->toContain('stream | pg_restore -U ${POSTGRES_USER} -d "$db"')
|
||||
->toContain('stream | psql -U ${POSTGRES_USER} -d "$db"')
|
||||
->and(strpos($command, 'kind=archive'))->toBeLessThan(strpos($command, 'dropdb'));
|
||||
});
|
||||
|
||||
test('dump-all PostgreSQL restore selects the client for the dump format', function (string $contents, bool $gzip, string $client) {
|
||||
expect(invokedPostgresRestoreClients($contents, $gzip))->toBe([$client]);
|
||||
test('dump-all PostgreSQL import text shows the client chosen by the dump format', function () {
|
||||
$component = importFormWithResource('App\Models\StandalonePostgresql');
|
||||
$component->updatedDumpAll(true);
|
||||
|
||||
expect($component->restoreCommandText)->toBe($component->buildRestoreCommand('<temp_backup_file>'));
|
||||
});
|
||||
|
||||
test('dump-all PostgreSQL restore selects the client for the dump format', function (string $contents, bool $gzip, string $restore, array $preflight) {
|
||||
// Check the archive, terminate sessions, list databases, recreate the target database, then restore.
|
||||
expect(invokedPostgresRestoreClients($contents, $gzip))->toBe([...$preflight, 'psql:', 'psql:', 'createdb:', $restore]);
|
||||
})->with([
|
||||
'custom archive' => ['PGDMP'.str_repeat("\0", 16), false, 'pg_restore'],
|
||||
'gzip custom archive' => ['PGDMP'.str_repeat("\0", 16), true, 'pg_restore'],
|
||||
'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", false, 'psql'],
|
||||
'gzip SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", true, 'psql'],
|
||||
'custom archive' => ['PGDMP'.str_repeat("\0", 16), false, 'pg_restore:PGDMP', ['pg_restore:PGDMP']],
|
||||
'gzip custom archive' => ['PGDMP'.str_repeat("\0", 16), true, 'pg_restore:PGDMP', ['pg_restore:PGDMP']],
|
||||
'plain SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", false, 'psql:-- Po', []],
|
||||
'gzip SQL' => ["-- PostgreSQL database dump\nSELECT 1;\n", true, 'psql:-- Po', []],
|
||||
]);
|
||||
|
||||
test('buildRestoreCommand handles MySQL without dumpAll', function () {
|
||||
@@ -135,8 +146,8 @@ test('buildRestoreCommand handles MySQL without dumpAll', function () {
|
||||
|
||||
$result = $component->buildRestoreCommand('/tmp/test.dump');
|
||||
|
||||
expect($result)->toContain('mysql -u $MYSQL_USER');
|
||||
expect($result)->toContain("(gunzip -cf '/tmp/test.dump' 2>/dev/null || cat '/tmp/test.dump') | mysql");
|
||||
expect($result)->toStartWith("backup='/tmp/test.dump'");
|
||||
expect($result)->toContain('stream | mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DATABASE');
|
||||
expect($result)->not->toContain("< '/tmp/test.dump'");
|
||||
});
|
||||
|
||||
@@ -146,18 +157,18 @@ test('buildRestoreCommand handles MariaDB without dumpAll', function () {
|
||||
|
||||
$result = $component->buildRestoreCommand('/tmp/test.dump');
|
||||
|
||||
expect($result)->toContain('mariadb -u $MARIADB_USER');
|
||||
expect($result)->toContain("(gunzip -cf '/tmp/test.dump' 2>/dev/null || cat '/tmp/test.dump') | mariadb");
|
||||
expect($result)->toStartWith("backup='/tmp/test.dump'");
|
||||
expect($result)->toContain('stream | mariadb -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE');
|
||||
expect($result)->not->toContain("< '/tmp/test.dump'");
|
||||
});
|
||||
|
||||
test('buildRestoreCommand always appends the MongoDB archive path', function (bool $dumpAll) {
|
||||
$component = importFormWithResource('App\Models\StandaloneMongodb');
|
||||
$component->dumpAll = $dumpAll;
|
||||
$component->mongodbRestoreCommand = 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive=';
|
||||
|
||||
$result = $component->buildRestoreCommand('/tmp/test.dump');
|
||||
|
||||
expect($result)->toStartWith("backup='/tmp/test.dump'");
|
||||
expect($result)->toContain('mongorestore');
|
||||
expect($result)->toContain("--archive='/tmp/test.dump'");
|
||||
expect($result)->toContain('restore --gzip --archive="$backup"');
|
||||
})->with([false, true]);
|
||||
|
||||
@@ -169,11 +169,12 @@ it('quotes dump all restore command temp paths with spaces', function (string $m
|
||||
$escapedTmpPath = escapeshellarg($tmpPath);
|
||||
$restoreCommand = $component->buildRestoreCommand($tmpPath);
|
||||
|
||||
// The path appears once, as a quoted assignment; the script only uses "$backup".
|
||||
expect($restoreCommand)
|
||||
->toContain("gunzip -cf {$escapedTmpPath}")
|
||||
->toContain("cat {$escapedTmpPath}")
|
||||
->not->toContain("gunzip -cf {$tmpPath}")
|
||||
->not->toContain("cat {$tmpPath}");
|
||||
->toStartWith("backup={$escapedTmpPath}\n")
|
||||
->and(substr_count($restoreCommand, $tmpPath))->toBe(1)
|
||||
->and($restoreCommand)->toContain('gunzip -c "$backup"')
|
||||
->toContain('cat "$backup"');
|
||||
})->with([
|
||||
'mariadb' => StandaloneMariadb::class,
|
||||
'mysql' => StandaloneMysql::class,
|
||||
|
||||
Reference in New Issue
Block a user