diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 8f4eb84be6..d17c4188ac 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -11,7 +11,9 @@ use App\Models\PrivateKey; use App\Models\Server; use App\Support\RemoteProcessCommand; use Carbon\Carbon; +use Illuminate\Contracts\Process\ProcessResult; use Illuminate\Database\Eloquent\Model; +use Illuminate\Process\Exceptions\ProcessTimedOutException; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\Process; @@ -137,6 +139,20 @@ function instant_scp_from_server(string $remoteSource, string $localDest, Server ); } +/** + * Runs an SSH command. The timeout error of the process contains the complete command line, and with it + * the secrets of the remote script (database passwords, S3 keys), so it is replaced with a generic error. + * Exit code 124 (as from `timeout`) marks it as a remote command failure, so the batch is not run again. + */ +function runSshProcess(string $sshCommand, int $timeout, ?string $input = null): ProcessResult +{ + try { + return Process::timeout($timeout)->input($input)->run($sshCommand); + } catch (ProcessTimedOutException) { + throw new RuntimeException("The command on the server timed out after {$timeout} seconds.", 124); + } +} + /** * Write content to a file on a server through SSH stdin. The content never becomes part of a command * line, so it is not limited by the 128 KiB per-argument limit of Linux. A non-root SSH user writes @@ -150,7 +166,7 @@ function instant_remote_write_file(Server $server, string $path, string $content SshRetryHandler::retry( function () use ($server, $remoteCommand, $content, $timeout) { $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, commandTimeout: $timeout); - $process = Process::timeout($timeout)->input($content)->run($sshCommand); + $process = runSshProcess($sshCommand, $timeout, $content); if ($process->exitCode() !== 0) { excludeCertainErrors($process->errorOutput(), $process->exitCode()); @@ -176,7 +192,7 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $ return SshRetryHandler::retry( function () use ($server, $command_string) { $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string); - $process = Process::timeout(30)->run($sshCommand); + $process = runSshProcess($sshCommand, 30); $output = trim($process->output()); $exitCode = $process->exitCode(); @@ -212,7 +228,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool return SshRetryHandler::retry( function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) { $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout); - $process = Process::timeout($effectiveTimeout)->run($sshCommand); + $process = runSshProcess($sshCommand, (int) $effectiveTimeout); $output = trim($process->output()); $exitCode = $process->exitCode(); diff --git a/tests/Feature/BackupTimeoutSecretRedactionTest.php b/tests/Feature/BackupTimeoutSecretRedactionTest.php new file mode 100644 index 0000000000..9cb4ec23f8 --- /dev/null +++ b/tests/Feature/BackupTimeoutSecretRedactionTest.php @@ -0,0 +1,115 @@ + false]); +}); + +/** + * The exception Process::run() throws when a command runs longer than its timeout. Symfony puts the + * complete command line into the message. + */ +function backupRedactionTimeoutException(string $command, int $timeout): ProcessTimedOutException +{ + $process = SymfonyProcess::fromShellCommandline($command); + $process->setTimeout($timeout); + + return new ProcessTimedOutException(new SymfonyTimeoutException($process, SymfonyTimeoutException::TYPE_GENERAL), new ProcessResult($process)); +} + +function backupRedactionServer(Team $team): Server +{ + $server = Server::factory()->create(['team_id' => $team->id]); + $server->update(['private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + + return $server->fresh(); +} + +it('does not put the command into the error when a remote command times out', function () { + config(['constants.ssh.max_retries' => 3, 'constants.ssh.retry_base_delay' => 0]); + $server = backupRedactionServer(Team::factory()->create()); + $attempts = 0; + Process::fake(function ($process) use (&$attempts) { + $attempts++; + + throw backupRedactionTimeoutException($process->command, 7200); + }); + + try { + instant_remote_process(["docker exec mysql mysqldump -u root -p'TopSecretPass' app > /tmp/x"], $server, timeout: 7200, disableMultiplexing: true); + $this->fail('The timeout was not reported.'); + } catch (Throwable $exception) { + expect($exception->getMessage())->toBe('The command on the server timed out after 7200 seconds.') + ->and($exception->getPrevious())->toBeNull(); + } + + expect($attempts)->toBe(1); +}); + +it('does not put database credentials into the backup execution or the failure notification when the dump times out', function () { + InstanceSettings::forceCreate(['id' => 0]); + Notification::fake(); + $team = Team::factory()->create(); + $server = backupRedactionServer($team); + $team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $database = create_standalone_mysql($environment->id, StandaloneDocker::where('server_id', $server->id)->firstOrFail(), [ + 'mysql_root_password' => 'TopSecretPass', + 'mysql_database' => 'app', + 'status' => 'running:healthy', + ]); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => StandaloneMysql::class, + 'database_id' => $database->id, + 'team_id' => $team->id, + 'databases_to_backup' => 'app', + 'timeout' => 600, + ]); + Process::fake(function ($process) { + if (str_contains($process->command, 'mysqldump')) { + throw backupRedactionTimeoutException($process->command, 600); + } + + return Process::result(output: ''); + }); + + try { + (new DatabaseBackupJob($backup))->handle(); + } catch (Throwable) { + // The job reports the failure after it records the execution. + } + + $execution = $backup->executions()->latest('id')->firstOrFail(); + expect($execution->status)->toBe('failed') + ->and($execution->message)->toBe('The command on the server timed out after 600 seconds.'); + Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => ! str_contains((string) $notification->output, 'TopSecretPass') + && str_contains((string) $notification->output, 'timed out after 600 seconds')); +});