From a836756e9b92b6d8391e15fecbfbc993943475f0 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:56:29 +0200 Subject: [PATCH] fix(backup): explain why a dump stopped without an error Closes #11549 Co-Authored-By: Claude Opus 5.5 --- app/Jobs/DatabaseBackupJob.php | 22 +++++ tests/Feature/BackupFailureExitCodeTest.php | 100 ++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 tests/Feature/BackupFailureExitCodeTest.php diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 2e6d71f18b..15b9ff4bba 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -460,6 +460,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue // Local backup failed: the partial file is not tracked, so retention would never remove it. deleteBackupsLocally($this->backup_location, $this->server); $failureMessage = $this->redact($this->error_output ?? $this->backup_output ?? $e->getMessage()); + $exitCodeReason = $this->exitCodeReason($e->getCode()); + if ($exitCodeReason !== null) { + $failureMessage = $exitCodeReason."\n\n".$failureMessage; + } if ($this->backup_log) { $this->backup_log->update([ 'status' => 'failed', @@ -816,6 +820,24 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * Explains exit codes of a dump that was stopped from outside. The dump tool cannot report these + * itself, so its output ends after the last progress line without an error. + */ + private function exitCodeReason(int|string $exitCode): ?string + { + return match ($exitCode) { + 137 => 'The backup stopped with exit code 137: the dump process or the database container was killed (SIGKILL). ' + .'This is usually the out-of-memory killer: the dump runs inside the database container and shares its memory limit. ' + .'Check the memory limit of the database and the kernel log (dmesg) on the server.', + 143 => 'The backup stopped with exit code 143: the dump process or the database container was terminated (SIGTERM), ' + .'for example because the database was stopped or restarted during the backup.', + 255 => 'The backup stopped with exit code 255: the SSH connection to the server closed. ' + .'Check the network connection and the SSH server log on the server.', + default => null, + }; + } + private function add_to_backup_output($output): void { if ($this->backup_output) { diff --git a/tests/Feature/BackupFailureExitCodeTest.php b/tests/Feature/BackupFailureExitCodeTest.php new file mode 100644 index 0000000000..cd5b8d492b --- /dev/null +++ b/tests/Feature/BackupFailureExitCodeTest.php @@ -0,0 +1,100 @@ + false, 'constants.ssh.retry_base_delay' => 0]); + Event::fake([BackupCreated::class]); + Notification::fake(); + InstanceSettings::unguarded(fn () => InstanceSettings::query()->firstOrCreate(['id' => 0])); + + $this->team = Team::factory()->create(); + $this->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]); + $server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $this->team->id])->id]); + $database = create_standalone_mongodb($environment->id, StandaloneDocker::where('server_id', $server->id)->firstOrFail(), [ + 'status' => 'running:healthy', + ]); + $this->backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => StandaloneMongodb::class, + 'database_id' => $database->id, + 'team_id' => $this->team->id, + 'databases_to_backup' => null, + ]); +}); + +/** + * mongodump writes its progress to stderr. When the dump process or the database container is killed, + * the output stops after the last progress line and there is no "Failed:" line (issue #11549). + */ +function interruptedMongodump(int $exitCode): void +{ + Process::fake(function ($process) use ($exitCode) { + if (str_contains($process->command, 'mongodump')) { + return Process::result( + errorOutput: "2026-08-29T15:44:00.555+0000\tdone dumping `evolution.leaverequests` (0 documents)\n2026-08-29T15:44:00.555+0000\twriting `evolution.printagents` to `archive on stdout`", + exitCode: $exitCode, + ); + } + + return Process::result(output: ''); + }); +} + +it('tells why a dump stopped without an error message', function (int $exitCode, string $reason) { + interruptedMongodump($exitCode); + + (new DatabaseBackupJob($this->backup))->handle(); + + $execution = $this->backup->executions()->latest('id')->firstOrFail(); + expect($execution->status)->toBe('failed') + ->and($execution->message)->toStartWith($reason) + ->and($execution->message)->toContain('writing `evolution.printagents` to `archive on stdout`'); + Notification::assertSentTo($this->team, BackupFailed::class, fn (BackupFailed $notification) => str_starts_with((string) $notification->output, $reason)); +})->with([ + 'dump or database killed' => [137, 'The backup stopped with exit code 137: the dump process or the database container was killed (SIGKILL).'], + 'dump terminated' => [143, 'The backup stopped with exit code 143: the dump process or the database container was terminated (SIGTERM)'], + 'SSH connection lost' => [255, 'The backup stopped with exit code 255: the SSH connection to the server closed.'], +]); + +it('keeps the error of a dump that reports its own failure', function () { + Process::fake(function ($process) { + if (str_contains($process->command, 'mongodump')) { + return Process::result(errorOutput: 'Failed: error connecting to db server: server returned error on SASL authentication step', exitCode: 1); + } + + return Process::result(output: ''); + }); + + (new DatabaseBackupJob($this->backup))->handle(); + + expect($this->backup->executions()->latest('id')->firstOrFail()->message) + ->toBe('Failed: error connecting to db server: server returned error on SASL authentication step'); +});