From 4f43c96e53e33d89b8b2954e3e590fb8f8fd6d80 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:21:34 +0200 Subject: [PATCH] fix(backups): treat removed containers as recovered after a volume backup When a container that a volume backup stopped no longer existed (for example after a redeploy), recovery kept it as remaining. The execution needed attention and every later run of the schedule was skipped. - Treat "no such container" or "no such object" from docker inspect as recovered; other failures still need attention --- app/Jobs/VolumeBackupRecoveryJob.php | 7 +- ...lumeBackupRecoveryMissingContainerTest.php | 171 ++++++++++++++++++ 2 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 tests/Feature/VolumeBackupRecoveryMissingContainerTest.php diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php index bd3310712e..cda0fdd046 100644 --- a/app/Jobs/VolumeBackupRecoveryJob.php +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -276,10 +276,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou $execution->update(['stop_container_ids' => $containers]); $remainingFile = $stateFile.'.remaining'; + // A container that no longer exists (for example after a redeploy) has nothing to restart; only + // other inspect or start failures keep the container for another recovery attempt. $script = 'status=0; : > '.escapeshellarg($remainingFile).'; ' .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do ' - .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) ' - .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; ' + .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>&1) ' + .'|| { case "$running" in *"o such object"*|*"o such container"*) continue ;; esac; ' + .'echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; ' .'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> ' .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; ' .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile) diff --git a/tests/Feature/VolumeBackupRecoveryMissingContainerTest.php b/tests/Feature/VolumeBackupRecoveryMissingContainerTest.php new file mode 100644 index 0000000000..fe5190a7b1 --- /dev/null +++ b/tests/Feature/VolumeBackupRecoveryMissingContainerTest.php @@ -0,0 +1,171 @@ + 'null', 'constants.ssh.mux_enabled' => false, 'constants.ssh.max_retries' => 1]); + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + + $this->fakeDockerDir = sys_get_temp_dir().'/prr-fix-fake-docker-'.uniqid(); + File::ensureDirectoryExists($this->fakeDockerDir); + $this->dockerLog = $this->fakeDockerDir.'/calls.log'; + File::put($this->fakeDockerDir.'/docker', <<<'SH' +#!/bin/sh +echo "$*" >> "$(dirname "$0")/calls.log" +case "$1" in + inspect) + case "$4" in + aaaaaaaaaaaa) echo "Error: no such object: $4" >&2; exit 1 ;; + *) echo false ;; + esac ;; + start) + case "$2" in + bbbbbbbbbbbb) echo "Error response from daemon: cannot start" >&2; exit 1 ;; + esac ;; +esac +SH); + chmod($this->fakeDockerDir.'/docker', 0755); +}); + +afterEach(function () { + File::deleteDirectory($this->fakeDockerDir); + foreach (ScheduledVolumeBackupExecution::query()->get() as $execution) { + File::delete([VolumeBackupRecoveryJob::stateFile($execution), VolumeBackupRecoveryJob::stateFile($execution).'.remaining']); + } + Carbon::setTestNow(); +}); + +/** + * @return array{0: Server, 1: ScheduledVolumeBackup} + */ +function missingContainerRecoveryBackup(string $user): array +{ + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + 'user' => $user, + ]); + $server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + 'save_s3' => false, + 'stop_during_backup' => true, + ]); + + return [Server::find($server->id), $backup]; +} + +/** + * Runs the remote container recovery script with a local sh and a fake docker binary, as the server would. + */ +function fakeMissingContainerRemote(string $fakeDockerDir): void +{ + Process::fake(function ($process) use ($fakeDockerDir) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $lines = explode(PHP_EOL, $command); + $remote = implode(PHP_EOL, array_slice($lines, 1, -1)); + if (str_starts_with($remote, 'sudo ')) { + $remote = substr($remote, strlen('sudo ')); + } + if (! str_starts_with($remote, 'sh -c ') && ! str_starts_with($remote, 'cat ')) { + return Process::result(output: ''); + } + + $result = new SymfonyProcess(['sh', '-c', $remote], null, ['PATH' => $fakeDockerDir.':'.getenv('PATH')]); + $result->run(); + + return Process::result(output: $result->getOutput(), errorOutput: $result->getErrorOutput(), exitCode: $result->getExitCode()); + }); +} + +it('treats stopped containers that no longer exist as recovered and does not block the schedule', function (string $user) { + Carbon::setTestNow('2026-10-03 12:00:00'); + config(['constants.coolify.self_hosted' => true]); + [$server, $backup] = missingContainerRecoveryBackup($user); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + 'stop_recovery_pending' => true, + ]); + File::put(VolumeBackupRecoveryJob::stateFile($execution), MISSING_CONTAINER_GONE."\n".MISSING_CONTAINER_STOPPED."\n"); + fakeMissingContainerRemote($this->fakeDockerDir); + + VolumeBackupRecoveryJob::recoverWithBounds($execution); + + $execution->refresh(); + expect($execution->stop_recovery_pending)->toBeFalse() + ->and($execution->recovery_needs_attention)->toBeFalse() + ->and(File::exists(VolumeBackupRecoveryJob::stateFile($execution)))->toBeFalse() + ->and(File::get($this->dockerLog))->toContain('start '.MISSING_CONTAINER_STOPPED); + + Queue::fake(); + Cache::forget("scheduled-volume-backup:{$backup->id}"); + (new ScheduledJobManager)->handle(); + Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup)); +})->with(['root user' => ['root'], 'non-root user' => ['ubuntu']]); + +it('still needs attention when an existing container fails to start', function (string $user) { + [, $backup] = missingContainerRecoveryBackup($user); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + 'stop_recovery_pending' => true, + ]); + File::put(VolumeBackupRecoveryJob::stateFile($execution), MISSING_CONTAINER_GONE."\n".MISSING_CONTAINER_BROKEN."\n"); + fakeMissingContainerRemote($this->fakeDockerDir); + + VolumeBackupRecoveryJob::recoverWithBounds($execution); + + $execution->refresh(); + expect($execution->stop_recovery_pending)->toBeTrue() + ->and($execution->recovery_needs_attention)->toBeTrue() + ->and(trim(File::get(VolumeBackupRecoveryJob::stateFile($execution))))->toBe(MISSING_CONTAINER_BROKEN); +})->with(['root user' => ['root'], 'non-root user' => ['ubuntu']]);