fix(backups): treat removed containers as recovered after a volume backup

When a container that a volume backup stopped no longer existed (for example after a redeploy), recovery kept it as remaining. The execution needed attention and every later run of the schedule was skipped. - Treat "no such container" or "no such object" from docker inspect as recovered; other failures still need attention
2026-11-06 13:10:37 -04:00 · 2026-10-05 21:21:34 +02:00
parent b794da0983
commit 4f43c96e53
2 changed files with 176 additions and 2 deletions
@@ -276,10 +276,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
        $execution->update(['stop_container_ids' => $containers]);

        $remainingFile = $stateFile.'.remaining';
+        // A container that no longer exists (for example after a redeploy) has nothing to restart; only
+        // other inspect or start failures keep the container for another recovery attempt.
        $script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
            .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
-            .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) '
-            .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
+            .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>&1) '
+            .'|| { case "$running" in *"o such object"*|*"o such container"*) continue ;; esac; '
+            .'echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
            .'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> '
            .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
            .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
@@ -0,0 +1,171 @@
+<?php
+
+use App\Jobs\ScheduledJobManager;
+use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\File;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use Symfony\Component\Process\Process as SymfonyProcess;
+
+uses(RefreshDatabase::class);
+
+const MISSING_CONTAINER_GONE = 'aaaaaaaaaaaa';
+const MISSING_CONTAINER_BROKEN = 'bbbbbbbbbbbb';
+const MISSING_CONTAINER_STOPPED = 'cccccccccccc';
+
+beforeEach(function () {
+    Server::flushIdentityMap();
+    Notification::fake();
+    Storage::fake('ssh-keys');
+    Storage::fake('ssh-mux');
+    config(['broadcasting.default' => 'null', 'constants.ssh.mux_enabled' => false, 'constants.ssh.max_retries' => 1]);
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+    $this->fakeDockerDir = sys_get_temp_dir().'/prr-fix-fake-docker-'.uniqid();
+    File::ensureDirectoryExists($this->fakeDockerDir);
+    $this->dockerLog = $this->fakeDockerDir.'/calls.log';
+    File::put($this->fakeDockerDir.'/docker', <<<'SH'
+#!/bin/sh
+echo "$*" >> "$(dirname "$0")/calls.log"
+case "$1" in
+  inspect)
+    case "$4" in
+      aaaaaaaaaaaa) echo "Error: no such object: $4" >&2; exit 1 ;;
+      *) echo false ;;
+    esac ;;
+  start)
+    case "$2" in
+      bbbbbbbbbbbb) echo "Error response from daemon: cannot start" >&2; exit 1 ;;
+    esac ;;
+esac
+SH);
+    chmod($this->fakeDockerDir.'/docker', 0755);
+});
+
+afterEach(function () {
+    File::deleteDirectory($this->fakeDockerDir);
+    foreach (ScheduledVolumeBackupExecution::query()->get() as $execution) {
+        File::delete([VolumeBackupRecoveryJob::stateFile($execution), VolumeBackupRecoveryJob::stateFile($execution).'.remaining']);
+    }
+    Carbon::setTestNow();
+});
+
+/**
+ * @return array{0: Server, 1: ScheduledVolumeBackup}
+ */
+function missingContainerRecoveryBackup(string $user): array
+{
+    $team = Team::factory()->create();
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+        'user' => $user,
+    ]);
+    $server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+    $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+    $application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ]);
+    $volume = LocalPersistentVolume::create([
+        'name' => 'app-data',
+        'mount_path' => '/data',
+        'resource_id' => $application->id,
+        'resource_type' => $application->getMorphClass(),
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => '* * * * *',
+        'enabled' => true,
+        'save_s3' => false,
+        'stop_during_backup' => true,
+    ]);
+
+    return [Server::find($server->id), $backup];
+}
+
+/**
+ * Runs the remote container recovery script with a local sh and a fake docker binary, as the server would.
+ */
+function fakeMissingContainerRemote(string $fakeDockerDir): void
+{
+    Process::fake(function ($process) use ($fakeDockerDir) {
+        $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+        $lines = explode(PHP_EOL, $command);
+        $remote = implode(PHP_EOL, array_slice($lines, 1, -1));
+        if (str_starts_with($remote, 'sudo ')) {
+            $remote = substr($remote, strlen('sudo '));
+        }
+        if (! str_starts_with($remote, 'sh -c ') && ! str_starts_with($remote, 'cat ')) {
+            return Process::result(output: '');
+        }
+
+        $result = new SymfonyProcess(['sh', '-c', $remote], null, ['PATH' => $fakeDockerDir.':'.getenv('PATH')]);
+        $result->run();
+
+        return Process::result(output: $result->getOutput(), errorOutput: $result->getErrorOutput(), exitCode: $result->getExitCode());
+    });
+}
+
+it('treats stopped containers that no longer exist as recovered and does not block the schedule', function (string $user) {
+    Carbon::setTestNow('2026-10-03 12:00:00');
+    config(['constants.coolify.self_hosted' => true]);
+    [$server, $backup] = missingContainerRecoveryBackup($user);
+    $execution = ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        'status' => 'failed',
+        'stop_recovery_pending' => true,
+    ]);
+    File::put(VolumeBackupRecoveryJob::stateFile($execution), MISSING_CONTAINER_GONE."\n".MISSING_CONTAINER_STOPPED."\n");
+    fakeMissingContainerRemote($this->fakeDockerDir);
+
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+    $execution->refresh();
+    expect($execution->stop_recovery_pending)->toBeFalse()
+        ->and($execution->recovery_needs_attention)->toBeFalse()
+        ->and(File::exists(VolumeBackupRecoveryJob::stateFile($execution)))->toBeFalse()
+        ->and(File::get($this->dockerLog))->toContain('start '.MISSING_CONTAINER_STOPPED);
+
+    Queue::fake();
+    Cache::forget("scheduled-volume-backup:{$backup->id}");
+    (new ScheduledJobManager)->handle();
+    Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup));
+})->with(['root user' => ['root'], 'non-root user' => ['ubuntu']]);
+
+it('still needs attention when an existing container fails to start', function (string $user) {
+    [, $backup] = missingContainerRecoveryBackup($user);
+    $execution = ScheduledVolumeBackupExecution::create([
+        'scheduled_volume_backup_id' => $backup->id,
+        'status' => 'failed',
+        'stop_recovery_pending' => true,
+    ]);
+    File::put(VolumeBackupRecoveryJob::stateFile($execution), MISSING_CONTAINER_GONE."\n".MISSING_CONTAINER_BROKEN."\n");
+    fakeMissingContainerRemote($this->fakeDockerDir);
+
+    VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+    $execution->refresh();
+    expect($execution->stop_recovery_pending)->toBeTrue()
+        ->and($execution->recovery_needs_attention)->toBeTrue()
+        ->and(trim(File::get(VolumeBackupRecoveryJob::stateFile($execution))))->toBe(MISSING_CONTAINER_BROKEN);
+})->with(['root user' => ['root'], 'non-root user' => ['ubuntu']]);