fix(docker): make cleanup commands idempotent (#11463)

app/Actions/Application/StopApplicationOneServer.php

@@ -29,7 +29,7 @@ class StopApplicationOneServer
                         instant_remote_process(
                             [
                                 dockerStopCommand($timeout, $containerName, $server),
-                                "docker rm -f $containerName",
+                                dockerRemoveCommand($containerName),
                             ],
                             $server
                         );

app/Actions/Database/StopDatabaseProxy.php

@@ -24,10 +24,10 @@ class StopDatabaseProxy
     {
         $server = data_get($database, 'destination.server');
         $uuid = $database->uuid;
-        if ($database->getMorphClass() === \App\Models\ServiceDatabase::class) {
+        if ($database->getMorphClass() === ServiceDatabase::class) {
             $server = data_get($database, 'service.server');
         }
-        instant_remote_process(["docker rm -f {$uuid}-proxy"], $server);
+        instant_remote_process([dockerRemoveCommand("{$uuid}-proxy")], $server);
 
         $database->save();
 

app/Actions/Destination/RemoveStandaloneDockerNetwork.php

@@ -11,6 +11,6 @@ class RemoveStandaloneDockerNetwork
         $safeNetwork = escapeshellarg($destination->network);
 
         instant_remote_process(["docker network disconnect {$safeNetwork} coolify-proxy"], $destination->server, throwError: false);
-        instant_remote_process(["docker network rm -f {$safeNetwork}"], $destination->server);
+        instant_remote_process([dockerNetworkRemoveCommand($destination->network)], $destination->server);
     }
 }

app/Jobs/DatabaseBackupJob.php

@@ -798,7 +798,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
             $this->add_to_error_output($e->getMessage());
             throw $e;
         } finally {
-            $command = "docker rm -f backup-of-{$this->backup_log_uuid}";
+            $command = dockerRemoveCommand("backup-of-{$this->backup_log_uuid}");
             instant_remote_process([$command], $this->server, true, false, null, disableMultiplexing: true);
         }
     }

app/Listeners/ProxyStatusChangedNotification.php

@@ -61,7 +61,7 @@ class ProxyStatusChangedNotification implements ShouldQueueAfterCommit
 
         if ($status === 'created') {
             instant_remote_process([
-                'docker rm -f coolify-proxy',
+                dockerRemoveCommand('coolify-proxy'),
             ], $server);
         }
     }

app/Livewire/Destination/Show.php

@@ -81,7 +81,7 @@ class Show extends Component
                 }
                 $safeNetwork = escapeshellarg($this->destination->network);
                 instant_remote_process(["docker network disconnect {$safeNetwork} coolify-proxy"], $this->destination->server, throwError: false);
-                instant_remote_process(["docker network rm -f {$safeNetwork}"], $this->destination->server);
+                instant_remote_process([dockerNetworkRemoveCommand($this->destination->network)], $this->destination->server);
             }
             $this->destination->delete();
 

bootstrap/helpers/docker.php

@@ -267,7 +267,28 @@ function dockerStopCommand(int $timeout, string $containers, Server|string|null
 function dockerRemoveCommandWithTimeout(string $container, int $timeout = 60, int $killAfter = 10): string
 {
     $container = escapeShellValue($container);
-    $script = "if command -v timeout >/dev/null 2>&1; then timeout -k {$killAfter}s {$timeout}s docker rm -f {$container}; exit_code=\$?; else exit_code=124; fi; if [ \"\$exit_code\" -eq 124 ]; then echo '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__'; fi; exit \$exit_code";
+    $script = "if command -v timeout >/dev/null 2>&1; then output=\$(timeout -k {$killAfter}s {$timeout}s docker rm -f {$container} 2>&1); exit_code=\$?; else output=''; exit_code=124; fi; if [ \"\$exit_code\" -eq 124 ]; then echo '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__'; elif [ \"\$exit_code\" -ne 0 ] && printf '%s' \"\$output\" | grep -q 'No such container:'; then exit 0; elif [ \"\$exit_code\" -ne 0 ]; then printf '%s\\n' \"\$output\" >&2; else printf '%s\\n' \"\$output\"; fi; exit \$exit_code";
+
+    return 'bash -c '.escapeShellValue($script);
+}
+
+function dockerRemoveCommand(string $container): string
+{
+    $command = 'docker rm -f '.escapeShellValue($container);
+
+    return dockerCommandIgnoringError($command, 'No such container:');
+}
+
+function dockerNetworkRemoveCommand(string $network): string
+{
+    $command = 'docker network rm '.escapeShellValue($network);
+
+    return dockerCommandIgnoringError($command, 'network .* not found');
+}
+
+function dockerCommandIgnoringError(string $command, string $ignoredError): string
+{
+    $script = "output=\$({$command} 2>&1); exit_code=\$?; if [ \"\$exit_code\" -ne 0 ] && printf '%s' \"\$output\" | grep -Eq ".escapeShellValue($ignoredError)."; then exit 0; fi; if [ \"\$exit_code\" -ne 0 ]; then printf '%s\\n' \"\$output\" >&2; else printf '%s\\n' \"\$output\"; fi; exit \$exit_code";
 
     return 'bash -c '.escapeShellValue($script);
 }

tests/Unit/DockerRemoveWithTimeoutTest.php

@@ -58,6 +58,124 @@ it('escapes container names in bounded removal commands', function () {
     rmdir($directory);
 });
 
+it('succeeds when the container was already removed', function () {
+    $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+    mkdir($directory);
+    file_put_contents($directory.'/docker', "#!/bin/sh\necho 'Error response from daemon: No such container: container-name' >&2\nexit 1\n");
+    chmod($directory.'/docker', 0755);
+
+    $process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout('container-name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+    ]);
+    $process->run();
+
+    expect($process->isSuccessful())->toBeTrue();
+
+    unlink($directory.'/docker');
+    rmdir($directory);
+});
+
+it('reports a timeout when timeout output also says the container is missing', function () {
+    $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+    mkdir($directory);
+    file_put_contents($directory.'/timeout', "#!/bin/sh\necho 'Error response from daemon: No such container: container-name'\nexit 124\n");
+    chmod($directory.'/timeout', 0755);
+
+    $process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout('container-name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+    ]);
+    $process->run();
+
+    expect($process->getExitCode())->toBe(124)
+        ->and($process->getOutput())->toContain('__COOLIFY_CONTAINER_REMOVE_TIMEOUT__');
+
+    unlink($directory.'/timeout');
+    rmdir($directory);
+});
+
+it('fails when Docker cannot remove an existing container', function () {
+    $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+    mkdir($directory);
+    file_put_contents($directory.'/docker', "#!/bin/sh\necho 'Error response from daemon: removal already in progress' >&2\nexit 1\n");
+    chmod($directory.'/docker', 0755);
+
+    $process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout('container-name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+    ]);
+    $process->run();
+
+    expect($process->isSuccessful())->toBeFalse()
+        ->and($process->getErrorOutput())->toContain('removal already in progress');
+
+    unlink($directory.'/docker');
+    rmdir($directory);
+});
+
+it('makes regular container removal idempotent without hiding other failures', function () {
+    $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+    mkdir($directory);
+    file_put_contents($directory.'/docker', "#!/bin/sh\necho \"\$DOCKER_ERROR\" >&2\nexit 1\n");
+    chmod($directory.'/docker', 0755);
+
+    $missingContainer = new Process(['/bin/sh', '-c', dockerRemoveCommand('container name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+        'DOCKER_ERROR' => 'Error response from daemon: No such container: container name',
+    ]);
+    $missingContainer->run();
+    $realFailure = new Process(['/bin/sh', '-c', dockerRemoveCommand('container name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+        'DOCKER_ERROR' => 'Error response from daemon: removal already in progress',
+    ]);
+    $realFailure->run();
+
+    expect($missingContainer->isSuccessful())->toBeTrue()
+        ->and($realFailure->isSuccessful())->toBeFalse();
+
+    unlink($directory.'/docker');
+    rmdir($directory);
+});
+
+it('makes network removal idempotent without hiding other failures', function () {
+    $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+    mkdir($directory);
+    file_put_contents($directory.'/docker', "#!/bin/sh\necho \"\$DOCKER_ERROR\" >&2\nexit 1\n");
+    chmod($directory.'/docker', 0755);
+
+    $missingNetwork = new Process(['/bin/sh', '-c', dockerNetworkRemoveCommand('network name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+        'DOCKER_ERROR' => 'Error response from daemon: network network name not found',
+    ]);
+    $missingNetwork->run();
+    $realFailure = new Process(['/bin/sh', '-c', dockerNetworkRemoveCommand('network name')], env: [
+        'PATH' => $directory.':'.getenv('PATH'),
+        'DOCKER_ERROR' => 'Error response from daemon: network has active endpoints',
+    ]);
+    $realFailure->run();
+
+    expect($missingNetwork->isSuccessful())->toBeTrue()
+        ->and($realFailure->isSuccessful())->toBeFalse();
+
+    unlink($directory.'/docker');
+    rmdir($directory);
+});
+
+it('uses idempotent commands in strict cleanup paths', function () {
+    $root = dirname(__DIR__, 2);
+
+    expect(file_get_contents($root.'/app/Jobs/DatabaseBackupJob.php'))
+        ->toContain('dockerRemoveCommand("backup-of-{$this->backup_log_uuid}")')
+        ->and(file_get_contents($root.'/app/Actions/Database/StopDatabaseProxy.php'))
+        ->toContain('dockerRemoveCommand("{$uuid}-proxy")')
+        ->and(file_get_contents($root.'/app/Actions/Destination/RemoveStandaloneDockerNetwork.php'))
+        ->toContain('dockerNetworkRemoveCommand($destination->network)')
+        ->and(file_get_contents($root.'/app/Livewire/Destination/Show.php'))
+        ->toContain('dockerNetworkRemoveCommand($this->destination->network)')
+        ->and(file_get_contents($root.'/app/Listeners/ProxyStatusChangedNotification.php'))
+        ->toContain("dockerRemoveCommand('coolify-proxy')")
+        ->and(file_get_contents($root.'/app/Actions/Application/StopApplicationOneServer.php'))
+        ->toContain('dockerRemoveCommand($containerName)');
+});
+
 it('configures deferred removal attempts to outlive the shell timeout', function () {
     $job = new RemoveContainerJob(123, 'container-name');