Fix Race Condition in Container Removal (#130) (#218)

This PR resolves a race condition when removing a container immediately
after stopping it, caused by the `stop` command returning before the
container fully transitions to the stopped state (#130).

**Changes:**
- Enhanced `TestCLIRmRace.swift` with robust test logic and helper
methods (`containerExists`, `safeRemove`).
- Improved error handling to distinguish race conditions from successful
removals.
- Added exponential backoff retry logic for cleanup operations.
- Updated `CLITest.swift` with missing `doRemove` method.
- Fixed `BuilderStart.swift` to handle `.stopping` case.
- Improved error messages with container ID for better debugging.

**Testing:**
- ✅ All tests pass (`make test`, `make integration`).
- ✅ Verified on macOS 26.
- ✅ Race condition test validates success and failure scenarios.
- ✅ Code formatted (`make fmt`).

Hopefully, this will pass the integration tests on GitHub.

Signed-off-by: ramsyana <47033578+ramsyana@users.noreply.github.com>

Sources/APIServer/Containers/ContainersService.swift

@@ -209,10 +209,10 @@ actor ContainersService {
         switch item.state {
         case .alive(let client):
             let state = try await client.state()
-            if state.status == .running {
+            if state.status == .running || state.status == .stopping {
                 throw ContainerizationError(
                     .invalidState,
-                    message: "container with ID \(id) is running"
+                    message: "container \(id) is not yet stopped and can not be deleted"
                 )
             }
             try self._cleanup(id: id, item: item)

Sources/CLI/Builder/BuilderStart.swift

@@ -125,6 +125,11 @@ extension Application {
                         return
                     }
                     try await existingContainer.delete()
+                case .stopping:
+                    throw ContainerizationError(
+                        .invalidState,
+                        message: "builder is stopping, please wait until it is fully stopped before proceeding"
+                    )
                 case .unknown:
                     break
                 }

Sources/ContainerClient/Core/RuntimeStatus.swift

@@ -24,4 +24,6 @@ public enum RuntimeStatus: String, CaseIterable, Sendable, Codable {
     case stopped
     /// The object is currently running.
     case running
+    /// The object is currently stopping.
+    case stopping
 }

Sources/Services/ContainerSandboxService/SandboxService.swift

@@ -321,8 +321,10 @@ public actor SandboxService {
         var cs: ContainerSnapshot?
 
         switch state {
-        case .created, .stopped(_), .starting, .booted, .stopping:
+        case .created, .stopped(_), .starting, .booted:
             status = .stopped
+        case .stopping:
+            status = .stopping
         case .running:
             let ctr = try getContainer()