From fbfe342f89ea91a604a18ba9d9d77eccbaa7b9ca Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:19:14 +0200 Subject: [PATCH] fix(databases): reserve start and restart atomically Two parallel start requests both passed the in-progress check and got 200, because the start activity is created later by the queued action. A per-database reservation (Cache::add, 600 s) is now taken before the check, so the second request gets 409. The action releases it when it creates its activity, skips, or fails. Restart, import, the deploy API, MCP, and Livewire use the same reservation. Co-Authored-By: Claude Opus 5.5 --- app/Actions/Database/RestartDatabase.php | 44 ++- app/Actions/Database/StartDatabase.php | 78 ++++- app/Actions/Database/StartDatabaseImport.php | 15 + .../Controllers/Api/DatabasesController.php | 15 +- app/Http/Controllers/Api/DeployController.php | 8 +- app/Mcp/Tools/Control.php | 10 +- app/Support/DatabaseOperationReservation.php | 72 +++++ .../Feature/DatabaseStartConcurrencyTest.php | 290 ++++++++++++++++++ .../StartDatabaseImportConcurrencyTest.php | 51 +++ 9 files changed, 554 insertions(+), 29 deletions(-) create mode 100644 app/Support/DatabaseOperationReservation.php diff --git a/app/Actions/Database/RestartDatabase.php b/app/Actions/Database/RestartDatabase.php index 3bb83e4187..3e8f6dec83 100644 --- a/app/Actions/Database/RestartDatabase.php +++ b/app/Actions/Database/RestartDatabase.php @@ -10,28 +10,44 @@ use App\Models\StandaloneMongodb; use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; +use App\Support\DatabaseOperationReservation; +use App\Support\ResourceStartActivity; use Lorisleiva\Actions\Concerns\AsAction; class RestartDatabase { use AsAction; - public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database) + /** + * @param string|null $reservation The token from StartDatabase::reserveOperation(). The request + * that queued this action holds the reservation; this action + * keeps it during the stop and releases it after the start. + */ + public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null) { - $server = $database->destination->server; - if (! $server->isFunctional()) { - return 'Server is not functional'; + $reservation ??= DatabaseOperationReservation::acquire($database->uuid); + if ($reservation === null) { + return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; } - $busyError = StartDatabase::operationInProgressError($database); - if ($busyError !== null) { - return $busyError; - } - $prerequisiteError = StartDatabase::prerequisiteError($database); - if ($prerequisiteError !== null) { - return $prerequisiteError; - } - StopDatabase::run($database, dockerCleanup: false); - return StartDatabase::run($database); + try { + $server = $database->destination->server; + if (! $server->isFunctional()) { + return 'Server is not functional'; + } + $busyError = StartDatabase::operationInProgressError($database, $reservation); + if ($busyError !== null) { + return $busyError; + } + $prerequisiteError = StartDatabase::prerequisiteError($database); + if ($prerequisiteError !== null) { + return $prerequisiteError; + } + StopDatabase::run($database, dockerCleanup: false); + + return StartDatabase::run($database, $reservation); + } finally { + DatabaseOperationReservation::release($database->uuid, $reservation); + } } } diff --git a/app/Actions/Database/StartDatabase.php b/app/Actions/Database/StartDatabase.php index 6bc06969d2..5e75d8d0b5 100644 --- a/app/Actions/Database/StartDatabase.php +++ b/app/Actions/Database/StartDatabase.php @@ -14,6 +14,7 @@ use App\Models\StandaloneMongodb; use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; +use App\Support\DatabaseOperationReservation; use App\Support\ResourceStartActivity; use Lorisleiva\Actions\Concerns\AsAction; use Lorisleiva\Actions\Decorators\JobDecorator; @@ -29,13 +30,31 @@ class StartDatabase $job->onQueue(deployment_queue()); } - public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): Activity|string + /** + * @param string|null $reservation The token from reserveOperation(). The request that queued + * this action holds the reservation; this action releases it. + */ + public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null): Activity|string + { + $reservation ??= DatabaseOperationReservation::acquire($database->uuid); + if ($reservation === null) { + return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; + } + + try { + return $this->queueStart($database, $reservation); + } finally { + DatabaseOperationReservation::release($database->uuid, $reservation); + } + } + + private function queueStart(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, string $reservation): Activity|string { $server = $database->destination->server; if (! $server->isFunctional()) { return 'Server is not functional'; } - $busyError = self::operationInProgressError($database); + $busyError = self::operationInProgressError($database, $reservation); if ($busyError !== null) { return $busyError; } @@ -88,16 +107,63 @@ class StartDatabase } /** - * Refuse a second start/restart while a start, restart or import of this database is - * queued or running. The UI hides Start/Restart in this state; this is the server-side check. - * Stale start activities do not block and are marked as failed, so the UI stays consistent. + * Reserve a start or restart before it is queued, so that a parallel request cannot also + * queue one before the queued action has created its activity. Pass the token to the + * queued action, which releases the reservation. + * + * @return string|null The reservation token, or null when another operation is in progress. */ - public static function operationInProgressError(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): ?string + public static function reserveOperation(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): ?string + { + $reservation = DatabaseOperationReservation::acquire($database->uuid); + if ($reservation === null) { + return null; + } + + if (self::operationInProgressError($database, $reservation) !== null) { + DatabaseOperationReservation::release($database->uuid, $reservation); + + return null; + } + + return $reservation; + } + + /** + * Queue a start or restart action with a reservation. Release the reservation if the + * action cannot be queued, because then no action will release it. + * + * @param class-string $action + */ + public static function dispatchReserved(string $action, StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, string $reservation): void + { + try { + $action::dispatch($database, $reservation); + } catch (Throwable $e) { + DatabaseOperationReservation::release($database->uuid, $reservation); + + throw $e; + } + } + + /** + * Refuse a second start/restart while a start, restart or import of this database is + * reserved, queued or running. The UI hides Start/Restart in this state; this is the + * server-side check. Stale start activities do not block and are marked as failed, so + * the UI stays consistent. + * + * @param string|null $reservation The token of the caller's own reservation, which does not block. + */ + public static function operationInProgressError(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null): ?string { if (blank($database->uuid)) { return null; } + if (DatabaseOperationReservation::isHeldByOther($database->uuid, $reservation)) { + return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; + } + $liveOperations = collect([ ResourceStartActivity::DATABASE_START_OPERATION, ResourceStartActivity::DATABASE_IMPORT_OPERATION, diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index aea98b0045..7439e9f1b0 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -10,6 +10,7 @@ use App\Support\DatabaseBackupFileValidator; use App\Support\DatabaseImport\DatabaseImportCommandBuilder; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; +use App\Support\DatabaseOperationReservation; use App\Support\ResourceStartActivity; use App\Support\ValidationPatterns; use Illuminate\Database\Eloquent\Model; @@ -59,9 +60,19 @@ class StartDatabaseImport throw new DatabaseImportException('A database import is already running.', 409); } + // A start or restart can be queued without an activity yet: its reservation blocks the import. + // The import holds the reservation until its own activity exists. + $reservation = null; + try { + $reservation = DatabaseOperationReservation::acquire($resource->uuid); + if ($reservation === null) { + throw new DatabaseImportException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE, 409); + } + return $this->startImport($resource, $source, $teamId, $server, $container, $network); } finally { + DatabaseOperationReservation::release($resource->uuid, $reservation); $lock->release(); } } @@ -72,6 +83,10 @@ class StartDatabaseImport if (ResourceStartActivity::failStale($active)->isNotEmpty()) { throw new DatabaseImportException('A database import is already running.', 409); } + $activeStarts = ResourceStartActivity::active($resource->uuid, ResourceStartActivity::DATABASE_START_OPERATION, $teamId); + if (ResourceStartActivity::failStale($activeStarts)->isNotEmpty()) { + throw new DatabaseImportException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE, 409); + } $operation = (string) Str::uuid(); $containerPath = "/tmp/restore_{$operation}"; diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index 2189a3e08c..7b04422384 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -22,6 +22,7 @@ use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\SwarmDocker; +use App\Support\ResourceStartActivity; use App\Support\ValidationPatterns; use Illuminate\Database\Eloquent\Model; use Illuminate\Http\JsonResponse; @@ -3264,10 +3265,11 @@ class DatabasesController extends Controller if (str($database->status)->contains('running')) { return response()->json(['message' => 'Database is already running.'], 400); } - if ($busyError = StartDatabase::operationInProgressError($database)) { - return response()->json(['message' => $busyError], 409); + $reservation = StartDatabase::reserveOperation($database); + if ($reservation === null) { + return response()->json(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE], 409); } - StartDatabase::dispatch($database); + StartDatabase::dispatchReserved(StartDatabase::class, $database, $reservation); auditLog('api.database.started', [ 'team_id' => $teamId, @@ -3454,10 +3456,11 @@ class DatabasesController extends Controller $this->authorize('manage', $database); - if ($busyError = StartDatabase::operationInProgressError($database)) { - return response()->json(['message' => $busyError], 409); + $reservation = StartDatabase::reserveOperation($database); + if ($reservation === null) { + return response()->json(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE], 409); } - RestartDatabase::dispatch($database); + StartDatabase::dispatchReserved(RestartDatabase::class, $database, $reservation); auditLog('api.database.restarted', [ 'team_id' => $teamId, diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php index a0f0cc1aed..c9029de94a 100644 --- a/app/Http/Controllers/Api/DeployController.php +++ b/app/Http/Controllers/Api/DeployController.php @@ -12,6 +12,7 @@ use App\Models\ApplicationPreview; use App\Models\Server; use App\Models\Service; use App\Models\Tag; +use App\Support\ResourceStartActivity; use Illuminate\Auth\Access\AuthorizationException; use Illuminate\Http\Request; use OpenApi\Attributes as OA; @@ -582,7 +583,12 @@ class DeployController extends Controller } catch (AuthorizationException $e) { return ['message' => 'Unauthorized to start this database.', 'deployment_uuid' => null]; } - StartDatabase::dispatch($resource); + $reservation = StartDatabase::reserveOperation($resource); + if ($reservation === null) { + $message = ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; + break; + } + StartDatabase::dispatchReserved(StartDatabase::class, $resource, $reservation); $resource->started_at ??= now(); $resource->save(); diff --git a/app/Mcp/Tools/Control.php b/app/Mcp/Tools/Control.php index 4b52d3d93f..aca6b46564 100644 --- a/app/Mcp/Tools/Control.php +++ b/app/Mcp/Tools/Control.php @@ -13,6 +13,7 @@ use App\Mcp\Concerns\BuildsResponse; use App\Mcp\Concerns\ResolvesResource; use App\Mcp\Concerns\ResolvesTeam; use App\Models\Application; +use App\Support\ResourceStartActivity; use Illuminate\Contracts\JsonSchema\JsonSchema; use Laravel\Mcp\Request; use Laravel\Mcp\Response; @@ -140,13 +141,18 @@ class Control extends Tool return ['message' => 'Database is already running.']; } + $reservation = StartDatabase::reserveOperation($database); + if ($reservation === null) { + throw new \RuntimeException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + } + if ($action === 'restart') { - RestartDatabase::dispatch($database); + StartDatabase::dispatchReserved(RestartDatabase::class, $database, $reservation); return ['message' => 'Database restart request queued.']; } - StartDatabase::dispatch($database); + StartDatabase::dispatchReserved(StartDatabase::class, $database, $reservation); return ['message' => 'Database starting request queued.']; } diff --git a/app/Support/DatabaseOperationReservation.php b/app/Support/DatabaseOperationReservation.php new file mode 100644 index 0000000000..936f5ab56f --- /dev/null +++ b/app/Support/DatabaseOperationReservation.php @@ -0,0 +1,72 @@ + $job->getAction() instanceof $actionClass)->last(); + expect($job)->not->toBeNull(); + + return $job->handle(); +} + +function databaseOperationReserved(string $databaseUuid): bool +{ + return Cache::has(DatabaseOperationReservation::key($databaseUuid)); +} + +describe('pending operation reservation', function () { + it('rejects a second API start while the first start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk() + ->assertJson(['message' => 'Database starting request queued.']); + + // The queued action did not run yet, so no activity exists. The reservation must block. + expect(databaseStartActivityCount($this->database->uuid))->toBe(0); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409) + ->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]); + + StartDatabase::assertPushed(1); + }); + + it('rejects an API restart while an API start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertStatus(409) + ->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]); + + RestartDatabase::assertNotPushed(); + }); + + it('rejects a second API restart and an API start while the first restart is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertOk() + ->assertJson(['message' => 'Database restarting request queued.']); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertStatus(409) + ->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + RestartDatabase::assertPushed(1); + StartDatabase::assertNotPushed(); + }); + + it('hands the reservation over to the start activity and allows a new start after it finished', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + expect(databaseOperationReserved($this->database->uuid))->toBeTrue(); + + $activity = runPushedDatabaseAction(StartDatabase::class); + + expect($activity)->toBeInstanceOf(Activity::class) + ->and(databaseOperationReserved($this->database->uuid))->toBeFalse(); + Queue::assertPushed(DatabaseStartJob::class, 1); + + // The queued start activity now blocks. + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + $activity->properties = $activity->properties->merge(['status' => ProcessStatus::FINISHED->value]); + $activity->save(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + }); + + it('stops and starts in the queued restart action and then releases the reservation', function () { + Queue::fake(); + StopDatabase::shouldRun()->once(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertOk(); + + $activity = runPushedDatabaseAction(RestartDatabase::class); + + expect($activity)->toBeInstanceOf(Activity::class) + ->and(databaseOperationReserved($this->database->uuid))->toBeFalse() + ->and(databaseStartActivityCount($this->database->uuid))->toBe(1); + }); + + it('releases the reservation when the queued action does not start the database', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + // For example a start that another path created without a reservation. + concurrentStartActivity($this->database->uuid, $this->team->id); + + expect(runPushedDatabaseAction(StartDatabase::class))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE) + ->and(databaseOperationReserved($this->database->uuid))->toBeFalse(); + Queue::assertNotPushed(DatabaseStartJob::class); + }); + + it('lets a new start through after a lost reservation expires', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $this->travel(DatabaseOperationReservation::TTL_SECONDS - 1)->seconds(); + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + $this->travel(2)->seconds(); + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + StartDatabase::assertPushed(2); + }); + + it('does not let an expired reservation start after a newer request took the reservation', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + $firstJob = Queue::pushed(JobDecorator::class)->first(); + + $this->travel(DatabaseOperationReservation::TTL_SECONDS + 1)->seconds(); + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + expect($firstJob->handle())->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE) + ->and(databaseOperationReserved($this->database->uuid))->toBeTrue() + ->and(databaseStartActivityCount($this->database->uuid))->toBe(0); + }); + + it('does not start from the UI while an API start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database]) + ->call('start') + ->assertDispatched('error', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + Queue::assertNotPushed(DatabaseStartJob::class); + expect(databaseStartActivityCount($this->database->uuid))->toBe(0); + }); + + it('does not stop the database on a UI restart while an API restart is still in the queue', function () { + Queue::fake(); + StopDatabase::shouldRun()->never(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertOk(); + + Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database]) + ->call('restart') + ->assertDispatched('error', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + }); + + it('releases its own reservation after a UI start created the activity', function () { + Bus::fake(); + + Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database]) + ->call('start') + ->assertNotDispatched('error'); + + Bus::assertDispatched(DatabaseStartJob::class); + expect(databaseOperationReserved($this->database->uuid))->toBeFalse() + ->and(databaseStartActivityCount($this->database->uuid))->toBe(1); + }); + + it('rejects an API start while a UI start holds the reservation', function () { + Queue::fake(); + $token = DatabaseOperationReservation::acquire($this->database->uuid); + expect($token)->toBeString(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + DatabaseOperationReservation::release($this->database->uuid, $token); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + }); + + it('does not release a reservation that another request holds', function () { + $token = DatabaseOperationReservation::acquire($this->database->uuid); + + DatabaseOperationReservation::release($this->database->uuid, 'another-token'); + + expect(databaseOperationReserved($this->database->uuid))->toBeTrue() + ->and(DatabaseOperationReservation::acquire($this->database->uuid))->toBeNull(); + DatabaseOperationReservation::release($this->database->uuid, $token); + expect(databaseOperationReserved($this->database->uuid))->toBeFalse(); + }); + + it('does not queue a database start through the deploy API while a start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson('/api/v1/deploy', ['uuid' => $this->database->uuid]) + ->assertOk() + ->assertJsonPath('deployments.0.message', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + StartDatabase::assertPushed(1); + }); + + it('does not queue a database start through the MCP control tool while a start is still in the queue', function (string $action) { + Queue::fake(); + InstanceSettings::query()->whereKey(0)->update(['is_mcp_server_enabled' => true]); + $this->team->update(['is_mcp_server_enabled' => true]); + Once::flush(); + $mcpToken = $this->user->createToken('mcp-deploy', ['read', 'deploy'])->plainTextToken; + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $response = $this->withHeaders([ + 'Content-Type' => 'application/json', + 'Accept' => 'application/json, text/event-stream', + 'Authorization' => 'Bearer '.$mcpToken, + ])->postJson('/mcp', [ + 'jsonrpc' => '2.0', + 'id' => 1, + 'method' => 'tools/call', + 'params' => [ + 'name' => 'control', + 'arguments' => (object) [ + 'resource' => 'database', + 'action' => $action, + 'uuid' => $this->database->uuid, + ], + ], + ]); + + $response->assertOk(); + expect($response->json('result.isError'))->toBeTrue() + ->and($response->json('result.content.0.text'))->toContain(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + StartDatabase::assertPushed(1); + RestartDatabase::assertNotPushed(); + })->with(['start', 'restart']); +}); + describe('start job', function () { beforeEach(function () { Event::fake([DatabaseStatusChanged::class]); diff --git a/tests/Feature/StartDatabaseImportConcurrencyTest.php b/tests/Feature/StartDatabaseImportConcurrencyTest.php index bab4f99b83..e238c2ae95 100644 --- a/tests/Feature/StartDatabaseImportConcurrencyTest.php +++ b/tests/Feature/StartDatabaseImportConcurrencyTest.php @@ -11,6 +11,7 @@ use App\Models\StandalonePostgresql; use App\Models\Team; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; +use App\Support\DatabaseOperationReservation; use App\Support\ResourceStartActivity; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Cache; @@ -191,3 +192,53 @@ test('an import interrupted by a Coolify restart is failed at boot and no longer expect(fn () => startImport($this->database, $this->team->id)) ->toThrow(DatabaseImportException::class, 'The server path is invalid.'); }); + +test('an import is rejected while a start or restart of the database waits in the queue', function () { + $token = DatabaseOperationReservation::acquire($this->database->uuid); + + try { + expect(fn () => startImport($this->database, $this->team->id)) + ->toThrow(DatabaseImportException::class, ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + } finally { + DatabaseOperationReservation::release($this->database->uuid, $token); + } +}); + +test('the 409 status is used when a pending start blocks an import', function () { + $token = DatabaseOperationReservation::acquire($this->database->uuid); + + try { + startImport($this->database, $this->team->id); + $this->fail('The import was not rejected.'); + } catch (DatabaseImportException $exception) { + expect($exception->status)->toBe(409); + } finally { + DatabaseOperationReservation::release($this->database->uuid, $token); + } +}); + +test('an import is rejected while a start of the database is running', function (ProcessStatus $status) { + Activity::create([ + 'log_name' => 'default', + 'description' => '[]', + 'properties' => [ + 'team_id' => $this->team->id, + 'type_uuid' => $this->database->uuid, + 'operation' => ResourceStartActivity::DATABASE_START_OPERATION, + 'status' => $status->value, + ], + ]); + + expect(fn () => startImport($this->database, $this->team->id)) + ->toThrow(DatabaseImportException::class, ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); +})->with([ + 'queued' => ProcessStatus::QUEUED, + 'in progress' => ProcessStatus::IN_PROGRESS, +]); + +test('an import holds the operation reservation and releases it afterwards', function () { + expect(fn () => startImport($this->database, $this->team->id)) + ->toThrow(DatabaseImportException::class, 'The server path is invalid.'); + + expect(Cache::has(DatabaseOperationReservation::key($this->database->uuid)))->toBeFalse(); +});