diff --git a/app/Actions/Database/RestartDatabase.php b/app/Actions/Database/RestartDatabase.php index 3bb83e4187..3e8f6dec83 100644 --- a/app/Actions/Database/RestartDatabase.php +++ b/app/Actions/Database/RestartDatabase.php @@ -10,28 +10,44 @@ use App\Models\StandaloneMongodb; use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; +use App\Support\DatabaseOperationReservation; +use App\Support\ResourceStartActivity; use Lorisleiva\Actions\Concerns\AsAction; class RestartDatabase { use AsAction; - public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database) + /** + * @param string|null $reservation The token from StartDatabase::reserveOperation(). The request + * that queued this action holds the reservation; this action + * keeps it during the stop and releases it after the start. + */ + public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null) { - $server = $database->destination->server; - if (! $server->isFunctional()) { - return 'Server is not functional'; + $reservation ??= DatabaseOperationReservation::acquire($database->uuid); + if ($reservation === null) { + return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; } - $busyError = StartDatabase::operationInProgressError($database); - if ($busyError !== null) { - return $busyError; - } - $prerequisiteError = StartDatabase::prerequisiteError($database); - if ($prerequisiteError !== null) { - return $prerequisiteError; - } - StopDatabase::run($database, dockerCleanup: false); - return StartDatabase::run($database); + try { + $server = $database->destination->server; + if (! $server->isFunctional()) { + return 'Server is not functional'; + } + $busyError = StartDatabase::operationInProgressError($database, $reservation); + if ($busyError !== null) { + return $busyError; + } + $prerequisiteError = StartDatabase::prerequisiteError($database); + if ($prerequisiteError !== null) { + return $prerequisiteError; + } + StopDatabase::run($database, dockerCleanup: false); + + return StartDatabase::run($database, $reservation); + } finally { + DatabaseOperationReservation::release($database->uuid, $reservation); + } } } diff --git a/app/Actions/Database/StartDatabase.php b/app/Actions/Database/StartDatabase.php index 6bc06969d2..5e75d8d0b5 100644 --- a/app/Actions/Database/StartDatabase.php +++ b/app/Actions/Database/StartDatabase.php @@ -14,6 +14,7 @@ use App\Models\StandaloneMongodb; use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; +use App\Support\DatabaseOperationReservation; use App\Support\ResourceStartActivity; use Lorisleiva\Actions\Concerns\AsAction; use Lorisleiva\Actions\Decorators\JobDecorator; @@ -29,13 +30,31 @@ class StartDatabase $job->onQueue(deployment_queue()); } - public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): Activity|string + /** + * @param string|null $reservation The token from reserveOperation(). The request that queued + * this action holds the reservation; this action releases it. + */ + public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null): Activity|string + { + $reservation ??= DatabaseOperationReservation::acquire($database->uuid); + if ($reservation === null) { + return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; + } + + try { + return $this->queueStart($database, $reservation); + } finally { + DatabaseOperationReservation::release($database->uuid, $reservation); + } + } + + private function queueStart(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, string $reservation): Activity|string { $server = $database->destination->server; if (! $server->isFunctional()) { return 'Server is not functional'; } - $busyError = self::operationInProgressError($database); + $busyError = self::operationInProgressError($database, $reservation); if ($busyError !== null) { return $busyError; } @@ -88,16 +107,63 @@ class StartDatabase } /** - * Refuse a second start/restart while a start, restart or import of this database is - * queued or running. The UI hides Start/Restart in this state; this is the server-side check. - * Stale start activities do not block and are marked as failed, so the UI stays consistent. + * Reserve a start or restart before it is queued, so that a parallel request cannot also + * queue one before the queued action has created its activity. Pass the token to the + * queued action, which releases the reservation. + * + * @return string|null The reservation token, or null when another operation is in progress. */ - public static function operationInProgressError(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): ?string + public static function reserveOperation(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): ?string + { + $reservation = DatabaseOperationReservation::acquire($database->uuid); + if ($reservation === null) { + return null; + } + + if (self::operationInProgressError($database, $reservation) !== null) { + DatabaseOperationReservation::release($database->uuid, $reservation); + + return null; + } + + return $reservation; + } + + /** + * Queue a start or restart action with a reservation. Release the reservation if the + * action cannot be queued, because then no action will release it. + * + * @param class-string $action + */ + public static function dispatchReserved(string $action, StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, string $reservation): void + { + try { + $action::dispatch($database, $reservation); + } catch (Throwable $e) { + DatabaseOperationReservation::release($database->uuid, $reservation); + + throw $e; + } + } + + /** + * Refuse a second start/restart while a start, restart or import of this database is + * reserved, queued or running. The UI hides Start/Restart in this state; this is the + * server-side check. Stale start activities do not block and are marked as failed, so + * the UI stays consistent. + * + * @param string|null $reservation The token of the caller's own reservation, which does not block. + */ + public static function operationInProgressError(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null): ?string { if (blank($database->uuid)) { return null; } + if (DatabaseOperationReservation::isHeldByOther($database->uuid, $reservation)) { + return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; + } + $liveOperations = collect([ ResourceStartActivity::DATABASE_START_OPERATION, ResourceStartActivity::DATABASE_IMPORT_OPERATION, diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index aea98b0045..7439e9f1b0 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -10,6 +10,7 @@ use App\Support\DatabaseBackupFileValidator; use App\Support\DatabaseImport\DatabaseImportCommandBuilder; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; +use App\Support\DatabaseOperationReservation; use App\Support\ResourceStartActivity; use App\Support\ValidationPatterns; use Illuminate\Database\Eloquent\Model; @@ -59,9 +60,19 @@ class StartDatabaseImport throw new DatabaseImportException('A database import is already running.', 409); } + // A start or restart can be queued without an activity yet: its reservation blocks the import. + // The import holds the reservation until its own activity exists. + $reservation = null; + try { + $reservation = DatabaseOperationReservation::acquire($resource->uuid); + if ($reservation === null) { + throw new DatabaseImportException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE, 409); + } + return $this->startImport($resource, $source, $teamId, $server, $container, $network); } finally { + DatabaseOperationReservation::release($resource->uuid, $reservation); $lock->release(); } } @@ -72,6 +83,10 @@ class StartDatabaseImport if (ResourceStartActivity::failStale($active)->isNotEmpty()) { throw new DatabaseImportException('A database import is already running.', 409); } + $activeStarts = ResourceStartActivity::active($resource->uuid, ResourceStartActivity::DATABASE_START_OPERATION, $teamId); + if (ResourceStartActivity::failStale($activeStarts)->isNotEmpty()) { + throw new DatabaseImportException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE, 409); + } $operation = (string) Str::uuid(); $containerPath = "/tmp/restore_{$operation}"; diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index 2189a3e08c..7b04422384 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -22,6 +22,7 @@ use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; use App\Models\SwarmDocker; +use App\Support\ResourceStartActivity; use App\Support\ValidationPatterns; use Illuminate\Database\Eloquent\Model; use Illuminate\Http\JsonResponse; @@ -3264,10 +3265,11 @@ class DatabasesController extends Controller if (str($database->status)->contains('running')) { return response()->json(['message' => 'Database is already running.'], 400); } - if ($busyError = StartDatabase::operationInProgressError($database)) { - return response()->json(['message' => $busyError], 409); + $reservation = StartDatabase::reserveOperation($database); + if ($reservation === null) { + return response()->json(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE], 409); } - StartDatabase::dispatch($database); + StartDatabase::dispatchReserved(StartDatabase::class, $database, $reservation); auditLog('api.database.started', [ 'team_id' => $teamId, @@ -3454,10 +3456,11 @@ class DatabasesController extends Controller $this->authorize('manage', $database); - if ($busyError = StartDatabase::operationInProgressError($database)) { - return response()->json(['message' => $busyError], 409); + $reservation = StartDatabase::reserveOperation($database); + if ($reservation === null) { + return response()->json(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE], 409); } - RestartDatabase::dispatch($database); + StartDatabase::dispatchReserved(RestartDatabase::class, $database, $reservation); auditLog('api.database.restarted', [ 'team_id' => $teamId, diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php index a0f0cc1aed..c9029de94a 100644 --- a/app/Http/Controllers/Api/DeployController.php +++ b/app/Http/Controllers/Api/DeployController.php @@ -12,6 +12,7 @@ use App\Models\ApplicationPreview; use App\Models\Server; use App\Models\Service; use App\Models\Tag; +use App\Support\ResourceStartActivity; use Illuminate\Auth\Access\AuthorizationException; use Illuminate\Http\Request; use OpenApi\Attributes as OA; @@ -582,7 +583,12 @@ class DeployController extends Controller } catch (AuthorizationException $e) { return ['message' => 'Unauthorized to start this database.', 'deployment_uuid' => null]; } - StartDatabase::dispatch($resource); + $reservation = StartDatabase::reserveOperation($resource); + if ($reservation === null) { + $message = ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE; + break; + } + StartDatabase::dispatchReserved(StartDatabase::class, $resource, $reservation); $resource->started_at ??= now(); $resource->save(); diff --git a/app/Mcp/Tools/Control.php b/app/Mcp/Tools/Control.php index 4b52d3d93f..aca6b46564 100644 --- a/app/Mcp/Tools/Control.php +++ b/app/Mcp/Tools/Control.php @@ -13,6 +13,7 @@ use App\Mcp\Concerns\BuildsResponse; use App\Mcp\Concerns\ResolvesResource; use App\Mcp\Concerns\ResolvesTeam; use App\Models\Application; +use App\Support\ResourceStartActivity; use Illuminate\Contracts\JsonSchema\JsonSchema; use Laravel\Mcp\Request; use Laravel\Mcp\Response; @@ -140,13 +141,18 @@ class Control extends Tool return ['message' => 'Database is already running.']; } + $reservation = StartDatabase::reserveOperation($database); + if ($reservation === null) { + throw new \RuntimeException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + } + if ($action === 'restart') { - RestartDatabase::dispatch($database); + StartDatabase::dispatchReserved(RestartDatabase::class, $database, $reservation); return ['message' => 'Database restart request queued.']; } - StartDatabase::dispatch($database); + StartDatabase::dispatchReserved(StartDatabase::class, $database, $reservation); return ['message' => 'Database starting request queued.']; } diff --git a/app/Support/DatabaseOperationReservation.php b/app/Support/DatabaseOperationReservation.php new file mode 100644 index 0000000000..936f5ab56f --- /dev/null +++ b/app/Support/DatabaseOperationReservation.php @@ -0,0 +1,72 @@ + $job->getAction() instanceof $actionClass)->last(); + expect($job)->not->toBeNull(); + + return $job->handle(); +} + +function databaseOperationReserved(string $databaseUuid): bool +{ + return Cache::has(DatabaseOperationReservation::key($databaseUuid)); +} + +describe('pending operation reservation', function () { + it('rejects a second API start while the first start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk() + ->assertJson(['message' => 'Database starting request queued.']); + + // The queued action did not run yet, so no activity exists. The reservation must block. + expect(databaseStartActivityCount($this->database->uuid))->toBe(0); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409) + ->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]); + + StartDatabase::assertPushed(1); + }); + + it('rejects an API restart while an API start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertStatus(409) + ->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]); + + RestartDatabase::assertNotPushed(); + }); + + it('rejects a second API restart and an API start while the first restart is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertOk() + ->assertJson(['message' => 'Database restarting request queued.']); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertStatus(409) + ->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + RestartDatabase::assertPushed(1); + StartDatabase::assertNotPushed(); + }); + + it('hands the reservation over to the start activity and allows a new start after it finished', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + expect(databaseOperationReserved($this->database->uuid))->toBeTrue(); + + $activity = runPushedDatabaseAction(StartDatabase::class); + + expect($activity)->toBeInstanceOf(Activity::class) + ->and(databaseOperationReserved($this->database->uuid))->toBeFalse(); + Queue::assertPushed(DatabaseStartJob::class, 1); + + // The queued start activity now blocks. + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + $activity->properties = $activity->properties->merge(['status' => ProcessStatus::FINISHED->value]); + $activity->save(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + }); + + it('stops and starts in the queued restart action and then releases the reservation', function () { + Queue::fake(); + StopDatabase::shouldRun()->once(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertOk(); + + $activity = runPushedDatabaseAction(RestartDatabase::class); + + expect($activity)->toBeInstanceOf(Activity::class) + ->and(databaseOperationReserved($this->database->uuid))->toBeFalse() + ->and(databaseStartActivityCount($this->database->uuid))->toBe(1); + }); + + it('releases the reservation when the queued action does not start the database', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + // For example a start that another path created without a reservation. + concurrentStartActivity($this->database->uuid, $this->team->id); + + expect(runPushedDatabaseAction(StartDatabase::class))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE) + ->and(databaseOperationReserved($this->database->uuid))->toBeFalse(); + Queue::assertNotPushed(DatabaseStartJob::class); + }); + + it('lets a new start through after a lost reservation expires', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $this->travel(DatabaseOperationReservation::TTL_SECONDS - 1)->seconds(); + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + $this->travel(2)->seconds(); + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + StartDatabase::assertPushed(2); + }); + + it('does not let an expired reservation start after a newer request took the reservation', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + $firstJob = Queue::pushed(JobDecorator::class)->first(); + + $this->travel(DatabaseOperationReservation::TTL_SECONDS + 1)->seconds(); + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + expect($firstJob->handle())->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE) + ->and(databaseOperationReserved($this->database->uuid))->toBeTrue() + ->and(databaseStartActivityCount($this->database->uuid))->toBe(0); + }); + + it('does not start from the UI while an API start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database]) + ->call('start') + ->assertDispatched('error', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + Queue::assertNotPushed(DatabaseStartJob::class); + expect(databaseStartActivityCount($this->database->uuid))->toBe(0); + }); + + it('does not stop the database on a UI restart while an API restart is still in the queue', function () { + Queue::fake(); + StopDatabase::shouldRun()->never(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/restart") + ->assertOk(); + + Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database]) + ->call('restart') + ->assertDispatched('error', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + }); + + it('releases its own reservation after a UI start created the activity', function () { + Bus::fake(); + + Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database]) + ->call('start') + ->assertNotDispatched('error'); + + Bus::assertDispatched(DatabaseStartJob::class); + expect(databaseOperationReserved($this->database->uuid))->toBeFalse() + ->and(databaseStartActivityCount($this->database->uuid))->toBe(1); + }); + + it('rejects an API start while a UI start holds the reservation', function () { + Queue::fake(); + $token = DatabaseOperationReservation::acquire($this->database->uuid); + expect($token)->toBeString(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertStatus(409); + + DatabaseOperationReservation::release($this->database->uuid, $token); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + }); + + it('does not release a reservation that another request holds', function () { + $token = DatabaseOperationReservation::acquire($this->database->uuid); + + DatabaseOperationReservation::release($this->database->uuid, 'another-token'); + + expect(databaseOperationReserved($this->database->uuid))->toBeTrue() + ->and(DatabaseOperationReservation::acquire($this->database->uuid))->toBeNull(); + DatabaseOperationReservation::release($this->database->uuid, $token); + expect(databaseOperationReserved($this->database->uuid))->toBeFalse(); + }); + + it('does not queue a database start through the deploy API while a start is still in the queue', function () { + Queue::fake(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson('/api/v1/deploy', ['uuid' => $this->database->uuid]) + ->assertOk() + ->assertJsonPath('deployments.0.message', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + + StartDatabase::assertPushed(1); + }); + + it('does not queue a database start through the MCP control tool while a start is still in the queue', function (string $action) { + Queue::fake(); + InstanceSettings::query()->whereKey(0)->update(['is_mcp_server_enabled' => true]); + $this->team->update(['is_mcp_server_enabled' => true]); + Once::flush(); + $mcpToken = $this->user->createToken('mcp-deploy', ['read', 'deploy'])->plainTextToken; + + $this->withHeaders(databaseStartApiHeaders($this->bearerToken)) + ->postJson("/api/v1/databases/{$this->database->uuid}/start") + ->assertOk(); + + $response = $this->withHeaders([ + 'Content-Type' => 'application/json', + 'Accept' => 'application/json, text/event-stream', + 'Authorization' => 'Bearer '.$mcpToken, + ])->postJson('/mcp', [ + 'jsonrpc' => '2.0', + 'id' => 1, + 'method' => 'tools/call', + 'params' => [ + 'name' => 'control', + 'arguments' => (object) [ + 'resource' => 'database', + 'action' => $action, + 'uuid' => $this->database->uuid, + ], + ], + ]); + + $response->assertOk(); + expect($response->json('result.isError'))->toBeTrue() + ->and($response->json('result.content.0.text'))->toContain(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + StartDatabase::assertPushed(1); + RestartDatabase::assertNotPushed(); + })->with(['start', 'restart']); +}); + describe('start job', function () { beforeEach(function () { Event::fake([DatabaseStatusChanged::class]); diff --git a/tests/Feature/StartDatabaseImportConcurrencyTest.php b/tests/Feature/StartDatabaseImportConcurrencyTest.php index bab4f99b83..e238c2ae95 100644 --- a/tests/Feature/StartDatabaseImportConcurrencyTest.php +++ b/tests/Feature/StartDatabaseImportConcurrencyTest.php @@ -11,6 +11,7 @@ use App\Models\StandalonePostgresql; use App\Models\Team; use App\Support\DatabaseImport\DatabaseImportException; use App\Support\DatabaseImport\DatabaseImportSource; +use App\Support\DatabaseOperationReservation; use App\Support\ResourceStartActivity; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Cache; @@ -191,3 +192,53 @@ test('an import interrupted by a Coolify restart is failed at boot and no longer expect(fn () => startImport($this->database, $this->team->id)) ->toThrow(DatabaseImportException::class, 'The server path is invalid.'); }); + +test('an import is rejected while a start or restart of the database waits in the queue', function () { + $token = DatabaseOperationReservation::acquire($this->database->uuid); + + try { + expect(fn () => startImport($this->database, $this->team->id)) + ->toThrow(DatabaseImportException::class, ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); + } finally { + DatabaseOperationReservation::release($this->database->uuid, $token); + } +}); + +test('the 409 status is used when a pending start blocks an import', function () { + $token = DatabaseOperationReservation::acquire($this->database->uuid); + + try { + startImport($this->database, $this->team->id); + $this->fail('The import was not rejected.'); + } catch (DatabaseImportException $exception) { + expect($exception->status)->toBe(409); + } finally { + DatabaseOperationReservation::release($this->database->uuid, $token); + } +}); + +test('an import is rejected while a start of the database is running', function (ProcessStatus $status) { + Activity::create([ + 'log_name' => 'default', + 'description' => '[]', + 'properties' => [ + 'team_id' => $this->team->id, + 'type_uuid' => $this->database->uuid, + 'operation' => ResourceStartActivity::DATABASE_START_OPERATION, + 'status' => $status->value, + ], + ]); + + expect(fn () => startImport($this->database, $this->team->id)) + ->toThrow(DatabaseImportException::class, ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE); +})->with([ + 'queued' => ProcessStatus::QUEUED, + 'in progress' => ProcessStatus::IN_PROGRESS, +]); + +test('an import holds the operation reservation and releases it afterwards', function () { + expect(fn () => startImport($this->database, $this->team->id)) + ->toThrow(DatabaseImportException::class, 'The server path is invalid.'); + + expect(Cache::has(DatabaseOperationReservation::key($this->database->uuid)))->toBeFalse(); +});