mirror of
https://github.com/coollabsio/coolify.git
synced 2026-10-06 22:17:23 -04:00
fix(databases): reserve start and restart atomically
Two parallel start requests both passed the in-progress check and got 200, because the start activity is created later by the queued action. A per-database reservation (Cache::add, 600 s) is now taken before the check, so the second request gets 409. The action releases it when it creates its activity, skips, or fails. Restart, import, the deploy API, MCP, and Livewire use the same reservation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e31c9b3926
commit
fbfe342f89
@@ -10,28 +10,44 @@ use App\Models\StandaloneMongodb;
|
||||
use App\Models\StandaloneMysql;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\StandaloneRedis;
|
||||
use App\Support\DatabaseOperationReservation;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
|
||||
class RestartDatabase
|
||||
{
|
||||
use AsAction;
|
||||
|
||||
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database)
|
||||
/**
|
||||
* @param string|null $reservation The token from StartDatabase::reserveOperation(). The request
|
||||
* that queued this action holds the reservation; this action
|
||||
* keeps it during the stop and releases it after the start.
|
||||
*/
|
||||
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null)
|
||||
{
|
||||
$server = $database->destination->server;
|
||||
if (! $server->isFunctional()) {
|
||||
return 'Server is not functional';
|
||||
$reservation ??= DatabaseOperationReservation::acquire($database->uuid);
|
||||
if ($reservation === null) {
|
||||
return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE;
|
||||
}
|
||||
$busyError = StartDatabase::operationInProgressError($database);
|
||||
if ($busyError !== null) {
|
||||
return $busyError;
|
||||
}
|
||||
$prerequisiteError = StartDatabase::prerequisiteError($database);
|
||||
if ($prerequisiteError !== null) {
|
||||
return $prerequisiteError;
|
||||
}
|
||||
StopDatabase::run($database, dockerCleanup: false);
|
||||
|
||||
return StartDatabase::run($database);
|
||||
try {
|
||||
$server = $database->destination->server;
|
||||
if (! $server->isFunctional()) {
|
||||
return 'Server is not functional';
|
||||
}
|
||||
$busyError = StartDatabase::operationInProgressError($database, $reservation);
|
||||
if ($busyError !== null) {
|
||||
return $busyError;
|
||||
}
|
||||
$prerequisiteError = StartDatabase::prerequisiteError($database);
|
||||
if ($prerequisiteError !== null) {
|
||||
return $prerequisiteError;
|
||||
}
|
||||
StopDatabase::run($database, dockerCleanup: false);
|
||||
|
||||
return StartDatabase::run($database, $reservation);
|
||||
} finally {
|
||||
DatabaseOperationReservation::release($database->uuid, $reservation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ use App\Models\StandaloneMongodb;
|
||||
use App\Models\StandaloneMysql;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\StandaloneRedis;
|
||||
use App\Support\DatabaseOperationReservation;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
use Lorisleiva\Actions\Decorators\JobDecorator;
|
||||
@@ -29,13 +30,31 @@ class StartDatabase
|
||||
$job->onQueue(deployment_queue());
|
||||
}
|
||||
|
||||
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): Activity|string
|
||||
/**
|
||||
* @param string|null $reservation The token from reserveOperation(). The request that queued
|
||||
* this action holds the reservation; this action releases it.
|
||||
*/
|
||||
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null): Activity|string
|
||||
{
|
||||
$reservation ??= DatabaseOperationReservation::acquire($database->uuid);
|
||||
if ($reservation === null) {
|
||||
return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE;
|
||||
}
|
||||
|
||||
try {
|
||||
return $this->queueStart($database, $reservation);
|
||||
} finally {
|
||||
DatabaseOperationReservation::release($database->uuid, $reservation);
|
||||
}
|
||||
}
|
||||
|
||||
private function queueStart(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, string $reservation): Activity|string
|
||||
{
|
||||
$server = $database->destination->server;
|
||||
if (! $server->isFunctional()) {
|
||||
return 'Server is not functional';
|
||||
}
|
||||
$busyError = self::operationInProgressError($database);
|
||||
$busyError = self::operationInProgressError($database, $reservation);
|
||||
if ($busyError !== null) {
|
||||
return $busyError;
|
||||
}
|
||||
@@ -88,16 +107,63 @@ class StartDatabase
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse a second start/restart while a start, restart or import of this database is
|
||||
* queued or running. The UI hides Start/Restart in this state; this is the server-side check.
|
||||
* Stale start activities do not block and are marked as failed, so the UI stays consistent.
|
||||
* Reserve a start or restart before it is queued, so that a parallel request cannot also
|
||||
* queue one before the queued action has created its activity. Pass the token to the
|
||||
* queued action, which releases the reservation.
|
||||
*
|
||||
* @return string|null The reservation token, or null when another operation is in progress.
|
||||
*/
|
||||
public static function operationInProgressError(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): ?string
|
||||
public static function reserveOperation(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database): ?string
|
||||
{
|
||||
$reservation = DatabaseOperationReservation::acquire($database->uuid);
|
||||
if ($reservation === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (self::operationInProgressError($database, $reservation) !== null) {
|
||||
DatabaseOperationReservation::release($database->uuid, $reservation);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return $reservation;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a start or restart action with a reservation. Release the reservation if the
|
||||
* action cannot be queued, because then no action will release it.
|
||||
*
|
||||
* @param class-string<StartDatabase|RestartDatabase> $action
|
||||
*/
|
||||
public static function dispatchReserved(string $action, StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, string $reservation): void
|
||||
{
|
||||
try {
|
||||
$action::dispatch($database, $reservation);
|
||||
} catch (Throwable $e) {
|
||||
DatabaseOperationReservation::release($database->uuid, $reservation);
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse a second start/restart while a start, restart or import of this database is
|
||||
* reserved, queued or running. The UI hides Start/Restart in this state; this is the
|
||||
* server-side check. Stale start activities do not block and are marked as failed, so
|
||||
* the UI stays consistent.
|
||||
*
|
||||
* @param string|null $reservation The token of the caller's own reservation, which does not block.
|
||||
*/
|
||||
public static function operationInProgressError(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, ?string $reservation = null): ?string
|
||||
{
|
||||
if (blank($database->uuid)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (DatabaseOperationReservation::isHeldByOther($database->uuid, $reservation)) {
|
||||
return ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE;
|
||||
}
|
||||
|
||||
$liveOperations = collect([
|
||||
ResourceStartActivity::DATABASE_START_OPERATION,
|
||||
ResourceStartActivity::DATABASE_IMPORT_OPERATION,
|
||||
|
||||
@@ -10,6 +10,7 @@ use App\Support\DatabaseBackupFileValidator;
|
||||
use App\Support\DatabaseImport\DatabaseImportCommandBuilder;
|
||||
use App\Support\DatabaseImport\DatabaseImportException;
|
||||
use App\Support\DatabaseImport\DatabaseImportSource;
|
||||
use App\Support\DatabaseOperationReservation;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use App\Support\ValidationPatterns;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
@@ -59,9 +60,19 @@ class StartDatabaseImport
|
||||
throw new DatabaseImportException('A database import is already running.', 409);
|
||||
}
|
||||
|
||||
// A start or restart can be queued without an activity yet: its reservation blocks the import.
|
||||
// The import holds the reservation until its own activity exists.
|
||||
$reservation = null;
|
||||
|
||||
try {
|
||||
$reservation = DatabaseOperationReservation::acquire($resource->uuid);
|
||||
if ($reservation === null) {
|
||||
throw new DatabaseImportException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE, 409);
|
||||
}
|
||||
|
||||
return $this->startImport($resource, $source, $teamId, $server, $container, $network);
|
||||
} finally {
|
||||
DatabaseOperationReservation::release($resource->uuid, $reservation);
|
||||
$lock->release();
|
||||
}
|
||||
}
|
||||
@@ -72,6 +83,10 @@ class StartDatabaseImport
|
||||
if (ResourceStartActivity::failStale($active)->isNotEmpty()) {
|
||||
throw new DatabaseImportException('A database import is already running.', 409);
|
||||
}
|
||||
$activeStarts = ResourceStartActivity::active($resource->uuid, ResourceStartActivity::DATABASE_START_OPERATION, $teamId);
|
||||
if (ResourceStartActivity::failStale($activeStarts)->isNotEmpty()) {
|
||||
throw new DatabaseImportException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE, 409);
|
||||
}
|
||||
|
||||
$operation = (string) Str::uuid();
|
||||
$containerPath = "/tmp/restore_{$operation}";
|
||||
|
||||
@@ -22,6 +22,7 @@ use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\SwarmDocker;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use App\Support\ValidationPatterns;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
@@ -3264,10 +3265,11 @@ class DatabasesController extends Controller
|
||||
if (str($database->status)->contains('running')) {
|
||||
return response()->json(['message' => 'Database is already running.'], 400);
|
||||
}
|
||||
if ($busyError = StartDatabase::operationInProgressError($database)) {
|
||||
return response()->json(['message' => $busyError], 409);
|
||||
$reservation = StartDatabase::reserveOperation($database);
|
||||
if ($reservation === null) {
|
||||
return response()->json(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE], 409);
|
||||
}
|
||||
StartDatabase::dispatch($database);
|
||||
StartDatabase::dispatchReserved(StartDatabase::class, $database, $reservation);
|
||||
|
||||
auditLog('api.database.started', [
|
||||
'team_id' => $teamId,
|
||||
@@ -3454,10 +3456,11 @@ class DatabasesController extends Controller
|
||||
|
||||
$this->authorize('manage', $database);
|
||||
|
||||
if ($busyError = StartDatabase::operationInProgressError($database)) {
|
||||
return response()->json(['message' => $busyError], 409);
|
||||
$reservation = StartDatabase::reserveOperation($database);
|
||||
if ($reservation === null) {
|
||||
return response()->json(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE], 409);
|
||||
}
|
||||
RestartDatabase::dispatch($database);
|
||||
StartDatabase::dispatchReserved(RestartDatabase::class, $database, $reservation);
|
||||
|
||||
auditLog('api.database.restarted', [
|
||||
'team_id' => $teamId,
|
||||
|
||||
@@ -12,6 +12,7 @@ use App\Models\ApplicationPreview;
|
||||
use App\Models\Server;
|
||||
use App\Models\Service;
|
||||
use App\Models\Tag;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use Illuminate\Auth\Access\AuthorizationException;
|
||||
use Illuminate\Http\Request;
|
||||
use OpenApi\Attributes as OA;
|
||||
@@ -582,7 +583,12 @@ class DeployController extends Controller
|
||||
} catch (AuthorizationException $e) {
|
||||
return ['message' => 'Unauthorized to start this database.', 'deployment_uuid' => null];
|
||||
}
|
||||
StartDatabase::dispatch($resource);
|
||||
$reservation = StartDatabase::reserveOperation($resource);
|
||||
if ($reservation === null) {
|
||||
$message = ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE;
|
||||
break;
|
||||
}
|
||||
StartDatabase::dispatchReserved(StartDatabase::class, $resource, $reservation);
|
||||
|
||||
$resource->started_at ??= now();
|
||||
$resource->save();
|
||||
|
||||
@@ -13,6 +13,7 @@ use App\Mcp\Concerns\BuildsResponse;
|
||||
use App\Mcp\Concerns\ResolvesResource;
|
||||
use App\Mcp\Concerns\ResolvesTeam;
|
||||
use App\Models\Application;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use Illuminate\Contracts\JsonSchema\JsonSchema;
|
||||
use Laravel\Mcp\Request;
|
||||
use Laravel\Mcp\Response;
|
||||
@@ -140,13 +141,18 @@ class Control extends Tool
|
||||
return ['message' => 'Database is already running.'];
|
||||
}
|
||||
|
||||
$reservation = StartDatabase::reserveOperation($database);
|
||||
if ($reservation === null) {
|
||||
throw new \RuntimeException(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
}
|
||||
|
||||
if ($action === 'restart') {
|
||||
RestartDatabase::dispatch($database);
|
||||
StartDatabase::dispatchReserved(RestartDatabase::class, $database, $reservation);
|
||||
|
||||
return ['message' => 'Database restart request queued.'];
|
||||
}
|
||||
|
||||
StartDatabase::dispatch($database);
|
||||
StartDatabase::dispatchReserved(StartDatabase::class, $database, $reservation);
|
||||
|
||||
return ['message' => 'Database starting request queued.'];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
<?php
|
||||
|
||||
namespace App\Support;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
/**
|
||||
* Reserves a database for one start, restart or import between the request and the moment
|
||||
* the operation has created its activity.
|
||||
*
|
||||
* The API and MCP queue StartDatabase/RestartDatabase, and the queued action creates the
|
||||
* activity later. Without a reservation, two parallel requests both find no activity and
|
||||
* both get "queued". Cache::add() is atomic (SET NX on Redis), so only one request can
|
||||
* hold the reservation. The holder releases it when its activity exists or when it stops.
|
||||
* The TTL releases the reservation of a lost job; it is the same as the limit after which
|
||||
* a queued start activity is stale.
|
||||
*/
|
||||
class DatabaseOperationReservation
|
||||
{
|
||||
public const TTL_SECONDS = ResourceStartActivity::QUEUED_STALE_AFTER_SECONDS;
|
||||
|
||||
public static function key(string $databaseUuid): string
|
||||
{
|
||||
return "database-operation-pending:{$databaseUuid}";
|
||||
}
|
||||
|
||||
/**
|
||||
* A database without a UUID (not saved) has nothing to reserve: the token is returned
|
||||
* without a cache entry, the same as operationInProgressError() does not block it.
|
||||
*
|
||||
* @return string|null The reservation token, or null when another operation holds the reservation.
|
||||
*/
|
||||
public static function acquire(?string $databaseUuid): ?string
|
||||
{
|
||||
$token = (string) Str::uuid();
|
||||
if (blank($databaseUuid)) {
|
||||
return $token;
|
||||
}
|
||||
|
||||
return Cache::add(self::key($databaseUuid), $token, self::TTL_SECONDS) ? $token : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a reservation exists that does not belong to the given token.
|
||||
*/
|
||||
public static function isHeldByOther(?string $databaseUuid, ?string $token = null): bool
|
||||
{
|
||||
if (blank($databaseUuid)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$holder = Cache::get(self::key($databaseUuid));
|
||||
|
||||
return $holder !== null && $holder !== $token;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the reservation only if the token still holds it, so that an expired
|
||||
* reservation does not remove the reservation of a newer request.
|
||||
*/
|
||||
public static function release(?string $databaseUuid, ?string $token): void
|
||||
{
|
||||
if ($token === null || blank($databaseUuid)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (Cache::get(self::key($databaseUuid)) === $token) {
|
||||
Cache::forget(self::key($databaseUuid));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,14 +17,17 @@ use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Support\DatabaseOperationReservation;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Bus;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Event;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Illuminate\Support\Once;
|
||||
use Illuminate\Support\Str;
|
||||
use Livewire\Livewire;
|
||||
use Lorisleiva\Actions\Decorators\JobDecorator;
|
||||
use Spatie\Activitylog\Models\Activity;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
@@ -207,6 +210,293 @@ describe('start and restart guards', function () {
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Run the queued StartDatabase/RestartDatabase action that the API pushed to the fake queue,
|
||||
* with the same parameters (including the reservation token) that the worker would use.
|
||||
*/
|
||||
function runPushedDatabaseAction(string $actionClass): mixed
|
||||
{
|
||||
$job = Queue::pushed(JobDecorator::class, fn (JobDecorator $job) => $job->getAction() instanceof $actionClass)->last();
|
||||
expect($job)->not->toBeNull();
|
||||
|
||||
return $job->handle();
|
||||
}
|
||||
|
||||
function databaseOperationReserved(string $databaseUuid): bool
|
||||
{
|
||||
return Cache::has(DatabaseOperationReservation::key($databaseUuid));
|
||||
}
|
||||
|
||||
describe('pending operation reservation', function () {
|
||||
it('rejects a second API start while the first start is still in the queue', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk()
|
||||
->assertJson(['message' => 'Database starting request queued.']);
|
||||
|
||||
// The queued action did not run yet, so no activity exists. The reservation must block.
|
||||
expect(databaseStartActivityCount($this->database->uuid))->toBe(0);
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertStatus(409)
|
||||
->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]);
|
||||
|
||||
StartDatabase::assertPushed(1);
|
||||
});
|
||||
|
||||
it('rejects an API restart while an API start is still in the queue', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/restart")
|
||||
->assertStatus(409)
|
||||
->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]);
|
||||
|
||||
RestartDatabase::assertNotPushed();
|
||||
});
|
||||
|
||||
it('rejects a second API restart and an API start while the first restart is still in the queue', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/restart")
|
||||
->assertOk()
|
||||
->assertJson(['message' => 'Database restarting request queued.']);
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/restart")
|
||||
->assertStatus(409)
|
||||
->assertJson(['message' => ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE]);
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertStatus(409);
|
||||
|
||||
RestartDatabase::assertPushed(1);
|
||||
StartDatabase::assertNotPushed();
|
||||
});
|
||||
|
||||
it('hands the reservation over to the start activity and allows a new start after it finished', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
expect(databaseOperationReserved($this->database->uuid))->toBeTrue();
|
||||
|
||||
$activity = runPushedDatabaseAction(StartDatabase::class);
|
||||
|
||||
expect($activity)->toBeInstanceOf(Activity::class)
|
||||
->and(databaseOperationReserved($this->database->uuid))->toBeFalse();
|
||||
Queue::assertPushed(DatabaseStartJob::class, 1);
|
||||
|
||||
// The queued start activity now blocks.
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertStatus(409);
|
||||
|
||||
$activity->properties = $activity->properties->merge(['status' => ProcessStatus::FINISHED->value]);
|
||||
$activity->save();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
});
|
||||
|
||||
it('stops and starts in the queued restart action and then releases the reservation', function () {
|
||||
Queue::fake();
|
||||
StopDatabase::shouldRun()->once();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/restart")
|
||||
->assertOk();
|
||||
|
||||
$activity = runPushedDatabaseAction(RestartDatabase::class);
|
||||
|
||||
expect($activity)->toBeInstanceOf(Activity::class)
|
||||
->and(databaseOperationReserved($this->database->uuid))->toBeFalse()
|
||||
->and(databaseStartActivityCount($this->database->uuid))->toBe(1);
|
||||
});
|
||||
|
||||
it('releases the reservation when the queued action does not start the database', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
// For example a start that another path created without a reservation.
|
||||
concurrentStartActivity($this->database->uuid, $this->team->id);
|
||||
|
||||
expect(runPushedDatabaseAction(StartDatabase::class))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE)
|
||||
->and(databaseOperationReserved($this->database->uuid))->toBeFalse();
|
||||
Queue::assertNotPushed(DatabaseStartJob::class);
|
||||
});
|
||||
|
||||
it('lets a new start through after a lost reservation expires', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
$this->travel(DatabaseOperationReservation::TTL_SECONDS - 1)->seconds();
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertStatus(409);
|
||||
|
||||
$this->travel(2)->seconds();
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
StartDatabase::assertPushed(2);
|
||||
});
|
||||
|
||||
it('does not let an expired reservation start after a newer request took the reservation', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
$firstJob = Queue::pushed(JobDecorator::class)->first();
|
||||
|
||||
$this->travel(DatabaseOperationReservation::TTL_SECONDS + 1)->seconds();
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
expect($firstJob->handle())->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE)
|
||||
->and(databaseOperationReserved($this->database->uuid))->toBeTrue()
|
||||
->and(databaseStartActivityCount($this->database->uuid))->toBe(0);
|
||||
});
|
||||
|
||||
it('does not start from the UI while an API start is still in the queue', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database])
|
||||
->call('start')
|
||||
->assertDispatched('error', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
|
||||
Queue::assertNotPushed(DatabaseStartJob::class);
|
||||
expect(databaseStartActivityCount($this->database->uuid))->toBe(0);
|
||||
});
|
||||
|
||||
it('does not stop the database on a UI restart while an API restart is still in the queue', function () {
|
||||
Queue::fake();
|
||||
StopDatabase::shouldRun()->never();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/restart")
|
||||
->assertOk();
|
||||
|
||||
Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database])
|
||||
->call('restart')
|
||||
->assertDispatched('error', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
});
|
||||
|
||||
it('releases its own reservation after a UI start created the activity', function () {
|
||||
Bus::fake();
|
||||
|
||||
Livewire::actingAs($this->user)->test(Heading::class, ['database' => $this->database])
|
||||
->call('start')
|
||||
->assertNotDispatched('error');
|
||||
|
||||
Bus::assertDispatched(DatabaseStartJob::class);
|
||||
expect(databaseOperationReserved($this->database->uuid))->toBeFalse()
|
||||
->and(databaseStartActivityCount($this->database->uuid))->toBe(1);
|
||||
});
|
||||
|
||||
it('rejects an API start while a UI start holds the reservation', function () {
|
||||
Queue::fake();
|
||||
$token = DatabaseOperationReservation::acquire($this->database->uuid);
|
||||
expect($token)->toBeString();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertStatus(409);
|
||||
|
||||
DatabaseOperationReservation::release($this->database->uuid, $token);
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
});
|
||||
|
||||
it('does not release a reservation that another request holds', function () {
|
||||
$token = DatabaseOperationReservation::acquire($this->database->uuid);
|
||||
|
||||
DatabaseOperationReservation::release($this->database->uuid, 'another-token');
|
||||
|
||||
expect(databaseOperationReserved($this->database->uuid))->toBeTrue()
|
||||
->and(DatabaseOperationReservation::acquire($this->database->uuid))->toBeNull();
|
||||
DatabaseOperationReservation::release($this->database->uuid, $token);
|
||||
expect(databaseOperationReserved($this->database->uuid))->toBeFalse();
|
||||
});
|
||||
|
||||
it('does not queue a database start through the deploy API while a start is still in the queue', function () {
|
||||
Queue::fake();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson('/api/v1/deploy', ['uuid' => $this->database->uuid])
|
||||
->assertOk()
|
||||
->assertJsonPath('deployments.0.message', ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
|
||||
StartDatabase::assertPushed(1);
|
||||
});
|
||||
|
||||
it('does not queue a database start through the MCP control tool while a start is still in the queue', function (string $action) {
|
||||
Queue::fake();
|
||||
InstanceSettings::query()->whereKey(0)->update(['is_mcp_server_enabled' => true]);
|
||||
$this->team->update(['is_mcp_server_enabled' => true]);
|
||||
Once::flush();
|
||||
$mcpToken = $this->user->createToken('mcp-deploy', ['read', 'deploy'])->plainTextToken;
|
||||
|
||||
$this->withHeaders(databaseStartApiHeaders($this->bearerToken))
|
||||
->postJson("/api/v1/databases/{$this->database->uuid}/start")
|
||||
->assertOk();
|
||||
|
||||
$response = $this->withHeaders([
|
||||
'Content-Type' => 'application/json',
|
||||
'Accept' => 'application/json, text/event-stream',
|
||||
'Authorization' => 'Bearer '.$mcpToken,
|
||||
])->postJson('/mcp', [
|
||||
'jsonrpc' => '2.0',
|
||||
'id' => 1,
|
||||
'method' => 'tools/call',
|
||||
'params' => [
|
||||
'name' => 'control',
|
||||
'arguments' => (object) [
|
||||
'resource' => 'database',
|
||||
'action' => $action,
|
||||
'uuid' => $this->database->uuid,
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$response->assertOk();
|
||||
expect($response->json('result.isError'))->toBeTrue()
|
||||
->and($response->json('result.content.0.text'))->toContain(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
StartDatabase::assertPushed(1);
|
||||
RestartDatabase::assertNotPushed();
|
||||
})->with(['start', 'restart']);
|
||||
});
|
||||
|
||||
describe('start job', function () {
|
||||
beforeEach(function () {
|
||||
Event::fake([DatabaseStatusChanged::class]);
|
||||
|
||||
@@ -11,6 +11,7 @@ use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Support\DatabaseImport\DatabaseImportException;
|
||||
use App\Support\DatabaseImport\DatabaseImportSource;
|
||||
use App\Support\DatabaseOperationReservation;
|
||||
use App\Support\ResourceStartActivity;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
@@ -191,3 +192,53 @@ test('an import interrupted by a Coolify restart is failed at boot and no longer
|
||||
expect(fn () => startImport($this->database, $this->team->id))
|
||||
->toThrow(DatabaseImportException::class, 'The server path is invalid.');
|
||||
});
|
||||
|
||||
test('an import is rejected while a start or restart of the database waits in the queue', function () {
|
||||
$token = DatabaseOperationReservation::acquire($this->database->uuid);
|
||||
|
||||
try {
|
||||
expect(fn () => startImport($this->database, $this->team->id))
|
||||
->toThrow(DatabaseImportException::class, ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
} finally {
|
||||
DatabaseOperationReservation::release($this->database->uuid, $token);
|
||||
}
|
||||
});
|
||||
|
||||
test('the 409 status is used when a pending start blocks an import', function () {
|
||||
$token = DatabaseOperationReservation::acquire($this->database->uuid);
|
||||
|
||||
try {
|
||||
startImport($this->database, $this->team->id);
|
||||
$this->fail('The import was not rejected.');
|
||||
} catch (DatabaseImportException $exception) {
|
||||
expect($exception->status)->toBe(409);
|
||||
} finally {
|
||||
DatabaseOperationReservation::release($this->database->uuid, $token);
|
||||
}
|
||||
});
|
||||
|
||||
test('an import is rejected while a start of the database is running', function (ProcessStatus $status) {
|
||||
Activity::create([
|
||||
'log_name' => 'default',
|
||||
'description' => '[]',
|
||||
'properties' => [
|
||||
'team_id' => $this->team->id,
|
||||
'type_uuid' => $this->database->uuid,
|
||||
'operation' => ResourceStartActivity::DATABASE_START_OPERATION,
|
||||
'status' => $status->value,
|
||||
],
|
||||
]);
|
||||
|
||||
expect(fn () => startImport($this->database, $this->team->id))
|
||||
->toThrow(DatabaseImportException::class, ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
|
||||
})->with([
|
||||
'queued' => ProcessStatus::QUEUED,
|
||||
'in progress' => ProcessStatus::IN_PROGRESS,
|
||||
]);
|
||||
|
||||
test('an import holds the operation reservation and releases it afterwards', function () {
|
||||
expect(fn () => startImport($this->database, $this->team->id))
|
||||
->toThrow(DatabaseImportException::class, 'The server path is invalid.');
|
||||
|
||||
expect(Cache::has(DatabaseOperationReservation::key($this->database->uuid)))->toBeFalse();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user