From 1c68eafa8499f448912abe4adcccf84a0253f58f Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:01:20 +0200 Subject: [PATCH] fix(backups): make overlapping and concurrent backup runs safe - Skip the occurrence of a backup whose previous run still holds the overlap lock, so stale recovery does not run it later; do not rerun a lost occurrence when a newer one already completed. - Skip a database backup while a start, restart, or import of the database is in progress, and record the skip in the execution list. - Give every backup file a unique name per run and keep the service database backup folder of earlier releases. - Record missed scheduled tasks and Docker cleanups as failed executions and notify the team. Co-Authored-By: Claude Opus 5.5 --- app/Jobs/DatabaseBackupJob.php | 74 ++++- app/Jobs/VolumeBackupJob.php | 3 +- app/Services/ScheduledJobDeliveryService.php | 125 +++++++- tests/Feature/BackupRunSafetyTest.php | 292 ++++++++++++++++++ tests/Feature/DatabaseDumpExitStatusTest.php | 2 +- .../ScheduledJobManagerDispatchTest.php | 59 ++++ 6 files changed, 534 insertions(+), 21 deletions(-) create mode 100644 tests/Feature/BackupRunSafetyTest.php diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 890e14462e..630e1153fc 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -22,6 +22,8 @@ use App\Rules\SafeWebhookUrl; use App\Services\ScheduledJobDeliveryService; use App\Support\BackupCompression; use App\Support\ClickhouseBackupCommand; +use App\Support\DatabaseOperationReservation; +use App\Support\ResourceStartActivity; use App\Support\ValidationPatterns; use Carbon\Carbon; use Illuminate\Bus\Queueable; @@ -29,10 +31,10 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; +use Spatie\Activitylog\Models\Activity; use Throwable; class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue @@ -91,7 +93,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue { $expireAfter = ($this->backup->timeout ?? 3600) + 300; - return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()]; + return [ScheduledJobDeliveryService::withoutOverlapping('database-backup-'.$this->backup->id, $this->occurrenceUuid)->expireAfter($expireAfter)->dontRelease()]; } public function handle(): void @@ -138,6 +140,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue return; } + if ($this->databaseOperationInProgress()) { + $this->recordSkippedBackup('Skipped: a start, restart or import of this database was in progress. A backup taken now could contain a partly restored database.'); + $this->logSkippedRun('database_operation_in_progress'); + + return; + } if (data_get($this->backup, 'database_type') === ServiceDatabase::class) { $databaseType = $this->database->databaseType(); $serviceUuid = $this->database->service->uuid; @@ -146,7 +154,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue if (! ValidationPatterns::isValidContainerName($this->container_name)) { throw new \Exception('Invalid database container name.'); } - $this->directory_name = $serviceName.'-'.str($this->database->name)->slug().'-'.$serviceUuid; + // The container name is validated above, so it is a safe folder name. Keep the folder of earlier releases. + $this->directory_name = $serviceName.'-'.$this->container_name; $escapedContainerName = escapeshellarg($this->container_name); if (str($databaseType)->contains('postgres')) { $commands[] = "docker exec {$escapedContainerName} env | grep POSTGRES_"; @@ -321,9 +330,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue try { if (str($databaseType)->contains('postgres')) { $fileDatabaseName = $this->backupFilenamePart($database); - $this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp'; + $this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp"; if ($this->backup->dump_all) { - $this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp.'.gz'; + $this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz"; } $this->backup_location = $this->backup_dir.$this->backup_file; $this->backup_log = ScheduledDatabaseBackupExecution::create([ @@ -347,7 +356,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } } $fileDatabaseName = $this->backupFilenamePart($databaseName); - $this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.tar.gz'; + $this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.tar.gz"; $this->backup_location = $this->backup_dir.$this->backup_file; $this->backup_log = ScheduledDatabaseBackupExecution::create([ 'uuid' => $this->backup_log_uuid, @@ -360,9 +369,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $this->backup_standalone_mongodb($database); } elseif (str($databaseType)->contains('mysql')) { $fileDatabaseName = $this->backupFilenamePart($database); - $this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp'; + $this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp"; if ($this->backup->dump_all) { - $this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp.'.gz'; + $this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz"; } $this->backup_location = $this->backup_dir.$this->backup_file; $this->backup_log = ScheduledDatabaseBackupExecution::create([ @@ -376,9 +385,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $this->backup_standalone_mysql($database); } elseif (str($databaseType)->contains('mariadb')) { $fileDatabaseName = $this->backupFilenamePart($database); - $this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp'; + $this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp"; if ($this->backup->dump_all) { - $this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp.'.gz'; + $this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz"; } $this->backup_location = $this->backup_dir.$this->backup_file; $this->backup_log = ScheduledDatabaseBackupExecution::create([ @@ -403,7 +412,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue BackupCreated::dispatch($this->team->id); $this->backup_standalone_clickhouse($database); } elseif ($this->database instanceof StandaloneSqlite) { - $this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp.'.gz'; + $this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz"; $this->backup_location = $this->backup_dir.$this->backup_file; $this->backup_log = ScheduledDatabaseBackupExecution::create([ 'uuid' => $this->backup_log_uuid, @@ -925,6 +934,49 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue ]); } + /** + * A start, restart or import of the database is reserved, queued or running. Stale activities do + * not count; they are failed by the start and import paths. + */ + private function databaseOperationInProgress(): bool + { + $uuid = $this->database->uuid; + if (blank($uuid)) { + return false; + } + + if (DatabaseOperationReservation::isHeldByOther($uuid)) { + return true; + } + + $liveOperation = collect([ResourceStartActivity::DATABASE_START_OPERATION, ResourceStartActivity::DATABASE_IMPORT_OPERATION]) + ->flatMap(fn (string $operation) => ResourceStartActivity::active($uuid, $operation)) + ->contains(fn (Activity $activity): bool => ! ResourceStartActivity::isStale($activity)); + + return $liveOperation + || ResourceStartActivity::latestRunning($uuid) !== null + || ($this->database instanceof ServiceDatabase && ResourceStartActivity::latestRunning($this->database->service->uuid) !== null); + } + + /** + * Show a backup run that did not take a backup in the execution list. The run is expected, so it + * sends no failure notification. + */ + private function recordSkippedBackup(string $message): void + { + $databaseNames = $this->backup->dump_all ? 'all' : data_get($this->backup, 'databases_to_backup'); + $this->backup_log_uuid = new_public_id(); + $this->backup_log = ScheduledDatabaseBackupExecution::create([ + 'uuid' => $this->backup_log_uuid, + 'database_name' => filled($databaseNames) ? Str::limit($databaseNames, 250) : null, + 'scheduled_database_backup_id' => $this->backup->id, + 'status' => 'failed', + 'message' => $message, + 'filename' => null, + 'local_storage_deleted' => false, + ]); + } + private function markStaleExecutionsAsFailed(): void { try { diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index fb7e3376b5..78f091bbe2 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -17,7 +17,6 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Database\Eloquent\Builder; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; @@ -42,7 +41,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue public function middleware(): array { return [ - (new WithoutOverlapping('volume-backup-'.$this->backup->id)) + ScheduledJobDeliveryService::withoutOverlapping('volume-backup-'.$this->backup->id, $this->occurrenceUuid) ->shared() ->expireAfter($this->timeout + 60) ->dontRelease(), diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.php index 84e8d4210f..330889c151 100644 --- a/app/Services/ScheduledJobDeliveryService.php +++ b/app/Services/ScheduledJobDeliveryService.php @@ -2,17 +2,25 @@ namespace App\Services; +use App\Events\DockerCleanupDone; +use App\Events\ScheduledTaskDone; use App\Jobs\DatabaseBackupJob; use App\Jobs\DockerCleanupJob; use App\Jobs\ScheduledTaskJob; use App\Jobs\VolumeBackupJob; +use App\Models\DockerCleanupExecution; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledJobDelivery; use App\Models\ScheduledJobState; use App\Models\ScheduledTask; +use App\Models\ScheduledTaskExecution; use App\Models\ScheduledVolumeBackup; use App\Models\Server; +use App\Models\Team; +use App\Notifications\ScheduledTask\TaskFailed; +use App\Notifications\Server\DockerCleanupFailed; use Cron\CronExpression; +use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; @@ -29,7 +37,8 @@ class ScheduledJobDeliveryService public const ENQUEUED_STALE_AFTER_MINUTES = 60; /** - * Job types that run once when they are late. A lost task or Docker cleanup is logged as missed. + * Job types that run once when they are late. A lost task or Docker cleanup is not run; it is + * recorded as a failed execution and the team is notified. */ private const LATE_RUN_JOB_TYPES = ['database-backup', 'volume-backup']; @@ -167,7 +176,7 @@ class ScheduledJobDeliveryService } /** - * Publish backups again whose queued job was not started, and log lost tasks and Docker + * Publish backups again whose queued job was not started, and report lost tasks and Docker * cleanups as missed. A lost backup is not run when a newer occurrence of the same schedule * exists, so a backup that runs longer than its interval does not build up a backlog. * @@ -196,10 +205,15 @@ class ScheduledJobDeliveryService private function recoverStaleEnqueuedOccurrence(ScheduledJobDelivery $occurrence): void { - $hasNewerOccurrence = ScheduledJobDelivery::query() + // complete() deletes the row of a finished occurrence, so the schedule state also counts. + $hasNewerOccurrence = ScheduledJobState::query() ->where('schedule_key', $occurrence->schedule_key) - ->where('scheduled_for', '>', $occurrence->scheduled_for) - ->exists(); + ->where('last_scheduled_for', '>', $occurrence->scheduled_for) + ->exists() + || ScheduledJobDelivery::query() + ->where('schedule_key', $occurrence->schedule_key) + ->where('scheduled_for', '>', $occurrence->scheduled_for) + ->exists(); $runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence; // Only one scheduler node can move each occurrence. @@ -217,6 +231,7 @@ class ScheduledJobDeliveryService $this->logOccurrence($occurrence->uuid, $hasNewerOccurrence ? 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists' : 'Scheduled occurrence missed: its queued job was not started', 'warning'); + $this->reportMissedOccurrence($occurrence); return; } @@ -226,6 +241,55 @@ class ScheduledJobDeliveryService $this->publish($occurrence); } + /** + * Show a missed task or Docker cleanup as a failed execution and send the failure notification. + * The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence, + * and claim() rejects the late job if it starts afterwards. + */ + private function reportMissedOccurrence(ScheduledJobDelivery $occurrence): void + { + $message = 'Skipped: the queued job did not start within '.self::ENQUEUED_STALE_AFTER_MINUTES.' minutes.'; + + try { + if ($occurrence->job_type === 'scheduled-task') { + $task = ScheduledTask::find($occurrence->resource_id); + if (! $task) { + return; + } + + ScheduledTaskExecution::create([ + 'scheduled_task_id' => $task->id, + 'status' => 'failed', + 'message' => $message, + 'retry_count' => 0, + 'finished_at' => now(), + ]); + ScheduledTaskDone::dispatch($task->team_id); + Team::find($task->team_id)?->notify(new TaskFailed($task, $message)); + } elseif ($occurrence->job_type === 'docker-cleanup') { + $server = Server::find($occurrence->resource_id); + if (! $server) { + return; + } + + $execution = DockerCleanupExecution::create([ + 'server_id' => $server->id, + 'status' => 'failed', + 'message' => $message, + 'finished_at' => now(), + ]); + event(new DockerCleanupDone($execution)); + $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}")); + } + } catch (\Throwable $e) { + Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [ + 'occurrence_uuid' => $occurrence->uuid, + 'schedule_key' => $occurrence->schedule_key, + 'error' => $e->getMessage(), + ]); + } + } + public function deleteOldOccurrences(): void { ScheduledJobDelivery::query() @@ -276,6 +340,53 @@ class ScheduledJobDeliveryService ->delete(); } + /** + * Skip an occurrence that no job has claimed, so stale recovery does not publish it again. + */ + public function skip(string $uuid, string $reason): void + { + $skipped = ScheduledJobDelivery::query() + ->where('uuid', $uuid) + ->whereIn('status', ['pending', 'enqueued']) + ->update([ + 'status' => 'skipped', + 'updated_at' => now(), + ]) === 1; + + if ($skipped) { + $this->logOccurrence($uuid, "Scheduled occurrence skipped: {$reason}", 'warning'); + } + } + + /** + * WithoutOverlapping for the job of a scheduled occurrence. When another run holds the lock, + * the job does not run and its occurrence is skipped, so stale recovery does not run it later. + */ + public static function withoutOverlapping(string $key, ?string $occurrenceUuid): WithoutOverlapping + { + return new class($key, $occurrenceUuid) extends WithoutOverlapping + { + public function __construct(string $key, private ?string $occurrenceUuid) + { + parent::__construct($key); + } + + public function handle($job, $next) + { + $started = false; + parent::handle($job, function ($job) use ($next, &$started) { + $started = true; + + return $next($job); + }); + + if (! $started && $this->occurrenceUuid !== null) { + app(ScheduledJobDeliveryService::class)->skip($this->occurrenceUuid, 'the previous run of this schedule is still running'); + } + } + }; + } + public function fail(string $uuid, string $claimToken): void { ScheduledJobDelivery::query() @@ -359,8 +470,8 @@ class ScheduledJobDeliveryService return false; } - // An open previous occurrence usually means the previous run is still running. Jobs with - // WithoutOverlapping(...)->dontRelease() then drop this one without running it. + // An open previous occurrence usually means the previous run is still running. Backup jobs + // then skip this one without running it (see withoutOverlapping()). $previousOpen = ScheduledJobDelivery::query() ->where('schedule_key', $occurrence->schedule_key) ->where('scheduled_for', '<', $occurrence->scheduled_for) diff --git a/tests/Feature/BackupRunSafetyTest.php b/tests/Feature/BackupRunSafetyTest.php new file mode 100644 index 0000000000..d1fb7480af --- /dev/null +++ b/tests/Feature/BackupRunSafetyTest.php @@ -0,0 +1,292 @@ + InstanceSettings::firstOrCreate(['id' => 0])); + + $this->team = Team::factory()->create(); + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id, + ]); + $this->server->settings()->update(['is_reachable' => true, 'is_usable' => true]); + $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $project->id]); +}); + +/** + * @return Collection The remote commands the job ran. + */ +function backupRunSafetyFakeRemoteCommands(array $outputs = []): Collection +{ + $commands = collect(); + Process::fake(function ($process) use ($commands, $outputs) { + $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; + $commands->push($command); + foreach ($outputs as $needle => $output) { + if (str_contains($command, $needle)) { + return Process::result(output: $output); + } + } + + return Process::result(output: str_contains($command, 'du -b') ? '128' : ''); + }); + + return $commands; +} + +function backupRunSafetySchedule(Model $database, Team $team, array $overrides = []): ScheduledDatabaseBackup +{ + $backup = new ScheduledDatabaseBackup; + $backup->forceFill([ + 'enabled' => true, + 'save_s3' => false, + 'frequency' => 'daily', + 'database_id' => $database->id, + 'database_type' => $database->getMorphClass(), + 'team_id' => $team->id, + ...$overrides, + ]); + $backup->save(); + + return $backup->fresh(); +} + +function backupRunSafetyOccurrence(string $scheduleKey, string $jobType, int $resourceId): ScheduledJobDelivery +{ + return ScheduledJobDelivery::create([ + 'schedule_key' => $scheduleKey, + 'scheduled_for' => now()->startOfMinute(), + 'job_type' => $jobType, + 'resource_id' => $resourceId, + 'status' => 'enqueued', + 'enqueued_at' => now(), + ]); +} + +function backupRunSafetyActivity(string $typeUuid, string $operation, ProcessStatus $status): Activity +{ + return Activity::create([ + 'log_name' => 'default', + 'description' => '[]', + 'properties' => [ + 'type_uuid' => $typeUuid, + 'operation' => $operation, + 'status' => $status->value, + ], + ]); +} + +// --- Overlapped occurrences (M5) --- + +it('skips a database backup occurrence when the previous run still holds the overlap lock, so recovery does not run it later', function () { + Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')); + $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']); + $backup = backupRunSafetySchedule($database, $this->team); + $occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id); + $job = new DatabaseBackupJob($backup, $occurrence->uuid); + $runningBackupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600); + $runningBackupLock->get(); + $commands = backupRunSafetyFakeRemoteCommands(); + + dispatch_sync($job); + $runningBackupLock->release(); + + expect($occurrence->fresh()->status)->toBe('skipped') + ->and(ScheduledDatabaseBackupExecution::query()->exists())->toBeFalse() + ->and($commands)->toBeEmpty(); + + Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1)); + Queue::fake(); + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + + Queue::assertNothingPushed(); +}); + +it('skips a volume backup occurrence when the previous run still holds the overlap lock', function () { + $volume = backupRunSafetyVolume($this->environment, $this->destination); + $backup = $volume->scheduledBackups()->create(['team_id' => $this->team->id, 'frequency' => 'daily', 'enabled' => true]); + $occurrence = backupRunSafetyOccurrence("scheduled-volume-backup:{$backup->id}", 'volume-backup', $backup->id); + $runningBackupLock = Cache::lock(VolumeBackupJob::lockKey($backup->id), 3600); + $runningBackupLock->get(); + $commands = backupRunSafetyFakeRemoteCommands(); + + dispatch_sync(new VolumeBackupJob($backup, $occurrence->uuid)); + $runningBackupLock->release(); + + expect($occurrence->fresh()->status)->toBe('skipped') + ->and(ScheduledVolumeBackupExecution::query()->exists())->toBeFalse() + ->and($commands)->toBeEmpty(); +}); + +it('does not run a lost backup occurrence again when a newer occurrence already completed', function () { + Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC')); + Queue::fake(); + $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']); + $backup = backupRunSafetySchedule($database, $this->team, ['frequency' => 'hourly']); + $scheduleKey = "scheduled-backup:{$backup->id}"; + $lost = ScheduledJobDelivery::create([ + 'schedule_key' => $scheduleKey, + 'scheduled_for' => Carbon::create(2026, 9, 17, 10, 0, 0, 'UTC'), + 'job_type' => 'database-backup', + 'resource_id' => $backup->id, + 'status' => 'enqueued', + 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1), + ]); + // The 11:00 occurrence ran and completed; complete() deleted its row. + ScheduledJobState::create(['schedule_key' => $scheduleKey, 'last_scheduled_for' => Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC')]); + + app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued(); + + Queue::assertNotPushed(DatabaseBackupJob::class); + expect($lost->fresh()->status)->toBe('failed'); +}); + +// --- Imports and starts (M10) --- + +it('does not back up a database while a start or import of it is in progress', function (Closure $startOperation) { + $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']); + $backup = backupRunSafetySchedule($database, $this->team); + $occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id); + $startOperation($database); + $commands = backupRunSafetyFakeRemoteCommands(); + + (new DatabaseBackupJob($backup, $occurrence->uuid))->handle(); + + $execution = ScheduledDatabaseBackupExecution::query()->sole(); + expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->toBeEmpty() + ->and($execution->status)->toBe('failed') + ->and($execution->message)->toContain('Skipped') + ->and($execution->filename)->toBeNull() + ->and($execution->finished_at)->not->toBeNull() + ->and(ScheduledJobDelivery::query()->whereKey($occurrence->id)->exists())->toBeFalse(); + Notification::assertNothingSent(); +})->with([ + 'reserved import, start or restart' => [fn (Model $database) => DatabaseOperationReservation::acquire($database->uuid)], + 'running import' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS)], + 'queued start' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_START_OPERATION, ProcessStatus::QUEUED)], +]); + +it('backs up a database when its last import is stale', function () { + $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']); + $backup = backupRunSafetySchedule($database, $this->team); + $import = backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS); + Activity::query()->whereKey($import->id)->update(['updated_at' => now()->subSeconds(ResourceStartActivity::importStaleAfterSeconds() + 60)]); + $commands = backupRunSafetyFakeRemoteCommands(); + + (new DatabaseBackupJob($backup))->handle(); + + expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->not->toBeEmpty() + ->and(ScheduledDatabaseBackupExecution::query()->sole()->status)->toBe('success'); +}); + +// --- Unique backup file names (M11) --- + +it('gives each backup run of the same database its own file in the same second', function (Closure $createDatabase, array $backupSettings, string $extension) { + Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')); + $database = $createDatabase($this->environment->id, $this->destination, ['status' => 'running:healthy']); + $first = backupRunSafetySchedule($database, $this->team, $backupSettings); + $second = backupRunSafetySchedule($database, $this->team, $backupSettings); + backupRunSafetyFakeRemoteCommands(); + + (new DatabaseBackupJob($first))->handle(); + (new DatabaseBackupJob($second))->handle(); + + $executions = ScheduledDatabaseBackupExecution::query()->orderBy('id')->get(); + expect($executions)->toHaveCount(2) + ->and($executions->pluck('status')->unique()->all())->toBe(['success']) + ->and($executions->pluck('filename')->unique())->toHaveCount(2); + foreach ($executions as $execution) { + expect($execution->filename)->toEndWith("-{$execution->uuid}{$extension}"); + } +})->with([ + 'postgresql' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), [], '.dmp'], + 'postgresql full dump' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), ['dump_all' => true], '.gz'], + 'mysql' => [fn (...$arguments) => create_standalone_mysql(...$arguments), [], '.dmp'], + 'mariadb' => [fn (...$arguments) => create_standalone_mariadb(...$arguments), [], '.dmp'], + 'mongodb' => [fn (...$arguments) => create_standalone_mongodb(...$arguments), [], '.tar.gz'], + 'sqlite' => [fn (...$arguments) => create_standalone_sqlite(...$arguments), [], '.gz'], +]); + +// --- Service database backup folder (M12) --- + +it('stores service database backups in the folder of earlier releases', function () { + $service = Service::factory()->create([ + 'name' => 'Rallly', + 'server_id' => $this->server->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + 'environment_id' => $this->environment->id, + ]); + $database = ServiceDatabase::create([ + 'service_id' => $service->id, + 'name' => 'rallly_db', + 'image' => 'postgres:16-alpine', + 'status' => 'running:healthy', + ]); + $backup = backupRunSafetySchedule($database, $this->team); + backupRunSafetyFakeRemoteCommands(['env | grep POSTGRES_' => "POSTGRES_USER=rallly\nPOSTGRES_DB=rallly"]); + + (new DatabaseBackupJob($backup))->handle(); + + expect(ScheduledDatabaseBackupExecution::query()->sole()->filename) + ->toContain("/rallly-rallly_db-{$service->uuid}/pg-dump-rallly-"); +}); + +function backupRunSafetyVolume(Environment $environment, StandaloneDocker $destination): LocalPersistentVolume +{ + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + return LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); +} diff --git a/tests/Feature/DatabaseDumpExitStatusTest.php b/tests/Feature/DatabaseDumpExitStatusTest.php index e614db4250..7a9382a74b 100644 --- a/tests/Feature/DatabaseDumpExitStatusTest.php +++ b/tests/Feature/DatabaseDumpExitStatusTest.php @@ -107,5 +107,5 @@ it('records a failed SQLite dump as failed and removes the partial backup file', $execution = $backup->executions()->latest('id')->firstOrFail(); expect($execution->status)->toBe('failed') ->and($execution->filename)->toBeNull() - ->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+\\.gz'/"); + ->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+-[a-z0-9]+\\.gz'/"); }); diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.php index 95f241b046..9c84560399 100644 --- a/tests/Feature/ScheduledJobManagerDispatchTest.php +++ b/tests/Feature/ScheduledJobManagerDispatchTest.php @@ -1,18 +1,22 @@ and($log)->toContain('Scheduled occurrence missed: its queued job was not started'); }); +it('records a missed scheduled task as a failed execution and notifies the team once', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC')); + Queue::fake(); + Notification::fake(); + $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']); + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + Team::find($task->team_id)->emailNotificationSettings->update(['smtp_enabled' => true, 'scheduled_task_failure_email_notifications' => true]); + $occurrence = createStaleEnqueuedOccurrence("scheduled-task:{$task->id}", 'scheduled-task', $task->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')); + + (new ScheduledJobManager)->handle(); + (new ScheduledJobManager)->handle(); + + $executions = ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->get(); + expect($executions)->toHaveCount(1) + ->and($executions->first()->status)->toBe('failed') + ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.') + ->and($executions->first()->finished_at)->not->toBeNull(); + Notification::assertSentToTimes(Team::find($task->team_id), TaskFailed::class, 1); + Notification::assertSentTo(Team::find($task->team_id), TaskFailed::class, fn (TaskFailed $notification) => $notification->task->is($task) + && $notification->output === 'Skipped: the queued job did not start within 60 minutes.'); + + $lateJob = new ScheduledTaskJob($task, $occurrence->uuid); + $lateJob->handle(); + + expect(ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->count())->toBe(1); +}); + +it('records a missed Docker cleanup as a failed execution and notifies the team once', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC')); + Queue::fake(); + Notification::fake(); + Event::fake([DockerCleanupDone::class]); + $server = createScheduledTaskApplication()->destination->server; + $server->team->emailNotificationSettings->update(['smtp_enabled' => true, 'docker_cleanup_failure_email_notifications' => true]); + $occurrence = createStaleEnqueuedOccurrence("docker-cleanup:{$server->id}", 'docker-cleanup', $server->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')); + + (new ScheduledJobManager)->handle(); + (new ScheduledJobManager)->handle(); + + $executions = DockerCleanupExecution::query()->where('server_id', $server->id)->get(); + expect($executions)->toHaveCount(1) + ->and($executions->first()->status)->toBe('failed') + ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.') + ->and($executions->first()->finished_at)->not->toBeNull() + ->and($occurrence->fresh()->status)->toBe('failed') + ->and(app(ScheduledJobDeliveryService::class)->claim($occurrence->uuid, 'original-job'))->toBeFalse(); + Notification::assertSentToTimes($server->team, DockerCleanupFailed::class, 1); + Event::assertDispatched(DockerCleanupDone::class, 1); +}); + it('leaves recently enqueued occurrences alone', function () { config(['constants.coolify.self_hosted' => true]); Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));