starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-12 15:53:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(backups): make overlapping and concurrent backup runs safe

Browse Source 
- Skip the occurrence of a backup whose previous run still holds the
  overlap lock, so stale recovery does not run it later; do not rerun a
  lost occurrence when a newer one already completed.
- Skip a database backup while a start, restart, or import of the database
  is in progress, and record the skip in the execution list.
- Give every backup file a unique name per run and keep the service
  database backup folder of earlier releases.
- Record missed scheduled tasks and Docker cleanups as failed executions
  and notify the team.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2026-10-02 16:01:20 +02:00
co-authored by Claude Opus 5.5
parent 7ec46faecb
commit 1c68eafa84
6 changed files with 534 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Jobs/DatabaseBackupJob.php
+63 -11
View File
@@ -22,6 +22,8 @@ use App\Rules\SafeWebhookUrl;
use App\Services\ScheduledJobDeliveryService;
use App\Support\BackupCompression;
use App\Support\ClickhouseBackupCommand;
use App\Support\DatabaseOperationReservation;
use App\Support\ResourceStartActivity;
use App\Support\ValidationPatterns;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
@@ -29,10 +31,10 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use Spatie\Activitylog\Models\Activity;
use Throwable;
class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
@@ -91,7 +93,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
{
$expireAfter = ($this->backup->timeout ?? 3600) + 300;
return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
return [ScheduledJobDeliveryService::withoutOverlapping('database-backup-'.$this->backup->id, $this->occurrenceUuid)->expireAfter($expireAfter)->dontRelease()];
}
public function handle(): void
@@ -138,6 +140,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
if ($this->databaseOperationInProgress()) {
$this->recordSkippedBackup('Skipped: a start, restart or import of this database was in progress. A backup taken now could contain a partly restored database.');
$this->logSkippedRun('database_operation_in_progress');
return;
}
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
$databaseType = $this->database->databaseType();
$serviceUuid = $this->database->service->uuid;
@@ -146,7 +154,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if (! ValidationPatterns::isValidContainerName($this->container_name)) {
throw new \Exception('Invalid database container name.');
}
$this->directory_name = $serviceName.'-'.str($this->database->name)->slug().'-'.$serviceUuid;
// The container name is validated above, so it is a safe folder name. Keep the folder of earlier releases.
$this->directory_name = $serviceName.'-'.$this->container_name;
$escapedContainerName = escapeshellarg($this->container_name);
if (str($databaseType)->contains('postgres')) {
$commands[] = "docker exec {$escapedContainerName} env | grep POSTGRES_";
@@ -321,9 +330,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
try {
if (str($databaseType)->contains('postgres')) {
$fileDatabaseName = $this->backupFilenamePart($database);
$this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';
$this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";
if ($this->backup->dump_all) {
$this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp.'.gz';
$this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -347,7 +356,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
$fileDatabaseName = $this->backupFilenamePart($databaseName);
$this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.tar.gz';
$this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.tar.gz";
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'uuid' => $this->backup_log_uuid,
@@ -360,9 +369,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_standalone_mongodb($database);
} elseif (str($databaseType)->contains('mysql')) {
$fileDatabaseName = $this->backupFilenamePart($database);
$this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';
$this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";
if ($this->backup->dump_all) {
$this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp.'.gz';
$this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -376,9 +385,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_standalone_mysql($database);
} elseif (str($databaseType)->contains('mariadb')) {
$fileDatabaseName = $this->backupFilenamePart($database);
$this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';
$this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";
if ($this->backup->dump_all) {
$this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp.'.gz';
$this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -403,7 +412,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
BackupCreated::dispatch($this->team->id);
$this->backup_standalone_clickhouse($database);
} elseif ($this->database instanceof StandaloneSqlite) {
$this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp.'.gz';
$this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'uuid' => $this->backup_log_uuid,
@@ -925,6 +934,49 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
]);
}
/**
* A start, restart or import of the database is reserved, queued or running. Stale activities do
* not count; they are failed by the start and import paths.
*/
private function databaseOperationInProgress(): bool
{
$uuid = $this->database->uuid;
if (blank($uuid)) {
return false;
}
if (DatabaseOperationReservation::isHeldByOther($uuid)) {
return true;
}
$liveOperation = collect([ResourceStartActivity::DATABASE_START_OPERATION, ResourceStartActivity::DATABASE_IMPORT_OPERATION])
->flatMap(fn (string $operation) => ResourceStartActivity::active($uuid, $operation))
->contains(fn (Activity $activity): bool => ! ResourceStartActivity::isStale($activity));
return $liveOperation
|| ResourceStartActivity::latestRunning($uuid) !== null
|| ($this->database instanceof ServiceDatabase && ResourceStartActivity::latestRunning($this->database->service->uuid) !== null);
}
/**
* Show a backup run that did not take a backup in the execution list. The run is expected, so it
* sends no failure notification.
*/
private function recordSkippedBackup(string $message): void
{
$databaseNames = $this->backup->dump_all ? 'all' : data_get($this->backup, 'databases_to_backup');
$this->backup_log_uuid = new_public_id();
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'uuid' => $this->backup_log_uuid,
'database_name' => filled($databaseNames) ? Str::limit($databaseNames, 250) : null,
'scheduled_database_backup_id' => $this->backup->id,
'status' => 'failed',
'message' => $message,
'filename' => null,
'local_storage_deleted' => false,
]);
}
private function markStaleExecutionsAsFailed(): void
{
try {
app/Jobs/VolumeBackupJob.php
+1 -2
View File
@@ -17,7 +17,6 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
@@ -42,7 +41,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
public function middleware(): array
{
return [
(new WithoutOverlapping('volume-backup-'.$this->backup->id))
ScheduledJobDeliveryService::withoutOverlapping('volume-backup-'.$this->backup->id, $this->occurrenceUuid)
->shared()
->expireAfter($this->timeout + 60)
->dontRelease(),
app/Services/ScheduledJobDeliveryService.php
+118 -7
View File
@@ -2,17 +2,25 @@
namespace App\Services;
use App\Events\DockerCleanupDone;
use App\Events\ScheduledTaskDone;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\ScheduledTaskJob;
use App\Jobs\VolumeBackupJob;
use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\ScheduledTaskExecution;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\Server\DockerCleanupFailed;
use Cron\CronExpression;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
@@ -29,7 +37,8 @@ class ScheduledJobDeliveryService
public const ENQUEUED_STALE_AFTER_MINUTES = 60;
/**
* Job types that run once when they are late. A lost task or Docker cleanup is logged as missed.
* Job types that run once when they are late. A lost task or Docker cleanup is not run; it is
* recorded as a failed execution and the team is notified.
*/
private const LATE_RUN_JOB_TYPES = ['database-backup', 'volume-backup'];
@@ -167,7 +176,7 @@ class ScheduledJobDeliveryService
}
/**
* Publish backups again whose queued job was not started, and log lost tasks and Docker
* Publish backups again whose queued job was not started, and report lost tasks and Docker
* cleanups as missed. A lost backup is not run when a newer occurrence of the same schedule
* exists, so a backup that runs longer than its interval does not build up a backlog.
*
@@ -196,10 +205,15 @@ class ScheduledJobDeliveryService
private function recoverStaleEnqueuedOccurrence(ScheduledJobDelivery $occurrence): void
{
$hasNewerOccurrence = ScheduledJobDelivery::query()
// complete() deletes the row of a finished occurrence, so the schedule state also counts.
$hasNewerOccurrence = ScheduledJobState::query()
->where('schedule_key', $occurrence->schedule_key)
->where('scheduled_for', '>', $occurrence->scheduled_for)
->exists();
->where('last_scheduled_for', '>', $occurrence->scheduled_for)
->exists()
|| ScheduledJobDelivery::query()
->where('schedule_key', $occurrence->schedule_key)
->where('scheduled_for', '>', $occurrence->scheduled_for)
->exists();
$runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence;
// Only one scheduler node can move each occurrence.
@@ -217,6 +231,7 @@ class ScheduledJobDeliveryService
$this->logOccurrence($occurrence->uuid, $hasNewerOccurrence
? 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists'
: 'Scheduled occurrence missed: its queued job was not started', 'warning');
$this->reportMissedOccurrence($occurrence);
return;
}
@@ -226,6 +241,55 @@ class ScheduledJobDeliveryService
$this->publish($occurrence);
}
/**
* Show a missed task or Docker cleanup as a failed execution and send the failure notification.
* The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,
* and claim() rejects the late job if it starts afterwards.
*/
private function reportMissedOccurrence(ScheduledJobDelivery $occurrence): void
{
$message = 'Skipped: the queued job did not start within '.self::ENQUEUED_STALE_AFTER_MINUTES.' minutes.';
try {
if ($occurrence->job_type === 'scheduled-task') {
$task = ScheduledTask::find($occurrence->resource_id);
if (! $task) {
return;
}
ScheduledTaskExecution::create([
'scheduled_task_id' => $task->id,
'status' => 'failed',
'message' => $message,
'retry_count' => 0,
'finished_at' => now(),
]);
ScheduledTaskDone::dispatch($task->team_id);
Team::find($task->team_id)?->notify(new TaskFailed($task, $message));
} elseif ($occurrence->job_type === 'docker-cleanup') {
$server = Server::find($occurrence->resource_id);
if (! $server) {
return;
}
$execution = DockerCleanupExecution::create([
'server_id' => $server->id,
'status' => 'failed',
'message' => $message,
'finished_at' => now(),
]);
event(new DockerCleanupDone($execution));
$server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}"));
}
} catch (\Throwable $e) {
Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [
'occurrence_uuid' => $occurrence->uuid,
'schedule_key' => $occurrence->schedule_key,
'error' => $e->getMessage(),
]);
}
}
public function deleteOldOccurrences(): void
{
ScheduledJobDelivery::query()
@@ -276,6 +340,53 @@ class ScheduledJobDeliveryService
->delete();
}
/**
* Skip an occurrence that no job has claimed, so stale recovery does not publish it again.
*/
public function skip(string $uuid, string $reason): void
{
$skipped = ScheduledJobDelivery::query()
->where('uuid', $uuid)
->whereIn('status', ['pending', 'enqueued'])
->update([
'status' => 'skipped',
'updated_at' => now(),
]) === 1;
if ($skipped) {
$this->logOccurrence($uuid, "Scheduled occurrence skipped: {$reason}", 'warning');
}
}
/**
* WithoutOverlapping for the job of a scheduled occurrence. When another run holds the lock,
* the job does not run and its occurrence is skipped, so stale recovery does not run it later.
*/
public static function withoutOverlapping(string $key, ?string $occurrenceUuid): WithoutOverlapping
{
return new class($key, $occurrenceUuid) extends WithoutOverlapping
{
public function __construct(string $key, private ?string $occurrenceUuid)
{
parent::__construct($key);
}
public function handle($job, $next)
{
$started = false;
parent::handle($job, function ($job) use ($next, &$started) {
$started = true;
return $next($job);
});
if (! $started && $this->occurrenceUuid !== null) {
app(ScheduledJobDeliveryService::class)->skip($this->occurrenceUuid, 'the previous run of this schedule is still running');
}
}
};
}
public function fail(string $uuid, string $claimToken): void
{
ScheduledJobDelivery::query()
@@ -359,8 +470,8 @@ class ScheduledJobDeliveryService
return false;
}
// An open previous occurrence usually means the previous run is still running. Jobs with
// WithoutOverlapping(...)->dontRelease() then drop this one without running it.
// An open previous occurrence usually means the previous run is still running. Backup jobs
// then skip this one without running it (see withoutOverlapping()).
$previousOpen = ScheduledJobDelivery::query()
->where('schedule_key', $occurrence->schedule_key)
->where('scheduled_for', '<', $occurrence->scheduled_for)
tests/Feature/BackupRunSafetyTest.php
+292
View File
@@ -0,0 +1,292 @@
<?php
use App\Enums\ProcessStatus;
use App\Events\BackupCreated;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\VolumeBackupJob;
use App\Models\Application;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\LocalPersistentVolume;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Services\ScheduledJobDeliveryService;
use App\Support\DatabaseOperationReservation;
use App\Support\ResourceStartActivity;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Facades\Storage;
use Spatie\Activitylog\Models\Activity;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
Storage::fake('ssh-keys');
Storage::fake('ssh-mux');
Event::fake([BackupCreated::class]);
Notification::fake();
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
$this->team = Team::factory()->create();
$this->server = Server::factory()->create([
'team_id' => $this->team->id,
'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id,
]);
$this->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
$this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
$project = Project::factory()->create(['team_id' => $this->team->id]);
$this->environment = Environment::factory()->create(['project_id' => $project->id]);
});
/**
* @return Collection<int, string> The remote commands the job ran.
*/
function backupRunSafetyFakeRemoteCommands(array $outputs = []): Collection
{
$commands = collect();
Process::fake(function ($process) use ($commands, $outputs) {
$command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
$commands->push($command);
foreach ($outputs as $needle => $output) {
if (str_contains($command, $needle)) {
return Process::result(output: $output);
}
}
return Process::result(output: str_contains($command, 'du -b') ? '128' : '');
});
return $commands;
}
function backupRunSafetySchedule(Model $database, Team $team, array $overrides = []): ScheduledDatabaseBackup
{
$backup = new ScheduledDatabaseBackup;
$backup->forceFill([
'enabled' => true,
'save_s3' => false,
'frequency' => 'daily',
'database_id' => $database->id,
'database_type' => $database->getMorphClass(),
'team_id' => $team->id,
...$overrides,
]);
$backup->save();
return $backup->fresh();
}
function backupRunSafetyOccurrence(string $scheduleKey, string $jobType, int $resourceId): ScheduledJobDelivery
{
return ScheduledJobDelivery::create([
'schedule_key' => $scheduleKey,
'scheduled_for' => now()->startOfMinute(),
'job_type' => $jobType,
'resource_id' => $resourceId,
'status' => 'enqueued',
'enqueued_at' => now(),
]);
}
function backupRunSafetyActivity(string $typeUuid, string $operation, ProcessStatus $status): Activity
{
return Activity::create([
'log_name' => 'default',
'description' => '[]',
'properties' => [
'type_uuid' => $typeUuid,
'operation' => $operation,
'status' => $status->value,
],
]);
}
// --- Overlapped occurrences (M5) ---
it('skips a database backup occurrence when the previous run still holds the overlap lock, so recovery does not run it later', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
$database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
$backup = backupRunSafetySchedule($database, $this->team);
$occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id);
$job = new DatabaseBackupJob($backup, $occurrence->uuid);
$runningBackupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
$runningBackupLock->get();
$commands = backupRunSafetyFakeRemoteCommands();
dispatch_sync($job);
$runningBackupLock->release();
expect($occurrence->fresh()->status)->toBe('skipped')
->and(ScheduledDatabaseBackupExecution::query()->exists())->toBeFalse()
->and($commands)->toBeEmpty();
Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1));
Queue::fake();
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
Queue::assertNothingPushed();
});
it('skips a volume backup occurrence when the previous run still holds the overlap lock', function () {
$volume = backupRunSafetyVolume($this->environment, $this->destination);
$backup = $volume->scheduledBackups()->create(['team_id' => $this->team->id, 'frequency' => 'daily', 'enabled' => true]);
$occurrence = backupRunSafetyOccurrence("scheduled-volume-backup:{$backup->id}", 'volume-backup', $backup->id);
$runningBackupLock = Cache::lock(VolumeBackupJob::lockKey($backup->id), 3600);
$runningBackupLock->get();
$commands = backupRunSafetyFakeRemoteCommands();
dispatch_sync(new VolumeBackupJob($backup, $occurrence->uuid));
$runningBackupLock->release();
expect($occurrence->fresh()->status)->toBe('skipped')
->and(ScheduledVolumeBackupExecution::query()->exists())->toBeFalse()
->and($commands)->toBeEmpty();
});
it('does not run a lost backup occurrence again when a newer occurrence already completed', function () {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
Queue::fake();
$database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
$backup = backupRunSafetySchedule($database, $this->team, ['frequency' => 'hourly']);
$scheduleKey = "scheduled-backup:{$backup->id}";
$lost = ScheduledJobDelivery::create([
'schedule_key' => $scheduleKey,
'scheduled_for' => Carbon::create(2026, 9, 17, 10, 0, 0, 'UTC'),
'job_type' => 'database-backup',
'resource_id' => $backup->id,
'status' => 'enqueued',
'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),
]);
// The 11:00 occurrence ran and completed; complete() deleted its row.
ScheduledJobState::create(['schedule_key' => $scheduleKey, 'last_scheduled_for' => Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC')]);
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
Queue::assertNotPushed(DatabaseBackupJob::class);
expect($lost->fresh()->status)->toBe('failed');
});
// --- Imports and starts (M10) ---
it('does not back up a database while a start or import of it is in progress', function (Closure $startOperation) {
$database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
$backup = backupRunSafetySchedule($database, $this->team);
$occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id);
$startOperation($database);
$commands = backupRunSafetyFakeRemoteCommands();
(new DatabaseBackupJob($backup, $occurrence->uuid))->handle();
$execution = ScheduledDatabaseBackupExecution::query()->sole();
expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->toBeEmpty()
->and($execution->status)->toBe('failed')
->and($execution->message)->toContain('Skipped')
->and($execution->filename)->toBeNull()
->and($execution->finished_at)->not->toBeNull()
->and(ScheduledJobDelivery::query()->whereKey($occurrence->id)->exists())->toBeFalse();
Notification::assertNothingSent();
})->with([
'reserved import, start or restart' => [fn (Model $database) => DatabaseOperationReservation::acquire($database->uuid)],
'running import' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS)],
'queued start' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_START_OPERATION, ProcessStatus::QUEUED)],
]);
it('backs up a database when its last import is stale', function () {
$database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
$backup = backupRunSafetySchedule($database, $this->team);
$import = backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS);
Activity::query()->whereKey($import->id)->update(['updated_at' => now()->subSeconds(ResourceStartActivity::importStaleAfterSeconds() + 60)]);
$commands = backupRunSafetyFakeRemoteCommands();
(new DatabaseBackupJob($backup))->handle();
expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->not->toBeEmpty()
->and(ScheduledDatabaseBackupExecution::query()->sole()->status)->toBe('success');
});
// --- Unique backup file names (M11) ---
it('gives each backup run of the same database its own file in the same second', function (Closure $createDatabase, array $backupSettings, string $extension) {
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
$database = $createDatabase($this->environment->id, $this->destination, ['status' => 'running:healthy']);
$first = backupRunSafetySchedule($database, $this->team, $backupSettings);
$second = backupRunSafetySchedule($database, $this->team, $backupSettings);
backupRunSafetyFakeRemoteCommands();
(new DatabaseBackupJob($first))->handle();
(new DatabaseBackupJob($second))->handle();
$executions = ScheduledDatabaseBackupExecution::query()->orderBy('id')->get();
expect($executions)->toHaveCount(2)
->and($executions->pluck('status')->unique()->all())->toBe(['success'])
->and($executions->pluck('filename')->unique())->toHaveCount(2);
foreach ($executions as $execution) {
expect($execution->filename)->toEndWith("-{$execution->uuid}{$extension}");
}
})->with([
'postgresql' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), [], '.dmp'],
'postgresql full dump' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), ['dump_all' => true], '.gz'],
'mysql' => [fn (...$arguments) => create_standalone_mysql(...$arguments), [], '.dmp'],
'mariadb' => [fn (...$arguments) => create_standalone_mariadb(...$arguments), [], '.dmp'],
'mongodb' => [fn (...$arguments) => create_standalone_mongodb(...$arguments), [], '.tar.gz'],
'sqlite' => [fn (...$arguments) => create_standalone_sqlite(...$arguments), [], '.gz'],
]);
// --- Service database backup folder (M12) ---
it('stores service database backups in the folder of earlier releases', function () {
$service = Service::factory()->create([
'name' => 'Rallly',
'server_id' => $this->server->id,
'destination_id' => $this->destination->id,
'destination_type' => $this->destination->getMorphClass(),
'environment_id' => $this->environment->id,
]);
$database = ServiceDatabase::create([
'service_id' => $service->id,
'name' => 'rallly_db',
'image' => 'postgres:16-alpine',
'status' => 'running:healthy',
]);
$backup = backupRunSafetySchedule($database, $this->team);
backupRunSafetyFakeRemoteCommands(['env | grep POSTGRES_' => "POSTGRES_USER=rallly\nPOSTGRES_DB=rallly"]);
(new DatabaseBackupJob($backup))->handle();
expect(ScheduledDatabaseBackupExecution::query()->sole()->filename)
->toContain("/rallly-rallly_db-{$service->uuid}/pg-dump-rallly-");
});
function backupRunSafetyVolume(Environment $environment, StandaloneDocker $destination): LocalPersistentVolume
{
$application = Application::factory()->create([
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
]);
return LocalPersistentVolume::create([
'name' => 'app-data',
'mount_path' => '/data',
'resource_id' => $application->id,
'resource_type' => $application->getMorphClass(),
]);
}
tests/Feature/DatabaseDumpExitStatusTest.php
+1 -1
View File
@@ -107,5 +107,5 @@ it('records a failed SQLite dump as failed and removes the partial backup file',
$execution = $backup->executions()->latest('id')->firstOrFail();
expect($execution->status)->toBe('failed')
->and($execution->filename)->toBeNull()
->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+\\.gz'/");
->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+-[a-z0-9]+\\.gz'/");
});
tests/Feature/ScheduledJobManagerDispatchTest.php
+59
View File
@@ -1,18 +1,22 @@
<?php
use App\Events\BackupCreated;
use App\Events\DockerCleanupDone;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\ScheduledJobManager;
use App\Jobs\ScheduledTaskJob;
use App\Models\Application;
use App\Models\DockerCleanupExecution;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\ScheduledTaskExecution;
use App\Models\Server;
use App\Models\ServerSetting;
use App\Models\Service;
@@ -20,6 +24,8 @@ use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\Server\DockerCleanupFailed;
use App\Services\ScheduledJobDeliveryService;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Queue\Middleware\WithoutOverlapping;
@@ -29,6 +35,7 @@ use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
@@ -691,6 +698,58 @@ it('logs a lost scheduled task occurrence as missed instead of running it late',
->and($log)->toContain('Scheduled occurrence missed: its queued job was not started');
});
it('records a missed scheduled task as a failed execution and notifies the team once', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
Queue::fake();
Notification::fake();
$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
Team::find($task->team_id)->emailNotificationSettings->update(['smtp_enabled' => true, 'scheduled_task_failure_email_notifications' => true]);
$occurrence = createStaleEnqueuedOccurrence("scheduled-task:{$task->id}", 'scheduled-task', $task->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
(new ScheduledJobManager)->handle();
(new ScheduledJobManager)->handle();
$executions = ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->get();
expect($executions)->toHaveCount(1)
->and($executions->first()->status)->toBe('failed')
->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
->and($executions->first()->finished_at)->not->toBeNull();
Notification::assertSentToTimes(Team::find($task->team_id), TaskFailed::class, 1);
Notification::assertSentTo(Team::find($task->team_id), TaskFailed::class, fn (TaskFailed $notification) => $notification->task->is($task)
&& $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
$lateJob = new ScheduledTaskJob($task, $occurrence->uuid);
$lateJob->handle();
expect(ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->count())->toBe(1);
});
it('records a missed Docker cleanup as a failed execution and notifies the team once', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
Queue::fake();
Notification::fake();
Event::fake([DockerCleanupDone::class]);
$server = createScheduledTaskApplication()->destination->server;
$server->team->emailNotificationSettings->update(['smtp_enabled' => true, 'docker_cleanup_failure_email_notifications' => true]);
$occurrence = createStaleEnqueuedOccurrence("docker-cleanup:{$server->id}", 'docker-cleanup', $server->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
(new ScheduledJobManager)->handle();
(new ScheduledJobManager)->handle();
$executions = DockerCleanupExecution::query()->where('server_id', $server->id)->get();
expect($executions)->toHaveCount(1)
->and($executions->first()->status)->toBe('failed')
->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
->and($executions->first()->finished_at)->not->toBeNull()
->and($occurrence->fresh()->status)->toBe('failed')
->and(app(ScheduledJobDeliveryService::class)->claim($occurrence->uuid, 'original-job'))->toBeFalse();
Notification::assertSentToTimes($server->team, DockerCleanupFailed::class, 1);
Event::assertDispatched(DockerCleanupDone::class, 1);
});
it('leaves recently enqueued occurrences alone', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));