mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-16 01:32:39 -04:00
The scheduled job manager checked every enabled schedule each minute and evaluated its cron expression in PHP. On Coolify Cloud one run took more than 60 seconds, so runs overlapped and minutes were skipped. Each schedule now stores its next due time: - next_run_at on scheduled database backups, scheduled tasks, and volume backups, and docker_cleanup_next_run_at on server settings (UTC, indexed) - model hooks calculate it when a schedule is created or its frequency or enabled flag changes; a server timezone change or a move to another server calculates it again for the affected schedules The `scheduled:dispatch` command runs the manager every minute from the scheduler, not from a queue. It selects only due rows and claims each occurrence with an atomic update of next_run_at, together with its delivery record, so parallel runs on several nodes never dispatch it twice. - database and volume backups run once when they are late; tasks and Docker cleanups more than 10 minutes late are logged as missed - a local time that occurs two times when the clocks go back runs once - the migration continues each schedule after its last recorded occurrence and drops scheduled_job_states Show the next run, in the server timezone, on the database backup, scheduled task, volume backup, and Docker cleanup pages, or "Invalid frequency" when the frequency cannot be calculated. scheduled:diagnostics now lists overdue schedules and open deliveries. Remove workarounds that are no longer needed: the queue overlap lock and its stale-lock repair, the catch-up window, and debug-only logs.
427 lines
17 KiB
PHP
427 lines
17 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Actions\Shared\DeleteScheduledVolumeBackup;
|
|
use App\Models\ScheduledDatabaseBackup;
|
|
use App\Models\ScheduledJobDelivery;
|
|
use App\Models\ScheduledTask;
|
|
use App\Models\ScheduledVolumeBackup;
|
|
use App\Models\ScheduledVolumeBackupExecution;
|
|
use App\Models\Server;
|
|
use App\Models\ServiceDatabase;
|
|
use App\Services\ScheduledJobDeliveryService;
|
|
use Carbon\CarbonImmutable;
|
|
use Illuminate\Database\Eloquent\Builder;
|
|
use Illuminate\Database\Eloquent\Model;
|
|
use Illuminate\Database\Eloquent\Relations\MorphTo;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
/**
|
|
* Dispatches the due scheduled backups, tasks, volume backups, and Docker cleanups.
|
|
*
|
|
* Each schedule stores its next due time in next_run_at. A run selects only due rows and claims
|
|
* each occurrence with an atomic update of next_run_at, so parallel runs on several nodes never
|
|
* dispatch the same occurrence twice. The `scheduled:dispatch` command runs it every minute.
|
|
*/
|
|
class ScheduledJobManager
|
|
{
|
|
private const CHUNK_SIZE = 100;
|
|
|
|
/**
|
|
* A task or Docker cleanup that is due longer than this is logged as missed and not run.
|
|
* Database and volume backups always run once, also when they are late.
|
|
*/
|
|
public const LATE_RUN_WINDOW_MINUTES = 10;
|
|
|
|
private CarbonImmutable $now;
|
|
|
|
private ScheduledJobDeliveryService $deliveries;
|
|
|
|
/** @var array{dispatched: int, skipped: int, missed: int} */
|
|
private array $counts = ['dispatched' => 0, 'skipped' => 0, 'missed' => 0];
|
|
|
|
public function handle(): void
|
|
{
|
|
$this->now = CarbonImmutable::now();
|
|
$this->deliveries = app(ScheduledJobDeliveryService::class);
|
|
$this->counts = ['dispatched' => 0, 'skipped' => 0, 'missed' => 0];
|
|
|
|
// Read the current server state, not servers cached by an earlier run in this process.
|
|
Server::flushIdentityMap();
|
|
|
|
Log::channel('scheduled')->info('ScheduledJobManager started', ['execution_time' => $this->now->toIso8601String()]);
|
|
|
|
$this->deliveries->publishPending();
|
|
$this->runStep('database backups', fn () => $this->processDatabaseBackups());
|
|
$this->runStep('scheduled tasks', fn () => $this->processScheduledTasks());
|
|
$this->runStep('volume backups', function () {
|
|
$this->recoverStoppedVolumeBackupContainers();
|
|
$this->processVolumeBackups();
|
|
});
|
|
$this->runStep('docker cleanups', fn () => $this->processDockerCleanups());
|
|
|
|
Log::channel('scheduled')->info('ScheduledJobManager completed', [
|
|
'execution_time' => $this->now->toIso8601String(),
|
|
'duration_ms' => $this->now->diffInMilliseconds(CarbonImmutable::now()),
|
|
...$this->counts,
|
|
]);
|
|
|
|
// The UI uses the heartbeat to show when the scheduler has stopped.
|
|
rescue(fn () => Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300), report: false);
|
|
}
|
|
|
|
private function runStep(string $name, callable $step): void
|
|
{
|
|
try {
|
|
$step();
|
|
} catch (\Throwable $e) {
|
|
Log::channel('scheduled-errors')->error("Failed to process {$name}", [
|
|
'error' => $e->getMessage(),
|
|
'trace' => $e->getTraceAsString(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
private function processDatabaseBackups(): void
|
|
{
|
|
$this->due(ScheduledDatabaseBackup::query())
|
|
->with([
|
|
'team.subscription',
|
|
'database' => fn (MorphTo $morphTo) => $morphTo->morphWith([
|
|
ServiceDatabase::class => ['service.destination.server.settings', 'service.destination.server.team.subscription'],
|
|
...array_fill_keys(STANDALONE_DATABASE_MODELS, ['destination.server.settings', 'destination.server.team.subscription']),
|
|
]),
|
|
])
|
|
->chunkById(self::CHUNK_SIZE, fn ($backups) => $backups->each(fn (ScheduledDatabaseBackup $backup) => $this->processItem(
|
|
'backup',
|
|
['backup_id' => $backup->id, 'database_id' => $backup->database_id, 'database_type' => $backup->database_type, 'team_id' => $backup->team_id],
|
|
fn () => $this->processDatabaseBackup($backup),
|
|
)));
|
|
}
|
|
|
|
private function processDatabaseBackup(ScheduledDatabaseBackup $backup): array
|
|
{
|
|
$server = $backup->server();
|
|
|
|
if (blank($backup->database) || blank($server)) {
|
|
$backup->delete();
|
|
|
|
return ['deleted' => blank($backup->database) ? 'database_deleted' : 'server_deleted'];
|
|
}
|
|
|
|
return $this->runOccurrence(
|
|
schedule: $backup,
|
|
frequency: $backup->frequency,
|
|
server: $server,
|
|
runLate: true,
|
|
skipReason: fn () => $this->serverSkipReason($server),
|
|
delivery: ['schedule_key' => "scheduled-backup:{$backup->id}", 'job_type' => 'database-backup', 'resource_id' => $backup->id],
|
|
);
|
|
}
|
|
|
|
private function processScheduledTasks(): void
|
|
{
|
|
$this->due(ScheduledTask::query())
|
|
->with([
|
|
'service.destination.server.settings',
|
|
'service.destination.server.team.subscription',
|
|
'application.destination.server.settings',
|
|
'application.destination.server.team.subscription',
|
|
])
|
|
->chunkById(self::CHUNK_SIZE, fn ($tasks) => $tasks->each(fn (ScheduledTask $task) => $this->processItem(
|
|
'task',
|
|
['task_id' => $task->id, 'task_name' => $task->name, 'team_id' => $task->team_id],
|
|
fn () => $this->processScheduledTask($task),
|
|
)));
|
|
}
|
|
|
|
private function processScheduledTask(ScheduledTask $task): array
|
|
{
|
|
$server = $task->server();
|
|
|
|
if (blank($server) || (! $task->service && ! $task->application)) {
|
|
$task->delete();
|
|
|
|
return ['deleted' => blank($server) ? 'server_deleted' : 'resource_deleted'];
|
|
}
|
|
|
|
return $this->runOccurrence(
|
|
schedule: $task,
|
|
frequency: $task->frequency,
|
|
server: $server,
|
|
runLate: false,
|
|
skipReason: fn () => $this->serverSkipReason($server) ?? $this->taskResourceSkipReason($task),
|
|
delivery: ['schedule_key' => "scheduled-task:{$task->id}", 'job_type' => 'scheduled-task', 'resource_id' => $task->id],
|
|
);
|
|
}
|
|
|
|
private function processVolumeBackups(): void
|
|
{
|
|
$this->due(ScheduledVolumeBackup::query())
|
|
->with(['backupable', 'team.subscription'])
|
|
->chunkById(self::CHUNK_SIZE, fn ($backups) => $backups->each(fn (ScheduledVolumeBackup $backup) => $this->processItem(
|
|
'volume_backup',
|
|
['backup_id' => $backup->id, 'backupable_type' => $backup->backupable_type, 'backupable_id' => $backup->backupable_id, 'team_id' => $backup->team_id],
|
|
fn () => $this->processVolumeBackup($backup),
|
|
)));
|
|
}
|
|
|
|
private function processVolumeBackup(ScheduledVolumeBackup $backup): array
|
|
{
|
|
$server = $backup->server();
|
|
|
|
// Wait without claiming: the backup runs late when the recovery is done.
|
|
if ($backup->executions()->where(fn (Builder $query) => $query->where('stop_recovery_pending', true)->orWhere('s3_cleanup_pending', true))->exists()) {
|
|
return ['waiting' => 'container_recovery_pending'];
|
|
}
|
|
|
|
if (! $backup->backupable) {
|
|
DeleteScheduledVolumeBackup::run($backup, $server);
|
|
|
|
return ['deleted' => 'resource_deleted'];
|
|
}
|
|
|
|
if (! $server) {
|
|
return ['waiting' => 'server_missing'];
|
|
}
|
|
|
|
return $this->runOccurrence(
|
|
schedule: $backup,
|
|
frequency: $backup->frequency,
|
|
server: $server,
|
|
runLate: true,
|
|
skipReason: fn () => $this->serverSkipReason($server),
|
|
delivery: ['schedule_key' => "scheduled-volume-backup:{$backup->id}", 'job_type' => 'volume-backup', 'resource_id' => $backup->id],
|
|
);
|
|
}
|
|
|
|
private function recoverStoppedVolumeBackupContainers(): void
|
|
{
|
|
ScheduledVolumeBackupExecution::query()
|
|
->where(fn (Builder $query) => $query
|
|
->where('stop_recovery_pending', true)
|
|
->orWhere('s3_cleanup_pending', true))
|
|
->chunkById(self::CHUNK_SIZE, function ($executions): void {
|
|
foreach ($executions as $execution) {
|
|
VolumeBackupRecoveryJob::dispatch($execution);
|
|
}
|
|
});
|
|
}
|
|
|
|
private function processDockerCleanups(): void
|
|
{
|
|
$query = Server::query()
|
|
->with('settings')
|
|
->whereNotNull('ip')
|
|
->where('ip', '!=', '')
|
|
->whereNotIn('ip', Server::PLACEHOLDER_IPS)
|
|
->whereHas('settings', fn (Builder $query) => $query
|
|
->whereNull('docker_cleanup_next_run_at')
|
|
->orWhere('docker_cleanup_next_run_at', '<=', $this->now));
|
|
|
|
if (isCloud()) {
|
|
$query->with('team.subscription')->where(fn (Builder $query) => $query
|
|
->where('team_id', 0)
|
|
->orWhereRelation('team.subscription', 'stripe_invoice_paid', true));
|
|
}
|
|
|
|
$query->chunkById(self::CHUNK_SIZE, fn ($servers) => $servers->each(fn (Server $server) => $this->processItem(
|
|
'docker_cleanup',
|
|
['server_id' => $server->id, 'server_name' => $server->name, 'team_id' => $server->team_id],
|
|
fn () => $this->runOccurrence(
|
|
schedule: $server->settings,
|
|
frequency: $server->settings->docker_cleanup_frequency ?? '0 0 * * *',
|
|
server: $server,
|
|
runLate: false,
|
|
skipReason: fn () => $this->serverSkipReason($server),
|
|
delivery: [
|
|
'schedule_key' => "docker-cleanup:{$server->id}",
|
|
'job_type' => 'docker-cleanup',
|
|
'resource_id' => $server->id,
|
|
'payload' => [
|
|
'delete_unused_volumes' => $server->settings->delete_unused_volumes,
|
|
'delete_unused_networks' => $server->settings->delete_unused_networks,
|
|
],
|
|
],
|
|
column: 'docker_cleanup_next_run_at',
|
|
),
|
|
)));
|
|
}
|
|
|
|
/**
|
|
* Enabled schedules that are due, or that have no next run yet.
|
|
*/
|
|
private function due(Builder $query): Builder
|
|
{
|
|
return $query
|
|
->where('enabled', true)
|
|
->where(fn (Builder $query) => $query->whereNull('next_run_at')->orWhere('next_run_at', '<=', $this->now));
|
|
}
|
|
|
|
/**
|
|
* Claim the due occurrence of a schedule and dispatch it, skip it, or log it as missed.
|
|
*
|
|
* @param array{schedule_key: string, job_type: string, resource_id: int, payload?: array<string, mixed>} $delivery
|
|
* @return array<string, mixed> The outcome, for the log.
|
|
*/
|
|
private function runOccurrence(
|
|
Model $schedule,
|
|
string $frequency,
|
|
Server $server,
|
|
bool $runLate,
|
|
callable $skipReason,
|
|
array $delivery,
|
|
string $column = 'next_run_at',
|
|
): array {
|
|
$timezone = data_get($server, 'settings.server_timezone');
|
|
$stored = $schedule->getRawOriginal($column);
|
|
|
|
if ($stored === null) {
|
|
// First run after the upgrade, a timezone change, or a move to another server.
|
|
$first = next_cron_run_at($frequency, $timezone, $this->now, includeCurrentMinute: true);
|
|
if ($first === null || ! $this->advance($schedule, $column, null, $first)) {
|
|
return $first === null ? ['invalid_frequency' => $frequency] : [];
|
|
}
|
|
if ($first->gt($this->now)) {
|
|
return [];
|
|
}
|
|
$stored = $first;
|
|
}
|
|
|
|
$dueAt = CarbonImmutable::instance($schedule->getAttribute($column))->utc();
|
|
$next = next_cron_run_at($frequency, $timezone, $this->now);
|
|
if ($next === null) {
|
|
return ['invalid_frequency' => $frequency];
|
|
}
|
|
// When the clocks go back, a local time occurs two times. Run it only once.
|
|
if ($this->sameLocalTime($next, $dueAt, $timezone)) {
|
|
$next = next_cron_run_at($frequency, $timezone, $next) ?? $next;
|
|
}
|
|
|
|
$minutesLate = (int) $dueAt->diffInMinutes($this->now);
|
|
if (! $runLate && $minutesLate > self::LATE_RUN_WINDOW_MINUTES) {
|
|
$outcome = ['missed' => $dueAt->toIso8601String(), 'minutes_late' => $minutesLate];
|
|
} elseif (($reason = $skipReason()) !== null) {
|
|
$outcome = ['skipped' => $reason];
|
|
} else {
|
|
$outcome = ['dispatched' => $dueAt->toIso8601String()];
|
|
}
|
|
|
|
// Claim and record the occurrence together, so a crash cannot claim it without a delivery.
|
|
$occurrence = DB::transaction(function () use ($schedule, $column, $stored, $next, $outcome, $delivery, $dueAt): ScheduledJobDelivery|bool {
|
|
if (! $this->advance($schedule, $column, $stored, $next)) {
|
|
return false;
|
|
}
|
|
|
|
if (! isset($outcome['dispatched'])) {
|
|
return true;
|
|
}
|
|
|
|
return $this->deliveries->create($delivery['schedule_key'], $dueAt, $delivery['job_type'], $delivery['resource_id'], $delivery['payload'] ?? []) ?? true;
|
|
});
|
|
|
|
if ($occurrence === false) {
|
|
return [];
|
|
}
|
|
|
|
if ($occurrence instanceof ScheduledJobDelivery) {
|
|
$this->deliveries->publish($occurrence);
|
|
}
|
|
|
|
return [...$outcome, 'server_id' => $server->id, 'next_run_at' => $next->toIso8601String()];
|
|
}
|
|
|
|
/**
|
|
* Move next_run_at from the stored value to $to. Only one process can win this update.
|
|
*/
|
|
private function advance(Model $schedule, string $column, mixed $from, CarbonImmutable $to): bool
|
|
{
|
|
$updated = $schedule->newQuery()
|
|
->whereKey($schedule->getKey())
|
|
->when($from === null, fn (Builder $query) => $query->whereNull($column), fn (Builder $query) => $query->where($column, $from))
|
|
->toBase()
|
|
->update([$column => $to]);
|
|
|
|
if ($updated === 1) {
|
|
$schedule->setAttribute($column, $to)->syncOriginalAttribute($column);
|
|
}
|
|
|
|
return $updated === 1;
|
|
}
|
|
|
|
private function sameLocalTime(CarbonImmutable $first, CarbonImmutable $second, ?string $timezone): bool
|
|
{
|
|
$timezone = filled($timezone) && validate_timezone($timezone) ? $timezone : config('app.timezone');
|
|
|
|
return $first->setTimezone($timezone)->format('Y-m-d H:i') === $second->setTimezone($timezone)->format('Y-m-d H:i');
|
|
}
|
|
|
|
private function serverSkipReason(Server $server): ?string
|
|
{
|
|
if (! $server->isFunctional()) {
|
|
return 'server_not_functional';
|
|
}
|
|
|
|
if (isCloud() && $server->team_id !== 0 && data_get($server->team?->subscription, 'stripe_invoice_paid', false) === false) {
|
|
return 'subscription_unpaid';
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private function taskResourceSkipReason(ScheduledTask $task): ?string
|
|
{
|
|
if ($task->application && ! str($task->application->status)->contains('running')) {
|
|
return 'application_not_running';
|
|
}
|
|
|
|
if ($task->service && ! str($task->service->status)->contains('running')) {
|
|
return 'service_not_running';
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Process one schedule and log its outcome. An error in one schedule never stops the others.
|
|
*
|
|
* @param array<string, mixed> $context
|
|
*/
|
|
private function processItem(string $type, array $context, callable $process): void
|
|
{
|
|
try {
|
|
$outcome = $process();
|
|
} catch (\Throwable $e) {
|
|
Log::channel('scheduled-errors')->error("Error processing {$type}", [...$context, 'error' => $e->getMessage()]);
|
|
|
|
return;
|
|
}
|
|
|
|
$label = ucfirst(str_replace('_', ' ', $type));
|
|
$context = [...$context, ...$outcome, 'execution_time' => $this->now->toIso8601String()];
|
|
|
|
match (true) {
|
|
isset($outcome['dispatched']) => $this->log('dispatched', 'info', "{$label} dispatched", $context),
|
|
isset($outcome['skipped']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['skipped'], ...$context]),
|
|
isset($outcome['deleted']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['deleted'], ...$context]),
|
|
isset($outcome['waiting']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['waiting'], ...$context]),
|
|
isset($outcome['missed']) => $this->log('missed', 'warning', "{$label} missed", $context),
|
|
isset($outcome['invalid_frequency']) => Log::channel('scheduled-errors')->warning("{$label} has an invalid frequency", $context),
|
|
default => null,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param 'dispatched'|'skipped'|'missed' $counter
|
|
* @param array<string, mixed> $context
|
|
*/
|
|
private function log(string $counter, string $level, string $message, array $context): void
|
|
{
|
|
$this->counts[$counter]++;
|
|
Log::channel('scheduled')->log($level, $message, $context);
|
|
}
|
|
}
|