diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.php index 61f26265a9..0b4f944a1d 100644 --- a/app/Console/Commands/ScheduledJobDiagnostics.php +++ b/app/Console/Commands/ScheduledJobDiagnostics.php @@ -4,6 +4,8 @@ namespace App\Console\Commands; use App\Models\DockerCleanupExecution; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledJobDelivery; +use App\Models\ScheduledJobState; use App\Models\ScheduledTask; use App\Models\Server; use App\Models\Team; @@ -15,10 +17,10 @@ use Illuminate\Support\Facades\Cache; class ScheduledJobDiagnostics extends Command { protected $signature = 'scheduled:diagnostics - {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all} + {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, deliveries, all} {--server= : Filter by server ID}'; - protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs'; + protected $description = 'Inspect scheduled job state, due schedules, and open deliveries without changing anything'; public function handle(): int { @@ -43,6 +45,10 @@ class ScheduledJobDiagnostics extends Command $this->inspectServerJobs($serverFilter); } + if (in_array($type, ['all', 'deliveries'])) { + $this->inspectDeliveries(); + } + return self::SUCCESS; } @@ -72,14 +78,14 @@ class ScheduledJobDiagnostics extends Command } $dedupKey = "docker-cleanup:{$server->id}"; - $cacheValue = Cache::get($dedupKey); + $cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String(); $timezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($timezone) === false) { $timezone = config('app.timezone'); } - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $lastExecution = DockerCleanupExecution::where('server_id', $server->id) ->latest() @@ -91,14 +97,14 @@ class ScheduledJobDiagnostics extends Command $timezone, $frequency, $dedupKey, - $cacheValue ?? '', + $cacheValue ?? 'never', $wouldFire ? 'YES' : 'no', $lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never', ]; } $this->table( - ['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'], + ['ID', 'Server', 'TZ', 'Frequency', 'Schedule Key', 'Last Scheduled For', 'Due Now', 'Last Execution'], $rows ); $this->newLine(); @@ -121,27 +127,27 @@ class ScheduledJobDiagnostics extends Command } $dedupKey = "scheduled-backup:{$backup->id}"; - $cacheValue = Cache::get($dedupKey); + $cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String(); $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone'); if (validate_timezone($timezone) === false) { $timezone = config('app.timezone'); } - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $rows[] = [ $backup->id, $backup->database_type ?? 'unknown', $server?->name ?? 'N/A', $frequency, - $cacheValue ?? '', + $cacheValue ?? 'never', $wouldFire ? 'YES' : 'no', ]; } $this->table( - ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'], + ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Last Scheduled For', 'Due Now'], $rows ); $this->newLine(); @@ -164,27 +170,27 @@ class ScheduledJobDiagnostics extends Command } $dedupKey = "scheduled-task:{$task->id}"; - $cacheValue = Cache::get($dedupKey); + $cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String(); $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone'); if (validate_timezone($timezone) === false) { $timezone = config('app.timezone'); } - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $rows[] = [ $task->id, $task->name, $server?->name ?? 'N/A', $frequency, - $cacheValue ?? '', + $cacheValue ?? 'never', $wouldFire ? 'YES' : 'no', ]; } $this->table( - ['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'], + ['Task ID', 'Name', 'Server', 'Frequency', 'Last Scheduled For', 'Due Now'], $rows ); $this->newLine(); @@ -215,7 +221,7 @@ class ScheduledJobDiagnostics extends Command } $cacheValue = Cache::get($dedupKey); - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $rows[] = [ $server->id, @@ -229,12 +235,31 @@ class ScheduledJobDiagnostics extends Command } $this->table( - ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'], + ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Due Now'], $rows ); $this->newLine(); } + /** + * Occurrences that are not finished. An old "enqueued" row usually means its queued job was lost. + */ + private function inspectDeliveries(): void + { + $this->info('=== Open and failed deliveries (failed ones are kept for 30 days) ==='); + + $this->table( + ['Job Type', 'Status', 'Count', 'Oldest Due (UTC)'], + ScheduledJobDelivery::query() + ->whereIn('status', ['pending', 'enqueued', 'claimed', 'failed']) + ->selectRaw('job_type, status, count(*) as total, min(scheduled_for) as oldest') + ->groupBy('job_type', 'status') + ->get() + ->map(fn ($row) => [$row->job_type, $row->status, $row->total, $row->oldest]) + ); + $this->newLine(); + } + private function getServers(?string $serverFilter): Collection { $query = Server::with('settings')->where('ip', '!=', '1.2.3.4'); diff --git a/tests/Feature/ScheduledJobDiagnosticsTest.php b/tests/Feature/ScheduledJobDiagnosticsTest.php new file mode 100644 index 0000000000..69293c7208 --- /dev/null +++ b/tests/Feature/ScheduledJobDiagnosticsTest.php @@ -0,0 +1,49 @@ + true]); +}); + +it('does not consume the dedup keys of the server manager checks', function () { + // Every check of ServerManagerJob is due at Sunday 00:00, including the weekly patch check. + Carbon::setTestNow(Carbon::create(2026, 10, 4, 0, 0, 0, 'UTC')); + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + + $this->artisan('scheduled:diagnostics', ['--type' => 'server-jobs'])->assertSuccessful(); + + expect(Cache::get("server-check:{$server->id}"))->toBeNull() + ->and(Cache::get("server-patch-check:{$server->id}"))->toBeNull() + ->and(shouldRunCronNow('0 0 * * 0', 'UTC', "server-patch-check:{$server->id}"))->toBeTrue(); +}); + +it('shows the occurrence state that the scheduled job manager records', function () { + Carbon::setTestNow(Carbon::create(2026, 10, 2, 12, 0, 0, 'UTC')); + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + ScheduledJobState::query()->create([ + 'schedule_key' => "docker-cleanup:{$server->id}", + 'last_scheduled_for' => Carbon::create(2026, 10, 2, 0, 0, 0, 'UTC'), + ]); + + $this->artisan('scheduled:diagnostics', ['--type' => 'docker-cleanup']) + ->expectsOutputToContain('2026-10-02T00:00:00') + ->assertSuccessful(); +});