From a218e8cdc6762fe5d2273de0da9388e43cd35d8c Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:07:30 +0200 Subject: [PATCH] fix(scheduler): make scheduled:diagnostics read-only and show the real state The command called shouldRunCronNow() with the dedup keys, and that helper writes the key it checks. For server-check, server-storage-check, and server-patch-check these are the keys of ServerManagerJob, so running the command in a minute when a check was due made the real check skip that run. It also read cache keys for backups, tasks, and Docker cleanups that the scheduled job manager no longer uses. Read scheduled_job_states instead, and list open and failed deliveries, where an old "enqueued" row shows a queued job that was lost. --- .../Commands/ScheduledJobDiagnostics.php | 57 +++++++++++++------ tests/Feature/ScheduledJobDiagnosticsTest.php | 49 ++++++++++++++++ 2 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 tests/Feature/ScheduledJobDiagnosticsTest.php diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.php index 61f26265a9..0b4f944a1d 100644 --- a/app/Console/Commands/ScheduledJobDiagnostics.php +++ b/app/Console/Commands/ScheduledJobDiagnostics.php @@ -4,6 +4,8 @@ namespace App\Console\Commands; use App\Models\DockerCleanupExecution; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledJobDelivery; +use App\Models\ScheduledJobState; use App\Models\ScheduledTask; use App\Models\Server; use App\Models\Team; @@ -15,10 +17,10 @@ use Illuminate\Support\Facades\Cache; class ScheduledJobDiagnostics extends Command { protected $signature = 'scheduled:diagnostics - {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all} + {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, deliveries, all} {--server= : Filter by server ID}'; - protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs'; + protected $description = 'Inspect scheduled job state, due schedules, and open deliveries without changing anything'; public function handle(): int { @@ -43,6 +45,10 @@ class ScheduledJobDiagnostics extends Command $this->inspectServerJobs($serverFilter); } + if (in_array($type, ['all', 'deliveries'])) { + $this->inspectDeliveries(); + } + return self::SUCCESS; } @@ -72,14 +78,14 @@ class ScheduledJobDiagnostics extends Command } $dedupKey = "docker-cleanup:{$server->id}"; - $cacheValue = Cache::get($dedupKey); + $cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String(); $timezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($timezone) === false) { $timezone = config('app.timezone'); } - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $lastExecution = DockerCleanupExecution::where('server_id', $server->id) ->latest() @@ -91,14 +97,14 @@ class ScheduledJobDiagnostics extends Command $timezone, $frequency, $dedupKey, - $cacheValue ?? '', + $cacheValue ?? 'never', $wouldFire ? 'YES' : 'no', $lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never', ]; } $this->table( - ['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'], + ['ID', 'Server', 'TZ', 'Frequency', 'Schedule Key', 'Last Scheduled For', 'Due Now', 'Last Execution'], $rows ); $this->newLine(); @@ -121,27 +127,27 @@ class ScheduledJobDiagnostics extends Command } $dedupKey = "scheduled-backup:{$backup->id}"; - $cacheValue = Cache::get($dedupKey); + $cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String(); $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone'); if (validate_timezone($timezone) === false) { $timezone = config('app.timezone'); } - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $rows[] = [ $backup->id, $backup->database_type ?? 'unknown', $server?->name ?? 'N/A', $frequency, - $cacheValue ?? '', + $cacheValue ?? 'never', $wouldFire ? 'YES' : 'no', ]; } $this->table( - ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'], + ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Last Scheduled For', 'Due Now'], $rows ); $this->newLine(); @@ -164,27 +170,27 @@ class ScheduledJobDiagnostics extends Command } $dedupKey = "scheduled-task:{$task->id}"; - $cacheValue = Cache::get($dedupKey); + $cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String(); $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone'); if (validate_timezone($timezone) === false) { $timezone = config('app.timezone'); } - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $rows[] = [ $task->id, $task->name, $server?->name ?? 'N/A', $frequency, - $cacheValue ?? '', + $cacheValue ?? 'never', $wouldFire ? 'YES' : 'no', ]; } $this->table( - ['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'], + ['Task ID', 'Name', 'Server', 'Frequency', 'Last Scheduled For', 'Due Now'], $rows ); $this->newLine(); @@ -215,7 +221,7 @@ class ScheduledJobDiagnostics extends Command } $cacheValue = Cache::get($dedupKey); - $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey); + $wouldFire = shouldRunCronNow($frequency, $timezone); $rows[] = [ $server->id, @@ -229,12 +235,31 @@ class ScheduledJobDiagnostics extends Command } $this->table( - ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'], + ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Due Now'], $rows ); $this->newLine(); } + /** + * Occurrences that are not finished. An old "enqueued" row usually means its queued job was lost. + */ + private function inspectDeliveries(): void + { + $this->info('=== Open and failed deliveries (failed ones are kept for 30 days) ==='); + + $this->table( + ['Job Type', 'Status', 'Count', 'Oldest Due (UTC)'], + ScheduledJobDelivery::query() + ->whereIn('status', ['pending', 'enqueued', 'claimed', 'failed']) + ->selectRaw('job_type, status, count(*) as total, min(scheduled_for) as oldest') + ->groupBy('job_type', 'status') + ->get() + ->map(fn ($row) => [$row->job_type, $row->status, $row->total, $row->oldest]) + ); + $this->newLine(); + } + private function getServers(?string $serverFilter): Collection { $query = Server::with('settings')->where('ip', '!=', '1.2.3.4'); diff --git a/tests/Feature/ScheduledJobDiagnosticsTest.php b/tests/Feature/ScheduledJobDiagnosticsTest.php new file mode 100644 index 0000000000..69293c7208 --- /dev/null +++ b/tests/Feature/ScheduledJobDiagnosticsTest.php @@ -0,0 +1,49 @@ + true]); +}); + +it('does not consume the dedup keys of the server manager checks', function () { + // Every check of ServerManagerJob is due at Sunday 00:00, including the weekly patch check. + Carbon::setTestNow(Carbon::create(2026, 10, 4, 0, 0, 0, 'UTC')); + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + + $this->artisan('scheduled:diagnostics', ['--type' => 'server-jobs'])->assertSuccessful(); + + expect(Cache::get("server-check:{$server->id}"))->toBeNull() + ->and(Cache::get("server-patch-check:{$server->id}"))->toBeNull() + ->and(shouldRunCronNow('0 0 * * 0', 'UTC', "server-patch-check:{$server->id}"))->toBeTrue(); +}); + +it('shows the occurrence state that the scheduled job manager records', function () { + Carbon::setTestNow(Carbon::create(2026, 10, 2, 12, 0, 0, 'UTC')); + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id, + ]); + ScheduledJobState::query()->create([ + 'schedule_key' => "docker-cleanup:{$server->id}", + 'last_scheduled_for' => Carbon::create(2026, 10, 2, 0, 0, 0, 'UTC'), + ]); + + $this->artisan('scheduled:diagnostics', ['--type' => 'docker-cleanup']) + ->expectsOutputToContain('2026-10-02T00:00:00') + ->assertSuccessful(); +});