starred/coolify
1
0
Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-12 15:53:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(scheduler): make scheduled:diagnostics read-only and show the real state

Browse Source 
The command called shouldRunCronNow() with the dedup keys, and that helper
writes the key it checks. For server-check, server-storage-check, and
server-patch-check these are the keys of ServerManagerJob, so running the
command in a minute when a check was due made the real check skip that run.

It also read cache keys for backups, tasks, and Docker cleanups that the
scheduled job manager no longer uses. Read scheduled_job_states instead, and
list open and failed deliveries, where an old "enqueued" row shows a queued
job that was lost.
This commit is contained in:
Andras Bacsai
2026-10-02 08:07:30 +02:00
parent 50be16be8f
commit a218e8cdc6
2 changed files with 90 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/ScheduledJobDiagnostics.php
+41 -16
View File
@@ -4,6 +4,8 @@ namespace App\Console\Commands;
use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
use App\Models\Server;
use App\Models\Team;
@@ -15,10 +17,10 @@ use Illuminate\Support\Facades\Cache;
class ScheduledJobDiagnostics extends Command
{
protected $signature = 'scheduled:diagnostics
{--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all}
{--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, deliveries, all}
{--server= : Filter by server ID}';
protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs';
protected $description = 'Inspect scheduled job state, due schedules, and open deliveries without changing anything';
public function handle(): int
{
@@ -43,6 +45,10 @@ class ScheduledJobDiagnostics extends Command
$this->inspectServerJobs($serverFilter);
}
if (in_array($type, ['all', 'deliveries'])) {
$this->inspectDeliveries();
}
return self::SUCCESS;
}
@@ -72,14 +78,14 @@ class ScheduledJobDiagnostics extends Command
}
$dedupKey = "docker-cleanup:{$server->id}";
$cacheValue = Cache::get($dedupKey);
$cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String();
$timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$wouldFire = shouldRunCronNow($frequency, $timezone);
$lastExecution = DockerCleanupExecution::where('server_id', $server->id)
->latest()
@@ -91,14 +97,14 @@ class ScheduledJobDiagnostics extends Command
$timezone,
$frequency,
$dedupKey,
$cacheValue ?? '<missing>',
$cacheValue ?? 'never',
$wouldFire ? 'YES' : 'no',
$lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never',
];
}
$this->table(
['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'],
['ID', 'Server', 'TZ', 'Frequency', 'Schedule Key', 'Last Scheduled For', 'Due Now', 'Last Execution'],
$rows
);
$this->newLine();
@@ -121,27 +127,27 @@ class ScheduledJobDiagnostics extends Command
}
$dedupKey = "scheduled-backup:{$backup->id}";
$cacheValue = Cache::get($dedupKey);
$cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String();
$timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$wouldFire = shouldRunCronNow($frequency, $timezone);
$rows[] = [
$backup->id,
$backup->database_type ?? 'unknown',
$server?->name ?? 'N/A',
$frequency,
$cacheValue ?? '<missing>',
$cacheValue ?? 'never',
$wouldFire ? 'YES' : 'no',
];
}
$this->table(
['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
['Backup ID', 'DB Type', 'Server', 'Frequency', 'Last Scheduled For', 'Due Now'],
$rows
);
$this->newLine();
@@ -164,27 +170,27 @@ class ScheduledJobDiagnostics extends Command
}
$dedupKey = "scheduled-task:{$task->id}";
$cacheValue = Cache::get($dedupKey);
$cacheValue = ScheduledJobState::query()->where('schedule_key', $dedupKey)->value('last_scheduled_for')?->toIso8601String();
$timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');
if (validate_timezone($timezone) === false) {
$timezone = config('app.timezone');
}
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$wouldFire = shouldRunCronNow($frequency, $timezone);
$rows[] = [
$task->id,
$task->name,
$server?->name ?? 'N/A',
$frequency,
$cacheValue ?? '<missing>',
$cacheValue ?? 'never',
$wouldFire ? 'YES' : 'no',
];
}
$this->table(
['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],
['Task ID', 'Name', 'Server', 'Frequency', 'Last Scheduled For', 'Due Now'],
$rows
);
$this->newLine();
@@ -215,7 +221,7 @@ class ScheduledJobDiagnostics extends Command
}
$cacheValue = Cache::get($dedupKey);
$wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);
$wouldFire = shouldRunCronNow($frequency, $timezone);
$rows[] = [
$server->id,
@@ -229,12 +235,31 @@ class ScheduledJobDiagnostics extends Command
}
$this->table(
['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'],
['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Due Now'],
$rows
);
$this->newLine();
}
/**
* Occurrences that are not finished. An old "enqueued" row usually means its queued job was lost.
*/
private function inspectDeliveries(): void
{
$this->info('=== Open and failed deliveries (failed ones are kept for 30 days) ===');
$this->table(
['Job Type', 'Status', 'Count', 'Oldest Due (UTC)'],
ScheduledJobDelivery::query()
->whereIn('status', ['pending', 'enqueued', 'claimed', 'failed'])
->selectRaw('job_type, status, count(*) as total, min(scheduled_for) as oldest')
->groupBy('job_type', 'status')
->get()
->map(fn ($row) => [$row->job_type, $row->status, $row->total, $row->oldest])
);
$this->newLine();
}
private function getServers(?string $serverFilter): Collection
{
$query = Server::with('settings')->where('ip', '!=', '1.2.3.4');
tests/Feature/ScheduledJobDiagnosticsTest.php
+49
View File
@@ -0,0 +1,49 @@
<?php
use App\Models\PrivateKey;
use App\Models\ScheduledJobState;
use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Cache;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
config(['constants.coolify.self_hosted' => true]);
});
it('does not consume the dedup keys of the server manager checks', function () {
// Every check of ServerManagerJob is due at Sunday 00:00, including the weekly patch check.
Carbon::setTestNow(Carbon::create(2026, 10, 4, 0, 0, 0, 'UTC'));
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
]);
$this->artisan('scheduled:diagnostics', ['--type' => 'server-jobs'])->assertSuccessful();
expect(Cache::get("server-check:{$server->id}"))->toBeNull()
->and(Cache::get("server-patch-check:{$server->id}"))->toBeNull()
->and(shouldRunCronNow('0 0 * * 0', 'UTC', "server-patch-check:{$server->id}"))->toBeTrue();
});
it('shows the occurrence state that the scheduled job manager records', function () {
Carbon::setTestNow(Carbon::create(2026, 10, 2, 12, 0, 0, 'UTC'));
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
]);
ScheduledJobState::query()->create([
'schedule_key' => "docker-cleanup:{$server->id}",
'last_scheduled_for' => Carbon::create(2026, 10, 2, 0, 0, 0, 'UTC'),
]);
$this->artisan('scheduled:diagnostics', ['--type' => 'docker-cleanup'])
->expectsOutputToContain('2026-10-02T00:00:00')
->assertSuccessful();
});