fix(upgrade): count only scheduled jobs running on this host

This commit is contained in:
Andras Bacsai
2026-10-28 22:13:40 +01:00
parent 24648d4b20
commit 57cdd92f7c
7 changed files with 88 additions and 6 deletions
@@ -3,12 +3,13 @@
namespace App\Models;
use App\Traits\HasS3Replicas;
use App\Traits\RecordsWorkerHost;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
class ScheduledDatabaseBackupExecution extends BaseModel
{
use HasS3Replicas;
use HasS3Replicas, RecordsWorkerHost;
protected static function booted(): void
{
+3
View File
@@ -2,6 +2,7 @@
namespace App\Models;
use App\Traits\RecordsWorkerHost;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use OpenApi\Attributes as OA;
@@ -22,6 +23,8 @@ use OpenApi\Attributes as OA;
)]
class ScheduledTaskExecution extends BaseModel
{
use RecordsWorkerHost;
protected $fillable = [
'scheduled_task_id',
'status',
@@ -3,12 +3,13 @@
namespace App\Models;
use App\Traits\HasS3Replicas;
use App\Traits\RecordsWorkerHost;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
class ScheduledVolumeBackupExecution extends BaseModel
{
use HasS3Replicas;
use HasS3Replicas, RecordsWorkerHost;
protected $fillable = [
'uuid',
+32
View File
@@ -0,0 +1,32 @@
<?php
namespace App\Traits;
use Illuminate\Database\Eloquent\Builder;
/**
* Stores the host that created the execution, so a host can count only the runs of its own workers.
* The host name is internal, so API and MCP responses do not show it.
*/
trait RecordsWorkerHost
{
protected static function bootRecordsWorkerHost(): void
{
static::creating(function (self $execution): void {
$execution->worker_host ??= gethostname();
});
}
protected function initializeRecordsWorkerHost(): void
{
$this->hidden[] = 'worker_host';
}
/**
* Rows without a host were created before the column existed, so they can belong to this host.
*/
public function scopeOnThisHost(Builder $query): Builder
{
return $query->where(fn (Builder $query) => $query->where('worker_host', gethostname())->orWhereNull('worker_host'));
}
}
+4 -4
View File
@@ -2936,20 +2936,20 @@ function isAnyDeploymentInprogress(bool $showAll = false)
}
/**
* Counts scheduled task, database backup, and volume backup runs that are still running.
* Counts scheduled task, database backup, and volume backup runs that are still running on this host.
* A run that started longer ago than its job timeout is stale: the worker has stopped it.
*/
function runningScheduledJobCount(): int
{
$startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false;
$tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get()
$tasks = ScheduledTaskExecution::with('scheduledTask')->onThisHost()->where('status', 'running')->get()
->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS));
$databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get()
$databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->onThisHost()->where('status', 'running')->get()
->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS));
$volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get()
$volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->onThisHost()->where('status', 'running')->get()
->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT));
return $tasks->count() + $databaseBackups->count() + $volumeBackups->count();
@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
private const TABLES = ['scheduled_task_executions', 'scheduled_database_backup_executions', 'scheduled_volume_backup_executions'];
public function up(): void
{
foreach (self::TABLES as $table) {
Schema::table($table, function (Blueprint $table) {
$table->string('worker_host')->nullable();
});
}
}
public function down(): void
{
foreach (self::TABLES as $table) {
Schema::table($table, function (Blueprint $table) {
$table->dropColumn('worker_host');
});
}
}
};
@@ -64,3 +64,20 @@ it('does not count running rows older than their job timeout', function () {
$this->travel(60)->minutes();
expect(runningScheduledJobCount())->toBe(0);
});
it('records the worker host on new runs and hides it from responses', function () {
$execution = ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']);
expect($execution->fresh()->worker_host)->toBe(gethostname())
->and($execution->toArray())->not->toHaveKey('worker_host');
});
it('counts only runs on this host and runs without a recorded host', function () {
ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']);
(new ScheduledTaskExecution(['scheduled_task_id' => $this->task->id, 'status' => 'running']))->forceFill(['worker_host' => 'other-host'])->save();
(new ScheduledDatabaseBackupExecution(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']))->forceFill(['worker_host' => 'other-host'])->save();
$this->volumeBackup->executions()->make()->forceFill(['worker_host' => 'other-host'])->save();
$this->volumeBackup->executions()->create()->forceFill(['worker_host' => null])->save();
expect(runningScheduledJobCount())->toBe(2);
});