diff --git a/app/Models/ScheduledDatabaseBackupExecution.php b/app/Models/ScheduledDatabaseBackupExecution.php index eb43a1a2e3..a11c581d9b 100644 --- a/app/Models/ScheduledDatabaseBackupExecution.php +++ b/app/Models/ScheduledDatabaseBackupExecution.php @@ -3,12 +3,13 @@ namespace App\Models; use App\Traits\HasS3Replicas; +use App\Traits\RecordsWorkerHost; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Eloquent\Relations\HasMany; class ScheduledDatabaseBackupExecution extends BaseModel { - use HasS3Replicas; + use HasS3Replicas, RecordsWorkerHost; protected static function booted(): void { diff --git a/app/Models/ScheduledTaskExecution.php b/app/Models/ScheduledTaskExecution.php index 8f496fc1e6..c5f1f393f7 100644 --- a/app/Models/ScheduledTaskExecution.php +++ b/app/Models/ScheduledTaskExecution.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Traits\RecordsWorkerHost; use Illuminate\Database\Eloquent\Relations\BelongsTo; use OpenApi\Attributes as OA; @@ -22,6 +23,8 @@ use OpenApi\Attributes as OA; )] class ScheduledTaskExecution extends BaseModel { + use RecordsWorkerHost; + protected $fillable = [ 'scheduled_task_id', 'status', diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php index 82dfe0ee3b..776b2b24fc 100644 --- a/app/Models/ScheduledVolumeBackupExecution.php +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -3,12 +3,13 @@ namespace App\Models; use App\Traits\HasS3Replicas; +use App\Traits\RecordsWorkerHost; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Eloquent\Relations\HasMany; class ScheduledVolumeBackupExecution extends BaseModel { - use HasS3Replicas; + use HasS3Replicas, RecordsWorkerHost; protected $fillable = [ 'uuid', diff --git a/app/Traits/RecordsWorkerHost.php b/app/Traits/RecordsWorkerHost.php new file mode 100644 index 0000000000..ac55770b65 --- /dev/null +++ b/app/Traits/RecordsWorkerHost.php @@ -0,0 +1,32 @@ +worker_host ??= gethostname(); + }); + } + + protected function initializeRecordsWorkerHost(): void + { + $this->hidden[] = 'worker_host'; + } + + /** + * Rows without a host were created before the column existed, so they can belong to this host. + */ + public function scopeOnThisHost(Builder $query): Builder + { + return $query->where(fn (Builder $query) => $query->where('worker_host', gethostname())->orWhereNull('worker_host')); + } +} diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index cd78aeee34..b31756dae9 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2936,20 +2936,20 @@ function isAnyDeploymentInprogress(bool $showAll = false) } /** - * Counts scheduled task, database backup, and volume backup runs that are still running. + * Counts scheduled task, database backup, and volume backup runs that are still running on this host. * A run that started longer ago than its job timeout is stale: the worker has stopped it. */ function runningScheduledJobCount(): int { $startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false; - $tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get() + $tasks = ScheduledTaskExecution::with('scheduledTask')->onThisHost()->where('status', 'running')->get() ->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS)); - $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get() + $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->onThisHost()->where('status', 'running')->get() ->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS)); - $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get() + $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->onThisHost()->where('status', 'running')->get() ->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT)); return $tasks->count() + $databaseBackups->count() + $volumeBackups->count(); diff --git a/database/migrations/2026_10_28_204953_add_worker_host_to_scheduled_job_executions.php b/database/migrations/2026_10_28_204953_add_worker_host_to_scheduled_job_executions.php new file mode 100644 index 0000000000..b90a9eb1c8 --- /dev/null +++ b/database/migrations/2026_10_28_204953_add_worker_host_to_scheduled_job_executions.php @@ -0,0 +1,28 @@ +string('worker_host')->nullable(); + }); + } + } + + public function down(): void + { + foreach (self::TABLES as $table) { + Schema::table($table, function (Blueprint $table) { + $table->dropColumn('worker_host'); + }); + } + } +}; diff --git a/tests/Feature/RunningScheduledJobCountTest.php b/tests/Feature/RunningScheduledJobCountTest.php index 37edc5d6bc..17cf9423eb 100644 --- a/tests/Feature/RunningScheduledJobCountTest.php +++ b/tests/Feature/RunningScheduledJobCountTest.php @@ -64,3 +64,20 @@ it('does not count running rows older than their job timeout', function () { $this->travel(60)->minutes(); expect(runningScheduledJobCount())->toBe(0); }); + +it('records the worker host on new runs and hides it from responses', function () { + $execution = ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + + expect($execution->fresh()->worker_host)->toBe(gethostname()) + ->and($execution->toArray())->not->toHaveKey('worker_host'); +}); + +it('counts only runs on this host and runs without a recorded host', function () { + ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']); + (new ScheduledTaskExecution(['scheduled_task_id' => $this->task->id, 'status' => 'running']))->forceFill(['worker_host' => 'other-host'])->save(); + (new ScheduledDatabaseBackupExecution(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']))->forceFill(['worker_host' => 'other-host'])->save(); + $this->volumeBackup->executions()->make()->forceFill(['worker_host' => 'other-host'])->save(); + $this->volumeBackup->executions()->create()->forceFill(['worker_host' => null])->save(); + + expect(runningScheduledJobCount())->toBe(2); +});