From fd85b740f5b635de03d79bb1ed92f581c9019e7d Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:13:11 +0200 Subject: [PATCH] feat(github-runners): add ephemeral self-hosted runners on build servers (#12054) Co-authored-by: Claude Opus 5.5 --- app/Actions/Server/InstallSysbox.php | 165 +++++ app/Console/Kernel.php | 2 + app/Enums/GithubRunnerDockerMode.php | 15 + app/Enums/GithubRunnerStatus.php | 49 ++ .../Controllers/Api/ServersController.php | 7 + app/Http/Controllers/Webhook/Github.php | 86 +++ app/Jobs/CleanupGithubRunnerJob.php | 53 ++ app/Jobs/GithubAppPermissionJob.php | 3 + app/Jobs/ProvisionGithubRunnerJob.php | 177 +++++ app/Jobs/ReconcileGithubRunnersJob.php | 153 ++++ .../Server/GithubRunnerExecutions.php | 76 ++ app/Livewire/Server/GithubRunners.php | 291 ++++++++ app/Livewire/Server/Show.php | 9 +- app/Livewire/Source/Github/Change.php | 18 + app/Livewire/Source/Github/Create.php | 16 +- app/Models/GithubApp.php | 45 ++ app/Models/GithubRunnerConfig.php | 99 +++ app/Models/GithubRunnerExecution.php | 104 +++ app/Models/Server.php | 21 +- app/Services/GithubRunner/GithubRunnerApi.php | 108 +++ .../GithubRunner/GithubRunnerContainer.php | 196 +++++ config/constants.php | 14 + ...hub_runner_fields_to_github_apps_table.php | 28 + ...001_create_github_runner_configs_table.php | 38 + ..._create_github_runner_executions_table.php | 51 ++ .../views/components/beta-badge.blade.php | 6 + .../views/components/server/sidebar.blade.php | 11 + .../server/github-runner-executions.blade.php | 55 ++ .../livewire/server/github-runners.blade.php | 187 +++++ .../livewire/source/github/change.blade.php | 31 +- .../livewire/source/github/create.blade.php | 22 +- .../source/github/permissions.blade.php | 30 +- routes/web.php | 2 + scripts/dev | 7 +- tests/Feature/GithubRunnerFlowTest.php | 493 +++++++++++++ tests/Feature/GithubRunnerSettingsTest.php | 676 ++++++++++++++++++ tests/Feature/ScheduleOnOneServerTest.php | 18 + tests/Feature/SentinelTokenValidationTest.php | 16 + tests/Unit/GithubRunnerContainerTest.php | 121 ++++ tests/Unit/InstallSysboxTest.php | 126 ++++ 40 files changed, 3605 insertions(+), 20 deletions(-) create mode 100644 app/Actions/Server/InstallSysbox.php create mode 100644 app/Enums/GithubRunnerDockerMode.php create mode 100644 app/Enums/GithubRunnerStatus.php create mode 100644 app/Jobs/CleanupGithubRunnerJob.php create mode 100644 app/Jobs/ProvisionGithubRunnerJob.php create mode 100644 app/Jobs/ReconcileGithubRunnersJob.php create mode 100644 app/Livewire/Server/GithubRunnerExecutions.php create mode 100644 app/Livewire/Server/GithubRunners.php create mode 100644 app/Models/GithubRunnerConfig.php create mode 100644 app/Models/GithubRunnerExecution.php create mode 100644 app/Services/GithubRunner/GithubRunnerApi.php create mode 100644 app/Services/GithubRunner/GithubRunnerContainer.php create mode 100644 database/migrations/2026_09_28_100000_add_github_runner_fields_to_github_apps_table.php create mode 100644 database/migrations/2026_09_28_100001_create_github_runner_configs_table.php create mode 100644 database/migrations/2026_09_28_100002_create_github_runner_executions_table.php create mode 100644 resources/views/components/beta-badge.blade.php create mode 100644 resources/views/livewire/server/github-runner-executions.blade.php create mode 100644 resources/views/livewire/server/github-runners.blade.php create mode 100644 tests/Feature/GithubRunnerFlowTest.php create mode 100644 tests/Feature/GithubRunnerSettingsTest.php create mode 100644 tests/Unit/GithubRunnerContainerTest.php create mode 100644 tests/Unit/InstallSysboxTest.php diff --git a/app/Actions/Server/InstallSysbox.php b/app/Actions/Server/InstallSysbox.php new file mode 100644 index 0000000000..dc1cd3dd53 --- /dev/null +++ b/app/Actions/Server/InstallSysbox.php @@ -0,0 +1,165 @@ +/dev/null || true'], $server), + (string) instant_remote_process(["docker network inspect bridge --format '{{range .IPAM.Config}}{{.Gateway}} {{.Subnet}}{{end}}'"], $server), + ); + + return remote_process(self::installCommands($architecture, $daemonConfig), $server); + } + + /** + * Sysbox must be registered in Docker, its services must run, and FUSE 3 must be available. + */ + public static function isInstalled(Server $server): bool + { + $runtimes = instant_remote_process([self::installedCheckCommand()], $server, false); + + return in_array(self::RUNTIME, explode(' ', trim((string) $runtimes)), true); + } + + /** + * Prints the Docker runtimes when the Sysbox services run and FUSE 3 is available. + */ + public static function installedCheckCommand(): string + { + return "systemctl is-active --quiet sysbox-mgr sysbox-fs && test -x /usr/bin/fusermount3 && docker info --format '{{range \$name, \$runtime := .Runtimes}}{{\$name}} {{end}}'"; + } + + public static function isKernelSupported(string $kernel): bool + { + if (! preg_match('/\A(\d+)\.(\d+)/', $kernel, $matches)) { + return false; + } + + return version_compare("{$matches[1]}.{$matches[2]}", self::MINIMUM_KERNEL, '>='); + } + + /** + * Returns the Docker daemon configuration with "bip" and "default-address-pools" set to the values that + * Docker already uses, or null when both are already set. + * + * @param string $bridge " " of the default bridge network, for example "172.17.0.1 172.17.0.0/16". + */ + public static function daemonConfigWithNetworkSettings(?string $currentConfig, string $bridge): ?string + { + $config = filled(trim((string) $currentConfig)) ? json_decode($currentConfig) : new \stdClass; + if (! $config instanceof \stdClass) { + throw new RuntimeException(self::DAEMON_CONFIG.' is not a valid JSON object. Fix it, then install Sysbox again.'); + } + + $hasBip = filled($config->bip ?? null); + $hasAddressPools = filled($config->{'default-address-pools'} ?? null); + if ($hasBip && $hasAddressPools) { + return null; + } + + if (! $hasBip) { + if (! preg_match('/\A(\d{1,3}(?:\.\d{1,3}){3}) \d{1,3}(?:\.\d{1,3}){3}\/(\d{1,2})\z/', trim($bridge), $matches)) { + throw new RuntimeException('Could not read the address of the default Docker bridge network.'); + } + $config->bip = "{$matches[1]}/{$matches[2]}"; + } + + if (! $hasAddressPools) { + $config->{'default-address-pools'} = self::dockerDefaultAddressPools(); + } + + // Sysbox checks both keys with a regex that expects one key per indented line. + return json_encode($config, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"; + } + + /** + * @return array + */ + public static function installCommands(string $architecture, ?string $daemonConfig): array + { + $version = config('constants.github_runner.sysbox.version'); + $checksum = config("constants.github_runner.sysbox.checksums.{$architecture}"); + $package = "/tmp/sysbox-ce_{$version}_{$architecture}.deb"; + $url = "https://github.com/nestybox/sysbox/releases/download/v{$version}/sysbox-ce_{$version}.linux_{$architecture}.deb"; + $apt = 'DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300'; + + $commands = ["echo 'Installing Sysbox {$version}.'"]; + if ($daemonConfig !== null) { + $commands[] = "echo 'Saving the current Docker network settings in ".self::DAEMON_CONFIG." so that Docker does not restart.'"; + $commands[] = 'mkdir -p /etc/docker'; + $commands[] = 'cp '.self::DAEMON_CONFIG.' '.self::DAEMON_CONFIG.'.before-sysbox 2>/dev/null || true'; + $commands[] = 'echo '.base64_encode($daemonConfig).' | base64 -d | tee '.self::DAEMON_CONFIG.' > /dev/null'; + } + + return [ + ...$commands, + "echo 'Downloading the Sysbox package.'", + "curl -fsSL -o {$package} {$url}", + "echo '{$checksum} {$package}' | sha256sum -c -", + "echo 'Installing the Sysbox package. Docker reloads its configuration, running containers are not restarted.'", + "{$apt} update", + // Sysbox needs fusermount3, but its "fuse" dependency is satisfied by FUSE 2 on Debian 12. + "{$apt} install -y fuse3 {$package}", + "rm -f {$package}", + 'systemctl is-active --quiet sysbox-mgr sysbox-fs', + 'test -x /usr/bin/fusermount3', + "docker info --format '{{range \$name, \$runtime := .Runtimes}}{{\$name}} {{end}}' | grep -qw ".self::RUNTIME, + "echo 'Sysbox is installed.'", + ]; + } + + /** + * The address pools that Docker uses when "default-address-pools" is not set. + * + * @return array + */ + private static function dockerDefaultAddressPools(): array + { + $pools = []; + foreach (range(17, 31) as $octet) { + $pools[] = ['base' => "172.{$octet}.0.0/16", 'size' => 16]; + } + $pools[] = ['base' => '192.168.0.0/16', 'size' => 20]; + + return $pools; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index c827586b49..4bf442d3bf 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -11,6 +11,7 @@ use App\Jobs\CleanupOrphanedPreviewContainersJob; use App\Jobs\CleanupStaleMultiplexedConnections; use App\Jobs\PullChangelog; use App\Jobs\PullTemplatesFromCDN; +use App\Jobs\ReconcileGithubRunnersJob; use App\Jobs\RegenerateSslCertJob; use App\Jobs\RevalidateUnusableS3StoragesJob; use App\Jobs\ServerManagerJob; @@ -65,6 +66,7 @@ class Kernel extends ConsoleKernel $this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer(); $this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer(); $this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer(); + $this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer(); if (isDev()) { // Instance Jobs diff --git a/app/Enums/GithubRunnerDockerMode.php b/app/Enums/GithubRunnerDockerMode.php new file mode 100644 index 0000000000..65c955b634 --- /dev/null +++ b/app/Enums/GithubRunnerDockerMode.php @@ -0,0 +1,15 @@ + + */ + public static function active(): array + { + return [self::Queued, self::Provisioning, self::Idle, self::Running]; + } + + /** + * States in which a runner container exists on a server and uses its capacity. + * + * @return array + */ + public static function occupying(): array + { + return [self::Provisioning, self::Idle, self::Running]; + } + + public function isActive(): bool + { + return in_array($this, self::active(), true); + } + + public function label(): string + { + return match ($this) { + self::Idle => 'Waiting for job', + self::TimedOut => 'Timed out', + default => ucfirst($this->value), + }; + } +} diff --git a/app/Http/Controllers/Api/ServersController.php b/app/Http/Controllers/Api/ServersController.php index ef69f6b424..b98dd42e7c 100644 --- a/app/Http/Controllers/Api/ServersController.php +++ b/app/Http/Controllers/Api/ServersController.php @@ -772,6 +772,13 @@ class ServersController extends Controller ], 422); } + if ($serverRole !== null && $serverRole !== ServerRole::BUILD && $server->hasEnabledGithubRunners()) { + return response()->json([ + 'message' => 'Validation failed.', + 'errors' => ['server_role' => ['Disable the GitHub runners before you change the role of this server.']], + ], 422); + } + if ($serverRole === ServerRole::DEPLOYMENT && ! ModelsServer::buildServers($teamId)->whereKeyNot($server->id)->exists()) { return response()->json([ 'message' => 'Validation failed.', diff --git a/app/Http/Controllers/Webhook/Github.php b/app/Http/Controllers/Webhook/Github.php index 4e9ef7da65..f1d5bb8cb9 100644 --- a/app/Http/Controllers/Webhook/Github.php +++ b/app/Http/Controllers/Webhook/Github.php @@ -2,20 +2,26 @@ namespace App\Http\Controllers\Webhook; +use App\Enums\GithubRunnerStatus; use App\Exceptions\InvalidWebhookPayloadException; use App\Http\Controllers\Controller; use App\Http\Controllers\Webhook\Concerns\DetectsSkipDeployCommits; use App\Http\Controllers\Webhook\Concerns\MatchesManualWebhookApplications; use App\Http\Controllers\Webhook\Concerns\ReadsWebhookPushPayload; +use App\Jobs\CleanupGithubRunnerJob; use App\Jobs\GithubAppPermissionJob; use App\Jobs\ProcessGithubPullRequestWebhook; +use App\Jobs\ProvisionGithubRunnerJob; use App\Models\Application; use App\Models\GithubApp; +use App\Models\GithubRunnerConfig; +use App\Models\GithubRunnerExecution; use App\Models\PrivateKey; use Exception; use Illuminate\Http\Exceptions\HttpResponseException; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; use Illuminate\Support\Str; @@ -255,6 +261,83 @@ class Github extends Controller } } + /** + * Handles GitHub Actions runner demand. Every lookup is scoped to the App whose signature was verified. + * GitHub can give a job to any idle runner with matching labels, so progress is matched by runner name. + */ + private function handleWorkflowJob(GithubApp $githubApp, Collection $payload) + { + $action = data_get($payload, 'action'); + $jobId = (int) data_get($payload, 'workflow_job.id', 0); + $runnerName = (string) data_get($payload, 'workflow_job.runner_name', ''); + if ($jobId <= 0) { + return response('Nothing to do. No workflow job found.'); + } + + $jobDetails = array_filter([ + 'workflow_job_id' => $jobId, + 'workflow_job_html_url' => data_get($payload, 'workflow_job.html_url'), + 'workflow_name' => data_get($payload, 'workflow_job.workflow_name'), + 'job_name' => data_get($payload, 'workflow_job.name'), + 'repository_full_name' => data_get($payload, 'repository.full_name'), + ], fn ($value) => filled($value)); + + if ($action === 'queued') { + $labels = array_values(array_filter((array) data_get($payload, 'workflow_job.labels', []), 'is_string')); + $matches = $githubApp->runnerConfigs() + ->where('is_enabled', true) + ->get() + ->contains(fn (GithubRunnerConfig $config) => $config->matchesLabels($labels)); + if (! $matches) { + return response('Nothing to do. No runner configuration matches the job labels.'); + } + + $execution = GithubRunnerExecution::createOrFirst( + ['github_app_id' => $githubApp->id, 'trigger_workflow_job_id' => $jobId], + [...$jobDetails, 'labels' => $labels, 'status' => GithubRunnerStatus::Queued, 'queued_at' => now()], + ); + if ($execution->wasRecentlyCreated) { + ProvisionGithubRunnerJob::dispatch($execution->id); + } + + return response('Runner queued.'); + } + + $execution = filled($runnerName) + ? GithubRunnerExecution::query()->where('github_app_id', $githubApp->id)->where('runner_name', $runnerName)->first() + : null; + + if ($action === 'in_progress' && $execution && in_array($execution->status, [GithubRunnerStatus::Provisioning, GithubRunnerStatus::Idle], true)) { + $execution->update([...$jobDetails, 'status' => GithubRunnerStatus::Running, 'started_at' => now()]); + + return response('Runner marked running.'); + } + + if ($action === 'completed') { + if ($execution) { + $execution->update([...$jobDetails, 'conclusion' => data_get($payload, 'workflow_job.conclusion')]); + if ($execution->isActive() || $execution->status === GithubRunnerStatus::Failed) { + $execution->finish(GithubRunnerStatus::Completed); + CleanupGithubRunnerJob::dispatch($execution->id); + } + + return response('Runner cleanup queued.'); + } + + // The job ended without a Coolify runner, so its queued runner is no longer needed. + GithubRunnerExecution::query() + ->where('github_app_id', $githubApp->id) + ->where('trigger_workflow_job_id', $jobId) + ->where('status', GithubRunnerStatus::Queued) + ->first() + ?->finish(GithubRunnerStatus::Cancelled, 'The job ended before a runner started.'); + + return response('Queued runner cancelled.'); + } + + return response('Nothing to do.'); + } + public function normal(Request $request) { try { @@ -306,6 +389,9 @@ class Github extends Controller return response('cool'); } + if ($x_github_event === 'workflow_job') { + return $this->handleWorkflowJob($github_app, $payload); + } if ($x_github_event === 'push') { $id = $this->webhookPayloadDatabaseId($payload, 'repository.id'); $branch = $this->webhookPushBranch(data_get($payload, 'ref')); diff --git a/app/Jobs/CleanupGithubRunnerJob.php b/app/Jobs/CleanupGithubRunnerJob.php new file mode 100644 index 0000000000..baca8504f0 --- /dev/null +++ b/app/Jobs/CleanupGithubRunnerJob.php @@ -0,0 +1,53 @@ +onQueue('high'); + } + + public function handle(): void + { + $execution = GithubRunnerExecution::query()->with(['server', 'githubApp'])->find($this->executionId); + if (! $execution) { + return; + } + + if ($execution->server) { + instant_remote_process(GithubRunnerContainer::cleanupCommands($execution->uuid), $execution->server, false); + } + + if ($execution->runner_id && $execution->githubApp) { + try { + (new GithubRunnerApi($execution->githubApp))->deleteRunner($execution->runner_id); + } catch (\Throwable $e) { + Log::warning('Could not remove GitHub runner', ['execution' => $execution->uuid, 'error' => $e->getMessage()]); + } + } + + ProvisionGithubRunnerJob::dispatchQueued($execution->github_app_id); + } +} diff --git a/app/Jobs/GithubAppPermissionJob.php b/app/Jobs/GithubAppPermissionJob.php index 319a202cf1..71920d6d76 100644 --- a/app/Jobs/GithubAppPermissionJob.php +++ b/app/Jobs/GithubAppPermissionJob.php @@ -45,6 +45,9 @@ class GithubAppPermissionJob implements ShouldBeEncrypted, ShouldQueue $this->github_app->metadata = data_get($permissions, 'metadata'); $this->github_app->pull_requests = data_get($permissions, 'pull_requests'); $this->github_app->administration = data_get($permissions, 'administration'); + $this->github_app->organization_self_hosted_runners = data_get($permissions, 'organization_self_hosted_runners'); + $this->github_app->actions = data_get($permissions, 'actions'); + $this->github_app->webhook_events = data_get($response, 'events', []); $this->github_app->save(); $this->github_app->makeVisible('client_secret')->makeVisible('webhook_secret'); diff --git a/app/Jobs/ProvisionGithubRunnerJob.php b/app/Jobs/ProvisionGithubRunnerJob.php new file mode 100644 index 0000000000..0ce1989aec --- /dev/null +++ b/app/Jobs/ProvisionGithubRunnerJob.php @@ -0,0 +1,177 @@ +onQueue('high'); + } + + /** + * Starts provisioning for the oldest queued executions of a GitHub App. + */ + public static function dispatchQueued(int $githubAppId, int $limit = 10): void + { + GithubRunnerExecution::query() + ->where('github_app_id', $githubAppId) + ->where('status', GithubRunnerStatus::Queued) + ->orderBy('id') + ->limit($limit) + ->pluck('id') + ->each(fn (int $id) => self::dispatch($id)); + } + + public function handle(): void + { + $execution = $this->reserveServer(); + if (! $execution) { + return; + } + + try { + $config = $execution->config; + $server = $execution->server; + $api = new GithubRunnerApi($execution->githubApp); + $runnerGroupId = $execution->githubApp->runner_group_id ?: $api->ensureRunnerGroup()['id']; + $container = new GithubRunnerContainer($execution, $config); + + instant_remote_process($container->prepareCommands(), $server, timeout: $this->timeout); + if ($container->usesDind()) { + $this->waitForDocker($container, $server); + } + + $runner = $api->generateJitConfig($container->name(), $config->registeredLabels(), $runnerGroupId); + $execution->update(['runner_id' => $runner['runner_id']]); + instant_remote_process($container->startCommands($runner['encoded_jit_config']), $server); + + $execution->update(['ready_at' => now()]); + GithubRunnerExecution::query() + ->whereKey($execution->id) + ->where('status', GithubRunnerStatus::Provisioning) + ->update(['status' => GithubRunnerStatus::Idle]); + } catch (\Throwable $e) { + $this->handleFailure($execution, $e); + } + } + + /** + * Removes the failed runner and queues the job again, or marks it failed after the last attempt. + * GitHub keeps the job queued, so a later runner with matching labels can still take it. + */ + private function handleFailure(GithubRunnerExecution $execution, \Throwable $e): void + { + $attempt = $execution->provision_attempts; + if ($attempt >= self::MAX_ATTEMPTS) { + $execution->finish(GithubRunnerStatus::Failed, "Could not start a runner after {$attempt} attempts: {$e->getMessage()}"); + CleanupGithubRunnerJob::dispatch($execution->id); + + return; + } + + // The retry keeps the execution uuid, so its Docker resources must be gone before it starts again. + (new CleanupGithubRunnerJob($execution->id))->handle(); + $execution->update([ + 'status' => GithubRunnerStatus::Queued, + 'github_runner_config_id' => null, + 'server_id' => null, + 'runner_name' => null, + 'runner_id' => null, + 'ready_at' => null, + 'error_message' => "Attempt {$attempt} of ".self::MAX_ATTEMPTS." failed: {$e->getMessage()}", + ]); + self::dispatch($execution->id)->delay(now()->addSeconds(self::RETRY_DELAY_SECONDS * $attempt)); + } + + /** + * Picks the least busy matching server and moves the execution to "provisioning". + * Row locks on the execution and on the App's configs stop parallel jobs from overbooking a server. + */ + private function reserveServer(): ?GithubRunnerExecution + { + return DB::transaction(function () { + $execution = GithubRunnerExecution::query()->whereKey($this->executionId)->lockForUpdate()->first(); + if (! $execution || $execution->status !== GithubRunnerStatus::Queued) { + return null; + } + + $config = GithubRunnerConfig::query() + ->where('github_app_id', $execution->github_app_id) + ->where('is_enabled', true) + ->orderBy('id') + ->lockForUpdate() + ->with('server.settings') + ->get() + ->filter(fn (GithubRunnerConfig $config) => $config->server !== null + && $config->matchesLabels($execution->labels ?? []) + && $config->server->isBuildServer() + && ! $config->server->isLocalhost() + && $config->server->isFunctional()) + ->map(fn (GithubRunnerConfig $config) => ['config' => $config, 'occupied' => $config->occupiedRunnerCount()]) + ->filter(fn (array $candidate) => $candidate['occupied'] < $candidate['config']->max_runners) + ->sortBy('occupied') + ->first()['config'] ?? null; + + if (! $config) { + return null; + } + + $execution->update([ + 'github_runner_config_id' => $config->id, + 'server_id' => $config->server_id, + 'status' => GithubRunnerStatus::Provisioning, + 'runner_name' => GithubRunnerContainer::nameFor($execution->uuid), + 'provision_attempts' => $execution->provision_attempts + 1, + ]); + + return $execution; + }); + } + + private function waitForDocker(GithubRunnerContainer $container, Server $server): void + { + for ($attempt = 0; $attempt < self::DOCKER_READY_ATTEMPTS; $attempt++) { + if (filled(instant_remote_process([$container->dockerReadyCommand()], $server, false))) { + return; + } + Sleep::for(1)->second(); + } + + throw new RuntimeException('The Docker sidecar did not start within '.self::DOCKER_READY_ATTEMPTS.' seconds.'); + } +} diff --git a/app/Jobs/ReconcileGithubRunnersJob.php b/app/Jobs/ReconcileGithubRunnersJob.php new file mode 100644 index 0000000000..459dd681c0 --- /dev/null +++ b/app/Jobs/ReconcileGithubRunnersJob.php @@ -0,0 +1,153 @@ +onQueue('high'); + } + + public function uniqueFor(): int + { + return $this->timeout; + } + + public function handle(): void + { + Server::query() + ->whereHas('githubRunnerConfig') + ->with('settings') + ->get() + ->filter(fn (Server $server) => $server->isFunctional()) + ->each(fn (Server $server) => $this->reconcileServer($server)); + + $this->reconcileQueued(); + } + + private function reconcileServer(Server $server): void + { + $output = instant_remote_process([GithubRunnerContainer::listCommand()], $server, false); + if ($output === null) { + return; + } + $containers = $this->parseContainers($output); + + $executions = GithubRunnerExecution::query() + ->where('server_id', $server->id) + ->whereIn('status', GithubRunnerStatus::occupying()) + ->with('config') + ->get(); + + foreach ($executions as $execution) { + $failure = $this->failureFor($execution, $containers->get($execution->uuid, [])); + if ($failure) { + $execution->finish(...$failure); + CleanupGithubRunnerJob::dispatch($execution->id); + } + } + + $containers->keys() + ->diff($executions->pluck('uuid')) + ->each(fn (string $uuid) => instant_remote_process(GithubRunnerContainer::cleanupCommands($uuid), $server, false)); + } + + /** + * @param array $containerStates container name => state + * @return array{0: GithubRunnerStatus, 1: string}|null + */ + private function failureFor(GithubRunnerExecution $execution, array $containerStates): ?array + { + if ($execution->status === GithubRunnerStatus::Provisioning) { + return $execution->updated_at->lt(now()->subMinutes(self::PROVISIONING_TIMEOUT_MINUTES)) + ? [GithubRunnerStatus::Failed, 'Provisioning did not finish in '.self::PROVISIONING_TIMEOUT_MINUTES.' minutes.'] + : null; + } + + $state = $containerStates[GithubRunnerContainer::nameFor($execution->uuid)] ?? null; + if (! in_array($state, ['running', 'restarting', 'paused'], true)) { + return [GithubRunnerStatus::Failed, 'The runner container stopped before GitHub reported a job result.']; + } + + $config = $execution->config; + if ($execution->status === GithubRunnerStatus::Idle && $config + && ($execution->ready_at ?? $execution->updated_at)->lt(now()->subMinutes($config->idle_timeout))) { + return [GithubRunnerStatus::Cancelled, "No job was assigned to the runner within {$config->idle_timeout} minutes."]; + } + + if ($execution->status === GithubRunnerStatus::Running && $config + && ($execution->started_at ?? $execution->updated_at)->lt(now()->subMinutes($config->job_timeout))) { + return [GithubRunnerStatus::TimedOut, "The job ran longer than {$config->job_timeout} minutes."]; + } + + return null; + } + + /** + * @return Collection> execution uuid => [container name => state] + */ + private function parseContainers(string $output): Collection + { + $containers = collect(); + foreach (preg_split('/\R/', trim($output)) as $line) { + $parts = preg_split('/\s+/', trim($line)); + if (count($parts) !== 3 || ! preg_match('/\A[a-z0-9]+\z/', $parts[0])) { + continue; + } + [$uuid, $name, $state] = $parts; + $containers->put($uuid, [...$containers->get($uuid, []), $name => $state]); + } + + return $containers; + } + + private function reconcileQueued(): void + { + $queued = GithubRunnerExecution::query() + ->where('status', GithubRunnerStatus::Queued) + ->get() + ->groupBy('github_app_id'); + + foreach ($queued as $githubAppId => $executions) { + $timeout = (int) (GithubRunnerConfig::query() + ->where('github_app_id', $githubAppId) + ->where('is_enabled', true) + ->max('capacity_wait_timeout') ?? 0); + + foreach ($executions as $execution) { + if (($execution->queued_at ?? $execution->created_at)->lt(now()->subMinutes($timeout))) { + $execution->finish(GithubRunnerStatus::TimedOut, "No build server had free capacity within {$timeout} minutes."); + } + } + + ProvisionGithubRunnerJob::dispatchQueued($githubAppId); + } + } +} diff --git a/app/Livewire/Server/GithubRunnerExecutions.php b/app/Livewire/Server/GithubRunnerExecutions.php new file mode 100644 index 0000000000..df74f2b955 --- /dev/null +++ b/app/Livewire/Server/GithubRunnerExecutions.php @@ -0,0 +1,76 @@ +authorize('view', $server); + $this->server = $server; + } + + /** + * Runners of this server, plus queued jobs of its GitHub App that wait for free capacity. + */ + #[Computed] + public function executions(): Collection + { + return $this->query()->latest('id')->limit(25)->get(); + } + + public function cancel(int $executionId) + { + $this->authorize('update', $this->server); + + try { + $execution = $this->query()->whereKey($executionId)->firstOrFail(); + if (! $execution->isActive()) { + return; + } + $hadServer = $execution->server_id !== null; + $execution->finish(GithubRunnerStatus::Cancelled, 'Cancelled by a user.'); + if ($hadServer) { + CleanupGithubRunnerJob::dispatch($execution->id); + } + unset($this->executions); + $this->dispatch('success', 'Runner cancelled.'); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + + private function query(): Builder + { + $githubAppId = $this->server->githubRunnerConfig()->value('github_app_id'); + + return GithubRunnerExecution::query()->where(function (Builder $query) use ($githubAppId) { + $query->where('server_id', $this->server->id); + if ($githubAppId) { + $query->orWhere(fn (Builder $queued) => $queued + ->whereNull('server_id') + ->where('status', GithubRunnerStatus::Queued) + ->where('github_app_id', $githubAppId)); + } + }); + } + + public function render() + { + return view('livewire.server.github-runner-executions'); + } +} diff --git a/app/Livewire/Server/GithubRunners.php b/app/Livewire/Server/GithubRunners.php new file mode 100644 index 0000000000..4b95ed67be --- /dev/null +++ b/app/Livewire/Server/GithubRunners.php @@ -0,0 +1,291 @@ + 'checkSysbox']; + + public function mount(string $server_uuid) + { + try { + $this->server = Server::ownedByCurrentTeam()->whereUuid($server_uuid)->firstOrFail(); + $this->authorize('view', $this->server); + if ($this->server->isLocalhost()) { + return redirect()->route('server.show', ['server_uuid' => $this->server->uuid]); + } + $this->parameters = get_route_parameters(); + $this->syncData(); + } catch (\Throwable) { + return redirect()->route('server.index'); + } + } + + protected function rules(): array + { + return [ + 'githubAppId' => ['required', 'integer', Rule::in($this->githubApps->pluck('id')->all())], + 'labels' => ['required', 'string', 'max:1000'], + 'maxRunners' => ['required', 'integer', 'min:1', 'max:32'], + 'dockerMode' => ['required', Rule::enum(GithubRunnerDockerMode::class)], + 'runnerImage' => ['nullable', 'string', 'max:255', new DockerImageFormat], + 'cpuLimit' => ['nullable', 'string', 'regex:/\A\d+(\.\d+)?\z/'], + 'memoryLimit' => ['nullable', 'string', 'regex:/\A\d+[bkmg]?\z/i'], + 'capacityWaitTimeout' => ['required', 'integer', 'min:1', 'max:1440'], + 'idleTimeout' => ['required', 'integer', 'min:1', 'max:1440'], + 'jobTimeout' => ['required', 'integer', 'min:1', 'max:7200'], + 'isDedicated' => ['required', 'boolean'], + ]; + } + + /** + * Organization GitHub Apps of the server's team. System-wide Apps of other teams are not allowed. + */ + #[Computed] + public function githubApps(): Collection + { + return GithubApp::query() + ->where('team_id', $this->server->team_id) + ->whereNotNull('app_id') + ->whereNotNull('installation_id') + ->whereNotNull('organization') + ->where('organization', '!=', '') + ->orderBy('name') + ->get(); + } + + #[Computed] + public function selectedApp(): ?GithubApp + { + return $this->githubApps->firstWhere('id', $this->githubAppId); + } + + #[Computed] + public function config(): ?GithubRunnerConfig + { + return $this->server->githubRunnerConfig()->first(); + } + + private function syncData(): void + { + $config = $this->config; + if (! $config) { + $apps = $this->githubApps; + $this->githubAppId = ($apps->first(fn (GithubApp $app) => $app->missingRunnerRequirements() === []) ?? $apps->first())?->id; + + return; + } + + $this->githubAppId = $config->github_app_id; + $this->labels = implode(', ', $config->labels ?? []); + $this->maxRunners = $config->max_runners; + $this->dockerMode = $config->docker_mode->value; + $this->runnerImage = $config->runner_image; + $this->cpuLimit = $config->cpu_limit; + $this->memoryLimit = $config->memory_limit; + $this->capacityWaitTimeout = $config->capacity_wait_timeout; + $this->idleTimeout = $config->idle_timeout; + $this->jobTimeout = $config->job_timeout; + $this->isDedicated = $config->is_dedicated; + } + + /** + * @return array + */ + private function parseLabels(): array + { + $labels = collect(explode(',', $this->labels)) + ->map(fn (string $label) => strtolower(trim($label))) + ->filter() + ->unique() + ->values(); + + foreach ($labels as $label) { + if (! preg_match('/\A[a-z0-9][a-z0-9._-]{0,99}\z/', $label)) { + throw ValidationException::withMessages(['labels' => "The label \"{$label}\" can only contain letters, numbers, dots, dashes, and underscores."]); + } + } + + $custom = $labels->reject(fn (string $label) => in_array($label, self::RESERVED_LABELS, true))->values(); + if ($custom->isEmpty()) { + throw ValidationException::withMessages(['labels' => 'Add at least one custom label, for example "coolify". Coolify only takes jobs that ask for a custom label.']); + } + + return $custom->all(); + } + + public function submit() + { + $this->authorize('update', $this->server); + $this->validate(); + $labels = $this->parseLabels(); + $githubApp = $this->selectedApp; + $this->authorize('update', $githubApp); + if ($this->dockerMode === GithubRunnerDockerMode::Sysbox->value && ! InstallSysbox::isInstalled($this->server)) { + $this->isSysboxInstalled = false; + throw ValidationException::withMessages(['dockerMode' => 'Sysbox is not installed on this server. Install it, then save again.']); + } + + try { + if (! $this->server->isBuildServer()) { + throw new \RuntimeException('GitHub Actions runners can only run on servers with the Build role.'); + } + + if ($this->config?->is_enabled ?? true) { + $this->prepareRunnerGroup($githubApp); + } + + GithubRunnerConfig::updateOrCreate(['server_id' => $this->server->id], [ + 'github_app_id' => $githubApp->id, + 'labels' => $labels, + 'max_runners' => $this->maxRunners, + 'docker_mode' => $this->dockerMode, + 'runner_image' => filled($this->runnerImage) ? trim($this->runnerImage) : null, + 'cpu_limit' => filled($this->cpuLimit) ? $this->cpuLimit : null, + 'memory_limit' => filled($this->memoryLimit) ? strtolower($this->memoryLimit) : null, + 'capacity_wait_timeout' => $this->capacityWaitTimeout, + 'idle_timeout' => $this->idleTimeout, + 'job_timeout' => $this->jobTimeout, + 'is_dedicated' => $this->isDedicated, + ]); + unset($this->config); + $this->syncData(); + $this->dispatch('success', 'GitHub runner settings saved.'); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + + /** + * Starts or stops taking workflow jobs on this server. The first enable saves the default settings. + * Disabling removes the idle runners so that GitHub cannot give them new jobs. Running jobs continue + * until they finish. + */ + public function toggleEnabled() + { + $this->authorize('update', $this->server); + + try { + $config = $this->config; + if (! $config) { + return $this->submit(); + } + + if ($config->is_enabled) { + $config->update(['is_enabled' => false]); + $config->executions()->where('status', GithubRunnerStatus::Idle)->get() + ->each(function (GithubRunnerExecution $execution) { + $execution->finish(GithubRunnerStatus::Cancelled, 'Runners were disabled on this server.'); + CleanupGithubRunnerJob::dispatch($execution->id); + }); + $this->dispatch('success', 'GitHub runners disabled. Running jobs continue until they finish.'); + } else { + if (! $this->server->isBuildServer()) { + throw new \RuntimeException('GitHub Actions runners can only run on servers with the Build role.'); + } + $this->authorize('update', $config->githubApp); + $this->prepareRunnerGroup($config->githubApp); + $config->update(['is_enabled' => true]); + ProvisionGithubRunnerJob::dispatchQueued($config->github_app_id); + $this->dispatch('success', 'GitHub runners enabled.'); + } + unset($this->config); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + + public function checkSysbox(): void + { + $this->authorize('view', $this->server); + $this->isSysboxInstalled = InstallSysbox::isInstalled($this->server); + } + + public function installSysbox() + { + $this->authorize('update', $this->server); + + try { + $activity = InstallSysbox::run($this->server); + $this->dispatch('sysbox-install-started'); + $this->dispatch('activityMonitor', $activity->id, 'sysboxInstalled'); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + + /** + * Creates the runner group when the App can run workflow jobs. An App that is not ready is saved anyway: + * the page lists what it misses, and the runner group is created when the first job is provisioned. + */ + private function prepareRunnerGroup(GithubApp $githubApp): void + { + if ($githubApp->missingRunnerRequirements() !== []) { + $this->dispatch('warning', 'The GitHub App is not ready for runners. Coolify cannot take workflow jobs until you update its permissions on GitHub.'); + + return; + } + + $group = (new GithubRunnerApi($githubApp))->ensureRunnerGroup(); + if ($group['is_default']) { + $this->dispatch('warning', 'Runners use the Default runner group of the organization. Check which repositories can use that group on GitHub.'); + } + } + + public function render() + { + return view('livewire.server.github-runners'); + } +} diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.php index d7e3852e30..707af97515 100644 --- a/app/Livewire/Server/Show.php +++ b/app/Livewire/Server/Show.php @@ -284,7 +284,7 @@ class Show extends Component $this->serverRole = $this->server->settings->effectiveServerRole()->value; $this->isMetricsEnabled = $this->server->settings->is_metrics_enabled; $this->sentinelToken = auth()->user()->can('update', $this->server) - ? $this->server->settings->sentinel_token + ? $this->server->settings->ensureValidSentinelToken() : ''; $this->sentinelMetricsRefreshRateSeconds = $this->server->settings->sentinel_metrics_refresh_rate_seconds; $this->sentinelMetricsHistoryDays = $this->server->settings->sentinel_metrics_history_days; @@ -456,6 +456,13 @@ class Show extends Component $newRole = ServerRole::from($this->serverRole); $currentRole = $this->server->settings()->firstOrFail()->effectiveServerRole(); + if ($newRole !== ServerRole::BUILD && $this->server->hasEnabledGithubRunners()) { + $this->serverRole = $currentRole->value; + $this->dispatch('error', 'Disable the GitHub runners before you change the role of this server.'); + + return; + } + if ($newRole === ServerRole::BUILD && ! $this->server->isEmpty()) { $this->serverRole = $currentRole->value; $this->dispatch('error', 'Move or remove the existing resources before you set this server to build only.'); diff --git a/app/Livewire/Source/Github/Change.php b/app/Livewire/Source/Github/Change.php index 9e354a2315..02a9f8385d 100644 --- a/app/Livewire/Source/Github/Change.php +++ b/app/Livewire/Source/Github/Change.php @@ -36,6 +36,8 @@ class Change extends Component public ?bool $administration = false; + public ?bool $github_runners = false; + public $parameters; public ?GithubApp $github_app = null; @@ -308,6 +310,10 @@ class Change extends Component // Sync data from model to properties $this->syncData(false); + if (! $this->github_app->app_id) { + $this->preview_deployment_permissions = request()->boolean('previews', true); + $this->github_runners = filled($this->github_app->organization) && request()->boolean('runners'); + } $this->isConnected = $this->github_app->isConnected(); // Override name with kebab case for display @@ -481,4 +487,16 @@ class Change extends Component return handleError($e, $this); } } + + /** + * After a delete, the confirmation modal refreshes the component before the redirect runs. + */ + public function render() + { + if (! $this->github_app) { + return '
'; + } + + return view('livewire.source.github.change'); + } } diff --git a/app/Livewire/Source/Github/Create.php b/app/Livewire/Source/Github/Create.php index 37eea732c3..88eb296e5d 100644 --- a/app/Livewire/Source/Github/Create.php +++ b/app/Livewire/Source/Github/Create.php @@ -26,6 +26,10 @@ class Create extends Component public bool $is_system_wide = false; + public bool $use_for_pull_request_previews = true; + + public bool $use_for_github_runners = false; + private bool $shouldDeriveApiUrlAfterHtmlUrlUpdate = false; public function mount() @@ -58,12 +62,16 @@ class Create extends Component $this->validate([ 'name' => 'required|string', - 'organization' => ['nullable', 'string', 'regex:/\A[^\s\/?#]+\z/'], + 'organization' => [$this->use_for_github_runners ? 'required' : 'nullable', 'string', 'regex:/\A[^\s\/?#]+\z/'], 'api_url' => ['required', 'string', 'url', SafeExternalUrl::forGitSource()], 'html_url' => ['required', 'string', 'url', SafeExternalUrl::forGitSource()], 'custom_user' => 'required|string', 'custom_port' => 'required|int', 'is_system_wide' => 'required|bool', + 'use_for_pull_request_previews' => 'required|bool', + 'use_for_github_runners' => 'required|bool', + ], [ + 'organization.required' => 'GitHub Actions runners are registered at organization level. Enter the organization.', ]); $payload = [ 'name' => $this->name, @@ -80,7 +88,11 @@ class Create extends Component session(['from' => session('from') + ['source_id' => $github_app->id]]); } - return redirectRoute($this, 'source.github.show', ['github_app_uuid' => $github_app->uuid]); + return redirectRoute($this, 'source.github.show', [ + 'github_app_uuid' => $github_app->uuid, + 'previews' => (int) $this->use_for_pull_request_previews, + 'runners' => (int) $this->use_for_github_runners, + ]); } catch (ValidationException $e) { throw $e; } catch (\Throwable $e) { diff --git a/app/Models/GithubApp.php b/app/Models/GithubApp.php index 96c7a2d39d..0052232fb1 100644 --- a/app/Models/GithubApp.php +++ b/app/Models/GithubApp.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Enums\GithubRunnerStatus; use App\Traits\Auditable; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Support\Facades\DB; @@ -35,6 +36,10 @@ class GithubApp extends BaseModel 'metadata', 'pull_requests', 'administration', + 'organization_self_hosted_runners', + 'actions', + 'webhook_events', + 'runner_group_id', ]; protected $appends = ['type']; @@ -43,6 +48,8 @@ class GithubApp extends BaseModel 'is_public' => 'boolean', 'is_system_wide' => 'boolean', 'type' => 'string', + 'webhook_events' => 'array', + 'runner_group_id' => 'integer', ]; protected $hidden = [ @@ -58,6 +65,15 @@ class GithubApp extends BaseModel throw new \Exception('You cannot delete this GitHub App because it is in use by '.$applications_count.' application(s). Delete them first.'); } + // Runner rows cascade with the App, so live runner containers would lose their cleanup and keep taking jobs. + $hasLiveRunners = $github_app->runnerConfigs()->where('is_enabled', true)->exists() + || GithubRunnerExecution::where('github_app_id', $github_app->id) + ->whereIn('status', GithubRunnerStatus::occupying()) + ->exists(); + if ($hasLiveRunners) { + throw new \Exception('You cannot delete this GitHub App because servers use it for GitHub Actions runners. Disable the runners on these servers and wait for running jobs to finish first.'); + } + $privateKey = $github_app->privateKey; if ($privateKey) { // Check if key is used by anything EXCEPT this GitHub app @@ -97,6 +113,35 @@ class GithubApp extends BaseModel return $this->belongsTo(PrivateKey::class); } + public function runnerConfigs() + { + return $this->hasMany(GithubRunnerConfig::class); + } + + /** + * Permissions and webhook events that GitHub Actions runners need but the App does not have. + * + * @return array + */ + public function missingRunnerRequirements(): array + { + $missing = []; + if (blank($this->organization)) { + $missing[] = 'An organization (runners are registered at organization level)'; + } + if ($this->organization_self_hosted_runners !== 'write') { + $missing[] = 'Organization permission "Self-hosted runners": write'; + } + if (! in_array($this->actions, ['read', 'write'], true)) { + $missing[] = 'Repository permission "Actions": read'; + } + if (! in_array('workflow_job', $this->webhook_events ?? [], true)) { + $missing[] = 'Webhook event "Workflow job"'; + } + + return $missing; + } + public function type(): Attribute { return Attribute::make( diff --git a/app/Models/GithubRunnerConfig.php b/app/Models/GithubRunnerConfig.php new file mode 100644 index 0000000000..7f848107c8 --- /dev/null +++ b/app/Models/GithubRunnerConfig.php @@ -0,0 +1,99 @@ + 'array', + 'is_enabled' => 'boolean', + 'is_dedicated' => 'boolean', + 'max_runners' => 'integer', + 'docker_mode' => GithubRunnerDockerMode::class, + 'capacity_wait_timeout' => 'integer', + 'idle_timeout' => 'integer', + 'job_timeout' => 'integer', + ]; + } + + public function server(): BelongsTo + { + return $this->belongsTo(Server::class); + } + + public function githubApp(): BelongsTo + { + return $this->belongsTo(GithubApp::class); + } + + public function executions(): HasMany + { + return $this->hasMany(GithubRunnerExecution::class); + } + + /** + * Labels sent to GitHub when a runner is registered. + * + * @return array + */ + public function registeredLabels(): array + { + return array_values(array_unique([...self::DEFAULT_LABELS, ...array_map('strtolower', $this->labels ?? [])])); + } + + /** + * A job matches when it asks for at least one custom label and every label it asks for is registered. + * Jobs that only ask for generic labels such as "self-hosted" are left for other runners. + * + * @param array $requestedLabels + */ + public function matchesLabels(array $requestedLabels): bool + { + $requested = array_map(fn ($label) => strtolower((string) $label), $requestedLabels); + $custom = array_map('strtolower', $this->labels ?? []); + + if (array_intersect($requested, $custom) === []) { + return false; + } + + return array_diff($requested, $this->registeredLabels()) === []; + } + + public function occupiedRunnerCount(): int + { + return $this->executions()->whereIn('status', GithubRunnerStatus::occupying())->count(); + } + + public function runnerImage(): string + { + return filled($this->runner_image) ? $this->runner_image : config('constants.github_runner.image'); + } +} diff --git a/app/Models/GithubRunnerExecution.php b/app/Models/GithubRunnerExecution.php new file mode 100644 index 0000000000..af334209d2 --- /dev/null +++ b/app/Models/GithubRunnerExecution.php @@ -0,0 +1,104 @@ + GithubRunnerStatus::class, + 'trigger_workflow_job_id' => 'integer', + 'workflow_job_id' => 'integer', + 'runner_id' => 'integer', + 'provision_attempts' => 'integer', + 'labels' => 'array', + 'queued_at' => 'datetime', + 'ready_at' => 'datetime', + 'started_at' => 'datetime', + 'completed_at' => 'datetime', + ]; + } + + public function githubApp(): BelongsTo + { + return $this->belongsTo(GithubApp::class); + } + + public function config(): BelongsTo + { + return $this->belongsTo(GithubRunnerConfig::class, 'github_runner_config_id'); + } + + public function server(): BelongsTo + { + return $this->belongsTo(Server::class); + } + + public function isActive(): bool + { + return $this->status->isActive(); + } + + /** + * Base name of the Docker resources (containers, network, volumes) of this runner. + */ + public function containerName(): string + { + return 'coolify-runner-'.$this->uuid; + } + + /** + * Marks the runner as finished and keeps the first completion time. + */ + public function finish(GithubRunnerStatus $status, ?string $errorMessage = null): void + { + $this->update([ + 'status' => $status, + 'error_message' => $errorMessage, + 'completed_at' => $this->completed_at ?? now(), + ]); + } + + public function duration(): ?string + { + if (! $this->started_at) { + return null; + } + + return $this->started_at->diffForHumans($this->completed_at ?? now(), true); + } + + public function workflowJobUrl(): ?string + { + $url = trim((string) $this->workflow_job_html_url); + + return str_starts_with($url, 'https://') ? $url : null; + } +} diff --git a/app/Models/Server.php b/app/Models/Server.php index 9f4e6667d8..28e00cc3a9 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -987,7 +987,8 @@ $siteAddress { /** * Usable dedicated (build-only) servers of a team. Servers with the combined role - * host deployments, so they are never picked as build servers. + * host deployments, so they are never picked as build servers. Servers dedicated to + * GitHub Actions runners are also left out. */ public static function buildServers($teamId): Builder { @@ -995,7 +996,8 @@ $siteAddress { ->whereRelation('settings', 'is_reachable', true) ->whereRelation('settings', 'is_usable', true) ->whereRelation('settings', 'is_swarm_worker', false) - ->whereRelation('settings', 'force_disabled', false); + ->whereRelation('settings', 'force_disabled', false) + ->whereDoesntHave('githubRunnerConfig', fn (Builder $config) => $config->where('is_enabled', true)->where('is_dedicated', true)); return self::whereServerRole($query, ServerRole::BUILD); } @@ -1507,6 +1509,21 @@ $siteAddress { return $standalone_docker->concat($swarm_docker); } + public function githubRunnerConfig() + { + return $this->hasOne(GithubRunnerConfig::class); + } + + public function githubRunnerExecutions() + { + return $this->hasMany(GithubRunnerExecution::class); + } + + public function hasEnabledGithubRunners(): bool + { + return $this->githubRunnerConfig()->where('is_enabled', true)->exists(); + } + public function standaloneDockers() { return $this->hasMany(StandaloneDocker::class); diff --git a/app/Services/GithubRunner/GithubRunnerApi.php b/app/Services/GithubRunner/GithubRunnerApi.php new file mode 100644 index 0000000000..8f25b18380 --- /dev/null +++ b/app/Services/GithubRunner/GithubRunnerApi.php @@ -0,0 +1,108 @@ +organization(); + + if ($this->githubApp->runner_group_id) { + $response = $this->client()->get("/orgs/{$org}/actions/runner-groups/{$this->githubApp->runner_group_id}"); + if ($response->successful()) { + return ['id' => (int) $this->githubApp->runner_group_id, 'is_default' => (bool) $response->json('default')]; + } + if ($response->status() !== 404) { + throw new RuntimeException('Could not read the runner group: '.$this->errorMessage($response)); + } + } + + $response = $this->client()->post("/orgs/{$org}/actions/runner-groups", [ + 'name' => 'Coolify '.$this->githubApp->uuid, + 'visibility' => 'private', + 'allows_public_repositories' => false, + ]); + if (! $response->successful() || ! $response->json('id')) { + throw new RuntimeException('Could not create a runner group: '.$this->errorMessage($response)); + } + $group = ['id' => (int) $response->json('id'), 'is_default' => false]; + + $this->githubApp->update(['runner_group_id' => $group['id']]); + + return $group; + } + + /** + * @param array $labels + * @return array{runner_id: int, encoded_jit_config: string} + */ + public function generateJitConfig(string $runnerName, array $labels, int $runnerGroupId): array + { + $response = $this->client()->post("/orgs/{$this->organization()}/actions/runners/generate-jitconfig", [ + 'name' => $runnerName, + 'runner_group_id' => $runnerGroupId, + 'labels' => $labels, + 'work_folder' => '_work', + ]); + + if (! $response->successful() || blank($response->json('encoded_jit_config'))) { + throw new RuntimeException('Could not register the runner on GitHub: '.$this->errorMessage($response)); + } + + return [ + 'runner_id' => (int) $response->json('runner.id'), + 'encoded_jit_config' => (string) $response->json('encoded_jit_config'), + ]; + } + + /** + * Removes a runner from GitHub. GitHub removes JIT runners itself after their job, so 404 is not an error. + */ + public function deleteRunner(int $runnerId): void + { + $response = $this->client()->delete("/orgs/{$this->organization()}/actions/runners/{$runnerId}"); + + if (! $response->successful() && $response->status() !== 404) { + throw new RuntimeException('Could not remove the runner from GitHub: '.$this->errorMessage($response)); + } + } + + private function client(): PendingRequest + { + return Http::GitHub($this->githubApp->api_url, generateGithubInstallationToken($this->githubApp))->timeout(30); + } + + private function organization(): string + { + $organization = trim((string) $this->githubApp->organization); + if ($organization === '') { + throw new RuntimeException('GitHub Actions runners need a GitHub App that belongs to an organization.'); + } + + return rawurlencode($organization); + } + + private function errorMessage(Response $response): string + { + return (string) ($response->json('message') ?? "HTTP {$response->status()}"); + } +} diff --git a/app/Services/GithubRunner/GithubRunnerContainer.php b/app/Services/GithubRunner/GithubRunnerContainer.php new file mode 100644 index 0000000000..591aadf067 --- /dev/null +++ b/app/Services/GithubRunner/GithubRunnerContainer.php @@ -0,0 +1,196 @@ +execution->uuid); + } + + public function dindName(): string + { + return $this->name().'-dind'; + } + + public function usesDind(): bool + { + return $this->config->docker_mode !== GithubRunnerDockerMode::None; + } + + /** + * Creates the network and volumes, pulls missing images, and starts the Docker sidecar. + * + * @return array + */ + public function prepareCommands(): array + { + $name = $this->name(); + $labels = $this->labelFlags(); + $commands = [ + "docker network create {$labels} {$name}", + $this->pullOrUseLocal($this->config->runnerImage()), + ]; + + if (! $this->usesDind()) { + return $commands; + } + + foreach (['sock', 'work', 'externals'] as $volume) { + $commands[] = "docker volume create {$labels} {$name}-{$volume}"; + } + $commands[] = $this->pullIfMissing(config('constants.github_runner.dind_image')); + $isolation = $this->config->docker_mode === GithubRunnerDockerMode::Sysbox + ? '--runtime='.InstallSysbox::RUNTIME + : '--privileged'; + $commands[] = "docker run -d {$isolation}" + ." --name {$this->dindName()} --network {$name} {$labels} {$this->limitFlags()}" + ." {$this->sharedVolumeFlags()} " + .escapeshellarg(config('constants.github_runner.dind_image')) + .' dockerd --host=unix:///var/run/docker.sock --group='.self::RUNNER_DOCKER_GID; + + return $commands; + } + + /** + * Prints the version of the sidecar Docker daemon once it accepts connections. + */ + public function dockerReadyCommand(): string + { + return "docker exec {$this->dindName()} docker version --format '{{.Server.Version}}'"; + } + + /** + * Writes the JIT config to a root-only env file, starts the runner, and deletes the file. + * The secret is never part of the docker run command line. + * + * @return array + */ + public function startCommands(string $encodedJitConfig): array + { + $name = $this->name(); + $envFile = $this->envFile(); + $environment = base64_encode("ACTIONS_RUNNER_INPUT_JITCONFIG={$encodedJitConfig}\n"); + + $commands = [ + 'mkdir -p '.self::SECRETS_DIRECTORY, + "echo {$environment} | base64 -d | tee {$envFile} > /dev/null", + "chmod 600 {$envFile}", + ]; + + $runnerFlags = "--name {$name} --network {$name} {$this->labelFlags()} {$this->limitFlags()} --env-file {$envFile}"; + if ($this->usesDind()) { + $commands[] = "docker exec {$this->dindName()} chown ".self::RUNNER_UID.':'.self::RUNNER_DOCKER_GID.' /home/runner/_work'; + $runnerFlags .= ' -e DOCKER_HOST=unix:///var/run/docker.sock -e RUNNER_WAIT_FOR_DOCKER_IN_SECONDS=120 '.$this->sharedVolumeFlags(); + } + $commands[] = "docker run -d {$runnerFlags} ".escapeshellarg($this->config->runnerImage()).' /home/runner/run.sh'; + $commands[] = "rm -f {$envFile}"; + + return $commands; + } + + /** + * Removes every Docker resource and the secret file of a runner. Missing resources are ignored. + * + * @return array + */ + public static function cleanupCommands(string $executionUuid): array + { + $name = self::nameFor($executionUuid); + + return [ + "docker rm -f -v {$name} {$name}-dind > /dev/null 2>&1 || true", + "docker volume rm {$name}-sock {$name}-work {$name}-externals > /dev/null 2>&1 || true", + "docker network rm {$name} > /dev/null 2>&1 || true", + 'rm -f '.self::SECRETS_DIRECTORY."/{$executionUuid}.env", + ]; + } + + /** + * Lists runner containers as " " lines. + */ + public static function listCommand(): string + { + return 'docker ps -a --filter label='.self::LABEL.'=true --format \'{{.Label "'.self::EXECUTION_LABEL.'"}} {{.Names}} {{.State}}\''; + } + + private function envFile(): string + { + return self::SECRETS_DIRECTORY."/{$this->execution->uuid}.env"; + } + + private function labelFlags(): string + { + return '--label '.self::LABEL.'=true --label '.self::EXECUTION_LABEL.'='.$this->execution->uuid; + } + + private function limitFlags(): string + { + $flags = ['--pids-limit '.self::PIDS_LIMIT]; + if (filled($this->config->cpu_limit)) { + $flags[] = '--cpus '.escapeshellarg($this->config->cpu_limit); + } + if (filled($this->config->memory_limit)) { + $memory = escapeshellarg($this->config->memory_limit); + $flags[] = "--memory {$memory} --memory-swap {$memory}"; + } + + return implode(' ', $flags); + } + + private function sharedVolumeFlags(): string + { + $name = $this->name(); + + return "-v {$name}-sock:/var/run -v {$name}-work:/home/runner/_work -v {$name}-externals:/home/runner/externals"; + } + + /** + * Pulls the newest version of a moving tag such as "latest", or keeps the local image when the + * registry cannot be reached. + */ + private function pullOrUseLocal(string $image): string + { + $image = escapeshellarg($image); + + return "docker pull {$image} || docker image inspect {$image} > /dev/null"; + } + + private function pullIfMissing(string $image): string + { + $image = escapeshellarg($image); + + return "docker image inspect {$image} > /dev/null 2>&1 || docker pull {$image}"; + } +} diff --git a/config/constants.php b/config/constants.php index dd2a41656b..d3934565f8 100644 --- a/config/constants.php +++ b/config/constants.php @@ -117,6 +117,20 @@ return [ 'connect_networks_interval_seconds' => env('PROXY_CONNECT_NETWORKS_INTERVAL_SECONDS', 3600), ], + 'github_runner' => [ + // GitHub stops sending jobs to runners that are more than 30 days behind, so the default follows + // "latest" and is pulled for every runner. Users can pin a tag in the runner settings. + 'image' => env('GITHUB_RUNNER_IMAGE', 'ghcr.io/actions/actions-runner:latest'), + 'dind_image' => env('GITHUB_RUNNER_DIND_IMAGE', 'docker:29.8-dind'), + 'sysbox' => [ + 'version' => '0.7.1', + 'checksums' => [ + 'amd64' => '9d6d5484f980d0a17f86c492c1262015c2afb66280bdb97215b79fde6a0261c5', + 'arm64' => '04ca894ae0b53f0fa54eaacc173ce40363c9a95ea5450f773716a84ef650a69b', + ], + ], + ], + 'webhooks' => [ 'feedback_discord_webhook' => env('FEEDBACK_DISCORD_WEBHOOK'), 'dev_webhook' => env('SERVEO_URL'), diff --git a/database/migrations/2026_09_28_100000_add_github_runner_fields_to_github_apps_table.php b/database/migrations/2026_09_28_100000_add_github_runner_fields_to_github_apps_table.php new file mode 100644 index 0000000000..539c0169b5 --- /dev/null +++ b/database/migrations/2026_09_28_100000_add_github_runner_fields_to_github_apps_table.php @@ -0,0 +1,28 @@ +string('organization_self_hosted_runners')->nullable()->after('administration'); + $table->string('actions')->nullable()->after('organization_self_hosted_runners'); + $table->json('webhook_events')->nullable()->after('actions'); + $table->unsignedBigInteger('runner_group_id')->nullable()->after('webhook_events'); + }); + } + + public function down(): void + { + Schema::table('github_apps', function (Blueprint $table) { + $table->dropColumn(['organization_self_hosted_runners', 'actions', 'webhook_events', 'runner_group_id']); + }); + } +}; diff --git a/database/migrations/2026_09_28_100001_create_github_runner_configs_table.php b/database/migrations/2026_09_28_100001_create_github_runner_configs_table.php new file mode 100644 index 0000000000..5c26eb65f1 --- /dev/null +++ b/database/migrations/2026_09_28_100001_create_github_runner_configs_table.php @@ -0,0 +1,38 @@ +id(); + $table->string('uuid')->unique(); + $table->foreignId('server_id')->unique()->constrained()->cascadeOnDelete(); + $table->foreignId('github_app_id')->constrained()->cascadeOnDelete(); + $table->json('labels'); + $table->boolean('is_enabled')->default(true); + $table->boolean('is_dedicated')->default(false); + $table->unsignedSmallInteger('max_runners')->default(2); + $table->string('docker_mode')->default('none'); + $table->string('runner_image')->nullable(); + $table->string('cpu_limit')->nullable(); + $table->string('memory_limit')->nullable(); + $table->unsignedInteger('capacity_wait_timeout')->default(60); + $table->unsignedInteger('idle_timeout')->default(10); + $table->unsignedInteger('job_timeout')->default(360); + $table->timestamps(); + }); + } + + public function down(): void + { + Schema::dropIfExists('github_runner_configs'); + } +}; diff --git a/database/migrations/2026_09_28_100002_create_github_runner_executions_table.php b/database/migrations/2026_09_28_100002_create_github_runner_executions_table.php new file mode 100644 index 0000000000..a8f3ad2b7d --- /dev/null +++ b/database/migrations/2026_09_28_100002_create_github_runner_executions_table.php @@ -0,0 +1,51 @@ +id(); + $table->string('uuid')->unique(); + $table->foreignId('github_app_id')->constrained()->cascadeOnDelete(); + $table->foreignId('github_runner_config_id')->nullable()->constrained()->nullOnDelete(); + $table->foreignId('server_id')->nullable()->constrained()->nullOnDelete(); + $table->string('status')->default('queued'); + $table->unsignedBigInteger('trigger_workflow_job_id'); + $table->unsignedBigInteger('workflow_job_id')->nullable(); + $table->string('workflow_job_html_url')->nullable(); + $table->string('workflow_name')->nullable(); + $table->string('job_name')->nullable(); + $table->string('repository_full_name')->nullable(); + $table->json('labels')->nullable(); + $table->string('runner_name')->nullable(); + $table->unsignedBigInteger('runner_id')->nullable(); + $table->string('conclusion')->nullable(); + $table->text('error_message')->nullable(); + $table->unsignedSmallInteger('provision_attempts')->default(0); + $table->timestamp('queued_at')->nullable(); + $table->timestamp('ready_at')->nullable(); + $table->timestamp('started_at')->nullable(); + $table->timestamp('completed_at')->nullable(); + $table->timestamps(); + + $table->unique(['github_app_id', 'trigger_workflow_job_id']); + $table->index(['github_app_id', 'runner_name']); + $table->index(['github_app_id', 'status']); + $table->index(['github_runner_config_id', 'status']); + }); + } + + public function down(): void + { + Schema::dropIfExists('github_runner_executions'); + } +}; diff --git a/resources/views/components/beta-badge.blade.php b/resources/views/components/beta-badge.blade.php new file mode 100644 index 0000000000..56c12a3771 --- /dev/null +++ b/resources/views/components/beta-badge.blade.php @@ -0,0 +1,6 @@ +merge(['class' => 'inline-flex items-center']) }}> + + Beta + + diff --git a/resources/views/components/server/sidebar.blade.php b/resources/views/components/server/sidebar.blade.php index e51d10a0ad..5418012621 100644 --- a/resources/views/components/server/sidebar.blade.php +++ b/resources/views/components/server/sidebar.blade.php @@ -135,6 +135,15 @@ 'group' => 'Operations', 'visible' => $server->isFunctional(), ], + [ + 'label' => 'GitHub Runners', + 'route' => 'server.github-runners', + 'active' => $activeMenu === 'github-runners', + 'icon' => 'play-circle', + 'group' => 'Operations', + 'visible' => ! $server->isLocalhost(), + 'beta' => true, + ], [ 'label' => 'Registries', 'route' => 'server.registries', @@ -274,6 +283,8 @@ @elseif ($menuItem['warning'] ?? false) + @elseif ($menuItem['beta'] ?? false) + @endif @if ($menuItem['active'] && isset($menuItem['children'])) diff --git a/resources/views/livewire/server/github-runner-executions.blade.php b/resources/views/livewire/server/github-runner-executions.blade.php new file mode 100644 index 0000000000..e89499303f --- /dev/null +++ b/resources/views/livewire/server/github-runner-executions.blade.php @@ -0,0 +1,55 @@ +
+ @forelse ($this->executions as $execution) +
+ +
+

+ {{ $execution->repository_full_name ?? 'Unknown repository' }} + @if ($execution->workflow_name || $execution->job_name) + + · {{ collect([$execution->workflow_name, $execution->job_name])->filter()->implode(' / ') }} + + @endif +

+

+ {{ $execution->created_at->diffForHumans() }} + @if ($execution->duration()) + · ran {{ $execution->duration() }} + @endif + @if ($execution->conclusion) + · {{ $execution->conclusion }} + @endif + @if ($execution->error_message) + · {{ $execution->error_message }} + @endif +

+
+
+ @if ($execution->workflowJobUrl()) + + Open + + + @endif + @if ($execution->isActive()) + @can('update', $server) + + Cancel + + @endcan + @endif +
+
+ @empty + + @endforelse +
diff --git a/resources/views/livewire/server/github-runners.blade.php b/resources/views/livewire/server/github-runners.blade.php new file mode 100644 index 0000000000..c98cc92477 --- /dev/null +++ b/resources/views/livewire/server/github-runners.blade.php @@ -0,0 +1,187 @@ +
+ + {{ data_get_str($server, 'name')->limit(10) }} > GitHub Runners | Coolify + + + + +
+ + +
+ @if (!$server->isBuildServer()) + + + + + General settings + + + + + + @elseif ($this->githubApps->isEmpty()) + + Runners are registered at organization level. Add a GitHub App that belongs to an organization + under Sources, and select "Run workflow jobs on build servers" when you register it. + + @elseif (! $this->config?->is_enabled) + + + + + + + + Enable runners + + + + + @else +
+ + + + + @can('update', $server) + + Disable runners + + @endcan + + +
+ + In the privileged Docker mode, a malicious workflow can take control of this server. + Use Isolated Docker (Sysbox) or No Docker to keep jobs away from the server. Coolify + limits the runner group to private repositories. + +
+ + @if ($this->selectedApp && $this->selectedApp->missingRunnerRequirements() !== []) + + Update the App on GitHub, then click Refetch on the App's Permissions page. Missing: +
    + @foreach ($this->selectedApp->missingRunnerRequirements() as $requirement) +
  • {{ $requirement }}
    • + @endforeach +
+ + @endif + +
+ + + +
+ + + +
+ + + + + +
+
+ @if ($isSysboxInstalled === true) + + Jobs get a private Docker daemon in an unprivileged container. + + @elseif ($isSysboxInstalled === false) + + Isolated Docker needs the Sysbox runtime. Coolify installs Sysbox + {{ config('constants.github_runner.sysbox.version') }} from the official package + (Debian and Ubuntu, kernel 5.12 or newer). Docker reloads its configuration, and + running containers are not restarted. + @can('update', $server) +
+ + Install Sysbox + +
+ @endcan + + @else + + Coolify checks if Sysbox is installed on this server. + + @endif +
+ + + +
+ + + +
+ + + @can('update', $server) + + Install Sysbox + + + + + @endcan + @endif + + @if ($server->isBuildServer() && $this->config?->is_enabled) + + + + @endif +
+
+
diff --git a/resources/views/livewire/source/github/change.blade.php b/resources/views/livewire/source/github/change.blade.php index 8f0a8cbbee..38cef644d9 100644 --- a/resources/views/livewire/source/github/change.blade.php +++ b/resources/views/livewire/source/github/change.blade.php @@ -260,19 +260,19 @@ @endphp
-
- -
+
@if (!isCloud() || isDev()) + +
-
- + -
+
@@ -352,7 +362,7 @@