mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-14 08:42:38 -04:00
419 lines
16 KiB
PHP
419 lines
16 KiB
PHP
<?php
|
|
|
|
namespace App\Actions\Development;
|
|
|
|
use Illuminate\Support\Facades\File;
|
|
use Illuminate\Support\Facades\Process;
|
|
use InvalidArgumentException;
|
|
use Lorisleiva\Actions\Concerns\AsAction;
|
|
use RuntimeException;
|
|
|
|
class StartDevelopmentQemuVm
|
|
{
|
|
use AsAction;
|
|
|
|
/**
|
|
* Start the VM of a profile, creating it only when its libvirt domain does not exist yet.
|
|
* With $fresh, only this profile's VM is destroyed, undefined, and deleted before it is created again.
|
|
* With $gpu, the NVIDIA GPU of the host is passed through to the VM. An existing VM is stopped to add it.
|
|
*/
|
|
public function handle(string $profileName, bool $fresh = false, bool $gpu = false): void
|
|
{
|
|
$this->ensureDevelopmentEnvironment();
|
|
$profile = config("development-qemu.profiles.{$profileName}");
|
|
|
|
if (! is_array($profile)) {
|
|
throw new InvalidArgumentException("Unknown development QEMU profile: {$profileName}");
|
|
}
|
|
|
|
ConfigureDevelopmentQemuHost::run();
|
|
$this->configureDhcpReservation($profile);
|
|
|
|
if ($fresh) {
|
|
$this->destroyVm($profile['domain']);
|
|
}
|
|
|
|
if (! $fresh && $this->domainExists($profile['domain'])) {
|
|
if ($gpu) {
|
|
$this->attachGpu($profile['domain']);
|
|
}
|
|
$this->startExistingVm($profile['domain']);
|
|
} else {
|
|
$this->createPreparedVm($profile, $gpu);
|
|
}
|
|
|
|
ConfigureDevelopmentQemuHost::run();
|
|
$this->waitForSsh($profile['ip']);
|
|
}
|
|
|
|
/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */
|
|
private function createPreparedVm(array $profile, bool $gpu): void
|
|
{
|
|
$preparedImage = $this->preparedImage($profile['template']);
|
|
|
|
if (! File::exists($preparedImage)) {
|
|
$this->createVm($profile, false);
|
|
$this->waitForShutOff($profile['domain'], 900);
|
|
$this->runOrFail('virsh undefine '.escapeshellarg($profile['domain']));
|
|
$this->runOrFail('mv '.escapeshellarg($this->vmDisk($profile['domain'])).' '.escapeshellarg($preparedImage));
|
|
if (File::exists($preparedImage)) {
|
|
File::chmod($preparedImage, 0644);
|
|
}
|
|
}
|
|
|
|
$this->createVm($profile, true, $gpu);
|
|
}
|
|
|
|
private function domainExists(string $domain): bool
|
|
{
|
|
return Process::run('virsh dominfo '.escapeshellarg($domain))->successful();
|
|
}
|
|
|
|
private function startExistingVm(string $domain): void
|
|
{
|
|
$state = trim(Process::run('virsh domstate '.escapeshellarg($domain))->output());
|
|
|
|
if ($state === 'running') {
|
|
return;
|
|
}
|
|
|
|
if ($state === 'paused') {
|
|
$this->runOrFail('virsh resume '.escapeshellarg($domain));
|
|
|
|
return;
|
|
}
|
|
|
|
$hasGpu = $this->hasGpu($domain);
|
|
$this->applyMemory($domain, $hasGpu);
|
|
|
|
if ($hasGpu) {
|
|
$this->releaseHostGpu();
|
|
}
|
|
|
|
$this->runOrFail('virsh start '.escapeshellarg($domain));
|
|
}
|
|
|
|
/**
|
|
* Sets the configured memory on a stopped VM, so VMs created with a larger size shrink on their next start.
|
|
*/
|
|
private function applyMemory(string $domain, bool $hasGpu): void
|
|
{
|
|
$memory = (int) config($hasGpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory').'M';
|
|
$this->runOrFail(sprintf('virsh setmaxmem %s %s --config', escapeshellarg($domain), $memory));
|
|
$this->runOrFail(sprintf('virsh setmem %s %s --config', escapeshellarg($domain), $memory));
|
|
}
|
|
|
|
private function hasGpu(string $domain): bool
|
|
{
|
|
return str_contains(Process::run('virsh dumpxml --inactive '.escapeshellarg($domain))->output(), '<hostdev ');
|
|
}
|
|
|
|
/**
|
|
* Adds the GPU to the persistent definition of an existing VM. A running VM is shut down first.
|
|
*/
|
|
private function attachGpu(string $domain): void
|
|
{
|
|
if ($this->hasGpu($domain)) {
|
|
return;
|
|
}
|
|
|
|
if (trim(Process::run('virsh domstate '.escapeshellarg($domain))->output()) !== 'shut off') {
|
|
$this->runOrFail('virsh shutdown '.escapeshellarg($domain));
|
|
$this->waitForShutOff($domain, 120);
|
|
}
|
|
|
|
foreach ($this->hostGpuDevices() as $device) {
|
|
$this->runOrFail(sprintf('virt-xml %s --add-device --hostdev %s', escapeshellarg($domain), escapeshellarg($device)));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns the libvirt node device names of every PCI function of the first NVIDIA display controller,
|
|
* for example pci_0000_01_00_0 (GPU) and pci_0000_01_00_1 (its audio device). All functions must move together.
|
|
*
|
|
* @return array<int, string>
|
|
*/
|
|
private function hostGpuDevices(): array
|
|
{
|
|
$devices = collect(explode("\n", trim(Process::run('lspci -Dnmm -d 10de:')->output())))
|
|
->filter()
|
|
->map(fn (string $line): array => str_getcsv($line, ' '));
|
|
$gpu = $devices->first(fn (array $device): bool => in_array($device[1] ?? null, ['0300', '0302'], true));
|
|
|
|
if ($gpu === null) {
|
|
throw new RuntimeException('No NVIDIA GPU found on this host.');
|
|
}
|
|
|
|
$slot = substr($gpu[0], 0, strrpos($gpu[0], '.'));
|
|
|
|
return $devices
|
|
->filter(fn (array $device): bool => str_starts_with($device[0], "{$slot}."))
|
|
->map(fn (array $device): string => 'pci_'.str_replace([':', '.'], '_', $device[0]))
|
|
->values()
|
|
->all();
|
|
}
|
|
|
|
/**
|
|
* nvidia-persistenced keeps the GPU open, which blocks libvirt from moving it to vfio-pci.
|
|
* Other users of the GPU (for example a desktop session) must stop first: if libvirt unbinds the
|
|
* nvidia driver while they hold the GPU open, the GPU stays unusable until the host reboots.
|
|
*/
|
|
private function releaseHostGpu(): void
|
|
{
|
|
Process::run('systemctl stop nvidia-persistenced');
|
|
|
|
$users = Process::run('fuser -v /dev/nvidia[0-9]* 2>&1');
|
|
|
|
if ($users->successful() && trim($users->output()) !== '') {
|
|
throw new RuntimeException("These host processes still use the NVIDIA GPU, so it cannot move to the VM. Stop them (for example the display manager), or remove the GPU from the VM with --fresh and without --gpu:\n".trim($users->output()));
|
|
}
|
|
}
|
|
|
|
private function destroyVm(string $domain): void
|
|
{
|
|
Process::run('virsh destroy '.escapeshellarg($domain));
|
|
Process::run('virsh undefine '.escapeshellarg($domain));
|
|
$this->deleteVmData($domain);
|
|
}
|
|
|
|
/** @param array{domain: string, template: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */
|
|
private function createVm(array $profile, bool $prepared, bool $gpu = false): void
|
|
{
|
|
$directory = config('development-qemu.storage_path');
|
|
File::ensureDirectoryExists($directory);
|
|
File::chmod($directory, 0777);
|
|
$this->moveLegacyFiles($directory);
|
|
$baseImage = $prepared ? $this->preparedImage($profile['template']) : "{$directory}/{$profile['image']}";
|
|
$disk = $this->vmDisk($profile['domain']);
|
|
$userData = "{$directory}/{$profile['domain']}-user-data.yaml";
|
|
$metaData = "{$directory}/{$profile['domain']}-meta-data.yaml";
|
|
$networkConfig = "{$directory}/{$profile['domain']}-network.yaml";
|
|
$seedImage = "{$directory}/{$profile['domain']}-seed.iso";
|
|
|
|
if (! $prepared && ! File::exists($baseImage)) {
|
|
$this->runOrFail(sprintf(
|
|
'curl --fail --location --output %s %s',
|
|
escapeshellarg($baseImage),
|
|
escapeshellarg($profile['image_url']),
|
|
));
|
|
}
|
|
|
|
if (! File::exists($disk)) {
|
|
$this->runOrFail(sprintf(
|
|
'qemu-img create -f qcow2 -F qcow2 -b %s %s %s',
|
|
escapeshellarg($baseImage),
|
|
escapeshellarg($disk),
|
|
escapeshellarg(config('development-qemu.disk_size')),
|
|
));
|
|
}
|
|
|
|
if (File::exists($baseImage)) {
|
|
File::chmod($baseImage, 0644);
|
|
}
|
|
|
|
if (File::exists($disk)) {
|
|
File::chmod($disk, 0666);
|
|
}
|
|
|
|
if (! $prepared) {
|
|
File::put($userData, $this->userData($profile));
|
|
File::put($metaData, "instance-id: {$profile['domain']}\nlocal-hostname: {$profile['template']}\n");
|
|
File::put($networkConfig, $this->networkConfig($profile));
|
|
$this->runOrFail(sprintf(
|
|
'xorriso -as mkisofs -V cidata -graft-points -o %s %s %s %s',
|
|
escapeshellarg($seedImage),
|
|
escapeshellarg('user-data='.$userData),
|
|
escapeshellarg('meta-data='.$metaData),
|
|
escapeshellarg('network-config='.$networkConfig),
|
|
));
|
|
}
|
|
|
|
$seedDisk = $prepared ? '' : ' --disk path='.escapeshellarg($seedImage).',format=raw,bus=virtio,readonly=on';
|
|
// Instance networks are isolated, so VMs of different instances may share the profile MAC.
|
|
$macCheck = config('development-qemu.instance') === null ? '' : ' --check mac_in_use=off';
|
|
$hostDevices = '';
|
|
|
|
if ($gpu) {
|
|
$hostDevices = collect($this->hostGpuDevices())->map(fn (string $device): string => ' --hostdev '.escapeshellarg($device))->implode('');
|
|
$this->releaseHostGpu();
|
|
}
|
|
|
|
$this->runOrFail(sprintf(
|
|
'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole'.$macCheck.$hostDevices,
|
|
escapeshellarg($profile['domain']),
|
|
config($gpu ? 'development-qemu.gpu_memory' : 'development-qemu.memory'),
|
|
config('development-qemu.vcpus'),
|
|
escapeshellarg($profile['os_variant']),
|
|
escapeshellarg($disk),
|
|
$seedDisk,
|
|
escapeshellarg(config('development-qemu.libvirt_network')),
|
|
escapeshellarg($profile['mac']),
|
|
));
|
|
}
|
|
|
|
private function vmDisk(string $domain): string
|
|
{
|
|
return config('development-qemu.storage_path')."/{$domain}.qcow2";
|
|
}
|
|
|
|
/**
|
|
* Prepared images are keyed by the instance-independent template name so every dev instance reuses them.
|
|
*/
|
|
private function preparedImage(string $template): string
|
|
{
|
|
return config('development-qemu.storage_path')."/{$template}-prepared.qcow2";
|
|
}
|
|
|
|
private function waitForShutOff(string $domain, int $timeoutSeconds): void
|
|
{
|
|
$check = 'while [ "$(virsh domstate '.escapeshellarg($domain).')" != "shut off" ]; do sleep 2; done';
|
|
$this->runOrFail("timeout {$timeoutSeconds} bash -c ".escapeshellarg($check));
|
|
}
|
|
|
|
private function moveLegacyFiles(string $directory): void
|
|
{
|
|
$legacyDirectory = storage_path('app/development-qemu');
|
|
|
|
if ($legacyDirectory === $directory || ! File::isDirectory($legacyDirectory)) {
|
|
return;
|
|
}
|
|
|
|
foreach (File::files($legacyDirectory) as $file) {
|
|
$destination = "{$directory}/{$file->getFilename()}";
|
|
|
|
if (! File::exists($destination)) {
|
|
File::move($file->getPathname(), $destination);
|
|
}
|
|
}
|
|
}
|
|
|
|
private function deleteVmData(string $domain): void
|
|
{
|
|
$directory = config('development-qemu.storage_path');
|
|
File::delete([
|
|
"{$directory}/{$domain}.qcow2",
|
|
"{$directory}/{$domain}-user-data.yaml",
|
|
"{$directory}/{$domain}-meta-data.yaml",
|
|
"{$directory}/{$domain}-network.yaml",
|
|
"{$directory}/{$domain}-seed.iso",
|
|
]);
|
|
}
|
|
|
|
/** @param array{user: string, provisioner: string} $profile */
|
|
private function userData(array $profile): string
|
|
{
|
|
$publicKey = config('development-qemu.public_key');
|
|
$adminGroup = $profile['provisioner'] === 'apt' ? 'sudo' : 'wheel';
|
|
$sudo = $profile['user'] === 'root' ? '' : " groups: [{$adminGroup}]\n sudo: ALL=(ALL) NOPASSWD:ALL\n";
|
|
$shell = $profile['provisioner'] === 'apk' ? '/bin/ash' : '/bin/bash';
|
|
$password = $profile['provisioner'] === 'apk'
|
|
? ' passwd: $6$dd2d71373a57c9ac$8.GUqZYlL/QqUmpUuupWfTuKQjNKT7kO31K5cp7OIY5SbBamlAVkJnBDYsIVimMaBrUtYfFjX3u6hzts3nKaD.'."\n lock_passwd: false"
|
|
: ' lock_passwd: true';
|
|
|
|
[$packages, $installDocker, $startDocker] = match ($profile['provisioner']) {
|
|
'apk' => [" - docker\n - docker-cli-buildx\n - docker-cli-compose\n - sudo", '', 'rc-update add cgroups boot && service cgroups start && rc-update add docker default && service docker start'],
|
|
default => [" - curl\n - sudo", "curl -fsSL https://get.docker.com -o /tmp/get-docker.sh\n sh /tmp/get-docker.sh\n ", 'systemctl enable --now docker'],
|
|
};
|
|
$addUserToDockerGroup = $profile['user'] === 'root' ? '' : ($profile['provisioner'] === 'apk'
|
|
? "addgroup {$profile['user']} docker"
|
|
: "usermod -aG docker {$profile['user']}");
|
|
|
|
return <<<YAML
|
|
#cloud-config
|
|
disable_root: false
|
|
users:
|
|
- name: {$profile['user']}
|
|
{$sudo} shell: {$shell}
|
|
{$password}
|
|
ssh_authorized_keys:
|
|
- {$publicKey}
|
|
package_update: true
|
|
packages:
|
|
{$packages}
|
|
runcmd:
|
|
- |
|
|
set -eu
|
|
{$installDocker}{$startDocker}
|
|
{$addUserToDockerGroup}
|
|
attempt=0
|
|
until docker info >/dev/null 2>&1; do
|
|
attempt=\$((attempt + 1))
|
|
[ "\$attempt" -lt 60 ] || exit 1
|
|
sleep 2
|
|
done
|
|
docker version
|
|
docker info
|
|
docker compose version
|
|
docker buildx version
|
|
touch /etc/cloud/cloud-init.disabled
|
|
poweroff
|
|
YAML;
|
|
}
|
|
|
|
/** @param array{mac: string} $profile */
|
|
private function networkConfig(array $profile): string
|
|
{
|
|
return <<<YAML
|
|
version: 2
|
|
ethernets:
|
|
default:
|
|
match:
|
|
macaddress: "{$profile['mac']}"
|
|
dhcp4: true
|
|
YAML;
|
|
}
|
|
|
|
/** @param array{template: string, ip: string, mac: string} $profile */
|
|
private function configureDhcpReservation(array $profile): void
|
|
{
|
|
$network = escapeshellarg(config('development-qemu.libvirt_network'));
|
|
$networkXml = Process::run("virsh net-dumpxml {$network}");
|
|
|
|
if ($networkXml->failed()) {
|
|
throw new RuntimeException(trim($networkXml->errorOutput()) ?: 'Unable to inspect the libvirt network.');
|
|
}
|
|
|
|
if (str_contains($networkXml->output(), $profile['mac']) && str_contains($networkXml->output(), $profile['ip'])) {
|
|
return;
|
|
}
|
|
|
|
$host = sprintf("<host mac='%s' name='%s' ip='%s'/>", $profile['mac'], $profile['template'], $profile['ip']);
|
|
$this->runOrFail("virsh net-update {$network} add-last ip-dhcp-host ".escapeshellarg($host).' --live --config');
|
|
}
|
|
|
|
private function waitForSsh(string $ip): void
|
|
{
|
|
$container = escapeshellarg(config('development-qemu.coolify_container'));
|
|
$probe = <<<'PHP'
|
|
$deadline = time() + 120;
|
|
do {
|
|
$socket = @fsockopen($argv[1], 22, $errorCode, $errorMessage, 1);
|
|
if (is_resource($socket)) {
|
|
fclose($socket);
|
|
exit(0);
|
|
}
|
|
sleep(1);
|
|
} while (time() < $deadline);
|
|
exit(1);
|
|
PHP;
|
|
$this->runOrFail("docker exec {$container} php -r ".escapeshellarg($probe).' '.escapeshellarg($ip));
|
|
}
|
|
|
|
private function runOrFail(string $command): void
|
|
{
|
|
$result = Process::forever()->run($command);
|
|
|
|
if ($result->failed()) {
|
|
throw new RuntimeException(trim($result->errorOutput()) ?: "Command failed: {$command}");
|
|
}
|
|
|
|
}
|
|
|
|
private function ensureDevelopmentEnvironment(): void
|
|
{
|
|
if (! in_array(config('app.env'), ['local', 'development', 'dev'], true)) {
|
|
throw new RuntimeException('QEMU VMs may only be managed in development environments.');
|
|
}
|
|
}
|
|
}
|