From 5f6fb5dcc209db5c226c28065b3b6abd2650e16a Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:45:29 +0200 Subject: [PATCH] feat(dev): run localhost dev server in a Lima VM on macOS Add a Lima backend to scripts/dev. Auto mode picks it on macOS when limactl (Lima >= 2.0) is installed, and COOLIFY_DEV_SERVER_BACKEND=lima forces it. Lima VMs are named coolify-dev--. Coolify connects to them through Lima's forwarded SSH port on host.docker.internal, and only guest ports 80/443 are forwarded to the host. dev:qemu:seed now takes --ip and --port options, so the seeded server can use a forwarded SSH address instead of the profile IP. Add tests for the Lima start flow and for seeding with an overridden address, and document the macOS setup in AGENTS.md. --- AGENTS.md | 6 +- .../Development/SeedDevelopmentQemuServer.php | 9 +- .../SeedDevelopmentQemuServerCommand.php | 4 +- scripts/dev | 151 +++++++++++++++++- tests/Feature/DevScriptTest.php | 75 +++++++++ tests/Feature/DevelopmentQemuVmTest.php | 18 +++ 6 files changed, 248 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e66b6737b1..b1fb6b0f3d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,8 +15,8 @@ For UI/UX design specifications, principles, and visual standards, consult the l Docker Compose-based dev setup with services: coolify (app, which also runs Reverb WebSockets and the terminal server), postgres, redis, vite, testing-host, mailpit, minio. ```bash -# One dev instance per git branch (containers, volumes, KVM VMs named after the branch) -./scripts/dev start [qemu-profile] # KVM VM as localhost when /dev/kvm + root/sudo, else testing-host +# One dev instance per git branch (containers, volumes, VMs named after the branch) +./scripts/dev start [qemu-profile] # localhost VM: KVM (Linux, /dev/kvm + root/sudo) or Lima (macOS, limactl), else testing-host ./scripts/dev stop # stop containers and VMs; data is kept for the next start ./scripts/dev run # start + follow logs, stop on exit (Jean run script) ./scripts/dev urls # all instances, URLs, ports, and checkouts @@ -29,7 +29,7 @@ Docker Compose-based dev setup with services: coolify (app, which also runs Reve docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d ``` -The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-` (`10.221..0/24`) and VMs `coolify-dev---`; VMs are reused, use `php artisan dev:qemu --fresh` to rebuild one. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=` to run another instance from the same checkout. +The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-` (`10.221..0/24`) and VMs `coolify-dev---`; VMs are reused, use `php artisan dev:qemu --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev--` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f `. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=` to run another instance from the same checkout. ## Testing the Self-Hosted Upgrade Process diff --git a/app/Actions/Development/SeedDevelopmentQemuServer.php b/app/Actions/Development/SeedDevelopmentQemuServer.php index 191d045360..ad385f878e 100644 --- a/app/Actions/Development/SeedDevelopmentQemuServer.php +++ b/app/Actions/Development/SeedDevelopmentQemuServer.php @@ -12,7 +12,10 @@ class SeedDevelopmentQemuServer { use AsAction; - public function handle(string $profileName, bool $removeOtherServers = true, bool $asLocalhost = false): Server + /** + * $ip and $port override the profile address, for example for a Lima VM whose SSH port is forwarded to the host. + */ + public function handle(string $profileName, bool $removeOtherServers = true, bool $asLocalhost = false, ?string $ip = null, int $port = 22): Server { $this->ensureDevelopmentEnvironment(); $profile = config("development-qemu.profiles.{$profileName}"); @@ -41,8 +44,8 @@ class SeedDevelopmentQemuServer $server->fill([ 'name' => $asLocalhost ? 'localhost' : $profile['name'], 'description' => 'Development QEMU virtual machine', - 'ip' => $profile['ip'], - 'port' => 22, + 'ip' => $ip ?? $profile['ip'], + 'port' => $port, 'user' => $profile['user'], 'team_id' => 0, 'private_key_id' => $privateKey->id, diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php index ce68b904f9..082ad83ae2 100644 --- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php +++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php @@ -7,7 +7,7 @@ use Illuminate\Console\Command; class SeedDevelopmentQemuServerCommand extends Command { - protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)}'; + protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port}'; protected $description = 'Seed one development QEMU server in the Coolify database'; @@ -19,7 +19,7 @@ class SeedDevelopmentQemuServerCommand extends Command return self::FAILURE; } - $server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost')); + $server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost'), $this->option('ip') ?: null, (int) $this->option('port')); $this->info("Seeded {$server->name} at {$server->ip}."); return self::SUCCESS; diff --git a/scripts/dev b/scripts/dev index 1f47900b54..24063dbf8d 100755 --- a/scripts/dev +++ b/scripts/dev @@ -2,7 +2,7 @@ # Manage the development instance of the current git branch. # # Usage: -# ./scripts/dev start [qemu-profile] # start this branch's instance (KVM VM or testing-host) +# ./scripts/dev start [qemu-profile] # start this branch's instance (KVM VM, Lima VM, or testing-host) # ./scripts/dev stop [qemu-profile] # stop containers and VMs; data is kept # ./scripts/dev run [qemu-profile] # start, follow logs, stop on exit (Jean) # ./scripts/dev urls # all instances, ports, and state @@ -10,15 +10,19 @@ # ./scripts/dev destroy # delete containers, volumes, VMs, and the port slot # ./scripts/dev teardown # destroy this worktree's instance if it exists (Jean) # -# Each branch is one instance: Compose project, container, volumes, and KVM VMs are +# Each branch is one instance: Compose project, container, volumes, and VMs are # named after it, so stop + start keeps the same data. Detached HEAD uses the # checkout directory name. Set COOLIFY_DEV_INSTANCE= to override. # +# Server backend (auto): KVM on Linux with /dev/kvm and root/sudo, Lima on macOS when +# limactl (Lima >= 2.0) is installed, else the testing-host container. Lima VMs are +# named coolify-dev--; rebuild one with: limactl delete -f +# # Ports: the main checkout uses 8000 (Reverb 6001, terminal 6002, db 5432, redis 6379, # vite 5173). Worktrees use 20000 + slot*10: app +0, Reverb +1, terminal +2, db +3, # redis +4, vite +5, mailpit +6/+7, minio +8/+9. # -# Env: COOLIFY_DEV_SERVER_BACKEND=auto|testing-host, COOLIFY_DEV_KVM_PROFILE= +# Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE= # set -euo pipefail @@ -34,7 +38,7 @@ QEMU_STORAGE="${DEVELOPMENT_QEMU_STORAGE_PATH:-/var/lib/libvirt/images/coolify-d LEGACY_CONTAINERS=(coolify coolify-db coolify-redis coolify-vite coolify-mail coolify-minio coolify-minio-init coolify-testing-host) usage() { - sed -n '2,22p' "$0" | sed 's/^# \?//' + sed -n '2,26p' "$0" | sed 's/^# \?//' } normalize_name() { @@ -170,15 +174,22 @@ compose() { select_backend() { backend="${COOLIFY_DEV_SERVER_BACKEND:-auto}" - if [[ "$backend" != "auto" && "$backend" != "testing-host" ]]; then - echo 'COOLIFY_DEV_SERVER_BACKEND must be auto or testing-host.' >&2 + if [[ "$backend" != "auto" && "$backend" != "lima" && "$backend" != "testing-host" ]]; then + echo 'COOLIFY_DEV_SERVER_BACKEND must be auto, lima, or testing-host.' >&2 exit 2 fi privileged=() if [[ $EUID -ne 0 ]]; then privileged=(sudo -n -E) fi - if [[ "$backend" == "auto" && -c /dev/kvm && -r /dev/kvm && -w /dev/kvm ]] && + if [[ "$backend" == "lima" ]]; then + command -v limactl >/dev/null || { + echo 'COOLIFY_DEV_SERVER_BACKEND=lima needs limactl (brew install lima).' >&2 + exit 2 + } + elif [[ "$backend" == "auto" && "$(uname -s)" == Darwin ]] && command -v limactl >/dev/null; then + backend=lima + elif [[ "$backend" == "auto" && -c /dev/kvm && -r /dev/kvm && -w /dev/kvm ]] && { [[ $EUID -eq 0 ]] || { command -v sudo >/dev/null && sudo -n -E true; }; }; then backend=kvm else @@ -247,6 +258,120 @@ vm_domains() { grep -E "^coolify-dev-${instance}--" || true } +# Lima VMs of an instance are named coolify-dev--; Lima names allow no "--". +lima_vm() { echo "coolify-dev-$(env_value "$instance" DEVELOPMENT_QEMU_SLOT)-$1"; } + +lima_vms() { + command -v limactl >/dev/null || return 0 + limactl list --format '{{.Name}} {{.Status}}' 2>/dev/null | + awk -v prefix="$(lima_vm '')" -v status="${1:-}" \ + 'index($1, prefix) == 1 && (status == "" || $2 == status) { print $1 }' +} + +# Same users, SSH key, and Docker setup as the KVM profiles in config/development-qemu.php. +# Provision scripts run on every boot, so each step is idempotent. The host directories +# are not mounted, and only guest ports 80 and 443 are forwarded to the host. +lima_config() { + local image user key + case "$1" in + ubuntu-root | ubuntu-non-root) image=ubuntu-24.04 ;; + debian-root | debian-non-root) image=debian-12 ;; + centos-root | centos-non-root) image=centos-stream-9 ;; + alpine-root | alpine-non-root) image=alpine-3.23 ;; + *) + echo "Unknown profile '$1'. Use a profile from config/development-qemu.php." >&2 + return 1 + ;; + esac + user=root + [[ "$1" == *-non-root ]] && user=coolify + key="$(sed -n "s/^ *'public_key' => '\(.*\)',\$/\1/p" config/development-qemu.php 2>/dev/null)" + if [[ -z "$key" ]]; then + echo 'Cannot read public_key from config/development-qemu.php.' >&2 + return 1 + fi + cat </dev/null; then + if command -v apk >/dev/null; then + apk add docker docker-cli-buildx docker-cli-compose sudo + else + curl -fsSL https://get.docker.com | sh + fi + fi + if command -v systemctl >/dev/null; then + systemctl enable --now docker + else + # OpenRC can report an error for a start during boot; the runlevels start both services. + rc-update add cgroups boot + rc-update add docker default + rc-service cgroups start || true + rc-service docker start || true + fi + home=/root + if [ "\$user" != root ]; then + # A fixed high UID: Lima creates its own user later with the host UID (Alpine reboots first). + if ! id "\$user" >/dev/null 2>&1; then + if command -v useradd >/dev/null; then + useradd -m -u 2000 -s /bin/bash "\$user" + else + adduser -D -u 2000 -s /bin/ash "\$user" + fi + echo "\$user:*" | chpasswd -e + fi + echo "\$user ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/90-coolify-dev + chmod 440 /etc/sudoers.d/90-coolify-dev + if command -v usermod >/dev/null; then usermod -aG docker "\$user"; else addgroup "\$user" docker; fi + home="/home/\$user" + fi + mkdir -p "\$home/.ssh" + grep -qxF "\$key" "\$home/.ssh/authorized_keys" 2>/dev/null || echo "\$key" >>"\$home/.ssh/authorized_keys" + chmod 700 "\$home/.ssh" + chmod 600 "\$home/.ssh/authorized_keys" + chown -R "\$user:" "\$home/.ssh" +YAML +} + +# Coolify connects over the SSH port that Lima forwards to the host loopback, which +# host.docker.internal reaches in Docker Desktop, OrbStack, and Colima. +start_lima_vm() { + local vm config port + vm="$(lima_vm "$profile")" + if ! limactl list --quiet 2>/dev/null | grep -qxF "$vm"; then + config="$(lima_config "$profile")" + limactl create --tty=false --name="$vm" - <<<"$config" + fi + limactl start --tty=false "$vm" + port="$(limactl list --format '{{.SSHLocalPort}}' "$vm")" + compose "$instance" exec -T coolify php artisan dev:qemu:seed "$profile" --as-localhost \ + --ip=host.docker.internal --port="$port" +} + start_backend() { stop_other_checkout_instances echo "==> Instance ${instance} → $(env_value "$instance" APP_URL) (backend: ${backend})" @@ -266,6 +391,9 @@ start_backend() { DEVELOPMENT_QEMU_DOCKER_NETWORK="$(project_name "$instance")" \ DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$(project_name "$instance")" \ "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost + elif [[ "$backend" == "lima" ]]; then + echo "Using Lima profile: $profile" + start_lima_vm else compose "$instance" exec -T coolify php artisan db:seed --class=ServerSeeder --force fi @@ -279,6 +407,12 @@ stop_backend() { "${privileged[@]}" virsh --connect qemu:///system shutdown "$domain" fi done + elif [[ "$backend" == "lima" ]]; then + for domain in $(lima_vms Running); do + if [[ -z "$profile_arg" || "$domain" == "$(lima_vm "$profile_arg")" ]]; then + limactl stop "$domain" + fi + done fi compose "$instance" stop } @@ -308,6 +442,9 @@ cmd_destroy() { tailscale_serve off subnet="$(docker network inspect "$(project_name "$name")" --format '{{(index .IPAM.Config 0).Subnet}}' 2>/dev/null || true)" compose "$name" --profile '*' down --remove-orphans --volumes + for domain in $(lima_vms); do + limactl delete --force "$domain" + done if command -v virsh >/dev/null; then for domain in $(vm_domains --all); do "${privileged[@]}" virsh --connect qemu:///system destroy "$domain" >/dev/null 2>&1 || true diff --git a/tests/Feature/DevScriptTest.php b/tests/Feature/DevScriptTest.php index dcc250cc1d..1864c012f9 100644 --- a/tests/Feature/DevScriptTest.php +++ b/tests/Feature/DevScriptTest.php @@ -14,6 +14,8 @@ beforeEach(function () { copy(base_path('scripts/dev'), $main.'/scripts/dev'); chmod($main.'/scripts/dev', 0755); copy(base_path('docker-compose.dev-multi.yml'), $main.'/docker-compose.dev-multi.yml'); + mkdir($main.'/config'); + copy(base_path('config/development-qemu.php'), $main.'/config/development-qemu.php'); file_put_contents($main.'/.gitignore', ".env\n.dev-instances/\n"); $git = Process::fromShellCommandline(implode(' && ', [ @@ -45,6 +47,23 @@ BASH; file_put_contents($this->devRoot.'/bin/'.$binary, $fake); chmod($this->devRoot.'/bin/'.$binary, 0755); } + + $this->limaConfig = $this->devRoot.'/lima.yaml'; + $this->fakeLima = function (): void { + file_put_contents($this->devRoot.'/bin/limactl', <<<'BASH' +#!/usr/bin/env bash +printf 'limactl %s\n' "$*" >> "$DEV_TEST_LOG" +case "$*" in + create*) cat > "$DEV_TEST_LIMA_CONFIG" ;; + *SSHLocalPort*) echo 53022 ;; + *'{{.Name}} {{.Status}}'*) printf '%s\n' 'coolify-dev-2-debian-root Running' 'coolify-dev-2-ubuntu-root Stopped' 'coolify-dev-21-debian-root Running' 'other Running' ;; +esac +exit 0 +BASH); + file_put_contents($this->devRoot.'/bin/uname', "#!/usr/bin/env bash\necho Darwin\n"); + chmod($this->devRoot.'/bin/limactl', 0755); + chmod($this->devRoot.'/bin/uname', 0755); + }; }); afterEach(function () { @@ -58,6 +77,7 @@ function runDevScript(string $directory, array $arguments, array $env = []): Pro 'DEV_TEST_LOG' => test()->devLog, 'DEVELOPMENT_QEMU_STORAGE_PATH' => test()->qemuStorage, 'COOLIFY_DEV_SERVER_BACKEND' => 'testing-host', + 'DEV_TEST_LIMA_CONFIG' => test()->limaConfig, ...$env, ]); $process->run(); @@ -225,6 +245,61 @@ BASH); ->not->toContain('--profile testing-host up'); }); +it('starts a lima vm as localhost on macos and seeds its forwarded ssh port', function () { + ($this->fakeLima)(); + runDevScript('main', ['start']); + file_put_contents($this->devLog, ''); + + $process = runDevScript('wt', ['start', 'debian-non-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto']); + + expect($process->isSuccessful())->toBeTrue($process->getErrorOutput()) + ->and(devScriptLog()) + ->toContain('limactl create --tty=false --name=coolify-dev-2-debian-non-root -') + ->toContain('limactl start --tty=false coolify-dev-2-debian-non-root') + ->toContain('exec -T coolify php artisan dev:qemu:seed debian-non-root --as-localhost --ip=host.docker.internal --port=53022') + ->not->toContain('--profile testing-host up') + ->not->toContain('dev:qemu debian-non-root') + ->and(file_get_contents($this->limaConfig)) + ->toContain('base: template:_images/debian-12') + ->toContain('mounts: []') + ->toContain("user='coolify'") + ->toContain(config('development-qemu.public_key')) + ->toContain('"$user ALL=(ALL) NOPASSWD:ALL"'); +}); + +it('rejects an unknown lima profile before it creates a vm', function () { + ($this->fakeLima)(); + + $process = runDevScript('wt', ['start', 'arch-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'lima']); + + expect($process->isSuccessful())->toBeFalse() + ->and($process->getErrorOutput())->toContain("Unknown profile 'arch-root'") + ->and(devScriptLog())->not->toContain('limactl create'); +}); + +it('stops and destroys only the lima vms of the instance slot', function () { + ($this->fakeLima)(); + runDevScript('main', ['start']); + runDevScript('wt', ['start']); + file_put_contents($this->devLog, ''); + + $stop = runDevScript('wt', ['stop'], ['COOLIFY_DEV_SERVER_BACKEND' => 'lima']); + $stopLog = devScriptLog(); + $destroy = runDevScript('main', ['destroy', 'fix-some_thing']); + + expect($stop->isSuccessful())->toBeTrue($stop->getErrorOutput()) + ->and($destroy->isSuccessful())->toBeTrue($destroy->getErrorOutput()) + ->and($stopLog) + ->toContain('limactl stop coolify-dev-2-debian-root') + ->not->toContain('limactl stop coolify-dev-2-ubuntu-root') + ->not->toContain('limactl stop coolify-dev-21-debian-root') + ->and(devScriptLog()) + ->toContain('limactl delete --force coolify-dev-2-debian-root') + ->toContain('limactl delete --force coolify-dev-2-ubuntu-root') + ->not->toContain('delete --force coolify-dev-21-debian-root') + ->not->toContain('delete --force other'); +}); + it('keeps fresh worktree instances writable and stable during the first composer install', function () { $initSetup = file_get_contents(base_path('docker/development/etc/s6-overlay/scripts/init-setup.sh')); diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.php index 9c2cd95d69..ad6d1d76d0 100644 --- a/tests/Feature/DevelopmentQemuVmTest.php +++ b/tests/Feature/DevelopmentQemuVmTest.php @@ -181,6 +181,24 @@ it('seeds localhost through the container command', function () { ->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10'); }); +it('seeds localhost with the forwarded ssh address of a lima vm', function () { + $this->seed(ServerSeeder::class); + + expect(Artisan::call('dev:qemu:seed', [ + 'profile' => 'debian-non-root', + '--as-localhost' => true, + '--ip' => 'host.docker.internal', + '--port' => '53022', + ]))->toBe(Command::SUCCESS); + + $server = Server::query()->findOrFail(0); + + expect($server->ip)->toBe('host.docker.internal') + ->and($server->port)->toBe(53022) + ->and($server->user)->toBe('coolify') + ->and(Server::query()->count())->toBe(1); +}); + it('replaces the seeded qemu server with the selected non-root equivalent', function () { SeedDevelopmentQemuServer::run('ubuntu-root'); $server = SeedDevelopmentQemuServer::run('ubuntu-non-root');