feat(dev): run localhost dev server in a Lima VM on macOS

Add a Lima backend to scripts/dev. Auto mode picks it on macOS when limactl (Lima >= 2.0) is installed, and COOLIFY_DEV_SERVER_BACKEND=lima forces it. Lima VMs are named coolify-dev-<slot>-<profile>. Coolify connects to them through Lima's forwarded SSH port on host.docker.internal, and only guest ports 80/443 are forwarded to the host. dev:qemu:seed now takes --ip and --port options, so the seeded server can use a forwarded SSH address instead of the profile IP. Add tests for the Lima start flow and for seeding with an overridden address, and document the macOS setup in AGENTS.md.
2026-11-06 13:10:37 -04:00 · 2026-09-28 09:45:29 +02:00
parent 2bb1757aab
commit 5f6fb5dcc2
6 changed files with 248 additions and 15 deletions
@@ -15,8 +15,8 @@ For UI/UX design specifications, principles, and visual standards, consult the l
 Docker Compose-based dev setup with services: coolify (app, which also runs Reverb WebSockets and the terminal server), postgres, redis, vite, testing-host, mailpit, minio.

 ```bash
-# One dev instance per git branch (containers, volumes, KVM VMs named after the branch)
-./scripts/dev start [qemu-profile]           # KVM VM as localhost when /dev/kvm + root/sudo, else testing-host
+# One dev instance per git branch (containers, volumes, VMs named after the branch)
+./scripts/dev start [qemu-profile]           # localhost VM: KVM (Linux, /dev/kvm + root/sudo) or Lima (macOS, limactl), else testing-host
 ./scripts/dev stop                           # stop containers and VMs; data is kept for the next start
 ./scripts/dev run                            # start + follow logs, stop on exit (Jean run script)
 ./scripts/dev urls                           # all instances, URLs, ports, and checkouts
@@ -29,7 +29,7 @@ Docker Compose-based dev setup with services: coolify (app, which also runs Reve
 docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
 ```

-The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.
+The main checkout serves its branch at `localhost:8000` (Reverb `6001`, terminal `6002`, db `5432`, redis `6379`, vite `5173`). Worktrees get a port block at `20000 + slot*10` (app `+0`, Reverb `+1`, terminal `+2`, db `+3`, redis `+4`, vite `+5`). Each instance has its own libvirt network `coolify-dev-<slot>` (`10.221.<slot>.0/24`) and VMs `coolify-dev-<branch>--<profile>`; VMs are reused, use `php artisan dev:qemu <profile> --fresh` to rebuild one. On macOS with Lima >= 2.0 (`brew install lima`), the localhost VM is the Lima instance `coolify-dev-<slot>-<profile>` (same profiles, users, and SSH key); Coolify reaches it through Lima's forwarded SSH port on `host.docker.internal`, and only guest ports 80/443 are forwarded to the Mac. Rebuild it with `limactl delete -f <vm>`. Force it with `COOLIFY_DEV_SERVER_BACKEND=lima`. If `APP_URL` in `.env` is a `*.ts.net` host, the browser ports are published with `tailscale serve`. Set `COOLIFY_DEV_INSTANCE=<name>` to run another instance from the same checkout.

 ## Testing the Self-Hosted Upgrade Process

@@ -12,7 +12,10 @@ class SeedDevelopmentQemuServer
 {
    use AsAction;

-    public function handle(string $profileName, bool $removeOtherServers = true, bool $asLocalhost = false): Server
+    /**
+     * $ip and $port override the profile address, for example for a Lima VM whose SSH port is forwarded to the host.
+     */
+    public function handle(string $profileName, bool $removeOtherServers = true, bool $asLocalhost = false, ?string $ip = null, int $port = 22): Server
    {
        $this->ensureDevelopmentEnvironment();
        $profile = config("development-qemu.profiles.{$profileName}");
@@ -41,8 +44,8 @@ class SeedDevelopmentQemuServer
        $server->fill([
            'name' => $asLocalhost ? 'localhost' : $profile['name'],
            'description' => 'Development QEMU virtual machine',
-            'ip' => $profile['ip'],
-            'port' => 22,
+            'ip' => $ip ?? $profile['ip'],
+            'port' => $port,
            'user' => $profile['user'],
            'team_id' => 0,
            'private_key_id' => $privateKey->id,
@@ -7,7 +7,7 @@ use Illuminate\Console\Command;

 class SeedDevelopmentQemuServerCommand extends Command
 {
-    protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)}';
+    protected $signature = 'dev:qemu:seed {profile : Profile key from config/development-qemu.php} {--keep-others} {--as-localhost : Use the VM for the localhost server (id 0)} {--ip= : SSH host instead of the profile IP} {--port=22 : SSH port}';

    protected $description = 'Seed one development QEMU server in the Coolify database';

@@ -19,7 +19,7 @@ class SeedDevelopmentQemuServerCommand extends Command
            return self::FAILURE;
        }

-        $server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost'));
+        $server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost'), $this->option('ip') ?: null, (int) $this->option('port'));
        $this->info("Seeded {$server->name} at {$server->ip}.");

        return self::SUCCESS;
@@ -2,7 +2,7 @@
 # Manage the development instance of the current git branch.
 #
 # Usage:
-#   ./scripts/dev start [qemu-profile]   # start this branch's instance (KVM VM or testing-host)
+#   ./scripts/dev start [qemu-profile]   # start this branch's instance (KVM VM, Lima VM, or testing-host)
 #   ./scripts/dev stop [qemu-profile]    # stop containers and VMs; data is kept
 #   ./scripts/dev run [qemu-profile]     # start, follow logs, stop on exit (Jean)
 #   ./scripts/dev urls                   # all instances, ports, and state
@@ -10,15 +10,19 @@
 #   ./scripts/dev destroy <name>         # delete containers, volumes, VMs, and the port slot
 #   ./scripts/dev teardown               # destroy this worktree's instance if it exists (Jean)
 #
-# Each branch is one instance: Compose project, container, volumes, and KVM VMs are
+# Each branch is one instance: Compose project, container, volumes, and VMs are
 # named after it, so stop + start keeps the same data. Detached HEAD uses the
 # checkout directory name. Set COOLIFY_DEV_INSTANCE=<name> to override.
 #
+# Server backend (auto): KVM on Linux with /dev/kvm and root/sudo, Lima on macOS when
+# limactl (Lima >= 2.0) is installed, else the testing-host container. Lima VMs are
+# named coolify-dev-<slot>-<profile>; rebuild one with: limactl delete -f <vm>
+#
 # Ports: the main checkout uses 8000 (Reverb 6001, terminal 6002, db 5432, redis 6379,
 # vite 5173). Worktrees use 20000 + slot*10: app +0, Reverb +1, terminal +2, db +3,
 # redis +4, vite +5, mailpit +6/+7, minio +8/+9.
 #
-# Env: COOLIFY_DEV_SERVER_BACKEND=auto|testing-host, COOLIFY_DEV_KVM_PROFILE=<profile>
+# Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE=<profile>
 #
 set -euo pipefail

@@ -34,7 +38,7 @@ QEMU_STORAGE="${DEVELOPMENT_QEMU_STORAGE_PATH:-/var/lib/libvirt/images/coolify-d
 LEGACY_CONTAINERS=(coolify coolify-db coolify-redis coolify-vite coolify-mail coolify-minio coolify-minio-init coolify-testing-host)

 usage() {
-  sed -n '2,22p' "$0" | sed 's/^# \?//'
+  sed -n '2,26p' "$0" | sed 's/^# \?//'
 }

 normalize_name() {
@@ -170,15 +174,22 @@ compose() {

 select_backend() {
  backend="${COOLIFY_DEV_SERVER_BACKEND:-auto}"
-  if [[ "$backend" != "auto" && "$backend" != "testing-host" ]]; then
-    echo 'COOLIFY_DEV_SERVER_BACKEND must be auto or testing-host.' >&2
+  if [[ "$backend" != "auto" && "$backend" != "lima" && "$backend" != "testing-host" ]]; then
+    echo 'COOLIFY_DEV_SERVER_BACKEND must be auto, lima, or testing-host.' >&2
    exit 2
  fi
  privileged=()
  if [[ $EUID -ne 0 ]]; then
    privileged=(sudo -n -E)
  fi
-  if [[ "$backend" == "auto" && -c /dev/kvm && -r /dev/kvm && -w /dev/kvm ]] &&
+  if [[ "$backend" == "lima" ]]; then
+    command -v limactl >/dev/null || {
+      echo 'COOLIFY_DEV_SERVER_BACKEND=lima needs limactl (brew install lima).' >&2
+      exit 2
+    }
+  elif [[ "$backend" == "auto" && "$(uname -s)" == Darwin ]] && command -v limactl >/dev/null; then
+    backend=lima
+  elif [[ "$backend" == "auto" && -c /dev/kvm && -r /dev/kvm && -w /dev/kvm ]] &&
    { [[ $EUID -eq 0 ]] || { command -v sudo >/dev/null && sudo -n -E true; }; }; then
    backend=kvm
  else
@@ -247,6 +258,120 @@ vm_domains() {
    grep -E "^coolify-dev-${instance}--" || true
 }

+# Lima VMs of an instance are named coolify-dev-<slot>-<profile>; Lima names allow no "--".
+lima_vm() { echo "coolify-dev-$(env_value "$instance" DEVELOPMENT_QEMU_SLOT)-$1"; }
+
+lima_vms() {
+  command -v limactl >/dev/null || return 0
+  limactl list --format '{{.Name}} {{.Status}}' 2>/dev/null |
+    awk -v prefix="$(lima_vm '')" -v status="${1:-}" \
+      'index($1, prefix) == 1 && (status == "" || $2 == status) { print $1 }'
+}
+
+# Same users, SSH key, and Docker setup as the KVM profiles in config/development-qemu.php.
+# Provision scripts run on every boot, so each step is idempotent. The host directories
+# are not mounted, and only guest ports 80 and 443 are forwarded to the host.
+lima_config() {
+  local image user key
+  case "$1" in
+    ubuntu-root | ubuntu-non-root) image=ubuntu-24.04 ;;
+    debian-root | debian-non-root) image=debian-12 ;;
+    centos-root | centos-non-root) image=centos-stream-9 ;;
+    alpine-root | alpine-non-root) image=alpine-3.23 ;;
+    *)
+      echo "Unknown profile '$1'. Use a profile from config/development-qemu.php." >&2
+      return 1
+      ;;
+  esac
+  user=root
+  [[ "$1" == *-non-root ]] && user=coolify
+  key="$(sed -n "s/^ *'public_key' => '\(.*\)',\$/\1/p" config/development-qemu.php 2>/dev/null)"
+  if [[ -z "$key" ]]; then
+    echo 'Cannot read public_key from config/development-qemu.php.' >&2
+    return 1
+  fi
+  cat <<YAML
+minimumLimaVersion: 2.0.0
+base: template:_images/${image}
+cpus: 2
+memory: 2GiB
+disk: 20GiB
+mounts: []
+containerd:
+  system: false
+  user: false
+portForwards:
+- guestPort: 80
+- guestPort: 443
+- guestIP: 0.0.0.0
+  guestPortRange: [1, 65535]
+  proto: any
+  ignore: true
+- guestPortRange: [1, 65535]
+  proto: any
+  ignore: true
+provision:
+- mode: system
+  script: |
+    #!/bin/sh
+    set -eu
+    user='${user}'
+    key='${key}'
+    if ! command -v docker >/dev/null; then
+      if command -v apk >/dev/null; then
+        apk add docker docker-cli-buildx docker-cli-compose sudo
+      else
+        curl -fsSL https://get.docker.com | sh
+      fi
+    fi
+    if command -v systemctl >/dev/null; then
+      systemctl enable --now docker
+    else
+      # OpenRC can report an error for a start during boot; the runlevels start both services.
+      rc-update add cgroups boot
+      rc-update add docker default
+      rc-service cgroups start || true
+      rc-service docker start || true
+    fi
+    home=/root
+    if [ "\$user" != root ]; then
+      # A fixed high UID: Lima creates its own user later with the host UID (Alpine reboots first).
+      if ! id "\$user" >/dev/null 2>&1; then
+        if command -v useradd >/dev/null; then
+          useradd -m -u 2000 -s /bin/bash "\$user"
+        else
+          adduser -D -u 2000 -s /bin/ash "\$user"
+        fi
+        echo "\$user:*" | chpasswd -e
+      fi
+      echo "\$user ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/90-coolify-dev
+      chmod 440 /etc/sudoers.d/90-coolify-dev
+      if command -v usermod >/dev/null; then usermod -aG docker "\$user"; else addgroup "\$user" docker; fi
+      home="/home/\$user"
+    fi
+    mkdir -p "\$home/.ssh"
+    grep -qxF "\$key" "\$home/.ssh/authorized_keys" 2>/dev/null || echo "\$key" >>"\$home/.ssh/authorized_keys"
+    chmod 700 "\$home/.ssh"
+    chmod 600 "\$home/.ssh/authorized_keys"
+    chown -R "\$user:" "\$home/.ssh"
+YAML
+}
+
+# Coolify connects over the SSH port that Lima forwards to the host loopback, which
+# host.docker.internal reaches in Docker Desktop, OrbStack, and Colima.
+start_lima_vm() {
+  local vm config port
+  vm="$(lima_vm "$profile")"
+  if ! limactl list --quiet 2>/dev/null | grep -qxF "$vm"; then
+    config="$(lima_config "$profile")"
+    limactl create --tty=false --name="$vm" - <<<"$config"
+  fi
+  limactl start --tty=false "$vm"
+  port="$(limactl list --format '{{.SSHLocalPort}}' "$vm")"
+  compose "$instance" exec -T coolify php artisan dev:qemu:seed "$profile" --as-localhost \
+    --ip=host.docker.internal --port="$port"
+}
+
 start_backend() {
  stop_other_checkout_instances
  echo "==> Instance ${instance} → $(env_value "$instance" APP_URL) (backend: ${backend})"
@@ -266,6 +391,9 @@ start_backend() {
      DEVELOPMENT_QEMU_DOCKER_NETWORK="$(project_name "$instance")" \
      DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$(project_name "$instance")" \
      "${privileged[@]}" php artisan dev:qemu "$profile" --as-localhost
+  elif [[ "$backend" == "lima" ]]; then
+    echo "Using Lima profile: $profile"
+    start_lima_vm
  else
    compose "$instance" exec -T coolify php artisan db:seed --class=ServerSeeder --force
  fi
@@ -279,6 +407,12 @@ stop_backend() {
        "${privileged[@]}" virsh --connect qemu:///system shutdown "$domain"
      fi
    done
+  elif [[ "$backend" == "lima" ]]; then
+    for domain in $(lima_vms Running); do
+      if [[ -z "$profile_arg" || "$domain" == "$(lima_vm "$profile_arg")" ]]; then
+        limactl stop "$domain"
+      fi
+    done
  fi
  compose "$instance" stop
 }
@@ -308,6 +442,9 @@ cmd_destroy() {
  tailscale_serve off
  subnet="$(docker network inspect "$(project_name "$name")" --format '{{(index .IPAM.Config 0).Subnet}}' 2>/dev/null || true)"
  compose "$name" --profile '*' down --remove-orphans --volumes
+  for domain in $(lima_vms); do
+    limactl delete --force "$domain"
+  done
  if command -v virsh >/dev/null; then
    for domain in $(vm_domains --all); do
      "${privileged[@]}" virsh --connect qemu:///system destroy "$domain" >/dev/null 2>&1 || true
@@ -14,6 +14,8 @@ beforeEach(function () {
    copy(base_path('scripts/dev'), $main.'/scripts/dev');
    chmod($main.'/scripts/dev', 0755);
    copy(base_path('docker-compose.dev-multi.yml'), $main.'/docker-compose.dev-multi.yml');
+    mkdir($main.'/config');
+    copy(base_path('config/development-qemu.php'), $main.'/config/development-qemu.php');
    file_put_contents($main.'/.gitignore', ".env\n.dev-instances/\n");

    $git = Process::fromShellCommandline(implode(' && ', [
@@ -45,6 +47,23 @@ BASH;
        file_put_contents($this->devRoot.'/bin/'.$binary, $fake);
        chmod($this->devRoot.'/bin/'.$binary, 0755);
    }
+
+    $this->limaConfig = $this->devRoot.'/lima.yaml';
+    $this->fakeLima = function (): void {
+        file_put_contents($this->devRoot.'/bin/limactl', <<<'BASH'
+#!/usr/bin/env bash
+printf 'limactl %s\n' "$*" >> "$DEV_TEST_LOG"
+case "$*" in
+  create*) cat > "$DEV_TEST_LIMA_CONFIG" ;;
+  *SSHLocalPort*) echo 53022 ;;
+  *'{{.Name}} {{.Status}}'*) printf '%s\n' 'coolify-dev-2-debian-root Running' 'coolify-dev-2-ubuntu-root Stopped' 'coolify-dev-21-debian-root Running' 'other Running' ;;
+esac
+exit 0
+BASH);
+        file_put_contents($this->devRoot.'/bin/uname', "#!/usr/bin/env bash\necho Darwin\n");
+        chmod($this->devRoot.'/bin/limactl', 0755);
+        chmod($this->devRoot.'/bin/uname', 0755);
+    };
 });

 afterEach(function () {
@@ -58,6 +77,7 @@ function runDevScript(string $directory, array $arguments, array $env = []): Pro
        'DEV_TEST_LOG' => test()->devLog,
        'DEVELOPMENT_QEMU_STORAGE_PATH' => test()->qemuStorage,
        'COOLIFY_DEV_SERVER_BACKEND' => 'testing-host',
+        'DEV_TEST_LIMA_CONFIG' => test()->limaConfig,
        ...$env,
    ]);
    $process->run();
@@ -225,6 +245,61 @@ BASH);
        ->not->toContain('--profile testing-host up');
 });

+it('starts a lima vm as localhost on macos and seeds its forwarded ssh port', function () {
+    ($this->fakeLima)();
+    runDevScript('main', ['start']);
+    file_put_contents($this->devLog, '');
+
+    $process = runDevScript('wt', ['start', 'debian-non-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'auto']);
+
+    expect($process->isSuccessful())->toBeTrue($process->getErrorOutput())
+        ->and(devScriptLog())
+        ->toContain('limactl create --tty=false --name=coolify-dev-2-debian-non-root -')
+        ->toContain('limactl start --tty=false coolify-dev-2-debian-non-root')
+        ->toContain('exec -T coolify php artisan dev:qemu:seed debian-non-root --as-localhost --ip=host.docker.internal --port=53022')
+        ->not->toContain('--profile testing-host up')
+        ->not->toContain('dev:qemu debian-non-root')
+        ->and(file_get_contents($this->limaConfig))
+        ->toContain('base: template:_images/debian-12')
+        ->toContain('mounts: []')
+        ->toContain("user='coolify'")
+        ->toContain(config('development-qemu.public_key'))
+        ->toContain('"$user ALL=(ALL) NOPASSWD:ALL"');
+});
+
+it('rejects an unknown lima profile before it creates a vm', function () {
+    ($this->fakeLima)();
+
+    $process = runDevScript('wt', ['start', 'arch-root'], ['COOLIFY_DEV_SERVER_BACKEND' => 'lima']);
+
+    expect($process->isSuccessful())->toBeFalse()
+        ->and($process->getErrorOutput())->toContain("Unknown profile 'arch-root'")
+        ->and(devScriptLog())->not->toContain('limactl create');
+});
+
+it('stops and destroys only the lima vms of the instance slot', function () {
+    ($this->fakeLima)();
+    runDevScript('main', ['start']);
+    runDevScript('wt', ['start']);
+    file_put_contents($this->devLog, '');
+
+    $stop = runDevScript('wt', ['stop'], ['COOLIFY_DEV_SERVER_BACKEND' => 'lima']);
+    $stopLog = devScriptLog();
+    $destroy = runDevScript('main', ['destroy', 'fix-some_thing']);
+
+    expect($stop->isSuccessful())->toBeTrue($stop->getErrorOutput())
+        ->and($destroy->isSuccessful())->toBeTrue($destroy->getErrorOutput())
+        ->and($stopLog)
+        ->toContain('limactl stop coolify-dev-2-debian-root')
+        ->not->toContain('limactl stop coolify-dev-2-ubuntu-root')
+        ->not->toContain('limactl stop coolify-dev-21-debian-root')
+        ->and(devScriptLog())
+        ->toContain('limactl delete --force coolify-dev-2-debian-root')
+        ->toContain('limactl delete --force coolify-dev-2-ubuntu-root')
+        ->not->toContain('delete --force coolify-dev-21-debian-root')
+        ->not->toContain('delete --force other');
+});
+
 it('keeps fresh worktree instances writable and stable during the first composer install', function () {
    $initSetup = file_get_contents(base_path('docker/development/etc/s6-overlay/scripts/init-setup.sh'));

@@ -181,6 +181,24 @@ it('seeds localhost through the container command', function () {
        ->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10');
 });

+it('seeds localhost with the forwarded ssh address of a lima vm', function () {
+    $this->seed(ServerSeeder::class);
+
+    expect(Artisan::call('dev:qemu:seed', [
+        'profile' => 'debian-non-root',
+        '--as-localhost' => true,
+        '--ip' => 'host.docker.internal',
+        '--port' => '53022',
+    ]))->toBe(Command::SUCCESS);
+
+    $server = Server::query()->findOrFail(0);
+
+    expect($server->ip)->toBe('host.docker.internal')
+        ->and($server->port)->toBe(53022)
+        ->and($server->user)->toBe('coolify')
+        ->and(Server::query()->count())->toBe(1);
+});
+
 it('replaces the seeded qemu server with the selected non-root equivalent', function () {
    SeedDevelopmentQemuServer::run('ubuntu-root');
    $server = SeedDevelopmentQemuServer::run('ubuntu-non-root');