diff --git a/.ai/lessons.md b/.ai/lessons.md index ee068cb71d..8b620a0887 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -54,3 +54,10 @@ ## Pass identities to Livewire actions - Pass record IDs to Livewire actions instead of display values, and resolve team-scoped records on the server. When JavaScript needs text, use `@js()` or `Js::from()`. +- Mark Livewire properties that select records or feed server-side lookups as `#[Locked]`; clients can change every other public property. + +## Keep host test runs away from the dev app cache +- The repository is bind-mounted into the dev `coolify` container. Tests that call `app:init` run `optimize` and write a testing config/route cache into `bootstrap/cache`, so the dev app returns 500. For broad host test runs, set `APP_CONFIG_CACHE`, `APP_ROUTES_CACHE`, `APP_EVENTS_CACHE`, `APP_SERVICES_CACHE`, and `APP_PACKAGES_CACHE` to a temporary directory. + +## Test the real runtime image +- Deployment shell commands run in the Alpine/BusyBox helper image and pass through the non-root sudo parser. Verify new flags and shell syntax in that image and with `parseCommandsByLineForSudo()`; faked command output hides both failures. diff --git a/app/Actions/Development/ConfigureDevelopmentQemuHost.php b/app/Actions/Development/ConfigureDevelopmentQemuHost.php index a999b77ada..7d6111d5c7 100644 --- a/app/Actions/Development/ConfigureDevelopmentQemuHost.php +++ b/app/Actions/Development/ConfigureDevelopmentQemuHost.php @@ -24,7 +24,7 @@ class ConfigureDevelopmentQemuHost private function installDependencies(): void { - $binaries = ['curl', 'docker', 'iptables', 'qemu-img', 'virsh', 'virt-install']; + $binaries = ['curl', 'docker', 'iptables', 'qemu-img', 'virsh', 'virt-install', 'xorriso']; $check = collect($binaries)->map(fn (string $binary) => 'command -v '.escapeshellarg($binary))->implode(' && '); if (Process::run($check)->successful()) { @@ -36,7 +36,7 @@ class ConfigureDevelopmentQemuHost } $this->runOrFail('apt-get update'); - $this->runOrFail('DEBIAN_FRONTEND=noninteractive apt-get install -y curl iptables libvirt-clients libvirt-daemon-system qemu-utils qemu-system-x86 virtinst'); + $this->runOrFail('DEBIAN_FRONTEND=noninteractive apt-get install -y curl iptables libvirt-clients libvirt-daemon-system qemu-utils qemu-system-x86 virtinst xorriso'); } private function configureLibvirtNetwork(): void diff --git a/app/Actions/Development/StartDevelopmentQemuVm.php b/app/Actions/Development/StartDevelopmentQemuVm.php index d6d9c00b2f..67d770e513 100644 --- a/app/Actions/Development/StartDevelopmentQemuVm.php +++ b/app/Actions/Development/StartDevelopmentQemuVm.php @@ -33,25 +33,39 @@ class StartDevelopmentQemuVm } } - $this->createVm($profile); + $preparedImage = $this->preparedImage($profile['domain']); + + if (! File::exists($preparedImage)) { + $this->createVm($profile, false); + $this->waitForPreparation($profile['domain']); + $this->runOrFail('virsh undefine '.escapeshellarg($profile['domain'])); + $this->runOrFail('mv '.escapeshellarg($this->vmDisk($profile['domain'])).' '.escapeshellarg($preparedImage)); + if (File::exists($preparedImage)) { + File::chmod($preparedImage, 0644); + } + } + + $this->createVm($profile, true); ConfigureDevelopmentQemuHost::run(); $this->waitForSsh($profile['ip']); } /** @param array{domain: string, ip: string, user: string, mac: string, image: string, image_url: string, os_variant: string, provisioner: string} $profile */ - private function createVm(array $profile): void + private function createVm(array $profile, bool $prepared): void { $directory = config('development-qemu.storage_path'); File::ensureDirectoryExists($directory); File::chmod($directory, 0777); $this->moveLegacyFiles($directory); - $baseImage = "{$directory}/{$profile['image']}"; - $disk = "{$directory}/{$profile['domain']}.qcow2"; + $baseImage = $prepared ? $this->preparedImage($profile['domain']) : "{$directory}/{$profile['image']}"; + $disk = $this->vmDisk($profile['domain']); $userData = "{$directory}/{$profile['domain']}-user-data.yaml"; + $metaData = "{$directory}/{$profile['domain']}-meta-data.yaml"; $networkConfig = "{$directory}/{$profile['domain']}-network.yaml"; + $seedImage = "{$directory}/{$profile['domain']}-seed.iso"; - if (! File::exists($baseImage)) { + if (! $prepared && ! File::exists($baseImage)) { $this->runOrFail(sprintf( 'curl --fail --location --output %s %s', escapeshellarg($baseImage), @@ -76,23 +90,50 @@ class StartDevelopmentQemuVm File::chmod($disk, 0666); } - File::put($userData, $this->userData($profile)); - File::put($networkConfig, $this->networkConfig($profile)); + if (! $prepared) { + File::put($userData, $this->userData($profile)); + File::put($metaData, "instance-id: {$profile['domain']}\nlocal-hostname: {$profile['domain']}\n"); + File::put($networkConfig, $this->networkConfig($profile)); + $this->runOrFail(sprintf( + 'xorriso -as mkisofs -V cidata -graft-points -o %s %s %s %s', + escapeshellarg($seedImage), + escapeshellarg('user-data='.$userData), + escapeshellarg('meta-data='.$metaData), + escapeshellarg('network-config='.$networkConfig), + )); + } + + $seedDisk = $prepared ? '' : ' --disk path='.escapeshellarg($seedImage).',format=raw,bus=virtio,readonly=on'; $this->runOrFail(sprintf( - 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio --network network=%s,model=virtio,mac=%s --cloud-init user-data=%s,network-config=%s,disable=on --noautoconsole', + 'virt-install --connect qemu:///system --name %s --memory %d --vcpus %d --import --os-variant %s --disk path=%s,format=qcow2,bus=virtio%s --network network=%s,model=virtio,mac=%s --noautoconsole', escapeshellarg($profile['domain']), config('development-qemu.memory'), config('development-qemu.vcpus'), escapeshellarg($profile['os_variant']), escapeshellarg($disk), + $seedDisk, escapeshellarg(config('development-qemu.libvirt_network')), escapeshellarg($profile['mac']), - escapeshellarg($userData), - escapeshellarg($networkConfig), )); } + private function vmDisk(string $domain): string + { + return config('development-qemu.storage_path')."/{$domain}.qcow2"; + } + + private function preparedImage(string $domain): string + { + return config('development-qemu.storage_path')."/{$domain}-prepared.qcow2"; + } + + private function waitForPreparation(string $domain): void + { + $check = 'while [ "$(virsh domstate '.escapeshellarg($domain).')" != "shut off" ]; do sleep 2; done'; + $this->runOrFail('timeout 900 bash -c '.escapeshellarg($check)); + } + private function moveLegacyFiles(string $directory): void { $legacyDirectory = storage_path('app/development-qemu'); @@ -116,7 +157,9 @@ class StartDevelopmentQemuVm File::delete([ "{$directory}/{$domain}.qcow2", "{$directory}/{$domain}-user-data.yaml", + "{$directory}/{$domain}-meta-data.yaml", "{$directory}/{$domain}-network.yaml", + "{$directory}/{$domain}-seed.iso", ]); } @@ -126,28 +169,48 @@ class StartDevelopmentQemuVm $publicKey = config('development-qemu.public_key'); $adminGroup = $profile['provisioner'] === 'apt' ? 'sudo' : 'wheel'; $sudo = $profile['user'] === 'root' ? '' : " groups: [{$adminGroup}]\n sudo: ALL=(ALL) NOPASSWD:ALL\n"; + $shell = $profile['provisioner'] === 'apk' ? '/bin/ash' : '/bin/bash'; + $password = $profile['provisioner'] === 'apk' + ? ' passwd: $6$dd2d71373a57c9ac$8.GUqZYlL/QqUmpUuupWfTuKQjNKT7kO31K5cp7OIY5SbBamlAVkJnBDYsIVimMaBrUtYfFjX3u6hzts3nKaD.'."\n lock_passwd: false" + : ' lock_passwd: true'; - [$packages, $startDocker] = match ($profile['provisioner']) { - 'apk' => [" - docker\n - sudo", 'rc-update add docker default && service docker start'], - 'rpm' => [" - curl\n - sudo", 'curl -fsSL https://get.docker.com | sh && systemctl enable --now docker'], - default => [" - docker.io\n - sudo", 'systemctl enable --now docker'], + [$packages, $installDocker, $startDocker] = match ($profile['provisioner']) { + 'apk' => [" - docker\n - docker-cli-buildx\n - docker-cli-compose\n - sudo", '', 'rc-update add cgroups boot && service cgroups start && rc-update add docker default && service docker start'], + default => [" - curl\n - sudo", "curl -fsSL https://get.docker.com -o /tmp/get-docker.sh\n sh /tmp/get-docker.sh\n ", 'systemctl enable --now docker'], }; - $addUserToDockerGroup = $profile['user'] === 'root' ? '' : "\n - usermod -aG docker {$profile['user']}"; + $addUserToDockerGroup = $profile['user'] === 'root' ? '' : ($profile['provisioner'] === 'apk' + ? "addgroup {$profile['user']} docker" + : "usermod -aG docker {$profile['user']}"); return <</dev/null 2>&1; do + attempt=\$((attempt + 1)) + [ "\$attempt" -lt 60 ] || exit 1 + sleep 2 + done + docker version + docker info + docker compose version + docker buildx version + touch /etc/cloud/cloud-init.disabled + poweroff YAML; } diff --git a/database/seeders/ApplicationSeeder.php b/database/seeders/ApplicationSeeder.php index 2a0273e0f9..779f1f7df2 100644 --- a/database/seeders/ApplicationSeeder.php +++ b/database/seeders/ApplicationSeeder.php @@ -20,7 +20,7 @@ class ApplicationSeeder extends Seeder 'name' => 'Docker Compose Example', 'repository_project_id' => 603035348, 'git_repository' => 'coollabsio/coolify-examples', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', 'base_directory' => '/docker-compose', 'docker_compose_location' => '/docker-compose-test.yaml', 'build_pack' => 'dockercompose', @@ -37,7 +37,7 @@ class ApplicationSeeder extends Seeder 'fqdn' => 'http://nodejs.127.0.0.1.sslip.io', 'repository_project_id' => 603035348, 'git_repository' => 'coollabsio/coolify-examples', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', 'base_directory' => '/nodejs', 'build_pack' => 'nixpacks', 'ports_exposes' => '3000', @@ -53,7 +53,7 @@ class ApplicationSeeder extends Seeder 'fqdn' => 'http://dockerfile.127.0.0.1.sslip.io', 'repository_project_id' => 603035348, 'git_repository' => 'coollabsio/coolify-examples', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', 'base_directory' => '/dockerfile', 'build_pack' => 'dockerfile', 'ports_exposes' => '80', @@ -68,7 +68,7 @@ class ApplicationSeeder extends Seeder 'name' => 'Pure Dockerfile Example', 'fqdn' => 'http://pure-dockerfile.127.0.0.1.sslip.io', 'git_repository' => 'coollabsio/coolify', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', 'git_commit_sha' => 'HEAD', 'build_pack' => 'dockerfile', 'ports_exposes' => '80', @@ -86,7 +86,7 @@ CMD ["nginx", "-g", "daemon off;"] 'uuid' => 'crashloop', 'name' => 'Crash Loop Example', 'git_repository' => 'coollabsio/coolify', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', 'git_commit_sha' => 'HEAD', 'build_pack' => 'dockerfile', 'ports_exposes' => '80', diff --git a/database/seeders/DevelopmentRailpackExamplesSeeder.php b/database/seeders/DevelopmentRailpackExamplesSeeder.php index ba5247698a..fa5d281ab8 100644 --- a/database/seeders/DevelopmentRailpackExamplesSeeder.php +++ b/database/seeders/DevelopmentRailpackExamplesSeeder.php @@ -285,13 +285,13 @@ class DevelopmentRailpackExamplesSeeder extends Seeder 'is_static' => true, 'is_spa' => true, ], - // Multi-language examples (only available on v4.x branch). + // Multi-language examples on the main branch. [ 'uuid' => 'railpack-python-flask', 'name' => 'Railpack Python Flask Example', 'base_directory' => '/flask', 'ports_exposes' => '5000', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', 'start_command' => 'flask run --host=0.0.0.0 --port=5000', ], [ @@ -299,63 +299,63 @@ class DevelopmentRailpackExamplesSeeder extends Seeder 'name' => 'Railpack Go Gin Example', 'base_directory' => '/go/gin', 'ports_exposes' => '3000', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-rust', 'name' => 'Railpack Rust Example', 'base_directory' => '/rust', 'ports_exposes' => '8000', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-laravel', 'name' => 'Railpack Laravel Example', 'base_directory' => '/laravel', 'ports_exposes' => '80', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-laravel-pure', 'name' => 'Railpack Laravel Pure Example', 'base_directory' => '/laravel-pure', 'ports_exposes' => '80', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-laravel-inertia', 'name' => 'Railpack Laravel Inertia Example', 'base_directory' => '/laravel-inertia', 'ports_exposes' => '80', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-symfony', 'name' => 'Railpack Symfony Example', 'base_directory' => '/symfony', 'ports_exposes' => '80', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-rails', 'name' => 'Railpack Ruby on Rails Example', 'base_directory' => '/rails-example', 'ports_exposes' => '3000', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-elixir-phoenix', 'name' => 'Railpack Elixir Phoenix Example', 'base_directory' => '/elixir-phoenix', 'ports_exposes' => '4000', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-bun', 'name' => 'Railpack Bun Example', 'base_directory' => '/bun', 'ports_exposes' => '3000', - 'git_branch' => 'v4.x', + 'git_branch' => 'main', ], [ 'uuid' => 'railpack-github-deploy-key', diff --git a/templates/service-templates-latest.json b/templates/service-templates-latest.json index c6fc73c60d..3098e52699 100644 --- a/templates/service-templates-latest.json +++ b/templates/service-templates-latest.json @@ -64,7 +64,7 @@ "category": "productivity", "logo": "svgs/alexandrie.svg", "minversion": "0.0.0", - "template_last_updated_at": "2026-04-05T13:36:24+02:00", + "template_last_updated_at": "2026-07-07T13:24:35+02:00", "port": "8200" }, "anythingllm": { @@ -1370,7 +1370,7 @@ "category": "productivity", "logo": "svgs/espocrm.svg", "minversion": "0.0.0", - "template_last_updated_at": "2026-04-06T11:35:16-05:00", + "template_last_updated_at": "2026-07-03T15:15:43+03:00", "port": "80" }, "evolution-api": { diff --git a/tests/Feature/ApplicationSeederTest.php b/tests/Feature/ApplicationSeederTest.php index f9c59f7a50..9f347c1bc0 100644 --- a/tests/Feature/ApplicationSeederTest.php +++ b/tests/Feature/ApplicationSeederTest.php @@ -36,4 +36,8 @@ it('seeds the default applications without railpack examples', function () { expect(Application::query()->where('build_pack', 'railpack')->exists())->toBeFalse(); expect(Application::query()->whereIn('uuid', ['railpack-nodejs', 'railpack-static'])->exists())->toBeFalse(); + + expect(Application::query() + ->whereIn('git_repository', ['coollabsio/coolify-examples', 'coollabsio/coolify']) + ->pluck('git_branch')->unique()->all())->toBe(['main']); }); diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.php index c0b0fe6fb0..f23653e7d4 100644 --- a/tests/Feature/DevelopmentQemuVmTest.php +++ b/tests/Feature/DevelopmentQemuVmTest.php @@ -20,6 +20,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Validator; +use Symfony\Component\Yaml\Yaml; uses(RefreshDatabase::class); @@ -103,6 +104,7 @@ it('automatically configures the qemu host', function () { ConfigureDevelopmentQemuHost::run(); + Process::assertRan(fn ($process) => str_contains($process->command, 'command -v') && str_contains($process->command, 'xorriso')); Process::assertRan(fn ($process) => str_contains($process->command, 'systemctl enable --now libvirtd')); Process::assertRan(fn ($process) => str_contains($process->command, 'virsh net-define')); Process::assertRan(fn ($process) => str_contains($process->command, 'virsh net-start')); @@ -255,6 +257,74 @@ it('can create a vm without a host database connection', function () { expect(DB::getQueryLog())->toBeEmpty(); Process::assertRan(fn ($process) => str_contains($process->command, 'virt-install')); Process::assertRan(fn ($process) => str_contains($process->command, 'iptables -I LIBVIRT_FWI')); + Process::assertRan(fn ($process) => str_contains($process->command, 'virsh domstate') && str_contains($process->command, 'shut off')); + Process::assertRan(fn ($process) => str_contains($process->command, 'mv ') && str_contains($process->command, 'prepared.qcow2')); + Process::assertRan(fn ($process) => str_contains($process->command, 'qemu-img create') && str_contains($process->command, 'prepared.qcow2')); + Process::assertRan(fn ($process) => str_contains($process->command, 'xorriso -as mkisofs -V cidata -graft-points')); + Process::assertRan(fn ($process) => str_contains($process->command, 'virt-install') && str_contains($process->command, 'bus=virtio,readonly=on')); + $userData = File::get(config('development-qemu.storage_path').'/coolify-dev-ubuntu-root-user-data.yaml'); + expect($userData)->toContain('docker compose version', 'docker buildx version', 'docker info', 'cloud-init.disabled'); + $cloudInit = Yaml::parse($userData); + expect($cloudInit['runcmd'][0])->toContain('https://get.docker.com', 'systemctl enable --now docker', 'poweroff'); +}); + +it('reuses a prepared vm image without reinstalling docker', function () { + $storagePath = sys_get_temp_dir().'/coolify-qemu-prepared-test-'.uniqid(); + config(['development-qemu.storage_path' => $storagePath]); + File::ensureDirectoryExists($storagePath); + File::put("{$storagePath}/coolify-dev-ubuntu-root-prepared.qcow2", 'prepared image'); + Process::fake([ + '* net-dumpxml *' => Process::result(output: ''), + '* network inspect *' => Process::result(output: "172.18.0.0/16\n"), + '*' => Process::result(), + ]); + + StartDevelopmentQemuVm::run('ubuntu-root'); + + Process::assertNotRan(fn ($process) => str_contains($process->command, 'curl --fail --location')); + Process::assertNotRan(fn ($process) => str_contains($process->command, 'virsh domstate')); + Process::assertRan(fn ($process) => str_contains($process->command, 'qemu-img create') && str_contains($process->command, 'coolify-dev-ubuntu-root-prepared.qcow2')); + Process::assertRan(fn ($process) => str_contains($process->command, 'virt-install') && ! str_contains($process->command, '--cloud-init')); + expect(File::exists("{$storagePath}/coolify-dev-ubuntu-root-prepared.qcow2"))->toBeTrue(); +}); + +it('prepares alpine with the compose and buildx packages for a non-root user', function () { + $storagePath = sys_get_temp_dir().'/coolify-qemu-alpine-test-'.uniqid(); + config(['development-qemu.storage_path' => $storagePath]); + Process::fake([ + '* net-dumpxml *' => Process::result(output: ''), + '* network inspect *' => Process::result(output: "172.18.0.0/16\n"), + '*' => Process::result(), + ]); + + StartDevelopmentQemuVm::run('alpine-non-root', false); + + $cloudInit = Yaml::parse(File::get("{$storagePath}/coolify-dev-alpine-non-root-user-data.yaml")); + expect($cloudInit['packages'])->toContain('docker', 'docker-cli-compose', 'docker-cli-buildx') + ->and($cloudInit['users'][0]['shell'])->toBe('/bin/ash') + ->and($cloudInit['users'][0]['lock_passwd'])->toBeFalse() + ->and($cloudInit['runcmd'][0])->toContain('service cgroups start', 'addgroup coolify docker', 'until docker info', 'docker compose version', 'docker buildx version'); +}); + +it('does not cache a vm when preparation does not finish', function () { + config(['development-qemu.storage_path' => sys_get_temp_dir().'/coolify-qemu-failed-preparation-'.uniqid()]); + Process::fake(function ($process) { + if (str_contains($process->command, 'net-dumpxml')) { + return Process::result(output: ''); + } + + if (str_contains($process->command, 'network inspect')) { + return Process::result(output: "172.18.0.0/16\n"); + } + + return str_contains($process->command, 'timeout 900 bash') + ? Process::result(exitCode: 124) + : Process::result(); + }); + + expect(fn () => StartDevelopmentQemuVm::run('ubuntu-root'))->toThrow(RuntimeException::class); + Process::assertNotRan(fn ($process) => str_contains($process->command, 'mv ') && str_contains($process->command, 'prepared.qcow2')); + Process::assertNotRan(fn ($process) => str_starts_with($process->command, 'virt-install ') && ! str_contains($process->command, 'readonly=on')); }); it('seeds through the coolify container when the host database is unavailable', function () { diff --git a/tests/Feature/DevelopmentRailpackExamplesSeederTest.php b/tests/Feature/DevelopmentRailpackExamplesSeederTest.php index a1661b225d..a54fddd66f 100644 --- a/tests/Feature/DevelopmentRailpackExamplesSeederTest.php +++ b/tests/Feature/DevelopmentRailpackExamplesSeederTest.php @@ -77,6 +77,11 @@ it('seeds every railpack example in the production environment on testing-host', $githubDeployKey = $applications->firstWhere('uuid', 'railpack-github-deploy-key'); $gitlabDeployKey = $applications->firstWhere('uuid', 'railpack-gitlab-deploy-key'); + expect($applications->where('git_repository', DevelopmentRailpackExamplesSeeder::GIT_REPOSITORY) + ->pluck('git_branch')->unique()->sort()->values()->all())->toBe(['main', 'next']) + ->and($applications->firstWhere('uuid', 'railpack-python-flask')->git_branch)->toBe('main') + ->and($nestjs->git_branch)->toBe('next'); + expect($nestjs->base_directory)->toBe('/node/nestjs') ->and($nestjs->build_command)->toBe('npm run build') ->and($nestjs->start_command)->toBe('npm run start:prod')