diff --git a/.ai/lessons.md b/.ai/lessons.md index c2137bdb38..edfabe8dcf 100644 --- a/.ai/lessons.md +++ b/.ai/lessons.md @@ -13,6 +13,7 @@ - Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open. - Browser `click('Text')` can match a hidden element first (for example inactive modal steps) and wait forever; use `button:visible:has-text("Text")`. - Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only. +- The helper image runs `docker compose` for deployments and for the Coolify upgrade itself. Before you bump its Docker CLI or Compose, test them against the oldest supported Docker Engine (for example a `docker:-dind` container); newer clients can reject data from older daemons. - Do not symlink `vendor` into another worktree. Composer resolves the base path from the real `vendor` path, so the tests load the app classes of the other checkout. Run `composer install --ignore-platform-req=ext-zip` in the worktree. - Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`. diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 449c9c55dd..57b19f91b8 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -268,7 +268,22 @@ trait ExecuteRemoteCommand $error = $this->redact_sensitive_info($error); - return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}"; + return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}".$this->dockerNetworkGatewayHint($error); + } + + /** + * Docker Engine 27.0 to 28.0.0 reports an IPv6 network gateway with its prefix length (fd00::1/64) until the + * daemon restarts, and newer Docker CLI and Compose versions fail to parse it. + * + * @see https://github.com/moby/moby/pull/49520 + */ + private function dockerNetworkGatewayHint(string $error): string + { + if (! preg_match('~ParseAddr\("[0-9A-Fa-f:.]+/\d+"\)~', $error)) { + return ''; + } + + return "\nDocker Engine on this server reports a Docker network gateway with a prefix length, which is a Docker Engine bug fixed in 28.0.1. Run 'systemctl restart docker' on the server (this restarts all containers) or upgrade Docker Engine, then deploy again."; } /** diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.sh index 88dd7ea093..4d594cb9e9 100644 --- a/other/nightly/upgrade.sh +++ b/other/nightly/upgrade.sh @@ -38,6 +38,47 @@ write_status() { echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE" } +# Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64). +# Docker Engine 27.0 to 28.0.0 does this for IPv6 until the daemon restarts (https://github.com/moby/moby/pull/49520). +# The Docker CLI and Compose in the helper image fail on such a network with a ParseAddr error. +network_gateway_with_prefix() { + docker network inspect "$1" --format '{{range .IPAM.Config}}{{.Gateway}} {{end}}' 2>&1 | grep -oE '[0-9A-Fa-f:.]+/[0-9]+' | head -n1 +} + +# Make sure the coolify network exists and the helper image can read it. +# It is created when starting Coolify with docker compose +ensure_coolify_network() { + log "Checking Docker network 'coolify'..." + if ! docker network inspect coolify >/dev/null 2>&1; then + log "Network 'coolify' does not exist, creating..." + if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then + log "Failed to create network with IPv6, trying without IPv6..." + docker network create --attachable coolify 2>/dev/null + log "Network 'coolify' created without IPv6" + elif [ -n "$(network_gateway_with_prefix coolify)" ]; then + # No container uses the new network yet, so it is safe to recreate it. + log "Docker reports the IPv6 gateway of network 'coolify' with a prefix length, recreating it without IPv6..." + docker network rm coolify >/dev/null 2>&1 + docker network create --attachable coolify 2>/dev/null + log "Network 'coolify' created without IPv6" + else + log "Network 'coolify' created with IPv6 support" + fi + return 0 + fi + + log "Network 'coolify' already exists" + local gateway + gateway=$(network_gateway_with_prefix coolify) + if [ -n "$gateway" ]; then + local message="Docker Engine $(docker version --format '{{.Server.Version}}' 2>/dev/null) reports the gateway of the 'coolify' network as ${gateway}, which Coolify cannot read (Docker Engine bug, fixed in 28.0.1). Run 'systemctl restart docker' (this restarts all containers) or upgrade Docker Engine, then upgrade Coolify again." + log "ERROR: ${message}" + write_status "error" "$message" + echo " ERROR: ${message}" + return 1 + fi +} + # Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time. pull_image() { local image="$1" @@ -177,21 +218,7 @@ fi log "Environment variables check complete" echo " Done." -# Make sure coolify network exists -# It is created when starting Coolify with docker compose -log "Checking Docker network 'coolify'..." -if ! docker network inspect coolify >/dev/null 2>&1; then - log "Network 'coolify' does not exist, creating..." - if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then - log "Failed to create network with IPv6, trying without IPv6..." - docker network create --attachable coolify 2>/dev/null - log "Network 'coolify' created without IPv6" - else - log "Network 'coolify' created with IPv6 support" - fi -else - log "Network 'coolify' already exists" -fi +ensure_coolify_network || exit 1 mkdir -p /data/coolify/images/{avatars,project-icons} chown -R 9999:root /data/coolify/images @@ -316,8 +343,16 @@ nohup bash -c " fi log 'Running docker compose up...' - docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 - log 'Docker compose up completed' + if docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1; then + log 'Docker compose up completed' + elif [ \"\$(docker inspect --format '{{.State.Running}}' coolify 2>/dev/null)\" = 'true' ]; then + # A slow first start (for example long migrations) can pass the wait timeout. Coolify is still starting. + log 'Docker compose up did not finish waiting, but the coolify container is running' + else + log 'ERROR: Docker compose up failed and the coolify container is not running' + write_status 'error' \"Coolify did not start. Check the upgrade log: \$LOGFILE\" + exit 1 + fi # Final log entry echo '' >>\"\$LOGFILE\" diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 987ce636a0..1e1e23cadd 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -38,6 +38,47 @@ write_status() { echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE" } +# Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64). +# Docker Engine 27.0 to 28.0.0 does this for IPv6 until the daemon restarts (https://github.com/moby/moby/pull/49520). +# The Docker CLI and Compose in the helper image fail on such a network with a ParseAddr error. +network_gateway_with_prefix() { + docker network inspect "$1" --format '{{range .IPAM.Config}}{{.Gateway}} {{end}}' 2>&1 | grep -oE '[0-9A-Fa-f:.]+/[0-9]+' | head -n1 +} + +# Make sure the coolify network exists and the helper image can read it. +# It is created when starting Coolify with docker compose +ensure_coolify_network() { + log "Checking Docker network 'coolify'..." + if ! docker network inspect coolify >/dev/null 2>&1; then + log "Network 'coolify' does not exist, creating..." + if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then + log "Failed to create network with IPv6, trying without IPv6..." + docker network create --attachable coolify 2>/dev/null + log "Network 'coolify' created without IPv6" + elif [ -n "$(network_gateway_with_prefix coolify)" ]; then + # No container uses the new network yet, so it is safe to recreate it. + log "Docker reports the IPv6 gateway of network 'coolify' with a prefix length, recreating it without IPv6..." + docker network rm coolify >/dev/null 2>&1 + docker network create --attachable coolify 2>/dev/null + log "Network 'coolify' created without IPv6" + else + log "Network 'coolify' created with IPv6 support" + fi + return 0 + fi + + log "Network 'coolify' already exists" + local gateway + gateway=$(network_gateway_with_prefix coolify) + if [ -n "$gateway" ]; then + local message="Docker Engine $(docker version --format '{{.Server.Version}}' 2>/dev/null) reports the gateway of the 'coolify' network as ${gateway}, which Coolify cannot read (Docker Engine bug, fixed in 28.0.1). Run 'systemctl restart docker' (this restarts all containers) or upgrade Docker Engine, then upgrade Coolify again." + log "ERROR: ${message}" + write_status "error" "$message" + echo " ERROR: ${message}" + return 1 + fi +} + # Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time. pull_image() { local image="$1" @@ -177,21 +218,7 @@ fi log "Environment variables check complete" echo " Done." -# Make sure coolify network exists -# It is created when starting Coolify with docker compose -log "Checking Docker network 'coolify'..." -if ! docker network inspect coolify >/dev/null 2>&1; then - log "Network 'coolify' does not exist, creating..." - if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then - log "Failed to create network with IPv6, trying without IPv6..." - docker network create --attachable coolify 2>/dev/null - log "Network 'coolify' created without IPv6" - else - log "Network 'coolify' created with IPv6 support" - fi -else - log "Network 'coolify' already exists" -fi +ensure_coolify_network || exit 1 mkdir -p /data/coolify/images/{avatars,project-icons} chown -R 9999:root /data/coolify/images @@ -316,8 +343,16 @@ nohup bash -c " fi log 'Running docker compose up...' - docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 - log 'Docker compose up completed' + if docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1; then + log 'Docker compose up completed' + elif [ \"\$(docker inspect --format '{{.State.Running}}' coolify 2>/dev/null)\" = 'true' ]; then + # A slow first start (for example long migrations) can pass the wait timeout. Coolify is still starting. + log 'Docker compose up did not finish waiting, but the coolify container is running' + else + log 'ERROR: Docker compose up failed and the coolify container is not running' + write_status 'error' \"Coolify did not start. Check the upgrade log: \$LOGFILE\" + exit 1 + fi # Final log entry echo '' >>\"\$LOGFILE\" diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.php index c6d104c9cb..a879ee286e 100644 --- a/tests/Feature/DeploymentSensitiveCommandFailureTest.php +++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php @@ -256,6 +256,33 @@ it('keeps the command in the error message for failed commands that are not sens ->not->toContain('[command hidden because it contains sensitive data]'); }); +it('explains how to fix a Docker network that Docker Engine reports with an IPv6 gateway prefix', function () { + [$job] = makeSensitiveCommandFailureJob(); + + Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1/64"): unexpected character, want colon (at "/64")', exitCode: 1)]); + + $exception = captureDeploymentException(fn () => $job->execute_remote_command([ + 'command' => 'docker compose up -d', + ])); + + expect($exception->getMessage()) + ->toContain('ParseAddr("fde4:75c5:b9d4::1/64")') + ->toContain('systemctl restart docker') + ->toContain('28.0.1'); +}); + +it('does not add the Docker network hint to other failures', function () { + [$job] = makeSensitiveCommandFailureJob(); + + Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1 "): unexpected character', exitCode: 1)]); + + $exception = captureDeploymentException(fn () => $job->execute_remote_command([ + 'command' => 'docker compose up -d', + ])); + + expect($exception->getMessage())->not->toContain('systemctl restart docker'); +}); + it('redacts locked values from the error output of failed commands', function () { [$job] = makeSensitiveCommandFailureJob([ ['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true], diff --git a/tests/Unit/UpgradeScriptContainerStartTest.php b/tests/Unit/UpgradeScriptContainerStartTest.php new file mode 100644 index 0000000000..efc16ad510 --- /dev/null +++ b/tests/Unit/UpgradeScriptContainerStartTest.php @@ -0,0 +1,63 @@ +>"\$LOGFILE" 2>&1 &$/ms', $source, $match); + $running = $coolifyRunning ? 'true' : 'false'; + + $harness = <</dev/null 2>&1', result_code: $exitCode); + $status = is_file("{$dir}/status") ? file_get_contents("{$dir}/status") : ''; + exec('rm -rf '.escapeshellarg($dir)); + + return ['exitCode' => $exitCode, 'status' => $status]; +} + +it('reports an error when the new Coolify containers do not start', function (string $script) { + $result = runUpgradeScriptContainerStart($script, composeExitCode: 1, coolifyRunning: false); + + expect($result['exitCode'])->toBe(1) + ->and($result['status'])->toStartWith('error|') + ->toContain('upgrade log'); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); + +it('completes the upgrade when the coolify container runs but is not healthy within the wait timeout', function (string $script) { + $result = runUpgradeScriptContainerStart($script, composeExitCode: 1, coolifyRunning: true); + + expect($result['exitCode'])->toBe(0) + ->and($result['status'])->toBe(''); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); + +it('completes the upgrade when docker compose up succeeds', function (string $script) { + $result = runUpgradeScriptContainerStart($script, composeExitCode: 0, coolifyRunning: true); + + expect($result['exitCode'])->toBe(0) + ->and($result['status'])->toBe(''); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); diff --git a/tests/Unit/UpgradeScriptCoolifyNetworkTest.php b/tests/Unit/UpgradeScriptCoolifyNetworkTest.php new file mode 100644 index 0000000000..ab3d497730 --- /dev/null +++ b/tests/Unit/UpgradeScriptCoolifyNetworkTest.php @@ -0,0 +1,93 @@ +, status: string} + */ +function runUpgradeScriptEnsureCoolifyNetwork(string $script, ?string $existingGateways, string $createdIpv6Gateway): array +{ + $dir = sys_get_temp_dir().'/upgrade-network-'.bin2hex(random_bytes(4)); + mkdir($dir); + touch("{$dir}/calls"); + touch("{$dir}/status"); + if ($existingGateways !== null) { + file_put_contents("{$dir}/gateways", $existingGateways); + } + $source = file_get_contents(getcwd().'/'.$script); + $functions = collect(['log', 'write_status', 'network_gateway_with_prefix', 'ensure_coolify_network']) + ->map(function (string $name) use ($source): string { + preg_match('/^'.$name.'\(\) \{.*?^\}\n/ms', $source, $match); + + return $match[0] ?? ''; + }) + ->implode(''); + + $harness = <<> "{$dir}/calls" + case "\$1 \$2" in + "network inspect") + [ -f "{$dir}/gateways" ] || { echo "Error: network coolify not found" >&2; return 1; } + cat "{$dir}/gateways" + ;; + "network create") + case "\$*" in + *--ipv6*) echo "172.19.0.1 {$createdIpv6Gateway} " > "{$dir}/gateways" ;; + *) echo "172.19.0.1 " > "{$dir}/gateways" ;; + esac + ;; + "network rm") rm -f "{$dir}/gateways" ;; + version*) echo 27.5.1 ;; + esac + } + {$functions} + ensure_coolify_network + BASH; + + exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode); + $calls = array_values(array_filter(explode("\n", file_get_contents("{$dir}/calls")), fn (string $call): bool => $call !== '' && ! str_starts_with($call, 'network inspect') && ! str_starts_with($call, 'version'))); + $status = file_get_contents("{$dir}/status"); + exec('rm -rf '.escapeshellarg($dir)); + + return ['exitCode' => $exitCode, 'calls' => $calls, 'status' => $status]; +} + +it('stops the upgrade before any container changes when the coolify network has a gateway with a prefix length', function (string $script) { + $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: '172.19.0.1 fd00:1::1/64 ', createdIpv6Gateway: 'unused'); + + expect($result['exitCode'])->toBe(1) + ->and($result['calls'])->toBe([]) + ->and($result['status'])->toStartWith('error|') + ->toContain('fd00:1::1/64') + ->toContain('systemctl restart docker'); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); + +it('keeps a readable coolify network', function (string $script) { + $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: '172.19.0.1 fd00:1::1 ', createdIpv6Gateway: 'unused'); + + expect($result)->toBe(['exitCode' => 0, 'calls' => [], 'status' => '']); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); + +it('creates the coolify network with IPv6', function (string $script) { + $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: null, createdIpv6Gateway: 'fd00:1::1'); + + expect($result)->toBe(['exitCode' => 0, 'calls' => ['network create --attachable --ipv6 coolify'], 'status' => '']); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']); + +it('recreates a new coolify network without IPv6 when Docker reports its gateway with a prefix length', function (string $script) { + $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: null, createdIpv6Gateway: 'fd00:1::1/64'); + + expect($result)->toBe([ + 'exitCode' => 0, + 'calls' => [ + 'network create --attachable --ipv6 coolify', + 'network rm coolify', + 'network create --attachable coolify', + ], + 'status' => '', + ]); +})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);