From 8c6774e17fa6f1553840500ce1679ddb31bb2b8c Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:31:02 +0100 Subject: [PATCH] fix(deploy): keep the helper Docker CLI on its buildx plugin A build-time HOME or DOCKER_CONFIG variable moved the Docker CLI in the helper away from /root/.docker/cli-plugins, so BuildKit builds failed with "buildx component is missing or broken". Force DOCKER_CONFIG=/root/.docker for the build command and for the helper when build secrets are passed. Closes #9726 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + app/Jobs/ApplicationDeploymentJob.php | 17 ++++++- ...ationDeploymentControlVarFilteringTest.php | 48 +++++++++++++++++- .../DeploymentSensitiveCommandFailureTest.php | 50 +++++++++++++++++++ 4 files changed, 112 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6176e44890..56b41167aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- Fix Docker builds that failed with "BuildKit is enabled but the buildx component is missing or broken" when the application has a build-time variable `HOME` or `DOCKER_CONFIG`. Coolify loads the build-time variables before it runs `docker build` in the helper container, so the variable moved the Docker CLI away from its plugins in `/root/.docker/cli-plugins`. The Docker CLI in the helper now always uses `/root/.docker`. A `HOME` build argument keeps its value ([#9726](https://github.com/coollabsio/coolify/issues/9726)) - Keep the trailing newline of multiline environment variables, for example an SSH private key that a Dockerfile mounts as a build secret. Before, Coolify removed the edge whitespace of every value, so OpenSSH rejected the key with "error in libcrypto". A value with more than one line is now saved exactly as you enter it, in the UI and in the API. Single-line values still lose their edge whitespace. Save a variable again to add the newline that Coolify removed before ([#9438](https://github.com/coollabsio/coolify/issues/9438)) - Use the same `COOLIFY_BRANCH` value in all parts of a preview deployment: `pull//head`. Before, a variable that refers to `${COOLIFY_BRANCH}` got the branch of the application (for example `main`), but the container got `pull//head`. Other build packs got `pull//head:pr--coolify`, and applications from a public repository got the branch of the application ([#9572](https://github.com/coollabsio/coolify/issues/9572)) - Send the **Resource status changes** notification when an application, preview, service, or database stops unexpectedly, for example after a crash or `docker kill`. Before, Coolify sent this notification only when it restarted the proxy or a database TCP proxy. Coolify waits 2 minutes and sends the notification only when the resource is still stopped, so a container that Docker restarts, or a deployment that replaces it, sends none. A stop, restart, or redeploy from Coolify sends no notification, and the same resource sends at most one notification in 15 minutes ([#9493](https://github.com/coollabsio/coolify/issues/9493)) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index d0b45e4f18..00ffdcd159 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -62,6 +62,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue private const BUILD_SCRIPT_PATH = '/artifacts/build.sh'; + /** + * The Docker CLI config directory of the helper image, which holds the buildx and compose plugins. + * Set explicitly, because a build-time HOME variable would otherwise move it. + */ + private const HELPER_DOCKER_CONFIG = '/root/.docker'; + private const NIXPACKS_PLAN_PATH = '/artifacts/thegameplan.json'; private const RAILPACK_REPOSITORY_CONFIG_PATH = 'railpack.json'; @@ -3214,7 +3220,11 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue instant_remote_process(["mkdir -p {$this->serverUserHomeDir}/.docker/buildx"], $this->server); $this->dockerConfigFileExists = instant_remote_process(["test -f {$this->serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $this->server); - $env_flags = $this->generate_docker_env_flags_for_secrets(); + // Build secrets put user variables such as HOME or DOCKER_CONFIG into the helper environment. + // DOCKER_CONFIG comes after them, so the Docker CLI keeps its plugins. Without build secrets the + // helper environment stays empty, so Docker Compose interpolates the user DOCKER_CONFIG from the env file. + $secretEnvFlags = $this->generate_docker_env_flags_for_secrets(); + $env_flags = filled($secretEnvFlags) ? "{$secretEnvFlags} -e DOCKER_CONFIG=".self::HELPER_DOCKER_CONFIG : ''; $buildxMetadataVolume = isDev() && $this->server->isLocalhost() ? '-v coolify-buildx:/root/.docker/buildx' : "-v {$this->serverUserHomeDir}/.docker/buildx:/root/.docker/buildx"; @@ -3245,7 +3255,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue [ $runCommand, 'hidden' => true, - 'skip_command_log' => filled($env_flags), + 'skip_command_log' => filled($secretEnvFlags), ], [ 'command' => executeInDocker($this->deployment_uuid, "mkdir -p {$this->basedir}"), @@ -4925,6 +4935,9 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); */ private function wrap_build_command_with_env_export(string $build_command): string { + // A build-time DOCKER_CONFIG must not move the Docker CLI away from its plugins (buildx, compose). + $build_command = 'export DOCKER_CONFIG='.self::HELPER_DOCKER_CONFIG." && {$build_command}"; + if (! $this->useBuildtimeEnvironmentLauncher) { return "cd {$this->workdir} && set -a && source ".self::BUILD_TIME_ENV_PATH." && set +a && {$build_command}"; } diff --git a/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php b/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php index ebdecee8da..a05610f9c6 100644 --- a/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php +++ b/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php @@ -497,7 +497,7 @@ it('loads shell variables and passes dotted variables through the build-time env expect($wrappedCommand) ->toContain(ApplicationDeploymentJob::BUILD_TIME_ENV_LAUNCHER_PATH) - ->toContain("/bin/bash -c 'printenv X.VALUE'") + ->toContain("/bin/bash -c 'export DOCKER_CONFIG=/root/.docker && printenv X.VALUE'") ->not->toContain('source '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH); file_put_contents($shellEnvironmentPath, $job->writtenArtifacts[ApplicationDeploymentJob::BUILD_TIME_SHELL_ENV_PATH]); @@ -546,10 +546,54 @@ it('keeps the original sourced environment path when build-time keys are shell s $wrappedCommand = invokeDeploymentJobMethod($job, $reflection, 'wrap_build_command_with_env_export', 'docker build .'); expect($wrappedCommand) - ->toContain('set -a && source '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH.' && set +a && docker build .') + ->toContain('set -a && source '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH.' && set +a && export DOCKER_CONFIG=/root/.docker && docker build .') ->not->toContain(ApplicationDeploymentJob::BUILD_TIME_ENV_LAUNCHER_PATH); }); +it('keeps the Docker CLI on the helper plugins when build-time variables set HOME or DOCKER_CONFIG', function (bool $withDottedVariable) { + [$application, $server] = makeDeploymentControlVarFixture(); + + createApplicationEnvironmentVariable($application, ['key' => 'HOME', 'value' => '/app']); + createApplicationEnvironmentVariable($application, ['key' => 'DOCKER_CONFIG', 'value' => '/app/.docker']); + if ($withDottedVariable) { + createApplicationEnvironmentVariable($application, ['key' => 'X.VALUE', 'value' => 'dotted']); + } + + $temporaryDirectory = sys_get_temp_dir().'/coolify-docker-config-'.str()->random(8); + expect(mkdir($temporaryDirectory))->toBeTrue(); + + try { + [$job, $reflection] = makeControlVarFilteringJob($application, $server, ['workdir' => $temporaryDirectory]); + invokeDeploymentJobMethod($job, $reflection, 'save_buildtime_environment_variables'); + + $localPaths = []; + foreach ($job->writtenArtifacts as $artifactPath => $contents) { + $localPath = $temporaryDirectory.'/'.basename($artifactPath); + $localPaths[$artifactPath] = $localPath; + file_put_contents($localPath, $contents); + } + foreach ($localPaths as $artifactPath => $localPath) { + file_put_contents($localPath, strtr(file_get_contents($localPath), $localPaths)); + } + + $wrappedCommand = invokeDeploymentJobMethod($job, $reflection, 'wrap_build_command_with_env_export', 'echo "$HOME|$DOCKER_CONFIG"'); + + $process = new Process(['/bin/bash', '-c', strtr($wrappedCommand, $localPaths)], env: ['HOME' => '/root']); + $process->mustRun(); + + // HOME keeps the user value for --build-arg HOME, but the Docker CLI finds buildx in /root/.docker/cli-plugins. + expect(trim($process->getOutput()))->toBe('/app|/root/.docker'); + } finally { + foreach (glob($temporaryDirectory.'/*') as $file) { + @unlink($file); + } + @rmdir($temporaryDirectory); + } +})->with([ + 'sourced build-time env file' => [false], + 'build-time env launcher' => [true], +]); + it('skips dotted keys from the Nixpacks plan without failing the deployment', function () { [$application, $server] = makeDeploymentControlVarFixture([ 'build_pack' => 'nixpacks', diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.php index 9e2b99e61a..fc062bd719 100644 --- a/tests/Feature/DeploymentSensitiveCommandFailureTest.php +++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php @@ -369,6 +369,56 @@ it('hides the helper container command when build secrets are passed as environm expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET); }); +it('starts the helper with its own DOCKER_CONFIG after build secret HOME and DOCKER_CONFIG variables', function () { + [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([ + ['key' => 'HOME', 'value' => '/app', 'is_runtime' => false], + ['key' => 'DOCKER_CONFIG', 'value' => '/app/.docker', 'is_runtime' => false], + ]); + $application->settings()->update(['use_build_secrets' => true]); + $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); + setSensitiveFailureJobProperties($job, $reflection, [ + 'application' => $application->fresh(), + 'destination' => $server->standaloneDockers()->firstOrFail(), + ]); + + $helperCommands = []; + Process::fake(function (PendingProcess $process) use (&$helperCommands) { + if (str_contains(remoteProcessScript($process), 'docker run -d')) { + $helperCommands[] = remoteProcessScript($process); + } + + return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK'); + }); + + invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'); + + // docker run uses the last -e value of a variable, so the helper Docker CLI keeps /root/.docker/cli-plugins. + expect($helperCommands)->toHaveCount(1) + ->and($helperCommands[0])->toContain('/app/.docker') + ->and(strrpos($helperCommands[0], '-e DOCKER_CONFIG=/root/.docker'))->toBeGreaterThan(strpos($helperCommands[0], '/app/.docker')); +}); + +it('shows the helper container command when no build secrets are passed', function () { + [$job, $reflection] = makeSensitiveCommandFailureJob(); + $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); + setSensitiveFailureJobProperties($job, $reflection, ['destination' => $server->standaloneDockers()->firstOrFail()]); + + Process::fake(function (PendingProcess $process) { + if (str_contains(remoteProcessScript($process), 'docker run -d')) { + return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125); + } + + return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK'); + }); + + $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image')); + + expect($exception->getMessage()) + ->toContain('docker run -d') + ->not->toContain('-e DOCKER_CONFIG=') + ->not->toContain('[command hidden because it contains sensitive data]'); +}); + it('hides the Railpack prepare command that passes build-time values', function () { [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],