From 4b9fe035dc3ece6e4a46f50548779156f39a1733 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:14:58 +0100 Subject: [PATCH] fix(deployment): trust the server CA bundle in the helper container BuildKit lets the Docker CLI in the helper fetch the registry token, and the helper trusts only its own CAs. Behind a TLS-inspecting proxy, builds failed with x509 errors although the server trusts the proxy CA. Mount the server CA bundle read-only into the helper. Closes #7568 --- app/Jobs/ApplicationDeploymentJob.php | 49 +++++++++++++++++-- .../DeploymentSensitiveCommandFailureTest.php | 34 +++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 7c6b6fd144..057b6ec158 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -68,6 +68,26 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue */ private const HELPER_DOCKER_CONFIG = '/root/.docker'; + /** + * System CA bundles of the supported server distributions, in the order that Go checks them. + * + * @var list + */ + private const HOST_CA_BUNDLE_PATHS = [ + '/etc/ssl/certs/ca-certificates.crt', + '/etc/pki/tls/certs/ca-bundle.crt', + '/etc/ssl/ca-bundle.pem', + '/etc/pki/tls/cacert.pem', + '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem', + '/etc/ssl/cert.pem', + ]; + + /** + * Go programs in the helper (the Docker CLI, which fetches registry tokens for BuildKit) read every file + * in /etc/ssl/certs, so this file adds the server CAs to the helper CAs. OpenSSL (git, curl) ignores it. + */ + private const HELPER_HOST_CA_BUNDLE_PATH = '/etc/ssl/certs/coolify-host-ca-certificates.crt'; + private const NIXPACKS_PLAN_PATH = '/artifacts/thegameplan.json'; private const RAILPACK_REPOSITORY_CONFIG_PATH = 'railpack.json'; @@ -3239,6 +3259,28 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * Mount the server CA bundle, so that the helper trusts the same registries as the Docker daemon, + * for example behind a TLS-inspecting proxy with a CA that is only installed on the server. + */ + private function hostCaBundleMount(): string + { + $paths = implode(' ', self::HOST_CA_BUNDLE_PATHS); + $bundle = instant_remote_process( + ["for f in {$paths}; do if [ -f \"\$f\" ]; then echo \"\$f\"; break; fi; done"], + $this->server, + throwError: false, + no_sudo: true, + ); + + if (! in_array($bundle, self::HOST_CA_BUNDLE_PATHS, true)) { + return ''; + } + + // Unlike -v, --mount does not create a missing source as a directory on the server. + return " --mount type=bind,source={$bundle},target=".self::HELPER_HOST_CA_BUNDLE_PATH.',readonly'; + } + private function prepare_builder_image(bool $firstTry = true) { $this->checkForCancellation(); @@ -3257,20 +3299,21 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $buildxMetadataVolume = isDev() && $this->server->isLocalhost() ? '-v coolify-buildx:/root/.docker/buildx' : "-v {$this->serverUserHomeDir}/.docker/buildx:/root/.docker/buildx"; + $hostCaBundleVolume = $this->hostCaBundleMount(); if ($this->use_build_server) { if ($this->dockerConfigFileExists === 'NOK') { throw new DeploymentException("Docker config file (~/.docker/config.json) of the SSH user ({$this->server->user}) not found on the build server ({$this->server->name}). Log in to your registry under Server → Docker Registries on the build server. On a non-root server, a login made with \"sudo docker login\" is saved for root and is not used for builds."); } - $runCommand = "docker run -d --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro {$buildxMetadataVolume} -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}"; + $runCommand = "docker run -d --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro {$buildxMetadataVolume}{$hostCaBundleVolume} -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}"; } else { // docker compose runs in this helper and reads seccomp profile files from the custom Docker options $seccompMounts = $this->pull_request_id === 0 ? seccompProfileMountFlags($this->application->custom_docker_run_options) : ''; if ($this->dockerConfigFileExists === 'OK') { $safeNetwork = escapeshellarg($this->destination->network); - $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro {$buildxMetadataVolume} -v /var/run/docker.sock:/var/run/docker.sock {$seccompMounts} {$helperImage}"; + $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro {$buildxMetadataVolume}{$hostCaBundleVolume} -v /var/run/docker.sock:/var/run/docker.sock {$seccompMounts} {$helperImage}"; } else { $safeNetwork = escapeshellarg($this->destination->network); - $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm {$buildxMetadataVolume} -v /var/run/docker.sock:/var/run/docker.sock {$seccompMounts} {$helperImage}"; + $runCommand = "docker run -d --network {$safeNetwork} --name {$this->deployment_uuid} {$env_flags} --rm {$buildxMetadataVolume}{$hostCaBundleVolume} -v /var/run/docker.sock:/var/run/docker.sock {$seccompMounts} {$helperImage}"; } } if ($firstTry) { diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.php index c95ff4a2be..66a4b54606 100644 --- a/tests/Feature/DeploymentSensitiveCommandFailureTest.php +++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php @@ -398,6 +398,40 @@ it('starts the helper with its own DOCKER_CONFIG after build secret HOME and DOC ->and(strrpos($helperCommands[0], '-e DOCKER_CONFIG=/root/.docker'))->toBeGreaterThan(strpos($helperCommands[0], '/app/.docker')); }); +it('mounts the server CA bundle into the helper so the Docker CLI can fetch registry tokens', function (string $detectedBundle, ?string $expectedMount) { + [$job, $reflection] = makeSensitiveCommandFailureJob(); + $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); + setSensitiveFailureJobProperties($job, $reflection, ['destination' => $server->standaloneDockers()->firstOrFail()]); + + $helperCommands = []; + Process::fake(function (PendingProcess $process) use (&$helperCommands, $detectedBundle) { + $script = remoteProcessScript($process); + if (str_contains($script, 'docker run -d')) { + $helperCommands[] = $script; + } + + return Process::result(output: match (true) { + str_contains($script, 'echo $HOME') => '/root', + str_contains($script, 'for f in /etc/ssl/certs/ca-certificates.crt') => $detectedBundle, + default => 'NOK', + }); + }); + + invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'); + + expect($helperCommands)->toHaveCount(1); + if ($expectedMount === null) { + expect($helperCommands[0])->not->toContain('coolify-host-ca-certificates.crt'); + } else { + expect($helperCommands[0])->toContain($expectedMount); + } +})->with([ + 'Debian and Ubuntu bundle' => ['/etc/ssl/certs/ca-certificates.crt', '--mount type=bind,source=/etc/ssl/certs/ca-certificates.crt,target=/etc/ssl/certs/coolify-host-ca-certificates.crt,readonly'], + 'RHEL bundle' => ['/etc/pki/tls/certs/ca-bundle.crt', '--mount type=bind,source=/etc/pki/tls/certs/ca-bundle.crt,target=/etc/ssl/certs/coolify-host-ca-certificates.crt,readonly'], + 'no bundle' => ['', null], + 'unknown output' => ['/root:/x -v /', null], +]); + it('shows the helper container command when no build secrets are passed', function () { [$job, $reflection] = makeSensitiveCommandFailureJob(); $server = readSensitiveFailureJobProperty($job, $reflection, 'server');