From 81045b2dc2f9080b499380b912ba7f2ffb31ad9c Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:30:42 +0200 Subject: [PATCH] fix(compose): deploy large Compose files such as Sentry self-hosted Sentry self-hosted (getsentry/self-hosted 26.8.0) could not be deployed as a Docker Compose application. Three separate problems blocked it: - YAML collection aliases: the first fix raised the limit to 256 only in Application::oldRawParser(). The safety check added in v4.4.0 (validateDockerComposeForInjection) and all other Compose parse calls still used the Symfony default of 128, so the Sentry file (133 aliases) failed with "Maximum number of collection aliases (128) exceeded". Add parseDockerComposeYaml() and use it for every Compose parse. - Argument list too long: the deployment job sent the generated Compose file as base64 inside the SSH command. Linux limits one argument to 128 KiB, and the Sentry file is about 370 KB. execute_remote_command() now accepts an 'input' key that streams content over SSH stdin, and the job writes the Compose file with it. - A list-style build arg without a value (build.args: [SENTRY_IMAGE]) merged into a map environment kept its numeric key, so Docker Compose rejected the file with "non-string key in ...environment: 0". It now becomes NAME with no value, as in Compose. Closes #11725 --- app/Actions/Service/StartService.php | 3 +- .../Api/ApplicationsController.php | 5 ++- .../Controllers/Api/ServicesController.php | 5 ++- app/Jobs/ApplicationDeploymentJob.php | 34 +++++++++++-------- .../Database/Sqlite/ConnectApplication.php | 2 +- app/Livewire/Project/New/DockerCompose.php | 3 +- app/Models/Application.php | 5 +-- app/Models/LocalFileVolume.php | 5 ++- app/Models/LocalPersistentVolume.php | 9 +++-- app/Models/Service.php | 3 +- app/Models/ServiceApplication.php | 5 ++- app/Traits/CalculatesExcludedStatus.php | 2 +- app/Traits/EnvironmentVariableProtection.php | 2 +- app/Traits/ExecuteRemoteCommand.php | 15 +++++--- bootstrap/helpers/docker.php | 4 +-- bootstrap/helpers/parsers.php | 28 ++++++++++----- bootstrap/helpers/services.php | 2 +- bootstrap/helpers/shared.php | 15 ++++---- ...licationComposeInjectionValidationTest.php | 2 +- .../DeploymentSensitiveCommandFailureTest.php | 29 +++++++--------- .../Unit/ConvertToKeyValueCollectionTest.php | 13 +++++++ tests/Unit/DockerComposeAliasLimitTest.php | 19 +++++++++++ 22 files changed, 125 insertions(+), 85 deletions(-) create mode 100644 tests/Unit/ConvertToKeyValueCollectionTest.php diff --git a/app/Actions/Service/StartService.php b/app/Actions/Service/StartService.php index aacf2bc0cc..eeb394e795 100644 --- a/app/Actions/Service/StartService.php +++ b/app/Actions/Service/StartService.php @@ -10,7 +10,6 @@ use App\Models\ServiceDatabase; use Illuminate\Support\Collection; use Lorisleiva\Actions\Concerns\AsAction; use Lorisleiva\Actions\Decorators\JobDecorator; -use Symfony\Component\Yaml\Yaml; class StartService { @@ -60,7 +59,7 @@ class StartService if (data_get($service, 'connect_to_docker_network')) { $compose = data_get($service, 'docker_compose', []); $safeNetwork = escapeshellarg($service->destination->network); - $serviceNames = data_get(Yaml::parse($compose), 'services', []); + $serviceNames = data_get(parseDockerComposeYaml($compose), 'services', []); foreach ($serviceNames as $serviceName => $serviceConfig) { $containerName = escapeshellarg("{$serviceName}-{$service->uuid}"); $commands[] = "docker network connect --alias {$containerName} {$safeNetwork} {$containerName} >/dev/null 2>&1 || true"; diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php index ec81538a0e..d995492e38 100644 --- a/app/Http/Controllers/Api/ApplicationsController.php +++ b/app/Http/Controllers/Api/ApplicationsController.php @@ -35,7 +35,6 @@ use Illuminate\Support\Facades\Validator; use Illuminate\Validation\Rule; use OpenApi\Attributes as OA; use Spatie\Url\Url; -use Symfony\Component\Yaml\Yaml; class ApplicationsController extends Controller { @@ -2764,7 +2763,7 @@ class ApplicationsController extends Controller $dockerComposeDomainsResponse = null; if ($isCompose) { try { - $compose = Yaml::parse($application->docker_compose_raw ?? ''); + $compose = parseDockerComposeYaml($application->docker_compose_raw ?? ''); } catch (\Throwable) { return response()->json([ 'message' => 'Validation failed.', @@ -3484,7 +3483,7 @@ class ApplicationsController extends Controller } } - $yaml = Yaml::parse($application->docker_compose_raw); + $yaml = parseDockerComposeYaml($application->docker_compose_raw); $services = data_get($yaml, 'services', []); $existingDockerComposeDomains = json_decode($application->docker_compose_domains ?? '[]', true) ?? []; $dockerComposeDomains->each(function ($domain) use ($services, $dockerComposeDomainsJson, $existingDockerComposeDomains) { diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.php index eccd726526..7411f8a472 100644 --- a/app/Http/Controllers/Api/ServicesController.php +++ b/app/Http/Controllers/Api/ServicesController.php @@ -26,7 +26,6 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Validator; use OpenApi\Attributes as OA; -use Symfony\Component\Yaml\Yaml; class ServicesController extends Controller { @@ -729,7 +728,7 @@ class ServicesController extends Controller ], ], 422); } - Yaml::parse($dockerComposeRaw); + parseDockerComposeYaml($dockerComposeRaw); // Validate for command injection BEFORE saving to database try { @@ -1278,7 +1277,7 @@ class ServicesController extends Controller ], ], 422); } - Yaml::parse($dockerComposeRaw); + parseDockerComposeYaml($dockerComposeRaw); // Validate for command injection BEFORE saving to database try { diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index c8641240cc..3a24d8bd50 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -163,7 +163,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue private $docker_compose; - private $docker_compose_base64; + private ?string $generated_docker_compose = null; private ?string $nixpacks_plan = null; @@ -831,11 +831,11 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $yaml = Yaml::dump(convertToArray($composeFile), 10); } - $this->docker_compose_base64 = base64_encode($yaml); + $this->generated_docker_compose = $yaml; $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}{$this->docker_compose_location} > /dev/null"), + "docker exec -i {$this->deployment_uuid} tee {$this->workdir}{$this->docker_compose_location} > /dev/null", + 'input' => $yaml, 'hidden' => true, - 'skip_command_log' => true, ]); // Modify Dockerfiles for ARGs and build secrets @@ -1253,7 +1253,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->server = $this->build_server; } } - if (isset($this->docker_compose_base64)) { + if (isset($this->generated_docker_compose)) { if ($this->use_build_server) { $this->server = $this->mainServer; } @@ -1274,8 +1274,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue "mkdir -p $mainDir", ], [ - "echo '{$this->docker_compose_base64}' | base64 -d | tee $composeFileName > /dev/null", - 'skip_command_log' => true, + "tee $composeFileName > /dev/null", + 'input' => $this->generated_docker_compose, ], [ "echo '{$readme}' | tee $mainDir/README.md > /dev/null", @@ -1717,9 +1717,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue // Generate SERVICE_NAME for dockercompose services from processed compose if ($this->application->settings->is_raw_compose_deployment_enabled) { - $dockerCompose = Yaml::parse($this->application->docker_compose_raw); + $dockerCompose = parseDockerComposeYaml($this->application->docker_compose_raw); } else { - $dockerCompose = Yaml::parse($this->application->docker_compose); + $dockerCompose = parseDockerComposeYaml($this->application->docker_compose); } $services = data_get($dockerCompose, 'services', []); foreach ($services as $serviceName => $_) { @@ -1787,7 +1787,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } // Generate SERVICE_NAME for dockercompose services - $rawDockerCompose = Yaml::parse($this->application->docker_compose_raw); + $rawDockerCompose = parseDockerComposeYaml($this->application->docker_compose_raw); $rawServices = data_get($rawDockerCompose, 'services', []); foreach ($rawServices as $rawServiceName => $_) { $envs->push('SERVICE_NAME_'.str($rawServiceName)->replace('-', '_')->replace('.', '_')->upper().'='.addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id)); @@ -2026,9 +2026,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue if ($this->pull_request_id === 0) { // Generate SERVICE_NAME for dockercompose services from processed compose if ($this->application->settings->is_raw_compose_deployment_enabled) { - $dockerCompose = Yaml::parse($this->application->docker_compose_raw); + $dockerCompose = parseDockerComposeYaml($this->application->docker_compose_raw); } else { - $dockerCompose = Yaml::parse($this->application->docker_compose); + $dockerCompose = parseDockerComposeYaml($this->application->docker_compose); } $services = data_get($dockerCompose, 'services', []); foreach ($services as $serviceName => $_) { @@ -2052,7 +2052,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } else { // Generate SERVICE_NAME for preview deployments - $rawDockerCompose = Yaml::parse($this->application->docker_compose_raw); + $rawDockerCompose = parseDockerComposeYaml($this->application->docker_compose_raw); $rawServices = data_get($rawDockerCompose, 'services', []); foreach ($rawServices as $rawServiceName => $_) { $envs_dict['SERVICE_NAME_'.str($rawServiceName)->replace('-', '_')->replace('.', '_')->upper()] = escapeBashEnvValue(addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id)); @@ -4100,8 +4100,12 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); } $this->docker_compose = Yaml::dump($docker_compose, 10); - $this->docker_compose_base64 = base64_encode($this->docker_compose); - $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true, 'skip_command_log' => true]); + $this->generated_docker_compose = $this->docker_compose; + $this->execute_remote_command([ + "docker exec -i {$this->deployment_uuid} tee {$this->workdir}/docker-compose.yaml > /dev/null", + 'input' => $this->docker_compose, + 'hidden' => true, + ]); } private function generate_local_persistent_volumes() diff --git a/app/Livewire/Project/Database/Sqlite/ConnectApplication.php b/app/Livewire/Project/Database/Sqlite/ConnectApplication.php index b2482378a7..d52ebc8573 100644 --- a/app/Livewire/Project/Database/Sqlite/ConnectApplication.php +++ b/app/Livewire/Project/Database/Sqlite/ConnectApplication.php @@ -241,7 +241,7 @@ class ConnectApplication extends Component private function composeVolumeSnippet(Application $application): string { try { - $services = data_get(Yaml::parse((string) $application->docker_compose_raw), 'services'); + $services = data_get(parseDockerComposeYaml((string) $application->docker_compose_raw), 'services'); } catch (\Throwable) { $services = null; } diff --git a/app/Livewire/Project/New/DockerCompose.php b/app/Livewire/Project/New/DockerCompose.php index 5a5385582b..80f9f481b4 100644 --- a/app/Livewire/Project/New/DockerCompose.php +++ b/app/Livewire/Project/New/DockerCompose.php @@ -7,7 +7,6 @@ use App\Models\Project; use App\Models\Service; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; -use Symfony\Component\Yaml\Yaml; class DockerCompose extends Component { @@ -38,7 +37,7 @@ class DockerCompose extends Component $this->validate([ 'dockerComposeRaw' => 'required', ]); - Yaml::parse($this->dockerComposeRaw); + parseDockerComposeYaml($this->dockerComposeRaw); // Validate for command injection BEFORE saving to database validateDockerComposeForInjection($this->dockerComposeRaw); diff --git a/app/Models/Application.php b/app/Models/Application.php index 23fb3e96e9..59e30f23c4 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -2138,10 +2138,7 @@ class Application extends BaseModel public function oldRawParser() { try { - $yaml = Yaml::parse( - $this->docker_compose_raw, - maxAliasesForCollections: self::MAX_DOCKER_COMPOSE_COLLECTION_ALIASES, - ); + $yaml = parseDockerComposeYaml($this->docker_compose_raw); } catch (\Exception $e) { throw new RuntimeException($e->getMessage()); } diff --git a/app/Models/LocalFileVolume.php b/app/Models/LocalFileVolume.php index 6f73eb8326..263bd8af45 100644 --- a/app/Models/LocalFileVolume.php +++ b/app/Models/LocalFileVolume.php @@ -11,7 +11,6 @@ use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Eloquent\Relations\MorphTo; use Illuminate\Support\Collection; use Illuminate\Support\Stringable; -use Symfony\Component\Yaml\Yaml; class LocalFileVolume extends BaseModel { @@ -691,7 +690,7 @@ class LocalFileVolume extends BaseModel } try { - $services = data_get(Yaml::parse($compose), 'services', []); + $services = data_get(parseDockerComposeYaml($compose), 'services', []); foreach ($services as $service) { foreach (data_get($service, 'volumes', []) as $volume) { if (is_string($volume)) { @@ -777,7 +776,7 @@ class LocalFileVolume extends BaseModel } // Parse the docker-compose content - $compose = Yaml::parse($actualService->docker_compose_raw); + $compose = parseDockerComposeYaml($actualService->docker_compose_raw); if (! isset($compose['services'])) { return false; } diff --git a/app/Models/LocalPersistentVolume.php b/app/Models/LocalPersistentVolume.php index cd251aa2d4..6f194d0323 100644 --- a/app/Models/LocalPersistentVolume.php +++ b/app/Models/LocalPersistentVolume.php @@ -6,7 +6,6 @@ use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Support\Str; -use Symfony\Component\Yaml\Yaml; class LocalPersistentVolume extends BaseModel { @@ -191,7 +190,7 @@ class LocalPersistentVolume extends BaseModel return true; } - $compose = Yaml::parse($composeContent); + $compose = parseDockerComposeYaml($composeContent); $services = data_get($compose, 'services', []); $topLevelVolumes = collect(data_get($compose, 'volumes') ?? []); @@ -246,7 +245,7 @@ class LocalPersistentVolume extends BaseModel return null; } - foreach (data_get(Yaml::parse($composeContent), 'volumes') ?? [] as $key => $declaration) { + foreach (data_get(parseDockerComposeYaml($composeContent), 'volumes') ?? [] as $key => $declaration) { $key = (string) $key; if (! isComposeExternalVolume($declaration)) { continue; @@ -293,7 +292,7 @@ class LocalPersistentVolume extends BaseModel } $resourceUuid = $resource instanceof Application ? $resource->uuid : data_get($resource, 'service.uuid'); - foreach (data_get(Yaml::parse($composeContent), 'volumes') ?? [] as $key => $declaration) { + foreach (data_get(parseDockerComposeYaml($composeContent), 'volumes') ?? [] as $key => $declaration) { if (! is_array($declaration) || isComposeExternalVolume($declaration)) { continue; } @@ -334,7 +333,7 @@ class LocalPersistentVolume extends BaseModel } // Parse the docker-compose content - $compose = Yaml::parse($actualService->docker_compose_raw); + $compose = parseDockerComposeYaml($actualService->docker_compose_raw); if (! isset($compose['services'])) { return false; } diff --git a/app/Models/Service.php b/app/Models/Service.php index c5ddeb5941..bcf9cf06ee 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -17,7 +17,6 @@ use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Support\Collection; use OpenApi\Attributes as OA; -use Symfony\Component\Yaml\Yaml; #[OA\Schema( description: 'Service model', @@ -1598,7 +1597,7 @@ class Service extends BaseModel // Generate SERVICE_NAME_* environment variables from docker-compose services if ($this->docker_compose) { try { - $dockerCompose = Yaml::parse($this->docker_compose); + $dockerCompose = parseDockerComposeYaml($this->docker_compose); $services = data_get($dockerCompose, 'services', []); foreach ($services as $serviceName => $_) { $envs->push('SERVICE_NAME_'.str($serviceName)->replace('-', '_')->replace('.', '_')->upper().'='.$serviceName); diff --git a/app/Models/ServiceApplication.php b/app/Models/ServiceApplication.php index db62005b0f..42dd88f169 100644 --- a/app/Models/ServiceApplication.php +++ b/app/Models/ServiceApplication.php @@ -10,7 +10,6 @@ use App\Traits\ReleasesManagedDnsRecords; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\SoftDeletes; -use Symfony\Component\Yaml\Yaml; class ServiceApplication extends BaseModel { @@ -367,7 +366,7 @@ class ServiceApplication extends BaseModel return $this->service->getRequiredPort(); } - $dockerCompose = Yaml::parse($dockerComposeRaw); + $dockerCompose = parseDockerComposeYaml($dockerComposeRaw); $serviceConfig = $dockerCompose['services'][$this->name] ?? null; if (! $serviceConfig) { return $this->service->getRequiredPort(); @@ -448,7 +447,7 @@ class ServiceApplication extends BaseModel */ private function getSavedLegacyRoutingPort(array $serviceConfig): ?int { - $savedCompose = Yaml::parse($this->service->docker_compose ?? ''); + $savedCompose = parseDockerComposeYaml($this->service->docker_compose ?? ''); $savedService = $savedCompose['services'][$this->name] ?? null; $image = $serviceConfig['image'] ?? null; if (! is_string($image) || $image === '' || ($savedService['image'] ?? null) !== $image) { diff --git a/app/Traits/CalculatesExcludedStatus.php b/app/Traits/CalculatesExcludedStatus.php index 5219878c00..21013dfef3 100644 --- a/app/Traits/CalculatesExcludedStatus.php +++ b/app/Traits/CalculatesExcludedStatus.php @@ -112,7 +112,7 @@ trait CalculatesExcludedStatus } try { - $dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw); + $dockerCompose = parseDockerComposeYaml($dockerComposeRaw); // Validate structure if (! is_array($dockerCompose)) { diff --git a/app/Traits/EnvironmentVariableProtection.php b/app/Traits/EnvironmentVariableProtection.php index ecc4849662..f2fea9355b 100644 --- a/app/Traits/EnvironmentVariableProtection.php +++ b/app/Traits/EnvironmentVariableProtection.php @@ -31,7 +31,7 @@ trait EnvironmentVariableProtection } try { - $dockerComposeData = Yaml::parse($dockerCompose); + $dockerComposeData = parseDockerComposeYaml($dockerCompose); $dockerEnvVars = data_get($dockerComposeData, 'services.*.environment'); foreach ($dockerEnvVars as $serviceEnvs) { diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 71f05e2cf2..449c9c55dd 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -95,6 +95,7 @@ trait ExecuteRemoteCommand $append = data_get($single_command, 'append', true); $command_hidden = data_get($single_command, 'command_hidden', false); $skip_command_log = data_get($single_command, 'skip_command_log', false); + $input = data_get($single_command, 'input'); $this->save = data_get($single_command, 'save'); if ($this->server->isNonRoot()) { if (str($command)->startsWith('docker exec')) { @@ -119,7 +120,7 @@ trait ExecuteRemoteCommand while ($attempt < $maxRetries && ! $commandExecuted) { try { - $this->executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors, $command_hidden, $skip_command_log); + $this->executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors, $command_hidden, $skip_command_log, $input); $commandExecuted = true; } catch (\RuntimeException|DeploymentException $e) { $lastError = $e; @@ -167,16 +168,20 @@ trait ExecuteRemoteCommand } /** - * Execute the actual command with process handling + * Execute the actual command with process handling. With $input, the content goes to the command over + * SSH stdin, because Linux limits one exec argument to 128 KiB and generateSshCommand() puts the whole + * script into one argument. */ - private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors, $command_hidden = false, $skip_command_log = false) + private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors, $command_hidden = false, $skip_command_log = false, ?string $input = null) { if ($command_hidden && ! $skip_command_log && isset($this->application_deployment_queue)) { $this->application_deployment_queue->addLogEntry('[CMD]: '.$this->redact_sensitive_info($command), hidden: true); } - $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); - $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append, $command_hidden, $skip_command_log) { + $remote_command = $input === null + ? SshMultiplexingHelper::generateSshCommand($this->server, $command) + : SshMultiplexingHelper::generateSshStdinCommand($this->server, $command); + $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->input($input)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append, $command_hidden, $skip_command_log) { // Sanitize output to ensure valid UTF-8 encoding before JSON encoding $sanitized_output = sanitize_utf8_text($output); $log_output = str($sanitized_output)->trim(); diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 143118c584..8cba703241 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -893,7 +893,7 @@ function dockerComposeServicePorts(?string $compose, ?string $serviceName): arra } try { - $services = data_get(Yaml::parse($compose), 'services', []); + $services = data_get(parseDockerComposeYaml($compose), 'services', []); } catch (Throwable) { return []; } @@ -1782,7 +1782,7 @@ function validateComposeFile(string $compose, int $server_id): string|Throwable if (! $server) { throw new Exception('Server not found'); } - $yaml_compose = Yaml::parse($compose); + $yaml_compose = parseDockerComposeYaml($compose); // Remove Coolify's custom fields before Docker validation $yaml_compose = stripCoolifyCustomFields($yaml_compose); diff --git a/bootstrap/helpers/parsers.php b/bootstrap/helpers/parsers.php index 5b435f0105..765ec995e8 100644 --- a/bootstrap/helpers/parsers.php +++ b/bootstrap/helpers/parsers.php @@ -15,8 +15,20 @@ use Illuminate\Support\Facades\File; use Illuminate\Support\Str; use Illuminate\Support\Stringable; use Spatie\Url\Url; +use Symfony\Component\Yaml\Exception\ParseException; use Symfony\Component\Yaml\Yaml; +/** + * Parses Docker Compose YAML with a higher collection alias limit than the Symfony default (128), + * because large Compose files (for example Sentry self-hosted) reuse YAML anchors many times. + * + * @throws ParseException If the YAML is invalid or exceeds the alias limit + */ +function parseDockerComposeYaml(string $compose): mixed +{ + return Yaml::parse($compose, maxAliasesForCollections: Application::MAX_DOCKER_COMPOSE_COLLECTION_ALIASES); +} + /** * Validates a Docker Compose YAML string for command injection vulnerabilities. * This should be called BEFORE saving to database to prevent malicious data from being stored. @@ -28,7 +40,7 @@ use Symfony\Component\Yaml\Yaml; function validateDockerComposeForInjection(string $composeYaml): void { try { - $parsed = Yaml::parse($composeYaml); + $parsed = parseDockerComposeYaml($composeYaml); } catch (Exception $e) { throw new Exception('Invalid YAML format: '.$e->getMessage(), 0, $e); } @@ -515,7 +527,7 @@ function composeExternalVolumeDockerNames(?string $compose): array return []; } try { - $volumes = data_get(Yaml::parse($compose), 'volumes'); + $volumes = data_get(parseDockerComposeYaml($compose), 'volumes'); } catch (Throwable) { return []; } @@ -542,7 +554,7 @@ function composeExternalVolumeMounts(?string $compose): array return []; } try { - $yaml = Yaml::parse($compose); + $yaml = parseDockerComposeYaml($compose); } catch (Throwable) { return []; } @@ -985,7 +997,7 @@ function removeComposeVolumeFieldsPreservingComments(string $source, array $clea $candidate = implode('', $result); try { - if (Yaml::parse($candidate) === $cleanedYaml) { + if (parseDockerComposeYaml($candidate) === $cleanedYaml) { return $candidate; } } catch (Exception) { @@ -1010,7 +1022,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int $isPullRequest = $pullRequestId == 0 ? false : true; $server = data_get($resource, 'destination.server'); try { - $yaml = Yaml::parse($compose); + $yaml = parseDockerComposeYaml($compose); } catch (Exception) { return collect([]); } @@ -2150,7 +2162,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int // This keeps the original user input clean while preventing content reapplication // Parse the original compose again to create a clean version without Coolify additions try { - $originalYaml = Yaml::parse($originalCompose); + $originalYaml = parseDockerComposeYaml($originalCompose); $originalYamlBeforeCleanup = $originalYaml; // Remove content, isDirectory, and is_directory from all volume definitions if (isset($originalYaml['services'])) { @@ -2197,7 +2209,7 @@ function serviceParser(Service $resource): Collection $server = data_get($resource, 'server'); try { - $yaml = Yaml::parse($compose); + $yaml = parseDockerComposeYaml($compose); } catch (Exception) { return collect([]); } @@ -3383,7 +3395,7 @@ function serviceParser(Service $resource): Collection // This keeps the original user input clean while preventing content reapplication // Parse the original compose again to create a clean version without Coolify additions try { - $originalYaml = Yaml::parse($originalCompose); + $originalYaml = parseDockerComposeYaml($originalCompose); $originalYamlBeforeCleanup = $originalYaml; // Remove content, isDirectory, and is_directory from all volume definitions if (isset($originalYaml['services'])) { diff --git a/bootstrap/helpers/services.php b/bootstrap/helpers/services.php index 4beec10305..ca6cd6014d 100644 --- a/bootstrap/helpers/services.php +++ b/bootstrap/helpers/services.php @@ -269,7 +269,7 @@ function updateCompose(ServiceApplication|ServiceDatabase $resource) if (! $dockerComposeRaw) { throw new Exception('No compose file found or not a valid YAML file.'); } - $dockerCompose = Yaml::parse($dockerComposeRaw); + $dockerCompose = parseDockerComposeYaml($dockerComposeRaw); // Switch Image $updatedImage = data_get_str($resource, 'image'); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 90063c7c28..8c7f03f59a 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1736,7 +1736,7 @@ function getTopLevelNetworks(Service|Application $resource): Collection if ($resource->getMorphClass() === Service::class) { if ($resource->docker_compose_raw) { try { - $yaml = Yaml::parse($resource->docker_compose_raw); + $yaml = parseDockerComposeYaml($resource->docker_compose_raw); } catch (Exception $e) { // If the docker-compose.yml file is not valid, we will return the network name as the key $topLevelNetworks = collect([ @@ -1802,7 +1802,7 @@ function getTopLevelNetworks(Service|Application $resource): Collection } } elseif ($resource->getMorphClass() === Application::class) { try { - $yaml = Yaml::parse($resource->docker_compose_raw); + $yaml = parseDockerComposeYaml($resource->docker_compose_raw); } catch (Exception $e) { // If the docker-compose.yml file is not valid, we will return the network name as the key $topLevelNetworks = collect([ @@ -2742,7 +2742,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $envComments = extractYamlEnvironmentComments($resource->docker_compose_raw); try { - $yaml = Yaml::parse($resource->docker_compose_raw); + $yaml = parseDockerComposeYaml($resource->docker_compose_raw); } catch (Exception $e) { throw new RuntimeException($e->getMessage()); } @@ -3534,7 +3534,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal } } elseif ($resource->getMorphClass() === Application::class) { try { - $yaml = Yaml::parse($resource->docker_compose_raw); + $yaml = parseDockerComposeYaml($resource->docker_compose_raw); } catch (Exception) { return; } @@ -4406,8 +4406,9 @@ function convertToKeyValueCollection($environment) $key = $parts[0]; $realValue = $parts[1] ?? ''; $changedEnvironment->put($key, $realValue); - } else { - $changedEnvironment->put($key, $value); + } elseif (is_string($value) && $value !== '') { + // A bare name (for example a list-style build arg) is the same as `NAME:` without a value. + $changedEnvironment->put($value, $changedEnvironment->get($value)); } } else { $changedEnvironment->put($key, $value); @@ -5052,7 +5053,7 @@ function extractHardcodedEnvironmentVariables(string $dockerComposeRaw): Collect } try { - $yaml = Yaml::parse($dockerComposeRaw); + $yaml = parseDockerComposeYaml($dockerComposeRaw); } catch (Exception $e) { // Malformed YAML - return empty collection return collect([]); diff --git a/tests/Feature/ApplicationComposeInjectionValidationTest.php b/tests/Feature/ApplicationComposeInjectionValidationTest.php index 3af9f50506..5e004fd232 100644 --- a/tests/Feature/ApplicationComposeInjectionValidationTest.php +++ b/tests/Feature/ApplicationComposeInjectionValidationTest.php @@ -373,7 +373,7 @@ test('the Compose deployment validates the repository file before any command us ->and($body)->not->toContain('$this->application->loadComposeFile(') ->and($loadPosition)->toBeLessThan(strpos($body, '$this->application->oldRawParser()')) ->and($loadPosition)->toBeLessThan(strpos($body, '$this->parseComposeFileForDeployment()')) - ->and($loadPosition)->toBeLessThan(strpos($body, 'base64 -d')) + ->and($loadPosition)->toBeLessThan(strpos($body, "'input' =>")) ->and($body)->toContain('"stat -c \'%F\' ".escapeshellarg($realPathInGit)'); }); diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.php index 3274b76e1b..448178fd51 100644 --- a/tests/Feature/DeploymentSensitiveCommandFailureTest.php +++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php @@ -411,7 +411,7 @@ it('hides the Nixpacks plan write command that contains build-time values', func 'not static' => [false], ]); -it('hides the compose file write command because compose files can contain secrets', function () { +it('streams the compose file over stdin so its content stays out of commands, errors, and logs', function () { [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob(); $server = readSensitiveFailureJobProperty($job, $reflection, 'server'); setSensitiveFailureJobProperties($job, $reflection, [ @@ -419,12 +419,10 @@ it('hides the compose file write command because compose files can contain secre 'production_image_name' => 'example/app:latest', ]); - $payloads = []; - Process::fake(function (PendingProcess $process) use (&$payloads) { - preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $process->command, $matches); - $composePayloads = array_filter($matches[0], fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), 'services:')); - if ($composePayloads !== []) { - $payloads = array_merge($payloads, $composePayloads); + $composeWrite = null; + Process::fake(function (PendingProcess $process) use (&$composeWrite) { + if (str_contains((string) $process->input, 'services:')) { + $composeWrite = $process; return Process::result(errorOutput: 'tee: docker-compose.yaml: No space left on device', exitCode: 1); } @@ -434,15 +432,14 @@ it('hides the compose file write command because compose files can contain secre $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'generate_compose_file')); - expect($payloads)->not->toBeEmpty(); - expect($exception->getMessage()) - ->toContain('[command hidden because it contains sensitive data]') - ->toContain('No space left on device'); - $storedLogs = (string) $queue->fresh()->logs; - foreach ($payloads as $payload) { - expect($exception->getMessage())->not->toContain($payload); - expect($storedLogs)->not->toContain($payload); - } + $compose = (string) readSensitiveFailureJobProperty($job, $reflection, 'docker_compose'); + expect($composeWrite)->not->toBeNull() + ->and($composeWrite->input)->toBe($compose) + ->and($composeWrite->command)->toEndWith(escapeshellarg('docker exec -i sensitive-failure-deployment tee /artifacts/sensitive-failure/docker-compose.yaml > /dev/null')) + ->and($composeWrite->command)->not->toContain('services:') + ->and($exception->getMessage())->toContain('No space left on device') + ->and($exception->getMessage())->not->toContain('services:') + ->and((string) $queue->fresh()->logs)->not->toContain('services:'); }); function readSensitiveFailureJobProperty(object $job, ReflectionClass $reflection, string $property): mixed diff --git a/tests/Unit/ConvertToKeyValueCollectionTest.php b/tests/Unit/ConvertToKeyValueCollectionTest.php new file mode 100644 index 0000000000..f7e81c99a3 --- /dev/null +++ b/tests/Unit/ConvertToKeyValueCollectionTest.php @@ -0,0 +1,13 @@ + '/etc/sentry', 'SENTRY_IMAGE' => 'getsentry/sentry']) + ->merge(['SENTRY_IMAGE', 'DOCKER_PLATFORM', 'FOO=bar']); + + expect(convertToKeyValueCollection($environment)->all())->toBe([ + 'SENTRY_CONF' => '/etc/sentry', + 'SENTRY_IMAGE' => 'getsentry/sentry', + 'DOCKER_PLATFORM' => null, + 'FOO' => 'bar', + ]); +}); diff --git a/tests/Unit/DockerComposeAliasLimitTest.php b/tests/Unit/DockerComposeAliasLimitTest.php index ad255d9b4d..7f0ecbc532 100644 --- a/tests/Unit/DockerComposeAliasLimitTest.php +++ b/tests/Unit/DockerComposeAliasLimitTest.php @@ -49,3 +49,22 @@ it('keeps a finite collection alias limit for Docker Compose files', function () $application->oldRawParser(); })->throws(RuntimeException::class, 'Maximum number of collection aliases (256) exceeded'); + +it('accepts Docker Compose files with more than 128 collection aliases during the safety check', function () { + $services = collect(range(1, 129)) + ->map(fn (int $index): string => " service-{$index}:\n <<: *defaults\n") + ->implode(''); + + validateDockerComposeForInjection("x-defaults: &defaults\n image: alpine:latest\nservices:\n{$services}"); + + expect(parseDockerComposeYaml("x-defaults: &defaults\n image: alpine:latest\nservices:\n{$services}")) + ->toHaveKey('services.service-129.image', 'alpine:latest'); +}); + +it('keeps a finite collection alias limit during the Docker Compose safety check', function () { + $services = collect(range(1, Application::MAX_DOCKER_COMPOSE_COLLECTION_ALIASES + 1)) + ->map(fn (int $index): string => " service-{$index}:\n <<: *defaults\n") + ->implode(''); + + validateDockerComposeForInjection("x-defaults: &defaults\n image: alpine:latest\nservices:\n{$services}"); +})->throws(Exception::class, 'Maximum number of collection aliases (256) exceeded');