From bc61f4860f6218fcc4100126a43dd1101ac7e06f Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:00:59 +0100 Subject: [PATCH] fix(railpack): serialize builder bootstrap before builds --- app/Actions/Server/CleanupDocker.php | 2 +- app/Jobs/ApplicationDeploymentJob.php | 2 +- bootstrap/helpers/docker.php | 15 ++++++ ...pplicationDeploymentRailpackConfigTest.php | 4 +- ...icationDeploymentRailpackEnvParityTest.php | 3 ++ tests/Unit/ParseCommandsByLineForSudoTest.php | 11 ++++ tests/Unit/RailpackBuilderLockTest.php | 29 +++++++++++ .../Browser/ApiTokenPermissionHelperTest.php | 50 +++++++++++++++++++ versions.json | 8 +-- 9 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 tests/v4/Browser/ApiTokenPermissionHelperTest.php diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 17b796a6df..a20d1485f8 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -197,7 +197,7 @@ class CleanupDocker implements ShouldBeUnique if (! $activeDeployment) { $commands[] = 'docker builder prune -af'; // The prune fails on a stopped builder, so start it first. - $commands[] = railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, 'docker buildx inspect --bootstrap coolify-railpack >/dev/null 2>&1; docker buildx prune --builder coolify-railpack -af'); + $commands[] = railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, railpackBuilderBootstrapScript().'; docker buildx prune --builder coolify-railpack -af'); } else { $cleanupLog[] = [ 'command' => self::BUILD_CACHE_KEPT_LOG_COMMAND, diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index a502ef7dc1..44d16791ae 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -3990,7 +3990,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue ." {$this->workdir}"; return 'DOCKER_CONFIG=/root/.docker docker buildx create --name coolify-railpack --driver docker-container 2>/dev/null || true' - ."\n".railpackBuilderLockedScript('('.$this->wrap_build_command_with_env_export($buildxBuildCommand).')'); + ."\n".railpackBuilderLockedScript(railpackBuilderBootstrapScript().'; ('.$this->wrap_build_command_with_env_export($buildxBuildCommand).')'); } private function decode_railpack_config(string $config, string $source): array diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index d3ca519766..4bdf9ca2d6 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -2257,6 +2257,21 @@ function railpackBuilderLockedScript(string $command): string .'exit $status'; } +/** + * Start the Railpack builder and wait until buildkitd accepts connections. Run it inside + * railpackBuilderLockedScript(), so that no other process stops the builder after it starts. + * + * buildx waits for buildkitd only when it starts the builder itself. A concurrent build that sees + * the container already running connects at once and fails while the BuildKit socket is missing. + * The lock makes the other builds wait until the first bootstrap is complete. + */ +function railpackBuilderBootstrapScript(): string +{ + return 'exec 8>/root/.docker/buildx/coolify-railpack-bootstrap.lock; flock -x 8; ' + .'DOCKER_CONFIG=/root/.docker docker buildx inspect --bootstrap coolify-railpack >/dev/null 2>&1; ' + .'flock -u 8'; +} + /** * The buildx metadata of a server, as the `docker run -v` source that the deployment helper uses. */ diff --git a/tests/Unit/ApplicationDeploymentRailpackConfigTest.php b/tests/Unit/ApplicationDeploymentRailpackConfigTest.php index 61deef08f2..30db6f5b0b 100644 --- a/tests/Unit/ApplicationDeploymentRailpackConfigTest.php +++ b/tests/Unit/ApplicationDeploymentRailpackConfigTest.php @@ -268,11 +268,13 @@ it('stops the railpack builder after the build when no other build uses it', fun $command = invokeRailpackMethod($job, $reflection, 'railpack_build_command', ['coollabsio/coolify:test', collect([])]); $sharedLock = strpos($command, 'flock -s 9'); + $bootstrap = strpos($command, railpackBuilderBootstrapScript()); $build = strpos($command, 'docker buildx build --builder coolify-railpack'); $stop = strpos($command, 'if flock -n -x 9; then DOCKER_CONFIG=/root/.docker docker buildx stop coolify-railpack'); expect($sharedLock)->not->toBeFalse() - ->and($build)->toBeGreaterThan($sharedLock) + ->and($bootstrap)->toBeGreaterThan($sharedLock) + ->and($build)->toBeGreaterThan($bootstrap) ->and($stop)->toBeGreaterThan($build) ->and($command)->toEndWith('exit $status'); }); diff --git a/tests/Unit/ApplicationDeploymentRailpackEnvParityTest.php b/tests/Unit/ApplicationDeploymentRailpackEnvParityTest.php index 0b7d5406c9..d62a8493ce 100644 --- a/tests/Unit/ApplicationDeploymentRailpackEnvParityTest.php +++ b/tests/Unit/ApplicationDeploymentRailpackEnvParityTest.php @@ -94,6 +94,7 @@ it('generates escaped railpack env args from resolved values and includes instal it('uses preview railpack environment variables for preview deployments', function () { $application = Mockery::mock(Application::class); $application->shouldReceive('getAttribute')->with('install_command')->andReturn(null); + $application->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['is_pr_deployments_public_enabled' => false]); $previewValue = Mockery::mock(EnvironmentVariable::class)->makePartial(); $previewValue->forceFill([ @@ -104,12 +105,14 @@ it('uses preview railpack environment variables for preview deployments', functi $previewValue->shouldReceive('getResolvedValueWithServer')->once()->with(Mockery::type(Server::class))->andReturn('preview-value'); $previewQuery = Mockery::mock(); + $previewQuery->shouldReceive('whereNotIn')->with('id', [])->once()->andReturnSelf(); $previewQuery->shouldReceive('withoutBuildpackControlVariables')->once()->andReturnSelf(); $previewQuery->shouldReceive('where')->with('is_buildtime', true)->once()->andReturnSelf(); $previewQuery->shouldReceive('get')->once()->andReturn(collect([])); $application->shouldReceive('environment_variables_preview')->once()->andReturn($previewQuery); $railpackPreviewQuery = Mockery::mock(); + $railpackPreviewQuery->shouldReceive('whereNotIn')->with('id', [])->once()->andReturnSelf(); $railpackPreviewQuery->shouldReceive('get')->once()->andReturn(collect([$previewValue])); $application->shouldReceive('railpack_environment_variables_preview')->once()->andReturn($railpackPreviewQuery); diff --git a/tests/Unit/ParseCommandsByLineForSudoTest.php b/tests/Unit/ParseCommandsByLineForSudoTest.php index 9ea9bf6a46..1c2f264485 100644 --- a/tests/Unit/ParseCommandsByLineForSudoTest.php +++ b/tests/Unit/ParseCommandsByLineForSudoTest.php @@ -885,3 +885,14 @@ test('railpack builder helper command gets sudo only outside the helper script', ." exit \$status' 2>/dev/null || sudo true" ); }); + +test('railpack builder bootstrap script gets no sudo inside the helper script', function () { + $command = railpackBuilderHelperCommand('$HOME/.docker/buildx', 'coolify-helper:1.0.0', railpackBuilderBootstrapScript().'; docker buildx prune --builder coolify-railpack -af'); + + $parsed = parseCommandsByLineForSudo(collect([$command]), $this->server)[0]; + + expect($parsed) + ->toStartWith('sudo docker run --rm ') + ->toContain(railpackBuilderBootstrapScript().'; docker buildx prune --builder coolify-railpack -af; status=$?;') + ->toEndWith("exit \$status' 2>/dev/null || sudo true"); +}); diff --git a/tests/Unit/RailpackBuilderLockTest.php b/tests/Unit/RailpackBuilderLockTest.php index 273104d6c4..c3f87c7277 100644 --- a/tests/Unit/RailpackBuilderLockTest.php +++ b/tests/Unit/RailpackBuilderLockTest.php @@ -41,3 +41,32 @@ test('a held shared lock prevents stopping the builder', function () { expect($process->getExitCode())->toBe(0); expect($process->getOutput())->toBe('built'); }); + +test('concurrent builds bootstrap the builder one at a time', function () { + $directory = sys_get_temp_dir().'/railpack-bootstrap-'.bin2hex(random_bytes(8)); + mkdir($directory); + + try { + // The fake bootstrap takes time, like buildkitd that starts but does not listen yet. + $setup = "docker() { if [ \"\$2\" = inspect ]; then echo start >> {$directory}/log; sleep 0.3; echo end >> {$directory}/log; fi; }; "; + $script = str_replace( + ['/root/.docker/buildx/coolify-railpack.lock', '/root/.docker/buildx/coolify-railpack-bootstrap.lock'], + [$directory.'/builder.lock', $directory.'/bootstrap.lock'], + $setup.railpackBuilderLockedScript(railpackBuilderBootstrapScript().'; true'), + ); + + $builds = [new Process(['bash', '-c', $script]), new Process(['bash', '-c', $script])]; + foreach ($builds as $build) { + $build->start(); + } + foreach ($builds as $build) { + $build->wait(); + expect($build->getExitCode())->toBe(0); + } + + expect(file($directory.'/log', FILE_IGNORE_NEW_LINES))->toBe(['start', 'end', 'start', 'end']); + } finally { + array_map('unlink', glob($directory.'/*')); + rmdir($directory); + } +}); diff --git a/tests/v4/Browser/ApiTokenPermissionHelperTest.php b/tests/v4/Browser/ApiTokenPermissionHelperTest.php new file mode 100644 index 0000000000..1d8deab328 --- /dev/null +++ b/tests/v4/Browser/ApiTokenPermissionHelperTest.php @@ -0,0 +1,50 @@ +resize(1024, 768)->assertSee('Selected permissions'); + + $page->script(<<<'JS' + () => [...document.querySelectorAll('button.listbox-trigger')] + .find(button => button.textContent.includes('Selected permissions')) + .click() + JS); + $page->wait(0.5); + + $page->script(<<<'JS' + () => document.querySelector('label[for^="permission-root"] .info-helper') + .dispatchEvent(new MouseEvent('mouseenter')) + JS); + $page->wait(0.5); + + $state = $page->script(<<<'JS' + () => { + const popup = [...document.querySelectorAll('.info-helper-popup')] + .find(element => element.textContent.includes('Full access to every API operation')); + const rect = popup.getBoundingClientRect(); + const points = [ + [rect.left + 4, rect.top + 4], + [rect.left + rect.width / 2, rect.top + rect.height / 2], + [rect.right - 4, rect.bottom - 4], + ]; + + return { + visible: rect.width > 0 && rect.height > 0, + onTop: points.every(([x, y]) => popup.contains(document.elementFromPoint(x, y))), + }; + } + JS); + + $page->screenshot(filename: 'api-token-permission-helper'); + + expect($state)->toBe(['visible' => true, 'onTop' => true]); +}); diff --git a/versions.json b/versions.json index 37c44741b7..fc0089d54a 100644 --- a/versions.json +++ b/versions.json @@ -7,14 +7,14 @@ "version": "4.6-rc.1" }, "helper": { - "version": "1.0.19" + "version": "1.0.18" }, "sentinel": { - "version": "1.1.3" + "version": "1.1.2" }, "sentinel-host": { - "version": "1.1.3", - "digest": "sha256:bca77d22355c9005185b438dad406747b4b776018215a0a4a75c88618d681ff7" + "version": "1.1.2", + "digest": "sha256:149d2097943716d30af9659eeeb11259601335442242ecb0b0331982a82a3bd2" } }, "traefik": {