From ddeb05280e57da6d69b778d0cdf1fd7b073a2a10 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:33:37 +0100 Subject: [PATCH] fix(deployments): prevent Git fetches from hanging (#12321) --- app/Jobs/ApplicationDeploymentJob.php | 6 +- app/Models/Application.php | 20 +++--- bootstrap/helpers/applications.php | 15 ++++ tests/Unit/DeployKeyDedicatedPathTest.php | 6 +- tests/Unit/GitImportStepsTest.php | 84 +++++++++++++++++++++++ tests/Unit/GitSubmoduleCredentialTest.php | 2 +- 6 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 tests/Unit/GitImportStepsTest.php diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index bdc84bc6a6..d954a62f7c 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -3171,11 +3171,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue private function gitLsRemoteCommand(string $lsRemoteRef, ?string $identityFile = null): string { - $sshCommand = "ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"; - - if ($identityFile !== null) { - $sshCommand .= " -i {$identityFile} -o IdentitiesOnly=yes"; - } + $sshCommand = gitSshCommand($this->customPort, $identityFile); return 'GIT_SSH_COMMAND="'.$sshCommand.'" git ls-remote '.escapeshellarg($this->fullRepoUrl).' '.escapeshellarg($lsRemoteRef); } diff --git a/app/Models/Application.php b/app/Models/Application.php index 22a8a67041..a183d412cc 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -1587,7 +1587,7 @@ class Application extends BaseModel // If shallow clone is enabled and we need a specific commit, // we need to fetch that specific commit with depth=1 if ($isShallowCloneEnabled) { - $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1"; + $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && echo 'Fetching the commit.' && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1"; } else { $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1"; } @@ -1601,10 +1601,12 @@ class Application extends BaseModel } // Add shallow submodules flag if shallow clone is enabled $submoduleFlags = $isShallowCloneEnabled ? '--depth=1' : ''; - $git_clone_command = "{$git_clone_command} {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi"; + $git_clone_command = "{$git_clone_command} echo 'Updating submodules.' && {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi"; } + // `git lfs pull` opens an SSH session even without LFS files, which can hang on hosts + // without LFS over SSH (Azure DevOps). `git lfs ls-files` is offline. if ($this->settings->is_git_lfs_enabled && ! $onlyCheckout) { - $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} lfs pull"; + $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && if {$gitCommand} lfs ls-files -n | grep -q .; then echo 'Downloading Git LFS objects.' && {$sshCommand} {$gitCommand} lfs pull; fi"; } return $git_clone_command; @@ -1714,7 +1716,8 @@ class Application extends BaseModel $private_key = base64_encode($private_key); $gitlabPort = $oauthSource->custom_port ?? 22; $escapedCustomRepository = str_replace("'", "'\\''", $customRepository); - $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'"; + $sshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation); + $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'"; $commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker); @@ -1759,7 +1762,8 @@ class Application extends BaseModel // When used with executeInDocker (which uses bash -c '...'), we need to escape for bash context // Replace ' with '\'' to safely escape within single-quoted bash strings $escapedCustomRepository = str_replace("'", "'\\''", $customRepository); - $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'"; + $sshCommand = gitSshCommand($customPort, $customSshKeyLocation); + $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'"; $commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker); @@ -1993,7 +1997,7 @@ class Application extends BaseModel $private_key = base64_encode($private_key); $gitlabPort = $oauthSource->custom_port ?? 22; $escapedCustomRepository = escapeshellarg($customRepository); - $gitlabSshCommand = "ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes"; + $gitlabSshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation); $gitlabGitSshCommand = "GIT_SSH_COMMAND=\"{$gitlabSshCommand}\""; $git_clone_command_base = "{$gitlabGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}"; if ($only_checkout) { @@ -2054,7 +2058,7 @@ class Application extends BaseModel } $private_key = base64_encode($private_key); $escapedCustomRepository = escapeshellarg($customRepository); - $deployKeySshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes"; + $deployKeySshCommand = gitSshCommand($customPort, $customSshKeyLocation); $deployKeyGitSshCommand = "GIT_SSH_COMMAND=\"{$deployKeySshCommand}\""; $git_clone_command_base = "{$deployKeyGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}"; if ($only_checkout) { @@ -2108,7 +2112,7 @@ class Application extends BaseModel $escapedCustomRepository = escapeshellarg($customRepository); $git_clone_command = "{$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}"; $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: true, commit: $commit, baseDir: $baseDir, onlyCheckout: $only_checkout); - $otherSshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa"; + $otherSshCommand = gitSshCommand($customPort, '/root/.ssh/id_rsa', identitiesOnly: false); if ($pull_request_id !== 0) { if ($git_type === 'gitlab') { diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 9e1f8867c5..c6d7a20a0c 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -553,3 +553,18 @@ function clone_application(Application $source, $destination, array $overrides = return $newApplication; } + +/** + * The SSH command that git uses to reach a repository. + * The keepalive options stop a stalled connection after about one minute; without them git waits until the deployment timeout. + */ +function gitSshCommand(int|string $port, ?string $identityFile = null, bool $identitiesOnly = true): string +{ + $command = "ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p {$port} -o Port={$port} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"; + + if ($identityFile !== null) { + $command .= " -i {$identityFile}".($identitiesOnly ? ' -o IdentitiesOnly=yes' : ''); + } + + return $command; +} diff --git a/tests/Unit/DeployKeyDedicatedPathTest.php b/tests/Unit/DeployKeyDedicatedPathTest.php index 40f6cecb4c..3586994bca 100644 --- a/tests/Unit/DeployKeyDedicatedPathTest.php +++ b/tests/Unit/DeployKeyDedicatedPathTest.php @@ -262,8 +262,8 @@ it('uses the per-deployment deploy key for pull request fetches', function () us $result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false); - expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify"); - expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify'); + expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify"); + expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify'); expectPrivateKeyMaterializationCommandsSkipLogging($result['commands']); }); @@ -282,6 +282,6 @@ it('does not force a missing per-deployment key for other repository pull reques $result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false); - expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify'); + expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify'); expectCommandListNotToContain($result['commands'], $keyPath); }); diff --git a/tests/Unit/GitImportStepsTest.php b/tests/Unit/GitImportStepsTest.php new file mode 100644 index 0000000000..e429ce37b9 --- /dev/null +++ b/tests/Unit/GitImportStepsTest.php @@ -0,0 +1,84 @@ +, 2: list} + */ +function runImportCommandWithStubGit(string $lfsFiles, bool $hasSubmodules = false): array +{ + $dir = sys_get_temp_dir().'/coolify-import-'.bin2hex(random_bytes(4)); + mkdir("{$dir}/bin", 0755, true); + file_put_contents("{$dir}/bin/git", <<> {$dir}/git.log + if [ "\$*" = "lfs ls-files -n" ]; then printf '%s' '{$lfsFiles}'; fi + SH); + chmod("{$dir}/bin/git", 0755); + if ($hasSubmodules) { + touch("{$dir}/.gitmodules"); + } + + $application = new Application; + $application->forceFill(['uuid' => 'test-app-uuid', 'git_commit_sha' => 'abc123']); + $settings = new ApplicationSetting; + $settings->is_git_shallow_clone_enabled = true; + $settings->is_git_submodules_enabled = true; + $settings->is_git_lfs_enabled = true; + $application->setRelation('settings', $settings); + + $command = $application->setGitImportSettings( + deployment_uuid: 'test-uuid', + git_clone_command: 'true', + gitSshCommand: 'ssh -o ConnectTimeout=30 -i /root/.ssh/id_rsa', + baseDir: $dir, + ); + + $process = Process::fromShellCommandline($command, env: ['PATH' => "{$dir}/bin:".getenv('PATH')]); + $process->run(); + $calls = file_exists("{$dir}/git.log") ? file("{$dir}/git.log", FILE_IGNORE_NEW_LINES) : []; + + exec('rm -rf '.escapeshellarg($dir)); + + return [$process->getExitCode(), $calls, explode("\n", trim($process->getOutput()))]; +} + +test('git lfs pull is skipped when the checkout has no LFS files', function () { + [$exitCode, $calls, $output] = runImportCommandWithStubGit(''); + + expect($exitCode)->toBe(0) + ->and($calls)->toBe([ + 'fetch --depth=1 origin abc123', + '-c advice.detachedHead=false checkout abc123', + 'lfs ls-files -n', + ]) + ->and($output)->toBe(['Fetching the commit.']); +}); + +test('each network step after the clone prints a log line before it runs', function () { + [$exitCode, $calls, $output] = runImportCommandWithStubGit('assets/logo.psd', hasSubmodules: true); + + expect($exitCode)->toBe(0) + ->and($calls)->toBe([ + 'fetch --depth=1 origin abc123', + '-c advice.detachedHead=false checkout abc123', + 'submodule sync', + 'submodule update --init --recursive --depth=1', + 'lfs ls-files -n', + 'lfs pull', + ]) + ->and($output)->toBe(['Fetching the commit.', 'Updating submodules.', 'Downloading Git LFS objects.']); +}); + +test('git ssh command stops a stalled connection instead of waiting for the deployment timeout', function () { + expect(gitSshCommand(2222, '/root/.ssh/key'))->toBe( + 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 2222 -o Port=2222 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/key -o IdentitiesOnly=yes' + ) + ->and(gitSshCommand(22))->not->toContain(' -i ') + ->and(gitSshCommand(22, '/root/.ssh/id_rsa', identitiesOnly: false))->toEndWith('-i /root/.ssh/id_rsa'); +}); diff --git a/tests/Unit/GitSubmoduleCredentialTest.php b/tests/Unit/GitSubmoduleCredentialTest.php index 11c28c9909..f4e8921a15 100644 --- a/tests/Unit/GitSubmoduleCredentialTest.php +++ b/tests/Unit/GitSubmoduleCredentialTest.php @@ -156,7 +156,7 @@ describe('Git submodule credential propagation', function () { exec_in_docker: false, ); - $sshCommand = 'ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes'; + $sshCommand = 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes'; // Commands are a list of ['command' => ..., 'hidden' => ...] entries; assert on the joined shell script $commands = collect($result['commands'])